# ========================================================================= # (c) Copyright 2022 # All rights reserved # Programs written by Haodi Jiang # Department of Computer Science # New Jersey Institute of Technology # University Heights, Newark, NJ 07102, USA # # Permission to use, copy, modify, and distribute this # software and its documentation for any purpose and without # fee is hereby granted, provided that this copyright # notice appears in all copies. Programmer(s) makes no # representations about the suitability of this # software for any purpose. It is provided "as is" without # express or implied warranty. # ========================================================================= from tensorflow.keras.models import Model from tensorflow.keras.layers import Activation, Conv2D, Dropout from tensorflow.keras.layers import MaxPooling2D, BatchNormalization from tensorflow.keras.layers import UpSampling2D from tensorflow.keras.layers import concatenate from tensorflow.keras.initializers import Constant from tensorflow.keras.layers import add from tensorflow.keras.layers import PReLU from layers.attention import PAM, CAM from tensorflow.keras import Input def conv3x3(x, out_filters, strides=(1, 1)): x = Conv2D(out_filters, 3, padding='same', strides=strides, use_bias=False, kernel_initializer='he_normal')(x) return x def Conv2d_BN(x, nb_filter, kernel_size, strides=(1, 1), padding='same', use_activation=True): x = Conv2D(nb_filter, kernel_size, padding=padding, strides=strides, kernel_initializer='he_normal')(x) x = BatchNormalization(axis=3)(x) if use_activation: x = PReLU(alpha_initializer=Constant(value=0.25))(x) return x else: return x def basic_Block(input, out_filters, strides=(1, 1), with_conv_shortcut=False): x = conv3x3(input, out_filters, strides) x = BatchNormalization(axis=3)(x) x = PReLU(alpha_initializer=Constant(value=0.25))(x) x = conv3x3(x, out_filters) x = BatchNormalization(axis=3)(x) if with_conv_shortcut: residual = Conv2D(out_filters, 1, strides=strides, use_bias=False, kernel_initializer='he_normal')(input) residual = BatchNormalization(axis=3)(residual) x = add([x, residual]) else: x = add([x, input]) x = PReLU(alpha_initializer=Constant(value=0.25))(x) return x def bottleneck_Block(input, out_filters, strides=(1, 1), dilation=(1, 1), with_conv_shortcut=False): expansion = 4 de_filters = int(out_filters / expansion) x = Conv2D(de_filters, 1, use_bias=False, kernel_initializer='he_normal')(input) x = BatchNormalization(axis=3)(x) x = PReLU(alpha_initializer=Constant(value=0.25))(x) x = Conv2D(de_filters, 3, strides=strides, padding='same', dilation_rate=dilation, use_bias=False, kernel_initializer='he_normal')(x) x = BatchNormalization(axis=3)(x) x = PReLU(alpha_initializer=Constant(value=0.25))(x) x = Conv2D(out_filters, 1, use_bias=False, kernel_initializer='he_normal')(x) x = BatchNormalization(axis=3)(x) if with_conv_shortcut: residual = Conv2D(out_filters, 1, strides=strides, use_bias=False, kernel_initializer='he_normal')(input) residual = BatchNormalization(axis=3)(residual) x = add([x, residual]) else: x = add([x, input]) x = PReLU(alpha_initializer=Constant(value=0.25))(x) return x # def danet_resnet101(height, width, channel): def danet_resnet101(inputs, b_label): # input1 = Input(shape=(height, width, channel),name = 'input_1') # conv1_1 = Conv2D(32, 7, strides=(2, 2), padding='same', use_bias=False, kernel_initializer='he_normal')(input1) # conv1_1 = BatchNormalization(axis=3)(conv1_1) # conv1_1 = PReLU(alpha_initializer=Constant(value=0.25))(conv1_1) # input2 = Input(shape=(height, width, channel),name = 'input_2') # conv1_3 = Conv2D(32, 7, strides=(2, 2), padding='same', use_bias=False, kernel_initializer='he_normal')(input2) # conv1_3 = BatchNormalization(axis=3)(conv1_3) # conv1_3 = PReLU(alpha_initializer=Constant(value=0.25))(conv1_3) # conv1_4 = concatenate([conv1_1,conv1_3],axis=-1) # conv1_2 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(conv1_4) # conv2_x 1/4 # conv2_1 = bottleneck_Block(conv1_2, 256, strides=(1, 1), with_conv_shortcut=True) conv1_4 = inputs conv1_2 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(conv1_4) conv2_1 = bottleneck_Block(conv1_2, 256, strides=(1, 1), with_conv_shortcut=True) conv2_2 = bottleneck_Block(conv2_1, 256) conv2_3 = bottleneck_Block(conv2_2, 256) # conv3_x 1/8 conv3_1 = bottleneck_Block(conv2_3, 512, strides=(2, 2), with_conv_shortcut=True) conv3_2 = bottleneck_Block(conv3_1, 512) conv3_3 = bottleneck_Block(conv3_2, 512) conv3_4 = bottleneck_Block(conv3_3, 512) # conv4_x 1/16 conv4_1 = bottleneck_Block(conv3_4, 1024, strides=(1, 1), dilation=(2, 2), with_conv_shortcut=True) conv4_2 = bottleneck_Block(conv4_1, 1024, dilation=(2, 2)) conv4_3 = bottleneck_Block(conv4_2, 1024, dilation=(2, 2)) conv4_4 = bottleneck_Block(conv4_3, 1024, dilation=(2, 2)) conv4_5 = bottleneck_Block(conv4_4, 1024, dilation=(2, 2)) conv4_6 = bottleneck_Block(conv4_5, 1024, dilation=(2, 2)) conv4_7 = bottleneck_Block(conv4_6, 1024, dilation=(2, 2)) conv4_8 = bottleneck_Block(conv4_7, 1024, dilation=(2, 2)) conv4_9 = bottleneck_Block(conv4_8, 1024, dilation=(2, 2)) conv4_10 = bottleneck_Block(conv4_9, 1024, dilation=(2, 2)) conv4_11 = bottleneck_Block(conv4_10, 1024, dilation=(2, 2)) conv4_12 = bottleneck_Block(conv4_11, 1024, dilation=(2, 2)) conv4_13 = bottleneck_Block(conv4_12, 1024, dilation=(2, 2)) conv4_14 = bottleneck_Block(conv4_13, 1024, dilation=(2, 2)) conv4_15 = bottleneck_Block(conv4_14, 1024, dilation=(2, 2)) conv4_16 = bottleneck_Block(conv4_15, 1024, dilation=(2, 2)) conv4_17 = bottleneck_Block(conv4_16, 1024, dilation=(2, 2)) conv4_18 = bottleneck_Block(conv4_17, 1024, dilation=(2, 2)) conv4_19 = bottleneck_Block(conv4_18, 1024, dilation=(2, 2)) conv4_20 = bottleneck_Block(conv4_19, 1024, dilation=(2, 2)) conv4_21 = bottleneck_Block(conv4_20, 1024, dilation=(2, 2)) conv4_22 = bottleneck_Block(conv4_21, 1024, dilation=(2, 2)) conv4_23 = bottleneck_Block(conv4_22, 1024, dilation=(2, 2)) # conv5_x 1/32 conv5_1 = bottleneck_Block(conv4_23, 2048, strides=(1, 1), dilation=(4, 4), with_conv_shortcut=True) conv5_2 = bottleneck_Block(conv5_1, 2048, dilation=(4, 4)) conv5_3 = bottleneck_Block(conv5_2, 2048, dilation=(4, 4)) # ATTENTION reduce_conv5_3 = Conv2D(512, 3, padding='same', use_bias=False, kernel_initializer='he_normal')(conv5_3) reduce_conv5_3 = BatchNormalization(axis=3)(reduce_conv5_3) reduce_conv5_3 = PReLU(alpha_initializer=Constant(value=0.25))(reduce_conv5_3) pam = PAM()(reduce_conv5_3) pam = Conv2D(512, 3, padding='same', use_bias=False, kernel_initializer='he_normal')(pam) pam = BatchNormalization(axis=3)(pam) pam = PReLU(alpha_initializer=Constant(value=0.25))(pam) pam = Dropout(0.5)(pam) pam = Conv2D(512, 3, padding='same', use_bias=False, kernel_initializer='he_normal')(pam) cam = CAM()(reduce_conv5_3) cam = Conv2D(512, 3, padding='same', use_bias=False, kernel_initializer='he_normal')(cam) cam = BatchNormalization(axis=3)(cam) cam = PReLU(alpha_initializer=Constant(value=0.25))(cam) cam = Dropout(0.5)(cam) cam = Conv2D(512, 3, padding='same', use_bias=False, kernel_initializer='he_normal')(cam) feature_sum = add([pam, cam]) feature_sum = Dropout(0.5)(feature_sum) feature_sum = Conv2d_BN(feature_sum, 512, 1) merge7 = concatenate([conv3_4, feature_sum], axis=3) conv7 = Conv2d_BN(merge7, 512, 3) conv7 = Conv2d_BN(conv7, 512, 3) up8 = Conv2d_BN(UpSampling2D(size=(2, 2))(conv7), 256, 2) merge8 = concatenate([conv2_3, up8], axis=3) conv8 = Conv2d_BN(merge8, 256, 3) conv8 = Conv2d_BN(conv8, 256, 3) up9 = Conv2d_BN(UpSampling2D(size=(2, 2))(conv8), 64, 2) merge9 = concatenate([conv1_4, up9], axis=3) conv9 = Conv2d_BN(merge9, 64, 3) conv9 = Conv2d_BN(conv9, 64, 3) up10 = Conv2d_BN(UpSampling2D(size=(2, 2))(conv9), 64, 2) conv10 = Conv2d_BN(up10, 64, 3) conv10 = Conv2d_BN(conv10, 64, 3) conv12 = Conv2D(16, 3, padding='same', use_bias=False, kernel_initializer='he_normal')(conv10) activation = Conv2D(1, 1, padding='same', use_bias=False, name = b_label, kernel_initializer='he_normal')(conv12) return activation # input = [input1,input2] # model = Model(inputs=input, outputs=activation) # return model def combined_model(height, width, channel): input1 = Input(shape=(height, width, channel), name='input_1') conv1_1 = Conv2D(32, 7, strides=(2, 2), padding='same', use_bias=False, kernel_initializer='he_normal')(input1) conv1_1 = BatchNormalization(axis=3)(conv1_1) conv1_1 = PReLU(alpha_initializer=Constant(value=0.25))(conv1_1) input2 = Input(shape=(height, width, channel), name='input_2') conv1_3 = Conv2D(32, 7, strides=(2, 2), padding='same', use_bias=False, kernel_initializer='he_normal')(input2) conv1_3 = BatchNormalization(axis=3)(conv1_3) conv1_3 = PReLU(alpha_initializer=Constant(value=0.25))(conv1_3) conv1_4 = concatenate([conv1_1, conv1_3], axis=-1) # conv1_2 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='same')(conv1_4) bx = danet_resnet101(conv1_4, 'b1') by = danet_resnet101(conv1_4, 'b2') input = [input1,input2] model = Model(inputs=input, outputs=[bx, by]) return model