-
Notifications
You must be signed in to change notification settings - Fork 14
/
RawTransformer_CNN&MLP
41 lines (40 loc) · 2.2 KB
/
RawTransformer_CNN&MLP
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
#Defining Model Architecture
def CNN_model():
#Trasnformer Encoding before feeding to CNN
text_in = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')
preprocess_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing')
encoder_in = preprocess_layer(text_in)
encode = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BERT_encoder')
outputs = encode(encoder_in)
network_layer= sequence_output = outputs["sequence_output"]
# 2 CNNLayers
network_layer = tf.keras.layers.Conv1D(32, (2), activation='relu')(network_layer) #CNN Layer 1
network_layer = tf.keras.layers.Conv1D(64, (2), activation='relu')(network_layer) #CNN Layer 2
network_layer = tf.keras.layers.GlobalMaxPool1D()(network_layer) #Global Maxpooling Layer
network_layer = tf.keras.layers.Dense(256, activation="relu")(network_layer) #DenseLayer
network_layer = tf.keras.layers.Dropout(0.1)(network_layer) #Dropout Layer
network_layer = tf.keras.layers.Dense(3, activation="softmax", name='classifier')(network_layer) #ClassificationLayer
return tf.keras.Model(text_input, network_layer)
def MLP_model(output_bias=None):
if output_bias is not None:
output_bias = tf.keras.initializers.Constant(output_bias)
#input layer
text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')
#preprocessing layer
preprocessing_layer = hub.KerasLayer(tfhub_handle_preprocess, name='preprocessing')
#giving input to bert model from the pre processed layer
encoder_inputs = preprocessing_layer(text_input)
#selecting the encoder
encoder = hub.KerasLayer(tfhub_handle_encoder, trainable=True, name='BERT_encoder')
#collecting the output of the bert encoder
outputs = encoder(encoder_inputs)
#collecting the pooled outputs
net = outputs['pooled_output']
#feeding the output to the multilayer perception
#1 DENSE LAYER
net = tf.keras.layers.Dense(512, activation="relu")(net)
#1 DROPUT LAYER
net = tf.keras.layers.Dropout(0.2)(net)
#1 OUTPUT LAYER WITH SOFTMAX
net = tf.keras.layers.Dense(3, activation="softmax", name='classifier', bias_initializer=output_bias)(net)
return tf.keras.Model(text_input, net)