Enhance embedding layer

tools/simnet/train/paddle/nets/mm_dnn.py

@@ -30,6 +30,18 @@ def __init__(self, config):
         else:
             logging.error("training mode not supported")
 
+    def embedding_layer(self, input, zero_pad=True, scale=True):
+        emb = fluid.layers.embedding(
+            input=input,
+            size=[self.vocab_size, self.emb_size],
+            is_sparse=True,
+            padding_idx=(0 if zero_pad else None),
+            param_attr=fluid.ParamAttr(name="word_embedding",
+                initializer=fluid.initializer.Xavier()))
+        if scale:
+            emb = emb * (self.emb_size ** 0.5)
+        return emb
+
     def bi_dynamic_lstm(self, input, hidden_size):
         fw_in_proj = fluid.layers.fc(input=input,
                                      size=4 * hidden_size,
@@ -52,7 +64,7 @@ def conv_pool_relu_layer(self, input, mask=None):
             input=emb_expanded,
             num_filters=self.kernel_size,
             stride=1,
-            padding=16,
+            padding=[self.seq_len1 / 2, self.seq_len2 / 2],
             filter_size=[self.seq_len1, self.seq_len2],
             bias_attr=fluid.ParamAttr(
                 initializer=fluid.initializer.Constant(0.1)))
@@ -87,16 +99,8 @@ def get_cross_mask(self, left_lens, right_lens):
         return cross_mask
 
     def predict(self, left, right):
-        left_emb = fluid.layers.embedding(
-            input=left,
-            size=[self.vocab_size, self.emb_size],
-            is_sparse=True,
-            param_attr=fluid.ParamAttr(name="word_embedding"))
-        right_emb = fluid.layers.embedding(
-            input=right,
-            size=[self.vocab_size, self.emb_size],
-            is_sparse=True,
-            param_attr=fluid.ParamAttr(name="word_embedding"))
+        left_emb = self.embedding_layer(left, zero_pad=True, scale=False)
+        right_emb = self.embedding_layer(right, zero_pad=True, scale=False)
 
         bi_left_outputs = self.bi_dynamic_lstm(
             input=left_emb, hidden_size=self.lstm_dim)

tools/simnet/train/paddle/paddle_simnet.py

@@ -105,14 +105,17 @@ def train(conf_dict):
         # Get batch data iterator
         batch_data = paddle.batch(reader, conf_dict["batch_size"], drop_last=False)
         start_time = time.time()
+        total_loss = 0.0
         for iter, data in enumerate(batch_data()):
             if len(data) < device_count:
                 continue
             avg_loss = parallel_executor.run(
                 [avg_cost.name], feed=feeder.feed(data))
-            if iter % 100 == 0:
+            total_loss += np.mean(avg_loss[0])
+            if (iter + 1) % 100 == 0:
                 print("epoch: %d, iter: %d, loss: %f" %
-                    (epoch_id, iter, np.mean(avg_loss[0])))
+                    (epoch_id, iter, total_loss / 100))
+                total_loss = 0.0
             losses.append(np.mean(avg_loss[0]))
         end_time = time.time()
         print("epoch: %d, loss: %f, used time: %d sec" %

tools/simnet/train/tf/utils/controler.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import sys
+import time
 
 import tensorflow as tf
 
@@ -88,6 +89,7 @@ def run_trainer(loss, optimizer, config):
         read_thread = tf.train.start_queue_runners(sess=sess, coord=coord)
         step = 0
         epoch_num = 1
+        start_time = time.time()
         while not coord.should_stop():
             try:
                 step += 1
@@ -98,10 +100,14 @@ def run_trainer(loss, optimizer, config):
                     print("loss: %f" % ((avg_cost / print_iter)))
                     avg_cost = 0.0
                 if step % epoch_iter == 0:
-                    print("save model epoch%d" % (epoch_num))
+                    end_time = time.time()
+                    print("save model epoch%d, used time: %d" % (epoch_num, 
+                          end_time - start_time))
                     save_path = saver.save(sess, 
                             "%s/%s.epoch%d" % (model_path, model_file, epoch_num))
                     epoch_num += 1
+                    start_time = time.time()
+
             except tf.errors.OutOfRangeError:
                 save_path = saver.save(sess, "%s/%s.final" % (model_path, model_file))
                 coord.request_stop()