From 0fa6c4cd8410beafe5d58464e15296896d9b4c5d Mon Sep 17 00:00:00 2001 From: MichSchli Date: Thu, 23 Mar 2017 09:57:15 +0100 Subject: [PATCH] Changes to loss function --- .idea/workspace.xml | 460 ++++++++++++------------ code/decoders/bilinear_diag.py | 4 +- code/decoders/complex.py | 3 +- code/encoders/message_gcns/gcn_basis.py | 9 +- run-train.sh | 4 +- settings/distmult.exp | 10 +- settings/gcn_basis.exp | 2 +- 7 files changed, 248 insertions(+), 244 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index dfbdc1e..58c94f7 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,7 +2,13 @@ + + + + + + @@ -211,6 +217,107 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -297,24 +404,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -765,14 +753,14 @@ - + - + @@ -802,7 +790,6 @@ - @@ -815,13 +802,6 @@ - - - - - - - @@ -941,13 +921,6 @@ - - - - - - - @@ -1025,120 +998,139 @@ - + - - + + + + + + + + + + + + + + + + - + - - - - - + + + + + + + + + + - - + + - + - - + + - + - - + + - + - + - - + + - + - - + + - + - - + + - + - - - + + + + + - + - - + + - + - - - - - + + + - + - - + + - + - - + + - + - - + + diff --git a/code/decoders/bilinear_diag.py b/code/decoders/bilinear_diag.py index 088c2d2..c2c3686 100644 --- a/code/decoders/bilinear_diag.py +++ b/code/decoders/bilinear_diag.py @@ -29,7 +29,9 @@ def get_loss(self, mode='train'): e1s, rs, e2s = self.compute_codes(mode=mode) energies = tf.reduce_sum(e1s * rs * e2s, 1) - return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(energies, self.Y)) + + weight = int(self.settings['NegativeSampleRate']) + return tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(energies, self.Y, weight)) def local_initialize_train(self): self.Y = tf.placeholder(tf.float32, shape=[None]) diff --git a/code/decoders/complex.py b/code/decoders/complex.py index 0f0e781..3a501ec 100644 --- a/code/decoders/complex.py +++ b/code/decoders/complex.py @@ -41,7 +41,8 @@ def get_loss(self, mode='train'): + tf.reduce_sum(e1s_r * rs_i * e2s_i, 1) \ - tf.reduce_sum(e1s_i * rs_i * e2s_r, 1) - return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(energies, self.Y)) + weight = int(self.settings['NegativeSampleRate']) + return tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(energies, self.Y, weight)) def local_initialize_train(self): self.Y = tf.placeholder(tf.float32, shape=[None]) diff --git a/code/encoders/message_gcns/gcn_basis.py b/code/encoders/message_gcns/gcn_basis.py index e482eda..358966a 100644 --- a/code/encoders/message_gcns/gcn_basis.py +++ b/code/encoders/message_gcns/gcn_basis.py @@ -85,7 +85,7 @@ def combine_messages(self, forward_messages, backward_messages, self_loop_messag collected_messages_f = tf.sparse_tensor_dense_matmul(mtr_f, forward_messages) collected_messages_b = tf.sparse_tensor_dense_matmul(mtr_b, backward_messages) - if mode=='train': + if False: #mode=='train': choice = tf.select(tf.random_uniform([50,1]) > 0.5, tf.ones([50,1], dtype=tf.int32), tf.zeros([50,1], dtype=tf.int32)) options = tf.constant([[2., 0.], [0., 2.]]) mixer = tf.nn.embedding_lookup(options, choice) @@ -99,3 +99,10 @@ def combine_messages(self, forward_messages, backward_messages, self_loop_messag new_embedding = tf.nn.relu(new_embedding) return new_embedding + + def local_get_regularization(self): + regularization = tf.reduce_mean(tf.square(self.W_forward)) + regularization += tf.reduce_mean(tf.square(self.W_backward)) + regularization += tf.reduce_mean(tf.square(self.W_self)) + + return 0.0 * regularization \ No newline at end of file diff --git a/run-train.sh b/run-train.sh index 7446375..34d136d 100644 --- a/run-train.sh +++ b/run-train.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -DATASET="DegreeData" -SETTINGS="complex.exp" +DATASET="FB-Toutanova" +SETTINGS="distmult.exp" SCRIPT_DIR=$(dirname $(readlink -f $0)) diff --git a/settings/distmult.exp b/settings/distmult.exp index a01d4ec..5ca285d 100644 --- a/settings/distmult.exp +++ b/settings/distmult.exp @@ -6,21 +6,23 @@ RegularizationParameter=0.01 [Shared] - CodeDimension=50 + CodeDimension=100 [Optimizer] MaxGradientNorm=1 - ReportTrainLossEvery=100 + ReportTrainLossEvery=5 + BatchSize=5000 + Minibatches=Yes [EarlyStopping] - CheckEvery=500 + CheckEvery=10 [Algorithm] Name=Adam learning_rate=0.01 [General] - NegativeSampleRate=1 + NegativeSampleRate=10 GraphSplitSize=15000 diff --git a/settings/gcn_basis.exp b/settings/gcn_basis.exp index 36a2cfe..4f809b5 100644 --- a/settings/gcn_basis.exp +++ b/settings/gcn_basis.exp @@ -1,7 +1,7 @@ [Encoder] Name=gcn_basis DropoutKeepProbability=1.0 - InternalEncoderDimension=50 + InternalEncoderDimension=100 NumberOfBasisFunctions=2 NumberOfLayers=1 UseInputTransform=No