From af76780339bcced1381a2dcafcdc42af9b828ead Mon Sep 17 00:00:00 2001 From: MichSchli Date: Wed, 10 May 2017 12:11:44 +0200 Subject: [PATCH] Fixed saving --- .idea/workspace.xml | 624 +++++++++++++++---------------- code/common/model_builder.py | 19 +- code/extras/dropover.py | 36 ++ code/model.py | 18 +- code/tools/make_split_dataset.py | 6 +- code/train.py | 32 +- settings/gcn_basis.exp | 18 +- 7 files changed, 422 insertions(+), 331 deletions(-) create mode 100644 code/extras/dropover.py diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 6d34870..4b80e63 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,15 +2,13 @@ - + - - - + + - - + + + + + + - - - - - @@ -712,8 +728,8 @@ - - + + @@ -733,15 +749,15 @@ - - + + - + @@ -789,41 +805,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -859,13 +840,6 @@ - - - - - - - @@ -873,13 +847,6 @@ - - - - - - - @@ -915,13 +882,6 @@ - - - - - - - @@ -936,13 +896,6 @@ - - - - - - - @@ -950,20 +903,6 @@ - - - - - - - - - - - - - - @@ -992,175 +931,236 @@ - + - - + + - + - - + + - + - - + + - + - - + + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + - + - - - + + - + - - - + + - + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + - + - - + + - + - - + + - + - - - - - + + + - + - - + + - + - - + + - + - - + + - - + + - + - - + + - + - - - + + + + + - + - - + + - + - - + + - + - - + + diff --git a/code/common/model_builder.py b/code/common/model_builder.py index 7674174..9fe7e3e 100644 --- a/code/common/model_builder.py +++ b/code/common/model_builder.py @@ -18,6 +18,7 @@ from extras.residual_layer import ResidualLayer from extras.highway_layer import HighwayLayer +from extras.dropover import DropoverLayer from extras.variational_encoding import VariationalEncoding @@ -147,6 +148,19 @@ def build_encoder(encoder_settings, triples): encoding = RandomEmbedding(input_shape, encoder_settings, next_component=graph) + elif encoder_settings['PartiallyRandomInput'] == 'Yes': + encoding1 = AffineTransform(input_shape, + encoder_settings, + next_component=graph, + onehot_input=True, + use_bias=True, + use_nonlinearity=False) + encoding2 = RandomEmbedding(input_shape, + encoder_settings, + next_component=graph) + encoding = DropoverLayer(input_shape, + next_component=encoding1, + next_component_2=encoding2) else: encoding = graph @@ -260,7 +274,10 @@ def apply_basis_gcn(encoder_settings, encoding, internal_shape, layers): for layer in range(layers): use_nonlinearity = layer < layers - 1 - if layer == 0 and encoder_settings['UseInputTransform'] == "No" and encoder_settings['RandomInput'] != "Yes" : + if layer == 0 \ + and encoder_settings['UseInputTransform'] == "No" \ + and encoder_settings['RandomInput'] == "No" \ + and encoder_settings['PartiallyRandomInput'] == "No" : onehot_input = True else: onehot_input = False diff --git a/code/extras/dropover.py b/code/extras/dropover.py new file mode 100644 index 0000000..a467410 --- /dev/null +++ b/code/extras/dropover.py @@ -0,0 +1,36 @@ +import numpy as np +import tensorflow as tf +from model import Model + +class DropoverLayer(Model): + vertex_embedding_function = {'train': None, 'test': None} + + def __init__(self, shape, next_component=None, next_component_2=None): + self.next_component = next_component + self.next_component_2 = next_component_2 + self.shape = shape + + def compute_vertex_embeddings(self, mode='train'): + if self.vertex_embedding_function[mode] is None and mode=='train': + code_1 = self.next_component.get_all_codes(mode=mode)[0] + code_2 = self.next_component_2.get_all_codes(mode=mode)[0] + + choice = tf.random_uniform(self.shape, -1, 1, dtype=tf.float32) + + self.vertex_embedding_function[mode] = tf.where(choice > 0, x=code_1, y=code_2) + elif mode=='test': + self.vertex_embedding_function[mode] = self.next_component.get_all_codes(mode=mode)[0] + + return self.vertex_embedding_function[mode] + + + def get_all_codes(self, mode='train'): + collected_messages = self.compute_vertex_embeddings(mode=mode) + + return collected_messages, None, collected_messages + + def get_all_subject_codes(self, mode='train'): + return self.compute_vertex_embeddings(mode=mode) + + def get_all_object_codes(self, mode='train'): + return self.compute_vertex_embeddings(mode=mode) diff --git a/code/model.py b/code/model.py index 3ef9f43..d8eb461 100644 --- a/code/model.py +++ b/code/model.py @@ -11,6 +11,7 @@ class Model: score_graph = None session = None next_component=None + save_iter=0 def __init__(self, next_component, settings): self.next_component = next_component @@ -25,6 +26,16 @@ def __init__(self, next_component, settings): def parse_settings(self): pass + def save(self): + variables_to_save = self.get_weights() + + if self.saver is None: + self.saver = tf.train.Saver(var_list=variables_to_save) + + self.saver.save(self.session, self.settings['ExperimentName'], global_step=self.save_iter) + self.save_iter += 1 + + ''' High-level functions: ''' @@ -47,7 +58,7 @@ def score_all_subjects(self, triplets): self.score_all_subjects_graph = self.predict_all_subject_scores() if self.needs_graph(): - d = {self.get_test_input_variables()[0]: self.train_triplets, + d = {self.get_test_input_variables()[0]: self.test_graph, self.get_test_input_variables()[1]: triplets} else: d = {self.get_test_input_variables()[0]: triplets} @@ -59,7 +70,7 @@ def score_all_objects(self, triplets): self.score_all_objects_graph = self.predict_all_object_scores() if self.needs_graph(): - d = {self.get_test_input_variables()[0]: self.train_triplets, + d = {self.get_test_input_variables()[0]: self.test_graph, self.get_test_input_variables()[1]: triplets} else: d = {self.get_test_input_variables()[0]: triplets} @@ -69,6 +80,9 @@ def score_all_objects(self, triplets): ''' ''' + def register_for_test(self, triplets): + self.test_graph = triplets + def preprocess(self, triplets): self.train_triplets = triplets pass #return self.__local_run_delegate__('preprocess', triplets) diff --git a/code/tools/make_split_dataset.py b/code/tools/make_split_dataset.py index 40af6ab..5d80c4d 100644 --- a/code/tools/make_split_dataset.py +++ b/code/tools/make_split_dataset.py @@ -12,9 +12,9 @@ args = parser.parse_args() -source_triplets = list(io.read_triplets('data/Toutanova-Split/train.txt')) -source_triplets_valid = list(io.read_triplets('data/Toutanova-Split/valid.txt')) -source_triplets_test = list(io.read_triplets('data/Toutanova-Split/test.txt')) +source_triplets = list(io.read_triplets('data/FB-Toutanova/train.txt')) +source_triplets_valid = list(io.read_triplets('data/FB-Toutanova/valid.txt')) +source_triplets_test = list(io.read_triplets('data/FB-Toutanova/test.txt')) source_entities = io.read_dictionary('data/FB15k/entities.dict') reversed_entities = {v: k for k, v in source_entities.items()} diff --git a/code/train.py b/code/train.py index 1f3236f..989514c 100644 --- a/code/train.py +++ b/code/train.py @@ -1,4 +1,6 @@ import argparse +import random + import tensorflow as tf from Converge.optimize import build_tensorflow from common import settings_reader, io, model_builder, optimizer_parameter_parser, evaluation, auxilliaries @@ -33,12 +35,33 @@ test_path = dataset + '/test_accuracy.txt' train_triplets = io.read_triplets_as_list(train_path, entities_path, relations_path) + valid_triplets = io.read_triplets_as_list(valid_path, entities_path, relations_path) test_triplets = io.read_triplets_as_list(test_path, entities_path, relations_path) + +train_triplets = np.array(train_triplets) +valid_triplets = np.array(valid_triplets) +test_triplets = np.array(test_triplets) + entities = io.read_dictionary(entities_path) relations = io.read_dictionary(relations_path) +''' +shuffled_rels = np.arange(len(relations)) +np.random.shuffle(shuffled_rels) + +known_rels = shuffled_rels[:int(len(relations)/2)] +target_rels = shuffled_rels[int(len(relations)/2):] + +known_train = train_triplets[np.where(np.in1d(train_triplets[:,1], known_rels))] +target_train = train_triplets[np.where(np.in1d(train_triplets[:,1], target_rels))] +known_valid = valid_triplets[np.where(np.in1d(valid_triplets[:,1], known_rels))] +target_valid = valid_triplets[np.where(np.in1d(valid_triplets[:,1], target_rels))] +known_test = test_triplets[np.where(np.in1d(test_triplets[:,1], known_rels))] +target_test = test_triplets[np.where(np.in1d(test_triplets[:,1], target_rels))] +''' + ''' Load general settings ''' @@ -74,7 +97,7 @@ ''' opp = optimizer_parameter_parser.Parser(optimizer_settings) -#opp.set_save_function(model.save) DISABLED SAVING +opp.set_save_function(model.save) scorer = evaluation.Scorer(evaluation_settings) scorer.register_data(train_triplets) @@ -82,7 +105,7 @@ scorer.register_data(test_triplets) scorer.register_degrees(train_triplets) scorer.register_model(model) -scorer.finalize_frequency_computation(train_triplets + valid_triplets + test_triplets) +scorer.finalize_frequency_computation(np.concatenate((train_triplets, valid_triplets, test_triplets), axis=0)) def score_validation_data(validation_data): score_summary = scorer.compute_scores(validation_data, verbose=False).get_summary() @@ -228,10 +251,9 @@ def t_func(x): #horrible hack!!! Initialize for training: ''' -graph = np.array(train_triplets) - # Hack for validation evaluation: -model.preprocess(graph) +model.preprocess(train_triplets) +model.register_for_test(train_triplets) model.initialize_train() diff --git a/settings/gcn_basis.exp b/settings/gcn_basis.exp index 5332e05..1eec3b7 100644 --- a/settings/gcn_basis.exp +++ b/settings/gcn_basis.exp @@ -1,8 +1,8 @@ [Encoder] Name=gcn_basis - DropoutKeepProbability=0.8 - InternalEncoderDimension=500 - NumberOfBasisFunctions=125 + DropoutKeepProbability=1.0 + InternalEncoderDimension=100 + NumberOfBasisFunctions=50 NumberOfLayers=1 UseInputTransform=Yes UseOutputTransform=No @@ -11,6 +11,7 @@ SkipConnections=Highway StoreEdgeData=No RandomInput=No + PartiallyRandomInput=No Concatenation=Yes [Decoder] @@ -18,14 +19,14 @@ RegularizationParameter=0.01 [Shared] - CodeDimension=500 + CodeDimension=100 [Optimizer] MaxGradientNorm=1 - ReportTrainLossEvery=50 + ReportTrainLossEvery=20 [EarlyStopping] - CheckEvery=200 + CheckEvery=100 BurninPhaseDuration=2500 [Algorithm] @@ -33,9 +34,10 @@ learning_rate=0.01 [General] - NegativeSampleRate=10 + NegativeSampleRate=1 + GraphBatchSize=30000 GraphSplitSize=0.5 - GraphBatchSize=10000 + ExperimentName=GcnTest [Evaluation] Metric=MRR \ No newline at end of file