From 863af30926326d6ac3026e3fbbe2d9ff2a9db4f3 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 6 May 2019 18:44:08 -0700 Subject: [PATCH 1/2] Fixed comment when word mappings file is missing, config for c1234yn variational flow --- .../evaluation/frozen_if_ffn_c1234_loss.yml | 2 +- .../frozen_if_vf_5ffn_c1234yn_5losses.yml | 497 ++++++++++++++++++ ptp/components/masking/string_to_mask.py | 25 +- ptp/components/mixins/word_mappings.py | 7 +- ptp/components/utils/word_mappings.py | 3 + 5 files changed, 510 insertions(+), 24 deletions(-) create mode 100644 configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml diff --git a/configs/vqa_med_2019/evaluation/frozen_if_ffn_c1234_loss.yml b/configs/vqa_med_2019/evaluation/frozen_if_ffn_c1234_loss.yml index 9407c13..2f00bae 100644 --- a/configs/vqa_med_2019/evaluation/frozen_if_ffn_c1234_loss.yml +++ b/configs/vqa_med_2019/evaluation/frozen_if_ffn_c1234_loss.yml @@ -71,7 +71,7 @@ validation: pipeline: - ################# PIPE 6: C1 + C2 + C3 questions ################# + ################# PIPE 6: C1 + C2 + C3 + C4 questions ################# # Answer encoding. pipe6_c1234_answer_indexer: diff --git a/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml b/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml new file mode 100644 index 0000000..eff5d97 --- /dev/null +++ b/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml @@ -0,0 +1,497 @@ +# Load config defining problems for training, validation and testing. +default_configs: + vqa_med_2019/default_vqa_med_2019.yml, + vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml, + vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml + +c123_hyperparameters: + # In here I am putting some of the hyperparameters from spreadsheet. + + question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize + # Accepted formats: a,b,c or [a,b,c] + # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all + + image_preprocessing: &image_preprocessing normalize + # Accepted formats: a,b,c or [a,b,c] + # none | random_affine | random_horizontal_flip | normalize | all + + # Final classifier: FFN. + answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [500] + + batch_size: &batch_size 256 + preload_images: &preload_images False + num_workers: &num_workers 4 + +# Training parameters: +training: + problem: + batch_size: *batch_size + categories: all + export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv + # Appy all preprocessing/data augmentations. + question_preprocessing: *question_preprocessing + image_preprocessing: *image_preprocessing + # Preload images. + preload_images: *preload_images + streams: + questions: tokenized_questions + sampler: + weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv + # Use workers for loading images. + dataloader: + num_workers: *num_workers + + # Optimizer parameters: + optimizer: + name: Adam + lr: 0.0001 + + # Terminal conditions: + terminal_conditions: + loss_stop: 1.0e-3 + episode_limit: 10000 + epoch_limit: -1 + +# Validation parameters: +validation: + partial_validation_interval: 100 + problem: + batch_size: *batch_size + categories: all + # Appy all preprocessing/data augmentations. + question_preprocessing: *question_preprocessing + image_preprocessing: *image_preprocessing + # Preload images: false, as we will need them only once, at the end. + preload_images: false + streams: + questions: tokenized_questions + dataloader: + num_workers: 1 + + +pipeline: + + ################# PIPE 6: masks for all 5 classifiers ################# + + # Add global variables. + c1234_global_publisher: + type: GlobalVariablePublisher + priority: 6.0 + # Add input_size to globals. + keys: [category_c1_word_to_ix, category_c2_word_to_ix, category_c3_word_to_ix, category_c4_word_to_ix, category_binary_word_to_ix] + values: [{"C1": 0}, {"C2": 0}, {"C3": 0}, {"C4": 0}, {"BINARY": 0}] + + # Sample masking based on categories. + c1_string_to_mask: + priority: 6.1 + type: StringToMask + globals: + word_mappings: category_c1_word_to_ix + streams: + # Input: predicted categories by the "question categorizer" pipeline. + strings: predicted_category_names + masks: c1_masks + + c2_string_to_mask: + priority: 6.2 + type: StringToMask + globals: + word_mappings: category_c2_word_to_ix + streams: + strings: predicted_category_names + masks: c2_masks + + c3_string_to_mask: + priority: 6.3 + type: StringToMask + globals: + word_mappings: category_c3_word_to_ix + streams: + strings: predicted_category_names + masks: c3_masks + + c4_string_to_mask: + priority: 6.4 + type: StringToMask + globals: + word_mappings: category_c4_word_to_ix + streams: + strings: predicted_category_names + masks: c4_masks + + binary_string_to_mask: + priority: 6.5 + type: StringToMask + globals: + word_mappings: category_binary_word_to_ix + streams: + strings: predicted_category_names + masks: binary_masks + + ################# PIPE 7: answers/targets for all 5 classifiers ################# + + c1_answer_indexer: + type: LabelIndexer + priority: 7.1 + data_folder: ~/data/vqa-med + word_mappings_file: answers.c1_without_yn.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: c1_without_yn_answers_ids + globals: + vocabulary_size: c1_without_yn_vocabulary_size + word_mappings: c1_without_yn_word_mappings + + c2_answer_indexer: + type: LabelIndexer + priority: 7.2 + data_folder: ~/data/vqa-med + word_mappings_file: answers.c2.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: c2_answers_ids + globals: + vocabulary_size: c2_vocabulary_size + word_mappings: c2_word_mappings + + c3_answer_indexer: + type: LabelIndexer + priority: 7.3 + data_folder: ~/data/vqa-med + word_mappings_file: answers.c3.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: c3_answers_ids + globals: + vocabulary_size: c3_vocabulary_size + word_mappings: c3_word_mappings + + c4_answer_indexer: + type: LabelIndexer + priority: 7.4 + data_folder: ~/data/vqa-med + word_mappings_file: answers.c4_without_yn.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: c4_without_yn_answers_ids + globals: + vocabulary_size: c4_without_yn_vocabulary_size + word_mappings: c4_without_yn_word_mappings + + binary_answer_indexer: + type: LabelIndexer + priority: 7.6 + data_folder: ~/data/vqa-med + word_mappings_file: answers.binary_yn.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: binary_yn_answers_ids + globals: + vocabulary_size: binary_yn_vocabulary_size + word_mappings: binary_yn_word_mappings + + # Answer encoding for all classes. + #c1234_answer_indexer: + # priority: 7.5 + # type: LabelIndexer + # data_folder: ~/data/vqa-med + # word_mappings_file: answers.all.word.mappings.csv + # # Export mappings and size to globals. + # export_word_mappings_to_globals: True + # streams: + # inputs: answers + # outputs: answers_ids + # globals: + # vocabulary_size: c1234_binary_yn_vocabulary_size + # word_mappings: c1234_binary_yn_word_mappings + + + ################# PIPE 8: C1 questions ################# + + # Model: FFN for C1 answering + c1_answer_classifier: + priority: 8.1 + type: FeedForwardNetwork + hidden: *answer_classifier_hidden_sizes_val + dropout_rate: 0.5 + streams: + inputs: concatenated_activations + predictions: c1_predictions + globals: + input_size: concatenated_activations_size + prediction_size: c1_without_yn_vocabulary_size + + c1_nllloss: + priority: 8.2 + type: NLLLoss + targets_dim: 1 + streams: + predictions: c1_predictions + targets: c1_without_yn_answers_ids + loss: c1_loss + + c1_precision_recall: + priority: 8.3 + type: PrecisionRecallStatistics + use_word_mappings: True + #show_class_scores: True + #show_confusion_matrix: True + streams: + predictions: c1_predictions + targets: c1_without_yn_answers_ids + globals: + word_mappings: c1_without_yn_word_mappings + statistics: + precision: c1_precision + recall: c1_recall + f1score: c1_f1score + + c1_prediction_decoder: + priority: 8.4 + type: WordDecoder + # Use the same word mappings as label indexer. + import_word_mappings_from_globals: True + streams: + inputs: c1_predictions + outputs: c1_without_yn_prediction_labels + globals: + word_mappings: c1_without_yn_word_mappings + + + ################# PIPE 9: C2 questions ################# + + # Model: FFN for C2 answering + c2_answer_classifier: + priority: 9.1 + type: FeedForwardNetwork + hidden: *answer_classifier_hidden_sizes_val + dropout_rate: 0.5 + streams: + inputs: concatenated_activations + predictions: c2_predictions + globals: + input_size: concatenated_activations_size + prediction_size: c2_vocabulary_size + + c2_nllloss: + priority: 9.2 + type: NLLLoss + targets_dim: 1 + streams: + predictions: c2_predictions + targets: c2_answers_ids + loss: c2_loss + + c2_precision_recall: + priority: 9.3 + type: PrecisionRecallStatistics + use_word_mappings: True + #show_class_scores: True + #show_confusion_matrix: True + streams: + predictions: c2_predictions + targets: c2_answers_ids + globals: + word_mappings: c2_word_mappings + statistics: + precision: c2_precision + recall: c2_recall + f1score: c2_f1score + + c2_prediction_decoder: + priority: 9.4 + type: WordDecoder + # Use the same word mappings as label indexer. + import_word_mappings_from_globals: True + streams: + inputs: c2_predictions + outputs: c2_prediction_labels + globals: + word_mappings: c2_word_mappings + + + ################# PIPE 10: C3 questions ################# + + # Model: FFN for c3 answering + c3_answer_classifier: + priority: 10.1 + type: FeedForwardNetwork + hidden: *answer_classifier_hidden_sizes_val + dropout_rate: 0.5 + streams: + inputs: concatenated_activations + predictions: c3_predictions + globals: + input_size: concatenated_activations_size + prediction_size: c3_vocabulary_size + + c3_nllloss: + priority: 10.2 + type: NLLLoss + targets_dim: 1 + streams: + predictions: c3_predictions + targets: c3_answers_ids + loss: c3_loss + + c3_precision_recall: + priority: 10.3 + type: PrecisionRecallStatistics + use_word_mappings: True + #show_class_scores: True + #show_confusion_matrix: True + streams: + predictions: c3_predictions + targets: c3_answers_ids + globals: + word_mappings: c3_word_mappings + statistics: + precision: c3_precision + recall: c3_recall + f1score: c3_f1score + + c3_prediction_decoder: + priority: 10.4 + type: WordDecoder + # Use the same word mappings as label indexer. + import_word_mappings_from_globals: True + streams: + inputs: c3_predictions + outputs: c3_prediction_labels + globals: + word_mappings: c3_word_mappings + + + ################# PIPE 11: C4 questions ################# + + # Model: FFN for C4 answering + c4_answer_classifier: + priority: 11.1 + type: FeedForwardNetwork + hidden: *answer_classifier_hidden_sizes_val + dropout_rate: 0.5 + streams: + inputs: concatenated_activations + predictions: c4_predictions + globals: + input_size: concatenated_activations_size + prediction_size: c4_without_yn_vocabulary_size + + c4_nllloss: + priority: 11.2 + type: NLLLoss + targets_dim: 1 + streams: + predictions: c4_predictions + targets: c4_without_yn_answers_ids + loss: c4_loss + + c4_precision_recall: + priority: 11.3 + type: PrecisionRecallStatistics + use_word_mappings: True + #show_class_scores: True + #show_confusion_matrix: True + streams: + predictions: c4_predictions + targets: c4_without_yn_answers_ids + globals: + word_mappings: c4_without_yn_word_mappings + statistics: + precision: c4_precision + recall: c4_recall + f1score: c4_f1score + + c4_prediction_decoder: + priority: 11.4 + type: WordDecoder + # Use the same word mappings as label indexer. + import_word_mappings_from_globals: True + streams: + inputs: c4_predictions + outputs: c4_without_yn_prediction_labels + globals: + word_mappings: c4_without_yn_word_mappings + + + ################# PIPE 12: BINARY Y/N questions ################# + + # Model: FFN for BINARY Y/N answering + binary_answer_classifier: + priority: 12.1 + type: FeedForwardNetwork + hidden: *answer_classifier_hidden_sizes_val + dropout_rate: 0.5 + streams: + inputs: concatenated_activations + predictions: binary_predictions + globals: + input_size: concatenated_activations_size + prediction_size: binary_yn_vocabulary_size + + binary_nllloss: + priority: 12.2 + type: NLLLoss + targets_dim: 1 + streams: + predictions: binary_predictions + targets: binary_yn_answers_ids + loss: binary_loss + + binary_precision_recall: + priority: 12.3 + type: PrecisionRecallStatistics + use_word_mappings: True + #show_class_scores: True + #show_confusion_matrix: True + streams: + predictions: binary_predictions + targets: binary_yn_answers_ids + globals: + word_mappings: binary_yn_word_mappings + statistics: + precision: binary_precision + recall: binary_recall + f1score: binary_f1score + + binary_prediction_decoder: + priority: 12.4 + type: WordDecoder + # Use the same word mappings as label indexer. + import_word_mappings_from_globals: True + streams: + inputs: binary_predictions + outputs: binary_prediction_labels + globals: + word_mappings: binary_yn_word_mappings + + ################# PIPE 13: MERGE ANSWERS ################# + + # Viewers. + viewer: + priority: 13.1 + type: StreamViewer + input_streams: + tokenized_questions, + category_names, predicted_category_names, + c1_masks, c2_masks, c3_masks, c4_masks, binary_masks, + answers, + c1_without_yn_prediction_labels, + c2_prediction_labels, + c3_prediction_labels, + c4_without_yn_prediction_labels, + binary_prediction_labels + + + +#: pipeline diff --git a/ptp/components/masking/string_to_mask.py b/ptp/components/masking/string_to_mask.py index 4723ff6..671cc70 100644 --- a/ptp/components/masking/string_to_mask.py +++ b/ptp/components/masking/string_to_mask.py @@ -25,7 +25,6 @@ class StringToMask(Component): """ Class responsible for producing masks for strings using the provided word mappings. - Additionally, it returns the associated string indices. """ def __init__(self, name, config): @@ -45,14 +44,10 @@ def __init__(self, name, config): # Get key mappings. self.key_strings = self.stream_keys["strings"] self.key_masks = self.stream_keys["masks"] - self.key_string_indices = self.stream_keys["string_indices"] # Retrieve word mappings from globals. self.word_to_ix = self.globals["word_mappings"] - # Get value from configuration. - self.out_of_vocabulary_value = self.config["out_of_vocabulary_value"] - def input_data_definitions(self): """ Function returns a dictionary with definitions of input data that are required by the component. @@ -70,8 +65,7 @@ def output_data_definitions(self): :return: Empty dictionary. """ return { - self.key_masks: DataDefinition([-1], [torch.Tensor], "Batch of masks [BATCH_SIZE]"), - self.key_string_indices: DataDefinition([-1], [torch.Tensor], "Batch of indices corresponging to inputs strings when using provided word mappings [BATCH_SIZE]") + self.key_masks: DataDefinition([-1], [torch.Tensor], "Batch of masks [BATCH_SIZE]") } @@ -91,28 +85,15 @@ def __call__(self, data_dict): masks = torch.zeros(len(strings), requires_grad=False).type(self.app_state.ByteTensor) - outputs_list = [] # Process samples 1 by 1. for i,sample in enumerate(strings): assert not isinstance(sample, (list,)), 'This encoder requires input sample to contain a single word' # Process single token. if sample in self.word_to_ix.keys(): - output_sample = self.word_to_ix[sample] masks[i] = 1 - else: - # Word out of vocabulary. - output_sample = self.out_of_vocabulary_value - outputs_list.append(output_sample) - # Transform to tensor. - output_indices = torch.tensor(outputs_list, requires_grad=False).type(self.app_state.LongTensor) - #print("strings ", strings) - #print("masks ", masks) - #print("indices ", output_indices) - - # Extend the dict by returned output streams. + # Extend the dict by returned stream with masks. data_dict.extend({ - self.key_masks: masks, - self.key_string_indices: output_indices + self.key_masks: masks }) diff --git a/ptp/components/mixins/word_mappings.py b/ptp/components/mixins/word_mappings.py index 2510c63..a6df45f 100644 --- a/ptp/components/mixins/word_mappings.py +++ b/ptp/components/mixins/word_mappings.py @@ -53,7 +53,12 @@ def __init__(self): #, name, class_type, config): assert (len(self.word_to_ix) > 0), "The word mappings imported from global variables are empty!" # We could also get vocabulary_size from globals... but what for;) - elif self.word_mappings_file != "" and os.path.exists(word_mappings_file_path) and not self.config['regenerate']: + elif self.word_mappings_file != "" and not self.config['regenerate']: + if not os.path.exists(word_mappings_file_path): + self.logger.warning("Cannot load word mappings from '{}' because the file does not exist".format(word_mappings_file_path)) + # This is a show stopper. + exit(-1) + # Try to load the preprocessed word mappings. self.word_to_ix = wm.load_word_mappings_from_csv_file(self.logger, self.data_folder, self.word_mappings_file) assert (len(self.word_to_ix) > 0), "The word mappings loaded from file are empty!" diff --git a/ptp/components/utils/word_mappings.py b/ptp/components/utils/word_mappings.py index 014c7a9..2f7e3a5 100644 --- a/ptp/components/utils/word_mappings.py +++ b/ptp/components/utils/word_mappings.py @@ -82,6 +82,9 @@ def load_word_mappings_from_csv_file(logger, folder, filename): """ file_path = os.path.join(os.path.expanduser(folder), filename) + if not os.path.exists(file_path): + logger.warning("Cannot load word mappings from '{}' because the file does not exist".format(file_path)) + with open(file_path, mode='rt') as csvfile: # Check the presence of the header. sniffer = csv.Sniffer() From 1061a0d486374cad6ec4ccc1f33c69ca7a2c2b6a Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 6 May 2019 19:21:36 -0700 Subject: [PATCH 2/2] None fix in attention.py --- ptp/components/models/vqa/attention.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptp/components/models/vqa/attention.py b/ptp/components/models/vqa/attention.py index 4a04b66..196b26e 100644 --- a/ptp/components/models/vqa/attention.py +++ b/ptp/components/models/vqa/attention.py @@ -149,7 +149,7 @@ def forward(self, data_dict): if(self.output_mode == 'Image'): # Output attention-weighted image encodings outputs = attention_enc_img - elif(self.output_mode == 'None'): + elif(self.output_mode == 'Fusion'): # Fusion -- Concatenate attention-weighted image encodings and question encodings. outputs = torch.cat([attention_enc_img, latent_q], dim=1) # print("outputs", outputs.shape)