From 3a714f5d2700f205780ec724201e219fc193a448 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 6 May 2019 20:18:21 -0700 Subject: [PATCH 1/6] Fix in problem - y/n categorization, added masking to VF --- .../frozen_if_vf_5ffn_c1234yn_5losses.yml | 30 +++++++++++++++---- ..._question_categorization_glove_rnn_ffn.yml | 4 +-- .../image_text_to_class/vqa_med_2019.py | 2 +- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml b/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml index eff5d97..fd5b0fa 100644 --- a/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml +++ b/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml @@ -18,7 +18,7 @@ c123_hyperparameters: # Final classifier: FFN. answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [500] - batch_size: &batch_size 256 + batch_size: &batch_size 512 preload_images: &preload_images False num_workers: &num_workers 4 @@ -235,7 +235,9 @@ pipeline: priority: 8.2 type: NLLLoss targets_dim: 1 + use_masking: True streams: + masks: c1_masks predictions: c1_predictions targets: c1_without_yn_answers_ids loss: c1_loss @@ -244,9 +246,11 @@ pipeline: priority: 8.3 type: PrecisionRecallStatistics use_word_mappings: True - #show_class_scores: True + show_class_scores: True #show_confusion_matrix: True + use_masking: True streams: + masks: c1_masks predictions: c1_predictions targets: c1_without_yn_answers_ids globals: @@ -287,7 +291,9 @@ pipeline: priority: 9.2 type: NLLLoss targets_dim: 1 + use_masking: True streams: + masks: c2_masks predictions: c2_predictions targets: c2_answers_ids loss: c2_loss @@ -296,9 +302,11 @@ pipeline: priority: 9.3 type: PrecisionRecallStatistics use_word_mappings: True - #show_class_scores: True + show_class_scores: True #show_confusion_matrix: True + use_masking: True streams: + masks: c2_masks predictions: c2_predictions targets: c2_answers_ids globals: @@ -339,7 +347,9 @@ pipeline: priority: 10.2 type: NLLLoss targets_dim: 1 + use_masking: True streams: + masks: c3_masks predictions: c3_predictions targets: c3_answers_ids loss: c3_loss @@ -348,9 +358,11 @@ pipeline: priority: 10.3 type: PrecisionRecallStatistics use_word_mappings: True - #show_class_scores: True + show_class_scores: True #show_confusion_matrix: True + use_masking: True streams: + masks: c3_masks predictions: c3_predictions targets: c3_answers_ids globals: @@ -391,7 +403,9 @@ pipeline: priority: 11.2 type: NLLLoss targets_dim: 1 + use_masking: True streams: + masks: c4_masks predictions: c4_predictions targets: c4_without_yn_answers_ids loss: c4_loss @@ -402,7 +416,9 @@ pipeline: use_word_mappings: True #show_class_scores: True #show_confusion_matrix: True + use_masking: True streams: + masks: c4_masks predictions: c4_predictions targets: c4_without_yn_answers_ids globals: @@ -443,7 +459,9 @@ pipeline: priority: 12.2 type: NLLLoss targets_dim: 1 + use_masking: True streams: + masks: binary_masks predictions: binary_predictions targets: binary_yn_answers_ids loss: binary_loss @@ -452,9 +470,11 @@ pipeline: priority: 12.3 type: PrecisionRecallStatistics use_word_mappings: True - #show_class_scores: True + show_class_scores: True #show_confusion_matrix: True + use_masking: True streams: + masks: binary_masks predictions: binary_predictions targets: binary_yn_answers_ids globals: diff --git a/configs/vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml b/configs/vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml index c144f57..0ae17d9 100644 --- a/configs/vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml +++ b/configs/vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml @@ -18,8 +18,8 @@ # 0.55: pipe_qc_category_decoder # 0.56: pipe_qc_category_accuracy -# Loaded checkpoint: 20190505_130406 -checkpoint: &checkpoint ~/image-clef-2019/experiments/q_categorization/20190505_130406/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt +# Loaded checkpoint: 20190506_200228 +checkpoint: &checkpoint ~/image-clef-2019/experiments/q_categorization/20190506_200228/checkpoints/vqa_med_question_categorization_rnn_ffn_best.pt pipeline: ################# PIPE: QUESTION CATEGORIZATION ################# diff --git a/ptp/components/problems/image_text_to_class/vqa_med_2019.py b/ptp/components/problems/image_text_to_class/vqa_med_2019.py index 8e94418..ed0d2e2 100644 --- a/ptp/components/problems/image_text_to_class/vqa_med_2019.py +++ b/ptp/components/problems/image_text_to_class/vqa_med_2019.py @@ -747,7 +747,7 @@ def __getitem__(self, index): # Question category related variables. # Check if this is binary question. - if self.predict_yes_no(item[self.key_answers]): + if self.predict_yes_no(item[self.key_questions]): data_dict[self.key_category_ids] = 4 # Binary. data_dict[self.key_category_names] = self.category_idx_to_word[4] else: From 6e36f79f230a687854b497ea7398c7435ea40e3c Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 6 May 2019 20:26:50 -0700 Subject: [PATCH 2/6] cleanup --- ...binary_vf_cat_rnn_shared_all_encoders_four_ffns_losses.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/configs/vqa_med_2019/vf/c1_c2_c3_binary_vf_cat_rnn_shared_all_encoders_four_ffns_losses.yml b/configs/vqa_med_2019/vf/c1_c2_c3_binary_vf_cat_rnn_shared_all_encoders_four_ffns_losses.yml index 762719d..c67e07e 100644 --- a/configs/vqa_med_2019/vf/c1_c2_c3_binary_vf_cat_rnn_shared_all_encoders_four_ffns_losses.yml +++ b/configs/vqa_med_2019/vf/c1_c2_c3_binary_vf_cat_rnn_shared_all_encoders_four_ffns_losses.yml @@ -284,7 +284,6 @@ pipeline: ################# PIPE 6: C2 question ################# - # Answer encoding for PIPE 6. pipe6_c2_answer_indexer: type: LabelIndexer priority: 6.1 @@ -310,7 +309,6 @@ pipeline: string_indices: predicted_c2_question_categories_indices # NOT USED masks: pipe6_c2_masks - # Model 4: FFN C1 answering pipe6_c2_ffn: priority: 6.3 type: FeedForwardNetwork @@ -503,7 +501,7 @@ pipeline: input_prediction_streams: [pipe5_c1_predictions, pipe6_c2_predictions, pipe7_c3_predictions, pipe8_binary_predictions] input_mask_streams: [pipe5_c1_masks, pipe6_c2_masks, pipe7_c3_masks, pipe8_binary_masks] input_word_mappings: [word_mappings_c1_without_yn, word_mappings_c2, word_mappings_c3, word_mappings_binary_yn] - globals: + globals: output_word_mappings: word_mappings_all_c1_c2_c3_binary streams: output_strings: pipe9_merged_predictions From 3b10d4df7e00957f366758f1f3767e36c6bc7f34 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 6 May 2019 21:00:27 -0700 Subject: [PATCH 3/6] Added component for merging --- .../frozen_if_vf_5ffn_c1234yn_5losses.yml | 64 ++++++++++++++----- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml b/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml index fd5b0fa..8633b43 100644 --- a/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml +++ b/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml @@ -201,19 +201,19 @@ pipeline: word_mappings: binary_yn_word_mappings # Answer encoding for all classes. - #c1234_answer_indexer: - # priority: 7.5 - # type: LabelIndexer - # data_folder: ~/data/vqa-med - # word_mappings_file: answers.all.word.mappings.csv - # # Export mappings and size to globals. - # export_word_mappings_to_globals: True - # streams: - # inputs: answers - # outputs: answers_ids - # globals: - # vocabulary_size: c1234_binary_yn_vocabulary_size - # word_mappings: c1234_binary_yn_word_mappings + c1234_answer_indexer: + priority: 7.7 + type: LabelIndexer + data_folder: ~/data/vqa-med + word_mappings_file: answers.all.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: answers_ids + globals: + vocabulary_size: c1234_binary_yn_vocabulary_size + word_mappings: c1234_binary_yn_word_mappings ################# PIPE 8: C1 questions ################# @@ -497,9 +497,42 @@ pipeline: ################# PIPE 13: MERGE ANSWERS ################# + # Merge predictions. + pipe9_merged_predictions: + type: JoinMaskedPredictions + priority: 13.1 + # Names of used input streams. + input_prediction_streams: [c1_predictions, c2_predictions, c3_predictions, c4_predictions, binary_predictions] + input_mask_streams: [c1_masks, c2_masks, c3_masks, c4_masks, binary_masks] + input_word_mappings: [c1_without_yn_word_mappings, c2_word_mappings, c3_word_mappings, c4_without_yn_word_mappings, binary_yn_word_mappings] + globals: + output_word_mappings: c1234_binary_yn_word_mappings + streams: + output_strings: predicted_answers + output_indices: merged_prediction_indices + + # Statistics. + pipe9_merged_precision_recall: + type: PrecisionRecallStatistics + priority: 13.2 + # Use prediction indices instead of distributions. + use_prediction_distributions: False + use_word_mappings: True + #show_class_scores: True + #show_confusion_matrix: True + globals: + word_mappings: c1234_binary_yn_word_mappings + streams: + targets: answers_ids + predictions: merged_prediction_indices + statistics: + precision: merged_precision + recall: merged_recall + f1score: merged_f1score + # Viewers. viewer: - priority: 13.1 + priority: 13.3 type: StreamViewer input_streams: tokenized_questions, @@ -510,7 +543,8 @@ pipeline: c2_prediction_labels, c3_prediction_labels, c4_without_yn_prediction_labels, - binary_prediction_labels + binary_prediction_labels, + predicted_answers From cb757d2d91a09cba0499978efafa29652b1814a3 Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 6 May 2019 21:55:19 -0700 Subject: [PATCH 4/6] VF with 5 losses and supporting networks --- ...zen_if_vf_5ffn_support_c1234yn_5losses.yml | 620 ++++++++++++++++++ 1 file changed, 620 insertions(+) create mode 100644 configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_support_c1234yn_5losses.yml diff --git a/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_support_c1234yn_5losses.yml b/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_support_c1234yn_5losses.yml new file mode 100644 index 0000000..976c623 --- /dev/null +++ b/configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_support_c1234yn_5losses.yml @@ -0,0 +1,620 @@ +# Load config defining problems for training, validation and testing. +default_configs: + vqa_med_2019/default_vqa_med_2019.yml, + vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml, + vqa_med_2019/frozen_pipelines/frozen_question_categorization_glove_rnn_ffn.yml + +c1234_hyperparameters: + # In here I am putting some of the hyperparameters from spreadsheet. + + question_preprocessing: &question_preprocessing lowercase, remove_punctuation, tokenize + # Accepted formats: a,b,c or [a,b,c] + # none | lowercase | remove_punctuation | tokenize | random_remove_stop_words | random_shuffle_words | all + + image_preprocessing: &image_preprocessing normalize + # Accepted formats: a,b,c or [a,b,c] + # none | random_affine | random_horizontal_flip | normalize | all + + # Hidden size of C1/C2/C3 "supporting fact" networks. + c123_support_hidden_sizes_val: &c123_support_hidden_sizes_val [500] + + # Size of output of C1/C2/C3 "supporting fact" networks. + c123_support_output_size_val: &c123_support_output_size_val 100 + + # Hidden size of C1/C2/C3 support networks. + binary_hidden_sizes_val: &binary_hidden_sizes_val [500] + + # C4 classifier hidden sizes. + c4_hidden_sizes_val: &c4_hidden_sizes_val [500, 500] + + # This comes from inputs fusion. + activation_size_val: &activation_size_val 1134 + + # Activation size + 3 * support output size + activation_support_size_val: &activation_support_size_val 1434 + + batch_size: &batch_size 512 + preload_images: &preload_images False + num_workers: &num_workers 4 + +# Training parameters: +training: + problem: + batch_size: *batch_size + categories: all + export_sample_weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv + # Appy all preprocessing/data augmentations. + question_preprocessing: *question_preprocessing + image_preprocessing: *image_preprocessing + # Preload images. + preload_images: *preload_images + streams: + questions: tokenized_questions + sampler: + weights: ~/data/vqa-med/answers.c1_c2_c3_c4_binary_yn.weights.csv + # Use workers for loading images. + dataloader: + num_workers: *num_workers + + # Optimizer parameters: + optimizer: + name: Adam + lr: 0.0001 + + # Terminal conditions: + terminal_conditions: + loss_stop: 1.0e-3 + episode_limit: 10000 + epoch_limit: -1 + +# Validation parameters: +validation: + partial_validation_interval: 100 + problem: + batch_size: *batch_size + categories: all + # Appy all preprocessing/data augmentations. + question_preprocessing: *question_preprocessing + image_preprocessing: *image_preprocessing + # Preload images: false, as we will need them only once, at the end. + preload_images: false + streams: + questions: tokenized_questions + dataloader: + num_workers: 1 + + +pipeline: + + ################# PIPE 6: masks for all 5 classifiers ################# + + # Add global variables. + c1234_global_publisher: + type: GlobalVariablePublisher + priority: 6.0 + # Add input_size to globals. + keys: [category_c1_word_to_ix, category_c2_word_to_ix, category_c3_word_to_ix, category_c4_word_to_ix, category_binary_word_to_ix, c123_support_output_size] + values: [{"C1": 0}, {"C2": 0}, {"C3": 0}, {"C4": 0}, {"BINARY": 0}, *c123_support_output_size_val] + + # Sample masking based on categories. + c1_string_to_mask: + priority: 6.1 + type: StringToMask + globals: + word_mappings: category_c1_word_to_ix + streams: + # Input: predicted categories by the "question categorizer" pipeline. + strings: predicted_category_names + masks: c1_masks + + c2_string_to_mask: + priority: 6.2 + type: StringToMask + globals: + word_mappings: category_c2_word_to_ix + streams: + strings: predicted_category_names + masks: c2_masks + + c3_string_to_mask: + priority: 6.3 + type: StringToMask + globals: + word_mappings: category_c3_word_to_ix + streams: + strings: predicted_category_names + masks: c3_masks + + c4_string_to_mask: + priority: 6.4 + type: StringToMask + globals: + word_mappings: category_c4_word_to_ix + streams: + strings: predicted_category_names + masks: c4_masks + + binary_string_to_mask: + priority: 6.5 + type: StringToMask + globals: + word_mappings: category_binary_word_to_ix + streams: + strings: predicted_category_names + masks: binary_masks + + ################# PIPE 7: answers/targets for all 5 classifiers ################# + + c1_answer_indexer: + type: LabelIndexer + priority: 7.1 + data_folder: ~/data/vqa-med + word_mappings_file: answers.c1_without_yn.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: c1_without_yn_answers_ids + globals: + vocabulary_size: c1_without_yn_vocabulary_size + word_mappings: c1_without_yn_word_mappings + + c2_answer_indexer: + type: LabelIndexer + priority: 7.2 + data_folder: ~/data/vqa-med + word_mappings_file: answers.c2.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: c2_answers_ids + globals: + vocabulary_size: c2_vocabulary_size + word_mappings: c2_word_mappings + + c3_answer_indexer: + type: LabelIndexer + priority: 7.3 + data_folder: ~/data/vqa-med + word_mappings_file: answers.c3.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: c3_answers_ids + globals: + vocabulary_size: c3_vocabulary_size + word_mappings: c3_word_mappings + + c4_answer_indexer: + type: LabelIndexer + priority: 7.4 + data_folder: ~/data/vqa-med + word_mappings_file: answers.c4_without_yn.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: c4_without_yn_answers_ids + globals: + vocabulary_size: c4_without_yn_vocabulary_size + word_mappings: c4_without_yn_word_mappings + + binary_answer_indexer: + type: LabelIndexer + priority: 7.6 + data_folder: ~/data/vqa-med + word_mappings_file: answers.binary_yn.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: binary_yn_answers_ids + globals: + vocabulary_size: binary_yn_vocabulary_size + word_mappings: binary_yn_word_mappings + + # Answer encoding for all classes. + c1234_answer_indexer: + priority: 7.7 + type: LabelIndexer + data_folder: ~/data/vqa-med + word_mappings_file: answers.all.word.mappings.csv + # Export mappings and size to globals. + export_word_mappings_to_globals: True + streams: + inputs: answers + outputs: answers_ids + globals: + vocabulary_size: c1234_binary_yn_vocabulary_size + word_mappings: c1234_binary_yn_word_mappings + + + ################# PIPE 8: C1 questions ################# + + c1_support_network: + priority: 8.1 + type: FeedForwardNetwork + hidden: *c123_support_hidden_sizes_val + dropout_rate: 0.5 + # This is support network, so we do not want softmax... do we? + use_logsoftmax: false + streams: + inputs: concatenated_activations + predictions: c1_support + globals: + input_size: concatenated_activations_size + prediction_size: c123_support_output_size + + c1_answer_classifier: + priority: 8.2 + type: FeedForwardNetwork + # No hidden layer, only projection to answer + softmax! + dropout_rate: 0.5 + streams: + inputs: c1_support + predictions: c1_predictions + globals: + input_size: c123_support_output_size + prediction_size: c1_without_yn_vocabulary_size + + c1_nllloss: + priority: 8.3 + type: NLLLoss + targets_dim: 1 + use_masking: True + streams: + masks: c1_masks + predictions: c1_predictions + targets: c1_without_yn_answers_ids + loss: c1_loss + + c1_precision_recall: + priority: 8.4 + type: PrecisionRecallStatistics + use_word_mappings: True + show_class_scores: True + #show_confusion_matrix: True + use_masking: True + streams: + masks: c1_masks + predictions: c1_predictions + targets: c1_without_yn_answers_ids + globals: + word_mappings: c1_without_yn_word_mappings + statistics: + precision: c1_precision + recall: c1_recall + f1score: c1_f1score + + c1_prediction_decoder: + priority: 8.5 + type: WordDecoder + # Use the same word mappings as label indexer. + import_word_mappings_from_globals: True + streams: + inputs: c1_predictions + outputs: c1_without_yn_prediction_labels + globals: + word_mappings: c1_without_yn_word_mappings + + + ################# PIPE 9: C2 questions ################# + + c2_support_network: + priority: 9.1 + type: FeedForwardNetwork + hidden: *c123_support_hidden_sizes_val + dropout_rate: 0.5 + # This is support network, so we do not want softmax... do we? + use_logsoftmax: false + streams: + inputs: concatenated_activations + predictions: c2_support + globals: + input_size: concatenated_activations_size + prediction_size: c123_support_output_size + + c2_answer_classifier: + priority: 9.2 + type: FeedForwardNetwork + # No hidden layer, only projection to answer + softmax! + dropout_rate: 0.5 + streams: + inputs: c2_support + predictions: c2_predictions + globals: + input_size: c123_support_output_size + prediction_size: c2_vocabulary_size + + c2_nllloss: + priority: 9.3 + type: NLLLoss + targets_dim: 1 + use_masking: True + streams: + masks: c2_masks + predictions: c2_predictions + targets: c2_answers_ids + loss: c2_loss + + c2_precision_recall: + priority: 9.4 + type: PrecisionRecallStatistics + use_word_mappings: True + show_class_scores: True + #show_confusion_matrix: True + use_masking: True + streams: + masks: c2_masks + predictions: c2_predictions + targets: c2_answers_ids + globals: + word_mappings: c2_word_mappings + statistics: + precision: c2_precision + recall: c2_recall + f1score: c2_f1score + + c2_prediction_decoder: + priority: 9.5 + type: WordDecoder + # Use the same word mappings as label indexer. + import_word_mappings_from_globals: True + streams: + inputs: c2_predictions + outputs: c2_prediction_labels + globals: + word_mappings: c2_word_mappings + + + ################# PIPE 10: C3 questions ################# + + c3_support_network: + priority: 10.1 + type: FeedForwardNetwork + hidden: *c123_support_hidden_sizes_val + dropout_rate: 0.5 + # This is support network, so we do not want softmax... do we? + use_logsoftmax: false + streams: + inputs: concatenated_activations + predictions: c3_support + globals: + input_size: concatenated_activations_size + prediction_size: c123_support_output_size + + c3_answer_classifier: + priority: 10.2 + type: FeedForwardNetwork + # No hidden layer, only projection to answer + softmax! + dropout_rate: 0.5 + streams: + inputs: c3_support + predictions: c3_predictions + globals: + input_size: c123_support_output_size + prediction_size: c3_vocabulary_size + + c3_nllloss: + priority: 10.3 + type: NLLLoss + targets_dim: 1 + use_masking: True + streams: + masks: c3_masks + predictions: c3_predictions + targets: c3_answers_ids + loss: c3_loss + + c3_precision_recall: + priority: 10.4 + type: PrecisionRecallStatistics + use_word_mappings: True + show_class_scores: True + #show_confusion_matrix: True + use_masking: True + streams: + masks: c3_masks + predictions: c3_predictions + targets: c3_answers_ids + globals: + word_mappings: c3_word_mappings + statistics: + precision: c3_precision + recall: c3_recall + f1score: c3_f1score + + c3_prediction_decoder: + priority: 10.5 + type: WordDecoder + # Use the same word mappings as label indexer. + import_word_mappings_from_globals: True + streams: + inputs: c3_predictions + outputs: c3_prediction_labels + globals: + word_mappings: c3_word_mappings + + + ################# PIPE 11: C4 questions ################# + + # Concatenate input fused activatons with three supporting streams. + activation_support_concat: + priority: 11.1 + type: Concatenation + input_streams: [concatenated_activations,c1_support,c2_support,c3_support] + # Concatenation + dim: 1 # default + input_dims: [[-1,*activation_size_val],[-1,*c123_support_output_size_val],[-1,*c123_support_output_size_val],[-1,*c123_support_output_size_val]] + output_dims: [-1,*activation_support_size_val] + streams: + outputs: concatenated_activations_supports + globals: + output_size: concatenated_activations_supports_size + + + # Model: FFN for C4 answering + c4_answer_classifier: + priority: 11.2 + type: FeedForwardNetwork + hidden: *c4_hidden_sizes_val + dropout_rate: 0.5 + streams: + inputs: concatenated_activations_supports + predictions: c4_predictions + globals: + input_size: concatenated_activations_supports_size + prediction_size: c4_without_yn_vocabulary_size + + c4_nllloss: + priority: 11.3 + type: NLLLoss + targets_dim: 1 + use_masking: True + streams: + masks: c4_masks + predictions: c4_predictions + targets: c4_without_yn_answers_ids + loss: c4_loss + + c4_precision_recall: + priority: 11.4 + type: PrecisionRecallStatistics + use_word_mappings: True + #show_class_scores: True + #show_confusion_matrix: True + use_masking: True + streams: + masks: c4_masks + predictions: c4_predictions + targets: c4_without_yn_answers_ids + globals: + word_mappings: c4_without_yn_word_mappings + statistics: + precision: c4_precision + recall: c4_recall + f1score: c4_f1score + + c4_prediction_decoder: + priority: 11.5 + type: WordDecoder + # Use the same word mappings as label indexer. + import_word_mappings_from_globals: True + streams: + inputs: c4_predictions + outputs: c4_without_yn_prediction_labels + globals: + word_mappings: c4_without_yn_word_mappings + + + ################# PIPE 12: BINARY Y/N questions ################# + + # Model: FFN for BINARY Y/N answering + binary_answer_classifier: + priority: 12.2 + type: FeedForwardNetwork + hidden: *binary_hidden_sizes_val + dropout_rate: 0.5 + streams: + inputs: concatenated_activations_supports + predictions: binary_predictions + globals: + input_size: concatenated_activations_supports_size + prediction_size: binary_yn_vocabulary_size + + binary_nllloss: + priority: 12.3 + type: NLLLoss + targets_dim: 1 + use_masking: True + streams: + masks: binary_masks + predictions: binary_predictions + targets: binary_yn_answers_ids + loss: binary_loss + + binary_precision_recall: + priority: 12.4 + type: PrecisionRecallStatistics + use_word_mappings: True + show_class_scores: True + #show_confusion_matrix: True + use_masking: True + streams: + masks: binary_masks + predictions: binary_predictions + targets: binary_yn_answers_ids + globals: + word_mappings: binary_yn_word_mappings + statistics: + precision: binary_precision + recall: binary_recall + f1score: binary_f1score + + binary_prediction_decoder: + priority: 12.5 + type: WordDecoder + # Use the same word mappings as label indexer. + import_word_mappings_from_globals: True + streams: + inputs: binary_predictions + outputs: binary_prediction_labels + globals: + word_mappings: binary_yn_word_mappings + + ################# PIPE 13: MERGE ANSWERS ################# + + # Merge predictions. + pipe9_merged_predictions: + type: JoinMaskedPredictions + priority: 13.1 + # Names of used input streams. + input_prediction_streams: [c1_predictions, c2_predictions, c3_predictions, c4_predictions, binary_predictions] + input_mask_streams: [c1_masks, c2_masks, c3_masks, c4_masks, binary_masks] + input_word_mappings: [c1_without_yn_word_mappings, c2_word_mappings, c3_word_mappings, c4_without_yn_word_mappings, binary_yn_word_mappings] + globals: + output_word_mappings: c1234_binary_yn_word_mappings + streams: + output_strings: predicted_answers + output_indices: merged_prediction_indices + + # Statistics. + pipe9_merged_precision_recall: + type: PrecisionRecallStatistics + priority: 13.2 + # Use prediction indices instead of distributions. + use_prediction_distributions: False + use_word_mappings: True + #show_class_scores: True + #show_confusion_matrix: True + globals: + word_mappings: c1234_binary_yn_word_mappings + streams: + targets: answers_ids + predictions: merged_prediction_indices + statistics: + precision: merged_precision + recall: merged_recall + f1score: merged_f1score + + # Viewers. + viewer: + priority: 13.3 + type: StreamViewer + input_streams: + tokenized_questions, + category_names, predicted_category_names, + c1_masks, c2_masks, c3_masks, c4_masks, binary_masks, + answers, + c1_without_yn_prediction_labels, + c2_prediction_labels, + c3_prediction_labels, + c4_without_yn_prediction_labels, + binary_prediction_labels, + predicted_answers + + + +#: pipeline From 9f1a9340036285ead1b9d51880f20d5c4956310a Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 6 May 2019 22:16:03 -0700 Subject: [PATCH 5/6] load model (hot)fix --- ptp/workers/processor.py | 9 +++++---- ptp/workers/trainer.py | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ptp/workers/processor.py b/ptp/workers/processor.py index 0abadc7..6974231 100644 --- a/ptp/workers/processor.py +++ b/ptp/workers/processor.py @@ -267,16 +267,17 @@ def setup_individual_experiment(self): msg = "'pipeline' section of the configuration file" else: pipeline_name = "" - # Try to load the model. + # Try to load the the whole pipeline. if pipeline_name != "": if path.isfile(pipeline_name): # Load parameters from checkpoint. self.pipeline.load(pipeline_name) else: raise Exception("Couldn't load the checkpoint {} indicated in the {}: file does not exist".format(pipeline_name, msg)) - - # Try to load the models parameters - one by one, if set so in the configuration file. - self.pipeline.load_models() + # If we succeeded, we do not want to load the models from the file anymore! + else: + # Try to load the models parameters - one by one, if set so in the configuration file. + self.pipeline.load_models() except KeyError: self.logger.error("File {} indicated in the {} seems not to be a valid model checkpoint".format(pipeline_name, msg)) diff --git a/ptp/workers/trainer.py b/ptp/workers/trainer.py index 1182574..56e72c6 100644 --- a/ptp/workers/trainer.py +++ b/ptp/workers/trainer.py @@ -279,9 +279,10 @@ def setup_experiment(self): self.pipeline.load(pipeline_name) else: raise Exception("Couldn't load the checkpoint {} indicated in the {}: file does not exist".format(pipeline_name, msg)) - - # Try to load the models parameters - one by one, if set so in the configuration file. - self.pipeline.load_models() + # If we succeeded, we do not want to load the models from the file anymore! + else: + # Try to load the models parameters - one by one, if set so in the configuration file. + self.pipeline.load_models() except KeyError: self.logger.error("File {} indicated in the {} seems not to be a valid model checkpoint".format(pipeline_name, msg)) From 99e84da66ffa0c06c6923cc5cb6d20de261ec5cb Mon Sep 17 00:00:00 2001 From: Tomasz Kornuta Date: Mon, 6 May 2019 22:30:23 -0700 Subject: [PATCH 6/6] parametrized pipe so it can be unfrozen easily --- ...n_input_fusion_glove_lstm_vgg_att_is_cat.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/configs/vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml b/configs/vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml index c3ae040..60a489b 100644 --- a/configs/vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml +++ b/configs/vqa_med_2019/frozen_pipelines/frozen_input_fusion_glove_lstm_vgg_att_is_cat.yml @@ -12,8 +12,9 @@ # globals: # * concatenated_activations_size -# "Inputs Fusion" -# 0.: +# Freeze the models in "Inputs Fusion". +freeze_models: &freeze_models True + checkpoint: &checkpoint ~/image-clef-2019/experiments/c4_encoders/20190504_202441/checkpoints/glove_lstm_vgg16_att_is_cat_ffn_c123_loss_best.pt # Loaded checkpoint: 20190504_202441 @@ -82,7 +83,7 @@ pipeline: load: file: *checkpoint model: pipe1_question_embeddings - freeze: True + freeze: *freeze_models ################### streams: inputs: tokenized_questions @@ -103,7 +104,7 @@ pipeline: load: file: *checkpoint model: pipe1_lstm - freeze: True + freeze: *freeze_models ################### streams: inputs: embedded_questions @@ -124,7 +125,7 @@ pipeline: load: file: *checkpoint model: image_encoder - freeze: True + freeze: *freeze_models ################### streams: inputs: images @@ -141,7 +142,7 @@ pipeline: load: file: *checkpoint model: image_size_encoder - freeze: True + freeze: *freeze_models ################### streams: inputs: image_sizes @@ -163,7 +164,7 @@ pipeline: load: file: *checkpoint model: question_image_fusion - freeze: True + freeze: *freeze_models ################### streams: image_encodings: feature_maps @@ -183,7 +184,7 @@ pipeline: load: file: *checkpoint model: question_image_ffn - freeze: True + freeze: *freeze_models ################### streams: inputs: fused_activations