Fix in problem - y/n categorization, added masking to VF

configs/vqa_med_2019/evaluation/frozen_if_vf_5ffn_c1234yn_5losses.yml

@@ -18,7 +18,7 @@ c123_hyperparameters:
   # Final classifier: FFN.
   answer_classifier_hidden_sizes_val: &answer_classifier_hidden_sizes_val [500]
 
-  batch_size: &batch_size 256
+  batch_size: &batch_size 512
   preload_images: &preload_images False
   num_workers: &num_workers 4
 
@@ -201,19 +201,19 @@ pipeline:
       word_mappings: binary_yn_word_mappings
 
   # Answer encoding for all classes.
-  #c1234_answer_indexer:
-  #  priority: 7.5
-  #  type: LabelIndexer
-  #  data_folder: ~/data/vqa-med
-  #  word_mappings_file: answers.all.word.mappings.csv
-  #  # Export mappings and size to globals.
-  #  export_word_mappings_to_globals: True
-  #  streams:
-  #    inputs: answers
-  #    outputs: answers_ids
-  #  globals:
-  #    vocabulary_size: c1234_binary_yn_vocabulary_size
-  #    word_mappings: c1234_binary_yn_word_mappings
+  c1234_answer_indexer:
+    priority: 7.7
+    type: LabelIndexer
+    data_folder: ~/data/vqa-med
+    word_mappings_file: answers.all.word.mappings.csv
+    # Export mappings and size to globals.
+    export_word_mappings_to_globals: True
+    streams:
+      inputs: answers
+      outputs: answers_ids
+    globals:
+      vocabulary_size: c1234_binary_yn_vocabulary_size
+      word_mappings: c1234_binary_yn_word_mappings
 
 
   ################# PIPE 8: C1 questions #################
@@ -235,7 +235,9 @@ pipeline:
     priority: 8.2
     type: NLLLoss
     targets_dim: 1
+    use_masking: True
     streams:
+      masks: c1_masks
       predictions: c1_predictions
       targets: c1_without_yn_answers_ids
       loss: c1_loss
@@ -244,9 +246,11 @@ pipeline:
     priority: 8.3
     type: PrecisionRecallStatistics
     use_word_mappings: True
-    #show_class_scores: True
+    show_class_scores: True
     #show_confusion_matrix: True
+    use_masking: True
     streams:
+      masks: c1_masks
       predictions: c1_predictions
       targets: c1_without_yn_answers_ids
     globals:
@@ -287,7 +291,9 @@ pipeline:
     priority: 9.2
     type: NLLLoss
     targets_dim: 1
+    use_masking: True
     streams:
+      masks: c2_masks
       predictions: c2_predictions
       targets: c2_answers_ids
       loss: c2_loss
@@ -296,9 +302,11 @@ pipeline:
     priority: 9.3
     type: PrecisionRecallStatistics
     use_word_mappings: True
-    #show_class_scores: True
+    show_class_scores: True
     #show_confusion_matrix: True
+    use_masking: True
     streams:
+      masks: c2_masks
       predictions: c2_predictions
       targets: c2_answers_ids
     globals:
@@ -339,7 +347,9 @@ pipeline:
     priority: 10.2
     type: NLLLoss
     targets_dim: 1
+    use_masking: True
     streams:
+      masks: c3_masks
       predictions: c3_predictions
       targets: c3_answers_ids
       loss: c3_loss
@@ -348,9 +358,11 @@ pipeline:
     priority: 10.3
     type: PrecisionRecallStatistics
     use_word_mappings: True
-    #show_class_scores: True
+    show_class_scores: True
     #show_confusion_matrix: True
+    use_masking: True
     streams:
+      masks: c3_masks
       predictions: c3_predictions
       targets: c3_answers_ids
     globals:
@@ -391,7 +403,9 @@ pipeline:
     priority: 11.2
     type: NLLLoss
     targets_dim: 1
+    use_masking: True
     streams:
+      masks: c4_masks
       predictions: c4_predictions
       targets: c4_without_yn_answers_ids
       loss: c4_loss
@@ -402,7 +416,9 @@ pipeline:
     use_word_mappings: True
     #show_class_scores: True
     #show_confusion_matrix: True
+    use_masking: True
     streams:
+      masks: c4_masks
       predictions: c4_predictions
       targets: c4_without_yn_answers_ids
     globals:
@@ -443,7 +459,9 @@ pipeline:
     priority: 12.2
     type: NLLLoss
     targets_dim: 1
+    use_masking: True
     streams:
+      masks: binary_masks
       predictions: binary_predictions
       targets: binary_yn_answers_ids
       loss: binary_loss
@@ -452,9 +470,11 @@ pipeline:
     priority: 12.3
     type: PrecisionRecallStatistics
     use_word_mappings: True
-    #show_class_scores: True
+    show_class_scores: True
     #show_confusion_matrix: True
+    use_masking: True
     streams:
+      masks: binary_masks
       predictions: binary_predictions
       targets: binary_yn_answers_ids
     globals:
@@ -477,9 +497,42 @@ pipeline:
 
   ################# PIPE 13: MERGE ANSWERS #################
 
+  # Merge predictions.
+  pipe9_merged_predictions:
+    type: JoinMaskedPredictions
+    priority: 13.1
+    # Names of used input streams.
+    input_prediction_streams: [c1_predictions, c2_predictions, c3_predictions, c4_predictions, binary_predictions]
+    input_mask_streams: [c1_masks, c2_masks, c3_masks, c4_masks, binary_masks]
+    input_word_mappings: [c1_without_yn_word_mappings, c2_word_mappings, c3_word_mappings, c4_without_yn_word_mappings, binary_yn_word_mappings]
+    globals: 
+      output_word_mappings: c1234_binary_yn_word_mappings
+    streams:
+      output_strings: predicted_answers
+      output_indices: merged_prediction_indices
+
+  # Statistics.
+  pipe9_merged_precision_recall:
+    type: PrecisionRecallStatistics
+    priority: 13.2
+    # Use prediction indices instead of distributions.
+    use_prediction_distributions: False
+    use_word_mappings: True
+    #show_class_scores: True
+    #show_confusion_matrix: True
+    globals:
+      word_mappings: c1234_binary_yn_word_mappings
+    streams:
+      targets: answers_ids
+      predictions: merged_prediction_indices
+    statistics:
+      precision: merged_precision
+      recall: merged_recall
+      f1score: merged_f1score
+
   # Viewers.
   viewer:
-    priority: 13.1
+    priority: 13.3
     type: StreamViewer
     input_streams:
       tokenized_questions,
@@ -490,7 +543,8 @@ pipeline:
       c2_prediction_labels,
       c3_prediction_labels,
       c4_without_yn_prediction_labels,
-      binary_prediction_labels
+      binary_prediction_labels,
+      predicted_answers