C1 variational flow

configs/default/components/text/label_indexer.yml

@@ -25,6 +25,10 @@ import_word_mappings_from_globals: False
 # Flag informing whether word mappings will be exported to globals (LOADED)
 export_word_mappings_to_globals: False
 
+# Value that will be used when word is out of vocavbulary (LOADED)
+# (Mask for that element will be 0 as well)
+out_of_vocabulary_value: -1
+
 streams: 
   ####################################################################
   # 2. Keymappings associated with INPUT and OUTPUT streams.

configs/vqa_med_2019/c1_classification/c1_classification_all_rnn_vgg16_concat.yml

@@ -0,0 +1,101 @@
+# Load config defining problems for training, validation and testing.
+default_configs: vqa_med_2019/c1_classification/default_c1_classification.yml
+
+pipeline:
+  name: vqa_med_c1_classification_all_rnn_vgg_concat
+
+  global_publisher:
+    type: GlobalVariablePublisher
+    priority: 0
+    # Add input_size to globals.
+    keys: [question_embeddings_output_size, image_size_encoder_input_size, image_size_encoder_output_size, image_encoder_output_size]
+    values: [100, 2, 10, 100]
+
+  # First subpipeline: question.
+  # Questions encoding.
+  question_tokenizer:
+    type: SentenceTokenizer
+    priority: 1.1
+    streams: 
+      inputs: questions
+      outputs: tokenized_questions
+
+  # Model 1: Embeddings
+  question_embeddings:
+    type: SentenceEmbeddings
+    priority: 1.2
+    embeddings_size: 50
+    pretrained_embeddings_file: glove.6B.50d.txt
+    data_folder: ~/data/vqa-med
+    word_mappings_file: questions.all.word.mappings.csv
+    streams:
+      inputs: tokenized_questions
+      outputs: embedded_questions      
+
+  # Model 2: RNN
+  question_lstm:
+    type: RecurrentNeuralNetwork
+    cell_type: LSTM
+    prediction_mode: Last
+    priority: 1.3
+    use_logsoftmax: False
+    initial_state_trainable: False
+    #num_layers: 5
+    hidden_size: 50
+    streams:
+      inputs: embedded_questions
+      predictions: question_activations
+    globals:
+      input_size: embeddings_size
+      prediction_size: question_embeddings_output_size
+
+  # 2nd subpipeline: image size.
+  # Model - image size classifier.
+  image_size_encoder:
+    type: FeedForwardNetwork 
+    priority: 2.1
+    streams:
+      inputs: image_sizes
+      predictions: image_size_activations
+    globals:
+      input_size: image_size_encoder_input_size
+      prediction_size: image_size_encoder_output_size
+
+  # 3rd subpipeline: image.
+  # Image encoder.
+  image_encoder:
+    type: TorchVisionWrapper
+    priority: 3.1
+    streams:
+      inputs: images
+      predictions: image_activations
+    globals:
+      prediction_size: image_encoder_output_size
+
+  # 4th subpipeline: concatenation + FF.
+  concat:
+    type: Concatenation
+    priority: 4.1
+    input_streams: [question_activations,image_size_activations,image_activations]
+    # Concatenation 
+    dim: 1 # default
+    input_dims: [[-1,100],[-1,10],[-1,100]]
+    output_dims: [-1,210]
+    streams:
+      outputs: concatenated_activations
+    globals:
+      output_size: output_size
+
+
+  classifier:
+    type: FeedForwardNetwork 
+    hidden_sizes: [100]
+    priority: 4.2
+    streams:
+      inputs: concatenated_activations
+    globals:
+      input_size: output_size
+      prediction_size: vocabulary_size_c1
+
+
+  #: pipeline

configs/vqa_med_2019/c1_classification/c1_classification_vf_question_rnn_separate_q_categorization.yml

@@ -0,0 +1,107 @@
+# Load config defining problems for training, validation and testing.
+default_configs: vqa_med_2019/c1_classification/default_c1_classification.yml
+
+pipeline:
+  name: vqa_med_c1_classification_vf_question_rnn_separate_q_categorization
+
+
+  ################# SHARED #################
+
+   # Questions encoding.
+  question_tokenizer:
+    type: SentenceTokenizer
+    priority: 0.2
+    streams: 
+      inputs: questions
+      outputs: tokenized_questions
+
+  ################# FLOW 0: CATEGORY #################
+
+  # Model 2: Embeddings
+  question_embeddings:
+    type: SentenceEmbeddings
+    priority: 0.3
+    # LOAD AND FREEZE
+    load:  ~/Documents/image-clef-2019/experiments/q_categorization/20190416_103111/checkpoints/vqa_med_question_categorization_rnn_best.pt
+    freeze: True
+    embeddings_size: 50
+    pretrained_embeddings_file: glove.6B.50d.txt
+    data_folder: ~/data/vqa-med
+    word_mappings_file: questions.all.word.mappings.csv
+    streams:
+      inputs: tokenized_questions
+      outputs: embedded_questions      
+
+  # Model 2: RNN
+  lstm:
+    type: RecurrentNeuralNetwork
+    cell_type: LSTM
+    # LOAD AND FREEZE
+    load:  ~/Documents/image-clef-2019/experiments/q_categorization/20190416_103111/checkpoints/vqa_med_question_categorization_rnn_best.pt
+    freeze: True
+    prediction_mode: Last
+    priority: 0.4
+    initial_state_trainable: False
+    streams:
+      inputs: embedded_questions
+      predictions: predicted_question_categories
+    globals:
+      input_size: embeddings_size
+      prediction_size: num_categories # C1,C2,C3,C4,BIN,UNK
+
+
+  category_decoder:
+    type: WordDecoder
+    priority: 0.5
+    # Use the same word mappings as label indexer.
+    import_word_mappings_from_globals: True
+    streams:
+      inputs: predicted_question_categories
+      outputs: predicted_question_categories_ids
+    globals:
+      vocabulary_size: num_categories
+      word_mappings: category_word_mappings
+
+  ################# FLOW C1: question #################
+
+  flow_c1_string_to_mask:
+    type: StringToMask
+    priority: 1.1
+    globals:
+      word_mappings: category_word_mappings
+    streams:
+      strings: predicted_question_categories_ids
+      string_indices: flow_c1_targets
+      masks: flow_c1_masks
+
+  # Model 1: Embeddings
+  flow_c1_question_embeddings:
+    type: SentenceEmbeddings
+    priority: 1.2
+    embeddings_size: 50
+    pretrained_embeddings_file: glove.6B.50d.txt
+    data_folder: ~/data/vqa-med
+    word_mappings_file: questions.all.word.mappings.csv
+    streams:
+      inputs: tokenized_questions
+      outputs: flow_c1_embedded_questions
+    globals:
+      embeddings_size: flow_c1_embeddings_size  
+
+  # Model 2: RNN
+  flow_c1_lstm:
+    type: RecurrentNeuralNetwork
+    cell_type: LSTM
+    prediction_mode: Last
+    priority: 1.3
+    initial_state_trainable: False
+    #num_layers: 5
+    hidden_size: 50
+    streams:
+      inputs: flow_c1_embedded_questions
+    globals:
+      input_size: flow_c1_embeddings_size
+      prediction_size: vocabulary_size_c1
+
+
+#: pipeline

configs/vqa_med_2019/c1_classification/default_c1_classification.yml

@@ -60,15 +60,15 @@ pipeline:
       loss: loss
 
   # Statistics.
-  accuracy:
-    type: AccuracyStatistics
-    priority: 100.1
-    streams:
-      targets: answers_ids
-
   batch_size:
     type: BatchSizeStatistics
-    priority: 100.2
+    priority: 100.1
+
+  #accuracy:
+  #  type: AccuracyStatistics
+  #  priority: 100.2
+  #  streams:
+  #    targets: answers_ids
 
   precision_recall:
     type: PrecisionRecallStatistics

configs/vqa_med_2019/c3_classification/default_c3_classification.yml

@@ -60,15 +60,15 @@ pipeline:
       loss: loss
 
   # Statistics.
-  accuracy:
-    type: AccuracyStatistics
-    priority: 100.1
-    streams:
-      targets: answers_ids
-
   batch_size:
     type: BatchSizeStatistics
-    priority: 100.2
+    priority: 100.1
+
+  #accuracy:
+  #  type: AccuracyStatistics
+  #  priority: 100.2
+  #  streams:
+  #    targets: answers_ids
 
   precision_recall:
     type: PrecisionRecallStatistics

configs/vqa_med_2019/question_categorization/default_question_categorization.yml

@@ -1,6 +1,11 @@
 # Load config defining problems for training, validation and testing.
 default_configs: vqa_med_2019/default_vqa_med_2019.yml
 
+training:
+  # settings parameters
+  terminal_conditions:
+    loss_stop: 1.0e-3
+
 pipeline:
 
   # Predictions decoder.
@@ -26,15 +31,15 @@ pipeline:
       loss: loss
 
   # Statistics.
-  accuracy:
-    type: AccuracyStatistics
-    priority: 100.1
-    streams:
-      targets: category_ids
-
   batch_size:
     type: BatchSizeStatistics
-    priority: 100.2
+    priority: 100.1
+
+  #accuracy:
+  #  type: AccuracyStatistics
+  #  priority: 100.2
+  #  streams:
+  #    targets: category_ids
 
   precision_recall:
     type: PrecisionRecallStatistics

configs/vqa_med_2019/question_categorization/question_categorization_rnn.yml

@@ -31,12 +31,10 @@ pipeline:
     prediction_mode: Last
     priority: 3
     initial_state_trainable: False
-    #num_layers: 5
-    #hidden_size: 1000
     streams:
       inputs: embedded_questions
     globals:
       input_size: embeddings_size
-      prediction_size: num_categories # C1,C2,C3,C4
+      prediction_size: num_categories # C1,C2,C3,C4,BIN,UNK
 
 #: pipeline