C4 answer-word classifiers, C123 - configs for evaluation

configs/default/components/models/sentence_embeddings.yml

@@ -40,7 +40,9 @@ fixed_padding: -1
 
 # File containing pretrained embeddings (LOADED)
 # Empty means that no embeddings will be loaded.
-# Options: '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt | glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled
+# Options: 
+# '' | glove.6B.50d.txt | glove.6B.100d.txt | glove.6B.200d.txt | glove.6B.300d.txt |
+# glove.42B.300d.txt | glove.840B.300d.txt | glove.twitter.27B.txt | mimic.fastText.no_clean.300d.pickled
 pretrained_embeddings_file: ''
 
 streams: 

configs/default/components/models/vqa/attention.yml

@@ -46,13 +46,13 @@ globals:
   # Size of the question encodings input (RETRIEVED)
   question_encoding_size: question_encoding_size
 
-  # Size of the output (RETRIEVED)
-  output_size: output_size
-
   ####################################################################
   # 4. Keymappings associated with GLOBAL variables that will be SET.
   ####################################################################
 
+  # Size of the output (SET)
+  output_size: output_size
+
   ####################################################################
   # 5. Keymappings associated with statistics that will be ADDED.
   ####################################################################

configs/default/components/models/vqa/multimodal_compact_bilinear_pooling.yml

@@ -4,6 +4,10 @@
 # 1. CONFIGURATION PARAMETERS that will be LOADED by the component.
 ####################################################################
 
+# Parameter denoting whether projection matrices are trainable (LOADED)
+# Setting flag that to true will result in trainable, dense (i.e. not "sketch") projection layers.
+trainable_projections: False
+
 streams: 
   ####################################################################
   # 2. Keymappings associated with INPUT and OUTPUT streams.

configs/default/components/transforms/reduce_tensor.yml

@@ -0,0 +1,47 @@
+# This file defines the default values for the ReduceTensor transformation.
+
+####################################################################
+# 1. CONFIGURATION PARAMETERS that will be LOADED by the component.
+####################################################################
+
+# Number of input dimensions, including batch (LOADED)
+num_inputs_dims: 2
+
+# Dimension along with the reduction will be applied (LOADED)
+reduction_dim: 1
+
+# Reduction type (LOADED)
+# Options: sum | mean | min | max | argmin | argmax
+reduction_type: sum
+
+# If True, the output tensor is of the same size as input, except dim where it is of size 1 (LOADED)
+keepdim: False
+
+streams: 
+  ####################################################################
+  # 2. Keymappings associated with INPUT and OUTPUT streams.
+  ####################################################################
+
+  # Stream containing input tensor (INPUT)
+  inputs: inputs
+
+  # Stream containing output tensor (OUTPUT)
+  outputs: outputs
+
+globals:
+  ####################################################################
+  # 3. Keymappings of variables that will be RETRIEVED from GLOBALS.
+  ####################################################################
+
+  # Size of the intput_item (GET)
+  # (last dimenstion)
+  input_size: input_size
+
+  ####################################################################
+  # 4. Keymappings associated with GLOBAL variables that will be SET.
+  ####################################################################
+
+  ####################################################################
+  # 5. Keymappings associated with statistics that will be ADDED.
+  ####################################################################
+

configs/vqa_med_2019/c2_classification/c2_classification_all_rnn_vgg16_mcb.yml

@@ -8,7 +8,7 @@ pipeline:
     type: GlobalVariablePublisher
     # Add input_size to globals.
     keys: [question_encoder_output_size, image_encoder_output_size, fused_image_question_activation_size]
-    values: [200, 1000, 100]
+    values: [200, 500, 100]
 
   ################# PIPE 0: question #################
   # Questions encoding.

configs/vqa_med_2019/c4_classification/c4_word_answer_glove_sum.yml

@@ -0,0 +1,91 @@
+# Load config defining problems for training, validation and testing.
+default_configs: vqa_med_2019/c4_classification/default_c4_classification.yml
+
+# Training parameters:
+training:
+  problem:
+    categories: C4
+    batch_size: 512
+    # In here we won't use images at all.
+    stream_images: False
+  dataloader:
+    num_workers: 0
+
+# Validation parameters:
+validation:
+  problem:
+    categories: C4
+    batch_size: 512
+    # In here we won't use images at all.
+    stream_images: False
+  dataloader:
+    num_workers: 0
+
+
+pipeline:
+
+  global_publisher:
+    priority: 0
+    type: GlobalVariablePublisher
+    # Add input_size to globals.
+    keys: [answer_word_embeddings_size]
+    values: [100]
+
+  # Answer encoding.
+  answer_tokenizer:
+    type: SentenceTokenizer
+    priority: 1.1
+    preprocessing: lowercase,remove_punctuation
+    remove_characters: [“,”,’]
+    streams: 
+      inputs: answers
+      outputs: tokenized_answer_words
+
+  # Model 1: Embeddings
+  answer_embeddings:
+    priority: 1.2
+    type: SentenceEmbeddings
+    embeddings_size: 100
+    pretrained_embeddings_file: glove.6B.100d.txt
+    data_folder: ~/data/vqa-med
+    word_mappings_file: answer_words.c4.preprocessed.word.mappings.csv
+    export_word_mappings_to_globals: True
+    streams:
+      inputs: tokenized_answer_words
+      outputs: encoded_answer_words
+    globals:
+      vocabulary_size: answer_words_vocabulary_size
+      word_mappings: answer_words_word_mappings
+
+  answer_reduction:
+    type: ReduceTensor
+    priority: 1.3
+    num_inputs_dims: 3
+    reduction_dim: 1
+    reduction_type: sum
+    keepdim: False
+    streams:
+      inputs: encoded_answer_words
+      outputs: reduced_answers
+    globals:
+      input_size: answer_word_embeddings_size
+
+  # Model.
+  classifier:
+    type: FeedForwardNetwork 
+    hidden_sizes: [500, 500]
+    dropout_rate: 0.5
+    priority: 3
+    streams:
+      inputs: reduced_answers
+    globals:
+      input_size: answer_word_embeddings_size
+      prediction_size: vocabulary_size_c4
+
+   # Viewers.
+  viewer:
+    type: StreamViewer
+    priority: 100.4
+    input_streams: answers, tokenized_answer_words, predicted_answers
+
+#: pipeline

configs/vqa_med_2019/c4_classification/c4_word_answer_mimic_sum.yml

@@ -0,0 +1,91 @@
+# Load config defining problems for training, validation and testing.
+default_configs: vqa_med_2019/c4_classification/default_c4_classification.yml
+
+# Training parameters:
+training:
+  problem:
+    categories: C4
+    batch_size: 512
+    # In here we won't use images at all.
+    stream_images: False
+  dataloader:
+    num_workers: 0
+
+# Validation parameters:
+validation:
+  problem:
+    categories: C4
+    batch_size: 512
+    # In here we won't use images at all.
+    stream_images: False
+  dataloader:
+    num_workers: 0
+
+
+pipeline:
+
+  global_publisher:
+    priority: 0
+    type: GlobalVariablePublisher
+    # Add input_size to globals.
+    keys: [answer_word_embeddings_size]
+    values: [300]
+
+  # Answer encoding.
+  answer_tokenizer:
+    type: SentenceTokenizer
+    priority: 1.1
+    preprocessing: lowercase,remove_punctuation
+    remove_characters: [“,”,’]
+    streams: 
+      inputs: answers
+      outputs: tokenized_answer_words
+
+  # Model 1: Embeddings
+  answer_embeddings:
+    priority: 1.2
+    type: SentenceEmbeddings
+    embeddings_size: 300
+    pretrained_embeddings_file: mimic.fastText.no_clean.300d.pickled
+    data_folder: ~/data/vqa-med
+    word_mappings_file: answer_words.c4.preprocessed.word.mappings.csv
+    export_word_mappings_to_globals: True
+    streams:
+      inputs: tokenized_answer_words
+      outputs: encoded_answer_words
+    globals:
+      vocabulary_size: answer_words_vocabulary_size
+      word_mappings: answer_words_word_mappings
+
+  answer_reduction:
+    type: ReduceTensor
+    priority: 1.3
+    num_inputs_dims: 3
+    reduction_dim: 1
+    reduction_type: sum
+    keepdim: False
+    streams:
+      inputs: encoded_answer_words
+      outputs: reduced_answers
+    globals:
+      input_size: answer_word_embeddings_size
+
+  # Model.
+  classifier:
+    type: FeedForwardNetwork 
+    hidden_sizes: [500, 500]
+    dropout_rate: 0.5
+    priority: 3
+    streams:
+      inputs: reduced_answers
+    globals:
+      input_size: answer_word_embeddings_size
+      prediction_size: vocabulary_size_c4
+
+   # Viewers.
+  viewer:
+    type: StreamViewer
+    priority: 100.4
+    input_streams: answers, tokenized_answer_words, predicted_answers
+
+#: pipeline

configs/vqa_med_2019/c4_classification/c4_word_answer_onehot_sum.yml

@@ -0,0 +1,91 @@
+# Load config defining problems for training, validation and testing.
+default_configs: vqa_med_2019/c4_classification/default_c4_classification.yml
+
+# Training parameters:
+training:
+  problem:
+    categories: C4
+    batch_size: 512
+    # In here we won't use images at all.
+    stream_images: False
+  dataloader:
+    num_workers: 0
+
+# Validation parameters:
+validation:
+  problem:
+    categories: C4
+    batch_size: 512
+    # In here we won't use images at all.
+    stream_images: False
+  dataloader:
+    num_workers: 0
+
+
+pipeline:
+  # Answer encoding.
+  answer_tokenizer:
+    type: SentenceTokenizer
+    priority: 1.1
+    preprocessing: lowercase,remove_punctuation
+    remove_characters: [“,”,’]
+    streams: 
+      inputs: answers
+      outputs: tokenized_answer_words
+
+  answer_onehot_encoder:
+    type: SentenceOneHotEncoder
+    priority: 1.2
+    data_folder: ~/data/vqa-med
+    word_mappings_file: answer_words.c4.preprocessed.word.mappings.csv
+    export_word_mappings_to_globals: True
+    streams:
+      inputs: tokenized_answer_words
+      outputs: encoded_answer_words
+    globals:
+      vocabulary_size: answer_words_vocabulary_size
+      word_mappings: answer_words_word_mappings
+
+  answer_to_tensor:
+    type: ListToTensor
+    priority: 1.3
+    num_inputs_dims: 3
+    streams:
+      inputs: encoded_answer_words
+      outputs: tensor_answer_words
+    globals:
+      input_size: answer_words_vocabulary_size
+
+
+  answer_reduction:
+    type: ReduceTensor
+    priority: 1.4
+    num_inputs_dims: 3
+    reduction_dim: 1
+    reduction_type: sum
+    keepdim: False
+    streams:
+      inputs: tensor_answer_words
+      outputs: reduced_answer_words
+    globals:
+      input_size: answer_words_vocabulary_size
+
+  # Model.
+  classifier:
+    type: FeedForwardNetwork 
+    hidden_sizes: [500, 500]
+    dropout_rate: 0.5
+    priority: 3
+    streams:
+      inputs: reduced_answer_words
+    globals:
+      input_size: answer_words_vocabulary_size
+      prediction_size: vocabulary_size_c4
+
+   # Viewers.
+  viewer:
+    type: StreamViewer
+    priority: 100.4
+    input_streams: answers, tokenized_answer_words, predicted_answers
+
+#: pipeline

configs/vqa_med_2019/c4_classification/default_c4_classification.yml

@@ -70,8 +70,8 @@ pipeline:
     type: PrecisionRecallStatistics
     priority: 100.3
     use_word_mappings: True
-    show_class_scores: True
-    show_confusion_matrix: True
+    #show_class_scores: True
+    #show_confusion_matrix: True
     streams:
       targets: answers_ids
     globals: