From e1622b468ff0e445ef47e63ca0733fe36d1bb209 Mon Sep 17 00:00:00 2001 From: tkornut Date: Fri, 31 May 2019 18:24:47 -0700 Subject: [PATCH 1/9] join masked fix, added image to class viewer, tested on mnist --- .../default/components/text/word_decoder.yml | 2 +- .../viewers/image_to_class_viewer.yml | 38 ++++++ configs/mnist/default_mnist.yml | 24 +++- ...nist_classification_vf_2lenet5_2losses.yml | 4 + ...n_vf_shared_convnet_2softmaxes_2losses.yml | 6 +- .../masking/join_masked_predictions.py | 2 +- .../image_to_class/image_to_class_problem.py | 37 ------ .../problems/image_to_class/mnist.py | 4 +- ptp/components/viewers/__init__.py | 2 + .../viewers/image_to_class_viewer.py | 123 ++++++++++++++++++ ptp/components/viewers/stream_viewer.py | 2 +- 11 files changed, 197 insertions(+), 47 deletions(-) create mode 100644 configs/default/components/viewers/image_to_class_viewer.yml create mode 100644 ptp/components/viewers/image_to_class_viewer.py diff --git a/configs/default/components/text/word_decoder.yml b/configs/default/components/text/word_decoder.yml index bb6e996..96064ec 100644 --- a/configs/default/components/text/word_decoder.yml +++ b/configs/default/components/text/word_decoder.yml @@ -36,7 +36,7 @@ streams: # Stream containing input tensor (INPUT) inputs: inputs - # Stream containing output tensor (OUTPUT) + # Stream containing output words (OUTPUT) outputs: outputs globals: diff --git a/configs/default/components/viewers/image_to_class_viewer.yml b/configs/default/components/viewers/image_to_class_viewer.yml new file mode 100644 index 0000000..b43b7bd --- /dev/null +++ b/configs/default/components/viewers/image_to_class_viewer.yml @@ -0,0 +1,38 @@ +# This file defines the default values for the ImageToClassViewer. + +#################################################################### +# 1. CONFIGURATION PARAMETERS that will be LOADED by the component. +#################################################################### + +# Number of sample that will be printed (LOADED) +# Default: -1 (means random) +sample_number: -1 + +streams: + #################################################################### + # 2. Keymappings associated with INPUT and OUTPUT streams. + #################################################################### + + # Stream containing inages (INPUT) + images: images + + # Stream containing target labels (strings) (INPUT) + label: labels + + # Stream containing predicted labels (strings) (INPUT) + answers: answers + + +globals: + #################################################################### + # 3. Keymappings of variables that will be RETRIEVED from GLOBALS. + #################################################################### + + #################################################################### + # 4. Keymappings associated with GLOBAL variables that will be SET. + #################################################################### + + #################################################################### + # 5. Keymappings associated with statistics that will be ADDED. + #################################################################### + diff --git a/configs/mnist/default_mnist.yml b/configs/mnist/default_mnist.yml index d8c849f..87e296a 100644 --- a/configs/mnist/default_mnist.yml +++ b/configs/mnist/default_mnist.yml @@ -50,20 +50,36 @@ pipeline: # Statistics. batch_size: - type: BatchSizeStatistics priority: 100.0 + type: BatchSizeStatistics accuracy: - type: AccuracyStatistics priority: 100.1 + type: AccuracyStatistics precision_recall: - type: PrecisionRecallStatistics priority: 100.2 + type: PrecisionRecallStatistics use_word_mappings: True show_class_scores: True globals: word_mappings: label_word_mappings - + answer_decoder: + priority: 100.3 + type: WordDecoder + import_word_mappings_from_globals: True + globals: + word_mappings: label_word_mappings + streams: + inputs: predictions + outputs: answers + + image_viewer: + priority: 100.4 + type: ImageToClassViewer + streams: + images: inputs + label: labels + answers: answers diff --git a/configs/mnist/mnist_classification_vf_2lenet5_2losses.yml b/configs/mnist/mnist_classification_vf_2lenet5_2losses.yml index 305df53..1d7d2a1 100644 --- a/configs/mnist/mnist_classification_vf_2lenet5_2losses.yml +++ b/configs/mnist/mnist_classification_vf_2lenet5_2losses.yml @@ -218,5 +218,9 @@ pipeline: recall: joined_recall f1score: joined_f1score + # "Fix" (overwrite) stream name in viewer. + image_viewer: + streams: + answers: merged_predictions #: pipeline diff --git a/configs/mnist/mnist_classification_vf_shared_convnet_2softmaxes_2losses.yml b/configs/mnist/mnist_classification_vf_shared_convnet_2softmaxes_2losses.yml index e934f64..7a9b51d 100644 --- a/configs/mnist/mnist_classification_vf_shared_convnet_2softmaxes_2losses.yml +++ b/configs/mnist/mnist_classification_vf_shared_convnet_2softmaxes_2losses.yml @@ -27,7 +27,7 @@ training: pipeline: # Disable components for "default" flow. - disable: nllloss, accuracy, precision_recall + disable: nllloss, accuracy, precision_recall, answer_decoder ################# SHARED ################# @@ -238,5 +238,9 @@ pipeline: recall: joined_recall f1score: joined_f1score + # "Fix" (overwrite) stream name in viewer. + image_viewer: + streams: + answers: merged_predictions #: pipeline diff --git a/ptp/components/masking/join_masked_predictions.py b/ptp/components/masking/join_masked_predictions.py index 0b418ea..07496b1 100644 --- a/ptp/components/masking/join_masked_predictions.py +++ b/ptp/components/masking/join_masked_predictions.py @@ -108,7 +108,7 @@ def output_data_definitions(self): """ return { self.key_output_indices: DataDefinition([-1], [torch.Tensor], "Batch of merged (output) indices [BATCH_SIZE]"), - self.key_output_strings: DataDefinition([-1], [torch.Tensor], "Batch of merged strings, corresponging to indices when using the provided word mappings [BATCH_SIZE]") + self.key_output_strings: DataDefinition([-1, 1], [list, str], "Batch of merged strings, corresponging to indices when using the provided word mappings [BATCH_SIZE] x [STRING]") } diff --git a/ptp/components/problems/image_to_class/image_to_class_problem.py b/ptp/components/problems/image_to_class/image_to_class_problem.py index b049322..94bcd08 100644 --- a/ptp/components/problems/image_to_class/image_to_class_problem.py +++ b/ptp/components/problems/image_to_class/image_to_class_problem.py @@ -17,7 +17,6 @@ __author__ = "Tomasz Kornuta, Younes Bouhadjar, Vincent Marois" -import numpy as np from ptp.components.problems.problem import Problem @@ -46,39 +45,3 @@ def __init__(self, name, class_type, config): # Call base class constructors. super(ImageToClassProblem, self).__init__(name, class_type, config) - def show_sample(self, data_dict, sample_number=0): - """ - Shows a sample from the batch. - - :param data_dict: ``DataDict`` containing inputs and targets. - :type data_dict: DataDict - - :param sample_number: Number of sample in batch (default: 0) - :type sample_number: int - - """ - import matplotlib.pyplot as plt - - # Unpack dict. - images, targets, labels = data_dict.values() - - # Get sample. - image = images[sample_number].cpu().detach().numpy() - target = targets[sample_number].cpu().detach().numpy() - label = labels[sample_number] - - # Reshape image. - if image.shape[0] == 1: - # This is a single channel image - get rid of this dimension - image = np.squeeze(image, axis=0) - else: - # More channels - move channels to axis2, according to matplotilb documentation. - # (X : array_like, shape (n, m) or (n, m, 3) or (n, m, 4)) - image = image.transpose(1, 2, 0) - - # Show data. - plt.title('Target class: {} ({})'.format(label, target)) - plt.imshow(image, interpolation='nearest', aspect='auto') - - # Plot! - plt.show() diff --git a/ptp/components/problems/image_to_class/mnist.py b/ptp/components/problems/image_to_class/mnist.py index 8d0d784..f6f75b6 100644 --- a/ptp/components/problems/image_to_class/mnist.py +++ b/ptp/components/problems/image_to_class/mnist.py @@ -21,10 +21,10 @@ import torch from torchvision import datasets, transforms -from ptp.components.problems.image_to_class.image_to_class_problem import ImageToClassProblem +from ptp.components.problems.problem import Problem from ptp.data_types.data_definition import DataDefinition -class MNIST(ImageToClassProblem): +class MNIST(Problem): """ Classic MNIST digit classification problem. diff --git a/ptp/components/viewers/__init__.py b/ptp/components/viewers/__init__.py index 0c18de7..62bdca5 100644 --- a/ptp/components/viewers/__init__.py +++ b/ptp/components/viewers/__init__.py @@ -1,5 +1,7 @@ +from .image_to_class_viewer import ImageToClassViewer from .stream_viewer import StreamViewer __all__ = [ + 'ImageToClassViewer', 'StreamViewer', ] diff --git a/ptp/components/viewers/image_to_class_viewer.py b/ptp/components/viewers/image_to_class_viewer.py new file mode 100644 index 0000000..70f7fca --- /dev/null +++ b/ptp/components/viewers/image_to_class_viewer.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) tkornuta, IBM Corporation 2019 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Tomasz Kornuta" + +import numpy as np +import torch +import matplotlib.pyplot as plt + +from ptp.configuration.config_parsing import get_value_list_from_dictionary +from ptp.components.component import Component +from ptp.data_types.data_definition import DataDefinition + + +class ImageToClassViewer(Component): + """ + Utility for displaying contents image along with label and prediction (a single sample from the batch). + """ + + def __init__(self, name, config): + """ + Initializes loss object. + + :param name: Loss name. + :type name: str + + :param config: Dictionary of parameters (read from the configuration ``.yaml`` file). + :type config: :py:class:`ptp.configuration.ConfigInterface` + + """ + # Call constructors of parent classes. + Component.__init__(self, name, ImageToClassViewer, config) + + # Get default key mappings. + self.key_indices = self.stream_keys["indices"] + self.key_images = self.stream_keys["images"] + self.key_labels = self.stream_keys["labels"] + self.key_answers = self.stream_keys["answers"] + + # Get sample number. + self.sample_number = self.config["sample_number"] + + + def input_data_definitions(self): + """ + Function returns a dictionary with definitions of input data that are required by the component. + + :return: dictionary containing input data definitions (each of type :py:class:`ptp.data_types.DataDefinition`). + """ + return { + self.key_indices: DataDefinition([-1, 1], [list, int], "Batch of sample indices [BATCH_SIZE] x [1]"), + self.key_images: DataDefinition([-1, -1, -1, -1], [torch.Tensor], "Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]"), + self.key_labels: DataDefinition([-1, 1], [list, str], "Batch of target labels, each being a single word [BATCH_SIZE] x [STRING]"), + self.key_answers: DataDefinition([-1, 1], [list, str], "Batch of predicted labels, each being a single word [BATCH_SIZE] x [STRING]") + } + + def output_data_definitions(self): + """ + Function returns a dictionary with definitions of output data produced the component. + + :return: dictionary containing output data definitions (each of type :py:class:`ptp.data_types.DataDefinition`). + """ + return { + } + + def __call__(self, data_dict): + """ + Shows a sample from the batch. + + :param data_dict: :py:class:`ptp.utils.DataDict` object. + + """ + # Use worker interval. + if self.app_state.episode % self.app_state.args.logging_interval == 0: + + # Get inputs + indices = data_dict[self.key_indices] + images = data_dict[self.key_images] + labels = data_dict[self.key_labels] + answers = data_dict[self.key_answers] + + # Get sample number. + if self.sample_number == -1: + # Random. + sample_number = np.random.randint(0, len(images)) + else: + sample_number = self.sample_number + + # Get "sample". + image = images[sample_number].cpu().data.numpy() + label = labels[sample_number] + answer = answers[sample_number] + + # Reshape image. + if image.shape[0] == 1: + # This is a single channel image - get rid of this dimension + image = np.squeeze(image, axis=0) + else: + # More channels - move channels to axis2, according to matplotilb documentation. + # (X : array_like, shape (n, m) or (n, m, 3) or (n, m, 4)) + image = image.transpose(1, 2, 0) + + # Show data. + plt.title('Sample: {} (index: {})\nPrediction: {} | Target: {}'.format(sample_number, indices[sample_number], answer, label)) + plt.imshow(image, interpolation='nearest', aspect='auto') + + # Plot! + plt.show() + + diff --git a/ptp/components/viewers/stream_viewer.py b/ptp/components/viewers/stream_viewer.py index 9d893cb..7c538ba 100644 --- a/ptp/components/viewers/stream_viewer.py +++ b/ptp/components/viewers/stream_viewer.py @@ -95,7 +95,7 @@ def __call__(self, data_dict): # Generate displayed string. absent_streams = [] - disp_str = "Showing selected streams for sample {}:\n".format(sample_number) + disp_str = "Showing selected streams for sample {} (index: {}):\n".format(sample_number, indices[sample_number]) for stream_key in self.input_stream_keys: if stream_key in data_dict.keys(): disp_str += " '{}': {}\n".format(stream_key, data_dict[stream_key][sample_number]) From 43e5a9f594abcaa3da003956a6c77e7614c78e20 Mon Sep 17 00:00:00 2001 From: tkornut Date: Fri, 31 May 2019 18:48:32 -0700 Subject: [PATCH 2/9] disabled visualization in mnist pipelines --- configs/mnist/default_mnist.yml | 7 +++++++ configs/mnist/mnist_classification_vf_2lenet5_2losses.yml | 7 +++++-- ...classification_vf_shared_convnet_2softmaxes_2losses.yml | 7 +++++-- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/configs/mnist/default_mnist.yml b/configs/mnist/default_mnist.yml index 87e296a..1f7a30a 100644 --- a/configs/mnist/default_mnist.yml +++ b/configs/mnist/default_mnist.yml @@ -42,6 +42,7 @@ test: #resize: [32, 32] pipeline: + disable: image_viewer # Loss nllloss: @@ -83,3 +84,9 @@ pipeline: images: inputs label: labels answers: answers + + stream_viewer: + priority: 100.5 + type: StreamViewer + input_streams: labels, answers + diff --git a/configs/mnist/mnist_classification_vf_2lenet5_2losses.yml b/configs/mnist/mnist_classification_vf_2lenet5_2losses.yml index 1d7d2a1..783283b 100644 --- a/configs/mnist/mnist_classification_vf_2lenet5_2losses.yml +++ b/configs/mnist/mnist_classification_vf_2lenet5_2losses.yml @@ -20,7 +20,7 @@ test: pipeline: # Disable components for "default" flow. - disable: nllloss, accuracy, precision_recall + disable: nllloss, accuracy, precision_recall, image_viewer # Add global variables. global_publisher: @@ -218,9 +218,12 @@ pipeline: recall: joined_recall f1score: joined_f1score - # "Fix" (overwrite) stream name in viewer. + # "Fix" (overwrite) stream names in viewers. image_viewer: streams: answers: merged_predictions + stream_viewer: + input_streams: labels, merged_predictions + #: pipeline diff --git a/configs/mnist/mnist_classification_vf_shared_convnet_2softmaxes_2losses.yml b/configs/mnist/mnist_classification_vf_shared_convnet_2softmaxes_2losses.yml index 7a9b51d..efea8a4 100644 --- a/configs/mnist/mnist_classification_vf_shared_convnet_2softmaxes_2losses.yml +++ b/configs/mnist/mnist_classification_vf_shared_convnet_2softmaxes_2losses.yml @@ -27,7 +27,7 @@ training: pipeline: # Disable components for "default" flow. - disable: nllloss, accuracy, precision_recall, answer_decoder + disable: nllloss, accuracy, precision_recall, answer_decoder, image_viewer ################# SHARED ################# @@ -238,9 +238,12 @@ pipeline: recall: joined_recall f1score: joined_f1score - # "Fix" (overwrite) stream name in viewer. + # "Fix" (overwrite) stream names in viewers. image_viewer: streams: answers: merged_predictions + stream_viewer: + input_streams: labels, merged_predictions + #: pipeline From 808f839e6f36ca1f71e0f7cef973a7fc5363217b Mon Sep 17 00:00:00 2001 From: tkornut Date: Fri, 31 May 2019 21:00:42 -0700 Subject: [PATCH 3/9] work on cifar - still wrong labels --- ...ifar100_classification_convnet_softmax.yml | 34 ++++++ configs/cifar100/default_cifar100.yml | 101 ++++++++++++++++++ .../problems/image_to_class/mnist.yml | 3 + configs/mnist/default_mnist.yml | 16 +-- .../problems/image_to_class/__init__.py | 5 +- .../image_to_class/image_to_class_problem.py | 47 -------- .../problems/image_to_class/mnist.py | 4 +- 7 files changed, 150 insertions(+), 60 deletions(-) create mode 100644 configs/cifar100/cifar100_classification_convnet_softmax.yml create mode 100644 configs/cifar100/default_cifar100.yml delete mode 100644 ptp/components/problems/image_to_class/image_to_class_problem.py diff --git a/configs/cifar100/cifar100_classification_convnet_softmax.yml b/configs/cifar100/cifar100_classification_convnet_softmax.yml new file mode 100644 index 0000000..c7a0442 --- /dev/null +++ b/configs/cifar100/cifar100_classification_convnet_softmax.yml @@ -0,0 +1,34 @@ +# Load config defining CIFAR100 problems for training, validation and testing. +default_configs: cifar100/default_cifar100.yml + +# Definition of the pipeline. +pipeline: + + # Model consisting of two components. + image_encoder: + priority: 1.1 + type: ConvNetEncoder + + # Reshape inputs + reshaper: + priority: 1.2 + type: ReshapeTensor + input_dims: [-1, 16, 2, 2] + output_dims: [-1, 64] + streams: + inputs: feature_maps + outputs: reshaped_maps + globals: + output_size: reshaped_maps_size + + # Image classifier. + classifier: + priority: 1.3 + type: FeedForwardNetwork + streams: + inputs: reshaped_maps + globals: + input_size: reshaped_maps_size + prediction_size: num_fine_classes + +#: pipeline diff --git a/configs/cifar100/default_cifar100.yml b/configs/cifar100/default_cifar100.yml new file mode 100644 index 0000000..4ffe3b2 --- /dev/null +++ b/configs/cifar100/default_cifar100.yml @@ -0,0 +1,101 @@ +# Training parameters: +training: + problem: + type: CIFAR100 + batch_size: &b 1 + use_train_data: True + # Use sampler that operates on a subset. + dataloader: + shuffle: False + #sampler: + # type: SubsetRandomSampler + # indices: [0, 55000] + # optimizer parameters: + optimizer: + type: Adam + lr: 0.0001 + # settings parameters + terminal_conditions: + loss_stop_threshold: 0.05 + early_stop_validations: -1 + episode_limit: 10000 + epoch_limit: 10 + +# Validation parameters: +validation: + #partial_validation_interval: 100 + problem: + type: CIFAR100 + batch_size: *b + use_train_data: True # True because we are splitting the training set to: validation and training + #resize: [32, 32] + # Use sampler that operates on a subset. + #sampler: + # type: SubsetRandomSampler + # indices: [55000, 60000] + +# Testing parameters: +test: + problem: + type: MNIST + batch_size: *b + use_train_data: False + #resize: [32, 32] + +pipeline: + #disable: image_viewer + + # Loss + nllloss: + type: NLLLoss + priority: 10.0 + streams: + targets: fine_targets + + # Statistics. + batch_size: + priority: 100.0 + type: BatchSizeStatistics + streams: + targets: fine_targets + + accuracy: + priority: 100.1 + type: AccuracyStatistics + streams: + targets: fine_targets + + + precision_recall: + priority: 100.2 + type: PrecisionRecallStatistics + use_word_mappings: True + #show_class_scores: True + globals: + word_mappings: fine_label_word_mappings + streams: + targets: fine_targets + + answer_decoder: + priority: 100.3 + type: WordDecoder + import_word_mappings_from_globals: True + globals: + word_mappings: fine_label_word_mappings + streams: + inputs: predictions + outputs: answers + + stream_viewer: + priority: 100.4 + type: StreamViewer + input_streams: coarse_targets, coarse_labels, fine_targets, fine_labels, answers + + image_viewer: + priority: 100.5 + type: ImageToClassViewer + streams: + images: inputs + labels: fine_labels + answers: coarse_labels + diff --git a/configs/default/components/problems/image_to_class/mnist.yml b/configs/default/components/problems/image_to_class/mnist.yml index eb82902..3dacab6 100644 --- a/configs/default/components/problems/image_to_class/mnist.yml +++ b/configs/default/components/problems/image_to_class/mnist.yml @@ -30,6 +30,9 @@ streams: # Stream containing targets (label ids) (OUTPUT) targets: targets + # Stream containing labels (words) (OUTPUT) + labels: labels + globals: #################################################################### # 3. Keymappings of variables that will be RETRIEVED from GLOBALS. diff --git a/configs/mnist/default_mnist.yml b/configs/mnist/default_mnist.yml index 1f7a30a..6cf6c94 100644 --- a/configs/mnist/default_mnist.yml +++ b/configs/mnist/default_mnist.yml @@ -16,7 +16,7 @@ training: # settings parameters terminal_conditions: loss_stop_threshold: 0.05 - early_stop_validations: 10 + early_stop_validations: -1 episode_limit: 10000 epoch_limit: 10 @@ -77,16 +77,16 @@ pipeline: inputs: predictions outputs: answers - image_viewer: + stream_viewer: priority: 100.4 + type: StreamViewer + input_streams: labels, answers + + image_viewer: + priority: 100.5 type: ImageToClassViewer streams: images: inputs - label: labels + labels: labels answers: answers - stream_viewer: - priority: 100.5 - type: StreamViewer - input_streams: labels, answers - diff --git a/ptp/components/problems/image_to_class/__init__.py b/ptp/components/problems/image_to_class/__init__.py index cad9c83..72b54db 100644 --- a/ptp/components/problems/image_to_class/__init__.py +++ b/ptp/components/problems/image_to_class/__init__.py @@ -1,9 +1,8 @@ #from .cifar10 import CIFAR10 -from .image_to_class_problem import ImageToClassProblem +from .cifar_100 import CIFAR100 from .mnist import MNIST __all__ = [ -# 'CIFAR10', - 'ImageToClassProblem', + 'CIFAR100', 'MNIST' ] diff --git a/ptp/components/problems/image_to_class/image_to_class_problem.py b/ptp/components/problems/image_to_class/image_to_class_problem.py deleted file mode 100644 index 94bcd08..0000000 --- a/ptp/components/problems/image_to_class/image_to_class_problem.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright (C) IBM Corporation 2018 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Tomasz Kornuta, Younes Bouhadjar, Vincent Marois" - - -from ptp.components.problems.problem import Problem - -class ImageToClassProblem(Problem): - """ - Abstract base class for image classification problems. - - Problem classes like MNIST & CIFAR10 inherits from it. - - Provides some basic features useful in all problems of such type. - - """ - - def __init__(self, name, class_type, config): - """ - Initializes problem. - - :param name: Problem name. - :type name: str - - :param class_type: Class type of the component. - - :param config: Dictionary of parameters (read from the configuration ``.yaml`` file). - :type config: :py:class:`ptp.configuration.ConfigInterface` - """ - # Call base class constructors. - super(ImageToClassProblem, self).__init__(name, class_type, config) - diff --git a/ptp/components/problems/image_to_class/mnist.py b/ptp/components/problems/image_to_class/mnist.py index f6f75b6..5d9c3fc 100644 --- a/ptp/components/problems/image_to_class/mnist.py +++ b/ptp/components/problems/image_to_class/mnist.py @@ -105,11 +105,11 @@ def __init__(self, name, config): # Class names. labels = 'Zero One Two Three Four Five Six Seven Eight Nine'.split(' ') + # Export to globals. word_to_ix = {labels[i]: i for i in range(10)} + self.globals["label_word_mappings"] = word_to_ix # Reverse mapping - for labels. self.ix_to_word = {value: key for (key, value) in word_to_ix.items()} - # Export to globals. - self.globals["label_word_mappings"] = word_to_ix def __len__(self): """ From a12f48f57dfa7afeb7e8c3a2bcb3619c371b46d7 Mon Sep 17 00:00:00 2001 From: tkornut Date: Fri, 31 May 2019 22:00:51 -0700 Subject: [PATCH 4/9] cifar still not working --- ptp/components/utils/io.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ptp/components/utils/io.py b/ptp/components/utils/io.py index 3c5dc30..2609fe4 100644 --- a/ptp/components/utils/io.py +++ b/ptp/components/utils/io.py @@ -25,16 +25,17 @@ import pickle -def load_pickle(logger, filename): +def load_pickle(logger, filename, encoding="ASCII"): """ Loads picke from file. :param logger: Logger object. :param filename: Absolute path along with the name of the file to be loaded. + :param encoding: file encoding (DEFAULT: "ASCII") """ try: with open(str(filename), 'rb') as f: - obj = pickle.load(f) + obj = pickle.load(f, encoding=encoding) logger.info('Loaded: %s', filename) except EOFError: From 12afbe4e136fbb4761a9decfd0ac87551a2a7fb3 Mon Sep 17 00:00:00 2001 From: tkornut Date: Fri, 31 May 2019 23:35:52 -0700 Subject: [PATCH 5/9] refined the names of categories, seem to be ok --- .../problems/image_to_class/cifar_100.yml | 65 +++++ .../problems/image_to_class/cifar_100.py | 251 ++++++++++++++++++ 2 files changed, 316 insertions(+) create mode 100644 configs/default/components/problems/image_to_class/cifar_100.yml create mode 100644 ptp/components/problems/image_to_class/cifar_100.py diff --git a/configs/default/components/problems/image_to_class/cifar_100.yml b/configs/default/components/problems/image_to_class/cifar_100.yml new file mode 100644 index 0000000..957b422 --- /dev/null +++ b/configs/default/components/problems/image_to_class/cifar_100.yml @@ -0,0 +1,65 @@ +# This file defines the default values for the CIFAR-100 problem. + +#################################################################### +# 1. CONFIGURATION PARAMETERS that will be LOADED by the component. +#################################################################### + +# Folder where problem will store data (LOADED) +data_folder: '~/data/cifar-100' + +# Defines the set that will be used used (LOADED) +# True: training set | False: test set. +use_train_data: True + +# Optional parameter (LOADED) +# When present, resizes the CIFAR images from [32,32] to [width, height] +#resize_image: [height, width] + +streams: + #################################################################### + # 2. Keymappings associated with INPUT and OUTPUT streams. + #################################################################### + + # Stream containing batch of indices (OUTPUT) + # Every problem MUST return that stream. + indices: indices + + # Stream containing batch of images (OUTPUT) + images: images + + # Streams containing targets (label ids) (OUTPUT) + coarse_targets: coarse_targets + fine_targets: fine_targets + + # Streams containing labels (words) (OUTPUT) + coarse_labels: coarse_labels + fine_labels: fine_labels + +globals: + #################################################################### + # 3. Keymappings of variables that will be RETRIEVED from GLOBALS. + #################################################################### + + #################################################################### + # 4. Keymappings associated with GLOBAL variables that will be SET. + #################################################################### + + # Width of the image (SET) + input_width: image_width + # Height of the image (SET) + input_height: image_height + # Depth of the image (SET) + input_depth: image_depth + + # Numbers of output classes (SET) + coarse_num_classes: coarse_num_classes + fine_num_classes: fine_num_classes + + # Labels (word-idx) mappings (SET) + coarse_label_word_mappings: coarse_label_word_mappings + fine_label_word_mappings: fine_label_word_mappings + + #################################################################### + # 5. Keymappings associated with statistics that will be ADDED. + #################################################################### + diff --git a/ptp/components/problems/image_to_class/cifar_100.py b/ptp/components/problems/image_to_class/cifar_100.py new file mode 100644 index 0000000..829c6e5 --- /dev/null +++ b/ptp/components/problems/image_to_class/cifar_100.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright (C) IBM Corporation 2018 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Tomasz Kornuta" + +import os +import numpy as np +import gzip +import torch +from torchvision import datasets, transforms + +from ptp.components.problems.problem import Problem +from ptp.data_types.data_definition import DataDefinition + +from ptp.components.utils.io import download, load_pickle + +class CIFAR100(Problem): + """ + Classic CIFAR-100 image classification problem. + + Reference page: http://www.cs.toronto.edu/~kriz/cifar.html + + + """ + + def __init__(self, name, config): + """ + Initializes the problem. + + .. warning:: + + Resizing images might cause a significant slow down in batch generation. + + :param name: Problem name. + :type name: str + + :param config: Dictionary of parameters (read from the configuration ``.yaml`` file). + :type config: :py:class:`ptp.configuration.ConfigInterface` + """ + + # Call base class constructors. + super(CIFAR100, self).__init__(name, CIFAR100, config) + + # Get default key mappings. + self.key_inputs = self.stream_keys["inputs"] + # Targets. + self.key_coarse_targets = self.stream_keys["coarse_targets"] + self.key_fine_targets = self.stream_keys["fine_targets"] + # Streams returning targets as words (labels). + self.key_coarse_labels = self.stream_keys["coarse_labels"] + self.key_fine_labels = self.stream_keys["fine_labels"] + + # Get absolute path. + data_folder = os.path.expanduser(self.config['data_folder']) + + # Retrieve parameters from the dictionary. + self.use_train_data = self.config['use_train_data'] + + # Add transformations depending on the resizing option. + if 'resize_image' in self.config: + # Check the desired size. + if len(self.config['resize_image']) != 2: + self.logger.error("'resize_image' field must contain 2 values: the desired height and width") + exit(-1) + + # Output image dimensions. + self.height = self.config['resize_image'][0] + self.width = self.config['resize_image'][1] + + # Up-scale and transform to tensors. + transform = transforms.Compose([transforms.Resize((self.height, self.width)), transforms.ToTensor()]) + + self.logger.warning('Upscaling the images to [{}, {}]. Slows down batch generation.'.format( + self.width, self.height)) + + else: + # Default CIFAR-100 settings. + self.width = 32 + self.height = 32 + # Simply turn to tensor. + transform = transforms.Compose([transforms.ToTensor()]) + + # Load the dataset. (PROBLEM WITH COARSE-TO-FINE LABEL MAPPING!) + self.dataset = datasets.CIFAR100(root=data_folder, train=self.use_train_data, download=True, transform=transform) + self.my_dataset = {} + fine_categories = set() + for image, fine_target in self.dataset: + fine_categories.add(fine_target) + #if fine_target == 26: + self.my_dataset[len(self.my_dataset)] = [image, fine_target] + + print(len(fine_categories)) + + fine_label_names = "apple aquarium_fish baby bear beaver bed bee beetle bicycle bottle \ +bowl boy bridge bus butterfly camel can castle caterpillar cattle chairq \ +chimpanzee clock cloud cockroach couch crab crocodileq cup dinosaur dolphin elephant flatfish forest fox girl hamster house kangaroo keyboard lamp \ +lawn_mower leopard lion lizard lobster man maple_treeq motorcycle mountainq mouse mushroom oak_tree \ +orange orchid otter palm_tree pear pickup_truck pine_tree plain plate poppy porcupineq possum rabbit raccoon ray road rocketq rose sea seal shark shrew skunk \ +skyscraper snail snake spider squirrel streetcar sunflower sweet_pepper table tank telephone \ +television tiger tractor train trout tulip turtle wardrobe whale willow_tree wolf woman worm".split(" ") + print(len(fine_label_names)) + + + fine_labels_from_file = "apple aquarium_fish baby bear beaver bed bee bow boy bridge bus \ +butterfly camel can castle caterpillar cattle chair chimpanzee clock cloud \ +cockroach couch crab crocodile cup dinosaur dolphin elephant flatfish forest \ +fox girl hamster house kangaroo keyboard lamp lawn_mower leopard lion \ +lizard lobster man maple_tree motorcycle mountain mouse mushroom oak_tree orange \ +orchid otter palm_tree pear pickup_truck pine_tree plain plate poppy porcupine \ +possum rabbit raccoon ray road rocket rose sea seal shark \ +shrew skunk skyscraper snail snake spider squirrel streetcar sunflower sweet_pepper \ +table tank telephone \ +actor train trout tulip turtle wardrobe whale willow_tree wolf woman worm".split(" ") + print(len(fine_labels_from_file)) + + coarse_labels_from_file = "aquatic_mammals fish flowers food_containers fruit_and_vegetables household_electrical_devices household_furniture insects \ +large_carnivores large_man-made_outdoor_things large_natural_outdoor_scenes large_omnivores_and_herbivores medium_mammal non-insect_invertebrates people reptiles small_mammals trees vehicles_1 vehicles_2".split(" ") + print(len(coarse_labels_from_file)) + for label in coarse_labels_from_file: + print(label) + + # Process labels. + all_labels = {"aquatic_mammals": "beaver, dolphin, otter, seal, whale".split(", "), + "fish": "aquarium_fish, flatfish, ray, shark, trout".split(", "), + "flowers": "orchid, poppy, rose, sunflower, tulip".split(", "), + "food_containers": "bottle, bowl, can, cup, plate".split(", "), + "fruit_and_vegetables": "apple, mushroom, orange, pear, sweet_pepper".split(", "), + "household_electrical_devices": "clock, keyboard, lamp, telephone, television".split(", "), + "household_furniture": "bed, chair, couch, table, wardrobe".split(", "), + "insects": "bee, beetle, butterfly, caterpillar, cockroach".split(", "), + "large_carnivores": "bear, leopard, lion, tiger, wolf".split(", "), + "large_man-made_outdoor_things": "bridge, castle, house, road, skyscraper".split(", "), + "large_natural_outdoor_scenes": "cloud, forest, mountain, plain, sea".split(", "), + "large_omnivores_and_herbivores": "camel, cattle, chimpanzee, elephant, kangaroo".split(", "), + "medium-sized_mammals": "fox, porcupine, possum, raccoon, skunk".split(", "), + "non-insect_invertebrates": "crab, lobster, snail, spider, worm".split(", "), + "people": "baby, boy, girl, man, woman".split(", "), + "reptiles": "crocodile, dinosaur, lizard, snake, turtle".split(", "), + "small_mammals": "hamster, mouse, rabbit, shrew, squirrel".split(", "), + "trees": "maple_tree, oak_tree, palm_tree, pine_tree, willow_tree".split(", "), + "vehicles_1": "bicycle, bus, motorcycle, pickup_truck, train".split(", "), + "vehicles_2": "lawn_mower, rocket, streetcar, tank, tractor".split(", ")} + + coarse_word_to_ix = {} + fine_to_coarse_mapping = {} + fine_labels = [] + for coarse_id, (key, values) in enumerate(all_labels.items()): + # Add mapping from coarse category name to coarse id. + coarse_word_to_ix[key] = coarse_id + # Add mappings from fine category names to coarse id. + for value in values: + fine_to_coarse_mapping[value] = coarse_id + # Add values to list of fine labels. + fine_labels.extend(values) + + # Sort fine labels. + fine_labels = sorted(fine_labels) + print(len(fine_labels)) + + # Generate fine word mappings. + fine_word_to_ix = {fine_labels[i]: i for i in range(len(fine_labels))} + # Export fine word mappings to globals. + self.globals["fine_label_word_mappings"] = fine_word_to_ix + # Reverse mapping - for labels. + self.fine_ix_to_word = {value: key for (key, value) in fine_word_to_ix.items()} + + # Export coarse word mappings to globals. + self.globals["coarse_label_word_mappings"] = coarse_word_to_ix + # Reverse mapping - for labels. + self.coarse_ix_to_word = {value: key for (key, value) in coarse_word_to_ix.items()} + + # Create fine to coarse id mapping. + self.fine_to_coarse_id_mapping = {} + for fine_label, fine_id in fine_word_to_ix.items(): + self.fine_to_coarse_id_mapping[fine_id] = fine_to_coarse_mapping[fine_label] + print(" {} ({}) : {} ".format(fine_label, fine_id, self.coarse_ix_to_word[fine_to_coarse_mapping[fine_label]])) + + # Set global variables - all dimensions ASIDE OF BATCH. + self.globals["num_coarse_classes"] = len(coarse_word_to_ix) + self.globals["num_fine_classes"] = len(fine_labels) + self.globals["image_width"] = self.width + self.globals["image_height"] = self.height + self.globals["image_depth"] = 3 + + def __len__(self): + """ + Returns the "size" of the "problem" (total number of samples). + + :return: The size of the problem. + """ + #return len(self.dataset) + return len(self.my_dataset) + + def output_data_definitions(self): + """ + Function returns a dictionary with definitions of output data produced the component. + + :return: dictionary containing output data definitions (each of type :py:class:`ptp.utils.DataDefinition`). + """ + return { + self.key_indices: DataDefinition([-1, 1], [list, int], "Batch of sample indices [BATCH_SIZE] x [1]"), + self.key_inputs: DataDefinition([-1, 3, self.height, self.width], [torch.Tensor], "Batch of images [BATCH_SIZE x IMAGE_DEPTH x IMAGE_HEIGHT x IMAGE_WIDTH]"), + self.key_coarse_targets: DataDefinition([-1], [torch.Tensor], "Batch of coarse targets, each being a single index [BATCH_SIZE]"), + self.key_coarse_labels: DataDefinition([-1, 1], [list, str], "Batch of coarse labels, each being a single word [BATCH_SIZE] x [STRING]"), + self.key_fine_targets: DataDefinition([-1], [torch.Tensor], "Batch of fine targets, each being a single index [BATCH_SIZE]"), + self.key_fine_labels: DataDefinition([-1, 1], [list, str], "Batch of fine labels, each being a single word [BATCH_SIZE] x [STRING]") + } + + + def __getitem__(self, index): + """ + Getter method to access the dataset and return a sample. + + :param index: index of the sample to return. + :type index: int + + :return: ``DataDict({'images','targets'})``, with: + + - images: Image, resized if ``self.resize`` is set, + - targets: Index of the target class + """ + # Get image and fine label id. + #image, fine_target = self.dataset.__getitem__(index) + image, fine_target = self.my_dataset[index] + + # Return data_dict. + data_dict = self.create_data_dict(index) + data_dict[self.key_inputs] = image + # Targets. + data_dict[self.key_coarse_targets] = self.fine_to_coarse_id_mapping[fine_target] + data_dict[self.key_fine_targets] = fine_target + # Labels. + data_dict[self.key_coarse_labels] = self.coarse_ix_to_word[self.fine_to_coarse_id_mapping[fine_target]] + data_dict[self.key_fine_labels] = self.fine_ix_to_word[fine_target] + + #print(data_dict) + return data_dict From 6920212f566d4aa6a74cd23a0433aab9f809a853 Mon Sep 17 00:00:00 2001 From: tkornut Date: Fri, 31 May 2019 23:48:42 -0700 Subject: [PATCH 6/9] Cleanup CIFAR100 --- configs/cifar100/default_cifar100.yml | 22 ++++----- .../problems/image_to_class/cifar_100.py | 45 ++----------------- 2 files changed, 14 insertions(+), 53 deletions(-) diff --git a/configs/cifar100/default_cifar100.yml b/configs/cifar100/default_cifar100.yml index 4ffe3b2..f4ad2e3 100644 --- a/configs/cifar100/default_cifar100.yml +++ b/configs/cifar100/default_cifar100.yml @@ -2,18 +2,18 @@ training: problem: type: CIFAR100 - batch_size: &b 1 + batch_size: &b 128 use_train_data: True # Use sampler that operates on a subset. - dataloader: - shuffle: False - #sampler: - # type: SubsetRandomSampler - # indices: [0, 55000] + #dataloader: + # shuffle: False + sampler: + type: SubsetRandomSampler + indices: [0, 45000] # optimizer parameters: optimizer: type: Adam - lr: 0.0001 + lr: 0.001 # settings parameters terminal_conditions: loss_stop_threshold: 0.05 @@ -30,9 +30,9 @@ validation: use_train_data: True # True because we are splitting the training set to: validation and training #resize: [32, 32] # Use sampler that operates on a subset. - #sampler: - # type: SubsetRandomSampler - # indices: [55000, 60000] + sampler: + type: SubsetRandomSampler + indices: [45000, 50000] # Testing parameters: test: @@ -43,7 +43,7 @@ test: #resize: [32, 32] pipeline: - #disable: image_viewer + disable: image_viewer # Loss nllloss: diff --git a/ptp/components/problems/image_to_class/cifar_100.py b/ptp/components/problems/image_to_class/cifar_100.py index 829c6e5..f6ad8b6 100644 --- a/ptp/components/problems/image_to_class/cifar_100.py +++ b/ptp/components/problems/image_to_class/cifar_100.py @@ -96,42 +96,6 @@ def __init__(self, name, config): # Load the dataset. (PROBLEM WITH COARSE-TO-FINE LABEL MAPPING!) self.dataset = datasets.CIFAR100(root=data_folder, train=self.use_train_data, download=True, transform=transform) - self.my_dataset = {} - fine_categories = set() - for image, fine_target in self.dataset: - fine_categories.add(fine_target) - #if fine_target == 26: - self.my_dataset[len(self.my_dataset)] = [image, fine_target] - - print(len(fine_categories)) - - fine_label_names = "apple aquarium_fish baby bear beaver bed bee beetle bicycle bottle \ -bowl boy bridge bus butterfly camel can castle caterpillar cattle chairq \ -chimpanzee clock cloud cockroach couch crab crocodileq cup dinosaur dolphin elephant flatfish forest fox girl hamster house kangaroo keyboard lamp \ -lawn_mower leopard lion lizard lobster man maple_treeq motorcycle mountainq mouse mushroom oak_tree \ -orange orchid otter palm_tree pear pickup_truck pine_tree plain plate poppy porcupineq possum rabbit raccoon ray road rocketq rose sea seal shark shrew skunk \ -skyscraper snail snake spider squirrel streetcar sunflower sweet_pepper table tank telephone \ -television tiger tractor train trout tulip turtle wardrobe whale willow_tree wolf woman worm".split(" ") - print(len(fine_label_names)) - - - fine_labels_from_file = "apple aquarium_fish baby bear beaver bed bee bow boy bridge bus \ -butterfly camel can castle caterpillar cattle chair chimpanzee clock cloud \ -cockroach couch crab crocodile cup dinosaur dolphin elephant flatfish forest \ -fox girl hamster house kangaroo keyboard lamp lawn_mower leopard lion \ -lizard lobster man maple_tree motorcycle mountain mouse mushroom oak_tree orange \ -orchid otter palm_tree pear pickup_truck pine_tree plain plate poppy porcupine \ -possum rabbit raccoon ray road rocket rose sea seal shark \ -shrew skunk skyscraper snail snake spider squirrel streetcar sunflower sweet_pepper \ -table tank telephone \ -actor train trout tulip turtle wardrobe whale willow_tree wolf woman worm".split(" ") - print(len(fine_labels_from_file)) - - coarse_labels_from_file = "aquatic_mammals fish flowers food_containers fruit_and_vegetables household_electrical_devices household_furniture insects \ -large_carnivores large_man-made_outdoor_things large_natural_outdoor_scenes large_omnivores_and_herbivores medium_mammal non-insect_invertebrates people reptiles small_mammals trees vehicles_1 vehicles_2".split(" ") - print(len(coarse_labels_from_file)) - for label in coarse_labels_from_file: - print(label) # Process labels. all_labels = {"aquatic_mammals": "beaver, dolphin, otter, seal, whale".split(", "), @@ -169,7 +133,6 @@ def __init__(self, name, config): # Sort fine labels. fine_labels = sorted(fine_labels) - print(len(fine_labels)) # Generate fine word mappings. fine_word_to_ix = {fine_labels[i]: i for i in range(len(fine_labels))} @@ -187,7 +150,7 @@ def __init__(self, name, config): self.fine_to_coarse_id_mapping = {} for fine_label, fine_id in fine_word_to_ix.items(): self.fine_to_coarse_id_mapping[fine_id] = fine_to_coarse_mapping[fine_label] - print(" {} ({}) : {} ".format(fine_label, fine_id, self.coarse_ix_to_word[fine_to_coarse_mapping[fine_label]])) + #print(" {} ({}) : {} ".format(fine_label, fine_id, self.coarse_ix_to_word[fine_to_coarse_mapping[fine_label]])) # Set global variables - all dimensions ASIDE OF BATCH. self.globals["num_coarse_classes"] = len(coarse_word_to_ix) @@ -202,8 +165,7 @@ def __len__(self): :return: The size of the problem. """ - #return len(self.dataset) - return len(self.my_dataset) + return len(self.dataset) def output_data_definitions(self): """ @@ -234,8 +196,7 @@ def __getitem__(self, index): - targets: Index of the target class """ # Get image and fine label id. - #image, fine_target = self.dataset.__getitem__(index) - image, fine_target = self.my_dataset[index] + image, fine_target = self.dataset.__getitem__(index) # Return data_dict. data_dict = self.create_data_dict(index) From 2f3a02de48665e769950f5da3c7182ee95407bc8 Mon Sep 17 00:00:00 2001 From: tkornut Date: Fri, 31 May 2019 23:57:40 -0700 Subject: [PATCH 7/9] adding matplotlib to setup --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index ca9a2e7..0f47e22 100644 --- a/setup.py +++ b/setup.py @@ -170,6 +170,7 @@ 'torchvision==0.2.1', 'torch==1.0.1', 'PyYAML', + 'matplotlib', 'requests' ], From e8a723676ae12048fed4b53aec388b5c11dc5fa4 Mon Sep 17 00:00:00 2001 From: tkornut Date: Tue, 4 Jun 2019 10:18:09 -0700 Subject: [PATCH 8/9] lgtm fixes --- ptp/components/problems/image_to_class/cifar_100.py | 4 ---- ptp/components/viewers/image_to_class_viewer.py | 1 - 2 files changed, 5 deletions(-) diff --git a/ptp/components/problems/image_to_class/cifar_100.py b/ptp/components/problems/image_to_class/cifar_100.py index f6ad8b6..a17f785 100644 --- a/ptp/components/problems/image_to_class/cifar_100.py +++ b/ptp/components/problems/image_to_class/cifar_100.py @@ -18,16 +18,12 @@ __author__ = "Tomasz Kornuta" import os -import numpy as np -import gzip import torch from torchvision import datasets, transforms from ptp.components.problems.problem import Problem from ptp.data_types.data_definition import DataDefinition -from ptp.components.utils.io import download, load_pickle - class CIFAR100(Problem): """ Classic CIFAR-100 image classification problem. diff --git a/ptp/components/viewers/image_to_class_viewer.py b/ptp/components/viewers/image_to_class_viewer.py index 70f7fca..aacea46 100644 --- a/ptp/components/viewers/image_to_class_viewer.py +++ b/ptp/components/viewers/image_to_class_viewer.py @@ -20,7 +20,6 @@ import torch import matplotlib.pyplot as plt -from ptp.configuration.config_parsing import get_value_list_from_dictionary from ptp.components.component import Component from ptp.data_types.data_definition import DataDefinition From 55464c4df46bb3d5bbc82dfa3056f9f6e675942f Mon Sep 17 00:00:00 2001 From: tkornut Date: Tue, 4 Jun 2019 10:20:29 -0700 Subject: [PATCH 9/9] update readme - cifar --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7151986..fd31e8d 100644 --- a/README.md +++ b/README.md @@ -22,10 +22,10 @@ In its core, to _accelerate the computations_ on their own, PTP relies on PyTorc The models are _agnostic_ to those operations and one indicates whether to use them in configuration files (data loaders) or by passing adequate run-time arguments (--gpu). **Datasets:** -PTP focuses on multi-modal reasoning combining vision and language. Currently it offers the following _Problems_ from both domains: +PTP focuses on multi-modal reasoning combining vision and language. Currently it offers the following _Problems_ from the following problem domains: * ImageCLEF VQA-Med 2019 (Visual Question Answering) - * MNIST (Image Classification) + * MNIST, CIFAR-100 (Image Classification) * WiLY (Language Identification) * WikiText-2 / WikiText-103 (Language Modelling) * ANKI (Machine Translation)