diff --git a/ptp/components/problems/image_text_to_class/clevr.py b/ptp/components/problems/image_text_to_class/clevr.py index 787154a..afc5fe3 100644 --- a/ptp/components/problems/image_text_to_class/clevr.py +++ b/ptp/components/problems/image_text_to_class/clevr.py @@ -52,7 +52,6 @@ class CLEVR(Problem): - Integer comparison (Equal, Less, More) For more details please refer to the associated _website or _paper for more details. - Test set with answers can be downloaded from a separate repository _repo. .. _website: https://cs.stanford.edu/people/jcjohns/clevr/ diff --git a/tests/__init__.py b/tests/__init__.py index c864ca0..a634db8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,8 +1,11 @@ from .application.pipeline_tests import TestPipeline +from .application.sampler_factory_tests import TestSamplerFactory +from .application.samplers_tests import TestkFoldRandomSampler, TestkFoldWeightedRandomSampler from .components.component_tests import TestComponent -from .components.clevr_tests import TestCLEVR -from .components.problem_tests import TestProblem +from .components.problems.clevr_tests import TestCLEVR +#from .components.problems.gqa_tests import TestGQA +from .components.problems.problem_tests import TestProblem from .configuration.config_interface_tests import TestConfigInterface from .configuration.config_registry_tests import TestConfigRegistry @@ -12,16 +15,17 @@ from .data_types.data_definition_tests import TestDataDefinition from .utils.app_state_tests import TestAppState -from .utils.sampler_factory_tests import TestSamplerFactory -from .utils.samplers_tests import TestkFoldRandomSampler, TestkFoldWeightedRandomSampler from .utils.statistics_tests import TestStatistics __all__ = [ # Application 'TestPipeline', + 'TestSamplerFactory', + 'TestkFoldRandomSampler', + 'TestkFoldWeightedRandomSampler', # Components 'TestComponent', - 'TestCLEVR', + #'TestGQA', 'TestProblem', # Configuration 'TestConfigRegistry', @@ -32,8 +36,5 @@ 'TestDataDefinition', # Utils 'TestAppState', - 'TestSamplerFactory', - 'TestkFoldRandomSampler', - 'TestkFoldWeightedRandomSampler', 'TestStatistics', ] diff --git a/tests/utils/sampler_factory_tests.py b/tests/application/sampler_factory_tests.py similarity index 98% rename from tests/utils/sampler_factory_tests.py rename to tests/application/sampler_factory_tests.py index 70e3ce5..7a79bd3 100644 --- a/tests/utils/sampler_factory_tests.py +++ b/tests/application/sampler_factory_tests.py @@ -64,7 +64,7 @@ def test_create_subset_random_sampler_range_str(self): def test_create_subset_random_sampler_list_of_indices(self): """ Tests whther SubsetRandomSampler accepts 'indices' with the option 3: list of indices. """ - yaml_list = yaml.load('[0, 2, 5, 10]') + yaml_list = yaml.safe_load('[0, 2, 5, 10]') config = ConfigInterface() config.add_default_params({'type': 'SubsetRandomSampler', 'indices': yaml_list}) diff --git a/tests/utils/samplers_tests.py b/tests/application/samplers_tests.py similarity index 100% rename from tests/utils/samplers_tests.py rename to tests/application/samplers_tests.py diff --git a/tests/components/clevr_tests.py b/tests/components/clevr_tests.py deleted file mode 100644 index f2bf218..0000000 --- a/tests/components/clevr_tests.py +++ /dev/null @@ -1,125 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) tkornuta, IBM Corporation 2019 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -__author__ = "Tomasz Kornuta" - -import unittest -from os import path - -from ptp.components.utils.io import check_file_existence -from ptp.components.problems.image_text_to_class.clevr import CLEVR -from ptp.data_types.data_definition import DataDefinition -from ptp.configuration.config_interface import ConfigInterface - - -class TestCLEVR(unittest.TestCase): - - def __init__(self, *args, **kwargs): - super(TestCLEVR, self).__init__(*args, **kwargs) - - # Check the existence of training set. - self.unittest_training_set = False # check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_train_questions.json') - # Check the existence of validation set. - self.unittest_validation_set = check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_val_questions.json') - # Check the existence of test set. - self.unittest_test_set = check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_test_questions.json') - - - def test_training_set(self): - """ - Tests the CLEVR training split. - - ..note: - Test is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_train_questions.json' is found. - """ - if not self.unittest_training_set: - return - # Empty config. - config = ConfigInterface() - config.add_config_params({"split": "training"}) - clevr = CLEVR("CLEVR", config) - - # Check dataset size. - self.assertEqual(len(clevr), 699989) - - # Check sample. - sample = clevr[0] - self.assertEqual(sample['indices'], 0) - self.assertEqual(sample['image_ids'], 'CLEVR_train_000000.png') - self.assertEqual(sample['question_type_ids'], 4) - self.assertEqual(sample['question_type_names'], 'greater_than') - self.assertEqual(sample['questions'], 'Are there more big green things than large purple shiny cubes?') - self.assertEqual(sample['answers'], 'yes') - - - def test_validation_set(self): - """ - Tests the CLEVR validation split. - - ..note: - Test is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_val_questions.json' is found. - """ - if not self.unittest_validation_set: - return - # Empty config. - config = ConfigInterface() - config.add_config_params({"split": "validation"}) - clevr = CLEVR("CLEVR", config) - - # Check dataset size. - self.assertEqual(len(clevr), 149991) - - # Check sample. - sample = clevr[0] - self.assertEqual(sample['indices'], 0) - self.assertEqual(sample['image_ids'], 'CLEVR_val_000000.png') - self.assertEqual(sample['question_type_ids'], 10) - self.assertEqual(sample['question_type_names'], 'exist') - self.assertEqual(sample['questions'], 'Are there any other things that are the same shape as the big metallic object?') - self.assertEqual(sample['answers'], 'no') - - - def test_test_set(self): - """ - Tests the CLEVR test split. - - ..note: - Test is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_test_questions.json' is found. - """ - if not self.unittest_test_set: - return - # Empty config. - config = ConfigInterface() - config.add_config_params({"split": "test"}) - clevr = CLEVR("CLEVR", config) - - # Check dataset size. - self.assertEqual(len(clevr), 149988) - - # Check sample. - sample = clevr[0] - self.assertEqual(sample['indices'], 0) - self.assertEqual(sample['image_ids'], 'CLEVR_test_000000.png') - self.assertEqual(sample['question_type_ids'], -1) - self.assertEqual(sample['question_type_names'], '') - self.assertEqual(sample['questions'], 'Is there anything else that is the same shape as the small brown matte object?') - self.assertEqual(sample['answers'], '') - - - - -#if __name__ == "__main__": -# unittest.main() \ No newline at end of file diff --git a/tests/components/problems/clevr_tests.py b/tests/components/problems/clevr_tests.py new file mode 100644 index 0000000..00d75af --- /dev/null +++ b/tests/components/problems/clevr_tests.py @@ -0,0 +1,169 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) tkornuta, IBM Corporation 2019 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__author__ = "Tomasz Kornuta" + +import unittest +from unittest.mock import MagicMock, patch +from os import path + +from ptp.components.utils.io import check_file_existence +from ptp.components.problems.image_text_to_class.clevr import CLEVR +from ptp.configuration.config_interface import ConfigInterface + + +class TestCLEVR(unittest.TestCase): + + def test_training_set(self): + """ + Tests the CLEVR training split. + + ..note: + Test is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_train_questions.json' is found. + """ + # Empty config. + config = ConfigInterface() + config.add_config_params({"split": "training"}) + + # Check the existence of test set. + if check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_train_questions.json'): + + # Create object. + clevr = CLEVR("CLEVR", config) + + # Check dataset size. + self.assertEqual(len(clevr), 699989) + + # Get sample. + sample = clevr[0] + + else: + dataset_content = [{'image_index': 0, 'program': [{'inputs': [], 'function': 'scene', 'value_inputs': []}, {'inputs': [0], 'function': 'filter_size', 'value_inputs': ['large']}, + {'inputs': [1], 'function': 'filter_color', 'value_inputs': ['green']}, {'inputs': [2], 'function': 'count', 'value_inputs': []}, + {'inputs': [], 'function': 'scene', 'value_inputs': []}, {'inputs': [4], 'function': 'filter_size', 'value_inputs': ['large']}, + {'inputs': [5], 'function': 'filter_color', 'value_inputs': ['purple']}, {'inputs': [6], 'function': 'filter_material', 'value_inputs': ['metal']}, + {'inputs': [7], 'function': 'filter_shape', 'value_inputs': ['cube']}, {'inputs': [8], 'function': 'count', 'value_inputs': []}, + {'inputs': [3, 9], 'function': 'greater_than', 'value_inputs': []}], 'question_index': 0, 'image_filename': 'CLEVR_train_000000.png', 'question_family_index': 2, + 'split': 'train', 'answer': 'yes', 'question': 'Are there more big green things than large purple shiny cubes?'}] + + # Mock up the load_dataset method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.load_dataset", MagicMock( side_effect = [ dataset_content ] )): + clevr = CLEVR("CLEVR", config) + + # Mock up the get_image method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.get_image", MagicMock( side_effect = [ "0" ] )): + sample = clevr[0] + + # Check sample. + self.assertEqual(sample['indices'], 0) + self.assertEqual(sample['image_ids'], 'CLEVR_train_000000.png') + self.assertEqual(sample['question_type_ids'], 4) + self.assertEqual(sample['question_type_names'], 'greater_than') + self.assertEqual(sample['questions'], 'Are there more big green things than large purple shiny cubes?') + self.assertEqual(sample['answers'], 'yes') + + + def test_validation_set(self): + """ + Tests the CLEVR validation split. + + ..note: + Test on real data is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_val_questions.json' is found. + """ + # Empty config. + config = ConfigInterface() + config.add_config_params({"split": "validation"}) + + # Check the existence of test set. + if check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_test_questions.json'): + + # Create object. + clevr = CLEVR("CLEVR", config) + + # Check dataset size. + self.assertEqual(len(clevr), 149991) + + # Get sample. + sample = clevr[0] + + else: + dataset_content = [{'image_index': 0, 'program': [{'inputs': [], 'function': 'scene', 'value_inputs': []}, {'inputs': [0], 'function': 'filter_size', 'value_inputs': ['large']}, + {'inputs': [1], 'function': 'filter_material', 'value_inputs': ['metal']}, {'inputs': [2], 'function': 'unique', 'value_inputs': []}, + {'inputs': [3], 'function': 'same_shape', 'value_inputs': []}, {'inputs': [4], 'function': 'exist', 'value_inputs': []}], + 'question_index': 0, 'image_filename': 'CLEVR_val_000000.png', 'question_family_index': 39, 'split': 'val', 'answer': 'no', 'question': 'Are there any other things that are the same shape as the big metallic object?'}] + + # Mock up the load_dataset method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.load_dataset", MagicMock( side_effect = [ dataset_content ] )): + clevr = CLEVR("CLEVR", config) + + # Mock up the get_image method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.get_image", MagicMock( side_effect = [ "0" ] )): + sample = clevr[0] + + # Check sample. + self.assertEqual(sample['indices'], 0) + self.assertEqual(sample['image_ids'], 'CLEVR_val_000000.png') + self.assertEqual(sample['question_type_ids'], 10) + self.assertEqual(sample['question_type_names'], 'exist') + self.assertEqual(sample['questions'], 'Are there any other things that are the same shape as the big metallic object?') + self.assertEqual(sample['answers'], 'no') + + + def test_test_set(self): + """ + Tests the CLEVR test split. + + ..note: + Test on real data is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_test_questions.json' is found. + """ + # Empty config. + config = ConfigInterface() + config.add_config_params({"split": "test"}) + + # Check the existence of test set. + if check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_test_questions.json'): + + # Create object. + clevr = CLEVR("CLEVR", config) + + # Check dataset size. + self.assertEqual(len(clevr), 149988) + + # Get sample. + sample = clevr[0] + + else: + dataset_content = [{'image_index': 0, 'split': 'test', 'image_filename': 'CLEVR_test_000000.png', 'question_index': 0, 'question': 'Is there anything else that is the same shape as the small brown matte object?'}] + + # Mock up the load_dataset method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.load_dataset", MagicMock( side_effect = [ dataset_content ] )): + clevr = CLEVR("CLEVR", config) + + # Mock up the get_image method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.get_image", MagicMock( side_effect = [ "0" ] )): + sample = clevr[0] + + # Check sample. + self.assertEqual(sample['indices'], 0) + self.assertEqual(sample['image_ids'], 'CLEVR_test_000000.png') + self.assertEqual(sample['question_type_ids'], -1) + self.assertEqual(sample['question_type_names'], '') + self.assertEqual(sample['questions'], 'Is there anything else that is the same shape as the small brown matte object?') + self.assertEqual(sample['answers'], '') + + +#if __name__ == "__main__": +# unittest.main() \ No newline at end of file diff --git a/tests/components/problem_tests.py b/tests/components/problems/problem_tests.py similarity index 100% rename from tests/components/problem_tests.py rename to tests/components/problems/problem_tests.py