From 73673f13d3405c38692bc9d437f8fd7f022384d3 Mon Sep 17 00:00:00 2001 From: tkornut Date: Wed, 5 Jun 2019 16:18:04 -0700 Subject: [PATCH 1/5] further reorganization of test/folders, fixed warning in yaml.safe_load --- tests/__init__.py | 17 +++++++++-------- .../sampler_factory_tests.py | 2 +- tests/{utils => application}/samplers_tests.py | 0 tests/components/{ => problems}/clevr_tests.py | 0 .../components/{ => problems}/problem_tests.py | 0 5 files changed, 10 insertions(+), 9 deletions(-) rename tests/{utils => application}/sampler_factory_tests.py (98%) rename tests/{utils => application}/samplers_tests.py (100%) rename tests/components/{ => problems}/clevr_tests.py (100%) rename tests/components/{ => problems}/problem_tests.py (100%) diff --git a/tests/__init__.py b/tests/__init__.py index c864ca0..9471702 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,8 +1,11 @@ from .application.pipeline_tests import TestPipeline +from .application.sampler_factory_tests import TestSamplerFactory +from .application.samplers_tests import TestkFoldRandomSampler, TestkFoldWeightedRandomSampler from .components.component_tests import TestComponent -from .components.clevr_tests import TestCLEVR -from .components.problem_tests import TestProblem +from .components.problems.clevr_tests import TestCLEVR +from .components.problems.gqa_tests import TestGQA +from .components.problems.problem_tests import TestProblem from .configuration.config_interface_tests import TestConfigInterface from .configuration.config_registry_tests import TestConfigRegistry @@ -12,16 +15,17 @@ from .data_types.data_definition_tests import TestDataDefinition from .utils.app_state_tests import TestAppState -from .utils.sampler_factory_tests import TestSamplerFactory -from .utils.samplers_tests import TestkFoldRandomSampler, TestkFoldWeightedRandomSampler from .utils.statistics_tests import TestStatistics __all__ = [ # Application 'TestPipeline', + 'TestSamplerFactory', + 'TestkFoldRandomSampler', + 'TestkFoldWeightedRandomSampler', # Components 'TestComponent', - 'TestCLEVR', + 'TestGQA', 'TestProblem', # Configuration 'TestConfigRegistry', @@ -32,8 +36,5 @@ 'TestDataDefinition', # Utils 'TestAppState', - 'TestSamplerFactory', - 'TestkFoldRandomSampler', - 'TestkFoldWeightedRandomSampler', 'TestStatistics', ] diff --git a/tests/utils/sampler_factory_tests.py b/tests/application/sampler_factory_tests.py similarity index 98% rename from tests/utils/sampler_factory_tests.py rename to tests/application/sampler_factory_tests.py index 70e3ce5..7a79bd3 100644 --- a/tests/utils/sampler_factory_tests.py +++ b/tests/application/sampler_factory_tests.py @@ -64,7 +64,7 @@ def test_create_subset_random_sampler_range_str(self): def test_create_subset_random_sampler_list_of_indices(self): """ Tests whther SubsetRandomSampler accepts 'indices' with the option 3: list of indices. """ - yaml_list = yaml.load('[0, 2, 5, 10]') + yaml_list = yaml.safe_load('[0, 2, 5, 10]') config = ConfigInterface() config.add_default_params({'type': 'SubsetRandomSampler', 'indices': yaml_list}) diff --git a/tests/utils/samplers_tests.py b/tests/application/samplers_tests.py similarity index 100% rename from tests/utils/samplers_tests.py rename to tests/application/samplers_tests.py diff --git a/tests/components/clevr_tests.py b/tests/components/problems/clevr_tests.py similarity index 100% rename from tests/components/clevr_tests.py rename to tests/components/problems/clevr_tests.py diff --git a/tests/components/problem_tests.py b/tests/components/problems/problem_tests.py similarity index 100% rename from tests/components/problem_tests.py rename to tests/components/problems/problem_tests.py From 247706dfd2fc33ec477d2a0bd6f4a0dc1c65de4a Mon Sep 17 00:00:00 2001 From: tkornut Date: Wed, 5 Jun 2019 17:54:12 -0700 Subject: [PATCH 2/5] Mocked up test data in case source json file is not present --- tests/components/problems/clevr_tests.py | 40 ++++++++++++++++-------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/tests/components/problems/clevr_tests.py b/tests/components/problems/clevr_tests.py index f2bf218..efc8419 100644 --- a/tests/components/problems/clevr_tests.py +++ b/tests/components/problems/clevr_tests.py @@ -17,11 +17,11 @@ __author__ = "Tomasz Kornuta" import unittest +from unittest.mock import MagicMock, patch from os import path from ptp.components.utils.io import check_file_existence from ptp.components.problems.image_text_to_class.clevr import CLEVR -from ptp.data_types.data_definition import DataDefinition from ptp.configuration.config_interface import ConfigInterface @@ -33,9 +33,7 @@ def __init__(self, *args, **kwargs): # Check the existence of training set. self.unittest_training_set = False # check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_train_questions.json') # Check the existence of validation set. - self.unittest_validation_set = check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_val_questions.json') - # Check the existence of test set. - self.unittest_test_set = check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_test_questions.json') + self.unittest_validation_set = False # check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_val_questions.json') def test_training_set(self): @@ -97,20 +95,36 @@ def test_test_set(self): Tests the CLEVR test split. ..note: - Test is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_test_questions.json' is found. + Test on real data is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_test_questions.json' is found. """ - if not self.unittest_test_set: - return # Empty config. config = ConfigInterface() config.add_config_params({"split": "test"}) - clevr = CLEVR("CLEVR", config) + + # Check the existence of test set. + if False: # check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_test_questions.json'): - # Check dataset size. - self.assertEqual(len(clevr), 149988) + # Create object. + clevr = CLEVR("CLEVR", config) + + # Check dataset size. + self.assertEqual(len(clevr), 149988) + + # Get sample. + sample = clevr[0] + + else: + test_content = [{'image_index': 0, 'split': 'test', 'image_filename': 'CLEVR_test_000000.png', 'question_index': 0, 'question': 'Is there anything else that is the same shape as the small brown matte object?'}] + + # Mock up the load_dataset method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.load_dataset", MagicMock( side_effect = [ test_content ] )): + clevr = CLEVR("CLEVR", config) + + # Mock up the get_image method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.get_image", MagicMock( side_effect = [ "0" ] )): + sample = clevr[0] # Check sample. - sample = clevr[0] self.assertEqual(sample['indices'], 0) self.assertEqual(sample['image_ids'], 'CLEVR_test_000000.png') self.assertEqual(sample['question_type_ids'], -1) @@ -121,5 +135,5 @@ def test_test_set(self): -#if __name__ == "__main__": -# unittest.main() \ No newline at end of file +if __name__ == "__main__": + unittest.main() \ No newline at end of file From c452f84b8cb31757819ba407bc112af69a06d532 Mon Sep 17 00:00:00 2001 From: tkornut Date: Wed, 5 Jun 2019 17:54:31 -0700 Subject: [PATCH 3/5] comment update --- ptp/components/problems/image_text_to_class/clevr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ptp/components/problems/image_text_to_class/clevr.py b/ptp/components/problems/image_text_to_class/clevr.py index 787154a..afc5fe3 100644 --- a/ptp/components/problems/image_text_to_class/clevr.py +++ b/ptp/components/problems/image_text_to_class/clevr.py @@ -52,7 +52,6 @@ class CLEVR(Problem): - Integer comparison (Equal, Less, More) For more details please refer to the associated _website or _paper for more details. - Test set with answers can be downloaded from a separate repository _repo. .. _website: https://cs.stanford.edu/people/jcjohns/clevr/ From e66c1eb8dcf3e8f177f1462268f0d41c256dd967 Mon Sep 17 00:00:00 2001 From: tkornut Date: Wed, 5 Jun 2019 18:12:33 -0700 Subject: [PATCH 4/5] CLEVR tests loading dataset and images mocked up --- tests/__init__.py | 4 +- tests/components/problems/clevr_tests.py | 82 ++++++++++++++++-------- 2 files changed, 59 insertions(+), 27 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 9471702..a634db8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -4,7 +4,7 @@ from .components.component_tests import TestComponent from .components.problems.clevr_tests import TestCLEVR -from .components.problems.gqa_tests import TestGQA +#from .components.problems.gqa_tests import TestGQA from .components.problems.problem_tests import TestProblem from .configuration.config_interface_tests import TestConfigInterface @@ -25,7 +25,7 @@ 'TestkFoldWeightedRandomSampler', # Components 'TestComponent', - 'TestGQA', + #'TestGQA', 'TestProblem', # Configuration 'TestConfigRegistry', diff --git a/tests/components/problems/clevr_tests.py b/tests/components/problems/clevr_tests.py index efc8419..313aea3 100644 --- a/tests/components/problems/clevr_tests.py +++ b/tests/components/problems/clevr_tests.py @@ -27,15 +27,6 @@ class TestCLEVR(unittest.TestCase): - def __init__(self, *args, **kwargs): - super(TestCLEVR, self).__init__(*args, **kwargs) - - # Check the existence of training set. - self.unittest_training_set = False # check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_train_questions.json') - # Check the existence of validation set. - self.unittest_validation_set = False # check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_val_questions.json') - - def test_training_set(self): """ Tests the CLEVR training split. @@ -43,18 +34,40 @@ def test_training_set(self): ..note: Test is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_train_questions.json' is found. """ - if not self.unittest_training_set: - return # Empty config. config = ConfigInterface() config.add_config_params({"split": "training"}) - clevr = CLEVR("CLEVR", config) - # Check dataset size. - self.assertEqual(len(clevr), 699989) + # Check the existence of test set. + if check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_train_questions.json'): + + # Create object. + clevr = CLEVR("CLEVR", config) + + # Check dataset size. + self.assertEqual(len(clevr), 699989) + + # Get sample. + sample = clevr[0] + + else: + dataset_content = [{'image_index': 0, 'program': [{'inputs': [], 'function': 'scene', 'value_inputs': []}, {'inputs': [0], 'function': 'filter_size', 'value_inputs': ['large']}, + {'inputs': [1], 'function': 'filter_color', 'value_inputs': ['green']}, {'inputs': [2], 'function': 'count', 'value_inputs': []}, + {'inputs': [], 'function': 'scene', 'value_inputs': []}, {'inputs': [4], 'function': 'filter_size', 'value_inputs': ['large']}, + {'inputs': [5], 'function': 'filter_color', 'value_inputs': ['purple']}, {'inputs': [6], 'function': 'filter_material', 'value_inputs': ['metal']}, + {'inputs': [7], 'function': 'filter_shape', 'value_inputs': ['cube']}, {'inputs': [8], 'function': 'count', 'value_inputs': []}, + {'inputs': [3, 9], 'function': 'greater_than', 'value_inputs': []}], 'question_index': 0, 'image_filename': 'CLEVR_train_000000.png', 'question_family_index': 2, + 'split': 'train', 'answer': 'yes', 'question': 'Are there more big green things than large purple shiny cubes?'}] + + # Mock up the load_dataset method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.load_dataset", MagicMock( side_effect = [ dataset_content ] )): + clevr = CLEVR("CLEVR", config) + + # Mock up the get_image method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.get_image", MagicMock( side_effect = [ "0" ] )): + sample = clevr[0] # Check sample. - sample = clevr[0] self.assertEqual(sample['indices'], 0) self.assertEqual(sample['image_ids'], 'CLEVR_train_000000.png') self.assertEqual(sample['question_type_ids'], 4) @@ -68,20 +81,39 @@ def test_validation_set(self): Tests the CLEVR validation split. ..note: - Test is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_val_questions.json' is found. + Test on real data is performed only if json file '~/data/CLEVR_v1.0/questions/CLEVR_val_questions.json' is found. """ - if not self.unittest_validation_set: - return # Empty config. config = ConfigInterface() config.add_config_params({"split": "validation"}) - clevr = CLEVR("CLEVR", config) - # Check dataset size. - self.assertEqual(len(clevr), 149991) + # Check the existence of test set. + if check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_test_questions.json'): + + # Create object. + clevr = CLEVR("CLEVR", config) + + # Check dataset size. + self.assertEqual(len(clevr), 149991) + + # Get sample. + sample = clevr[0] + + else: + dataset_content = [{'image_index': 0, 'program': [{'inputs': [], 'function': 'scene', 'value_inputs': []}, {'inputs': [0], 'function': 'filter_size', 'value_inputs': ['large']}, + {'inputs': [1], 'function': 'filter_material', 'value_inputs': ['metal']}, {'inputs': [2], 'function': 'unique', 'value_inputs': []}, + {'inputs': [3], 'function': 'same_shape', 'value_inputs': []}, {'inputs': [4], 'function': 'exist', 'value_inputs': []}], + 'question_index': 0, 'image_filename': 'CLEVR_val_000000.png', 'question_family_index': 39, 'split': 'val', 'answer': 'no', 'question': 'Are there any other things that are the same shape as the big metallic object?'}] + + # Mock up the load_dataset method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.load_dataset", MagicMock( side_effect = [ dataset_content ] )): + clevr = CLEVR("CLEVR", config) + + # Mock up the get_image method. + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.get_image", MagicMock( side_effect = [ "0" ] )): + sample = clevr[0] # Check sample. - sample = clevr[0] self.assertEqual(sample['indices'], 0) self.assertEqual(sample['image_ids'], 'CLEVR_val_000000.png') self.assertEqual(sample['question_type_ids'], 10) @@ -102,7 +134,7 @@ def test_test_set(self): config.add_config_params({"split": "test"}) # Check the existence of test set. - if False: # check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_test_questions.json'): + if check_file_existence(path.expanduser('~/data/CLEVR_v1.0/questions'),'CLEVR_test_questions.json'): # Create object. clevr = CLEVR("CLEVR", config) @@ -114,10 +146,10 @@ def test_test_set(self): sample = clevr[0] else: - test_content = [{'image_index': 0, 'split': 'test', 'image_filename': 'CLEVR_test_000000.png', 'question_index': 0, 'question': 'Is there anything else that is the same shape as the small brown matte object?'}] + dataset_content = [{'image_index': 0, 'split': 'test', 'image_filename': 'CLEVR_test_000000.png', 'question_index': 0, 'question': 'Is there anything else that is the same shape as the small brown matte object?'}] # Mock up the load_dataset method. - with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.load_dataset", MagicMock( side_effect = [ test_content ] )): + with patch( "ptp.components.problems.image_text_to_class.clevr.CLEVR.load_dataset", MagicMock( side_effect = [ dataset_content ] )): clevr = CLEVR("CLEVR", config) # Mock up the get_image method. From dabdac0832174445161df95db8613add7100cf7f Mon Sep 17 00:00:00 2001 From: tkornut Date: Wed, 5 Jun 2019 18:14:07 -0700 Subject: [PATCH 5/5] cleanup --- tests/components/problems/clevr_tests.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/components/problems/clevr_tests.py b/tests/components/problems/clevr_tests.py index 313aea3..00d75af 100644 --- a/tests/components/problems/clevr_tests.py +++ b/tests/components/problems/clevr_tests.py @@ -165,7 +165,5 @@ def test_test_set(self): self.assertEqual(sample['answers'], '') - - -if __name__ == "__main__": - unittest.main() \ No newline at end of file +#if __name__ == "__main__": +# unittest.main() \ No newline at end of file