From 3b3aad452fbd101762fecddf35cf6a5e83786889 Mon Sep 17 00:00:00 2001 From: Vikas Kedigehalli Date: Thu, 16 Feb 2017 16:55:20 -0800 Subject: [PATCH 1/4] Parameterized tests for standard coders in Python SDK --- .../coders/standard_coders_test.py | 45 +++++++------------ sdks/python/setup.py | 1 + 2 files changed, 17 insertions(+), 29 deletions(-) diff --git a/sdks/python/apache_beam/coders/standard_coders_test.py b/sdks/python/apache_beam/coders/standard_coders_test.py index d4179ebefdd5..299c1d68ea11 100644 --- a/sdks/python/apache_beam/coders/standard_coders_test.py +++ b/sdks/python/apache_beam/coders/standard_coders_test.py @@ -30,6 +30,20 @@ from apache_beam.utils.timestamp import Timestamp from apache_beam.transforms.window import IntervalWindow +from nose_parameterized import parameterized + +STANDARD_CODERS_YAML = os.path.join( + os.path.dirname(__file__), '..', 'tests', 'data', 'standard_coders.yaml') + + +def data(test_yaml): + if not os.path.exists(test_yaml): + raise ValueError('Could not find the test spec: %s' % test_yaml) + for ix, spec in enumerate(yaml.load_all(open(test_yaml))): + spec['index'] = ix + name = spec.get('name', spec['coder']['urn'].split(':')[-2]) + yield [name, spec] + class StandardCodersTest(unittest.TestCase): @@ -54,30 +68,8 @@ class StandardCodersTest(unittest.TestCase): 'urn:beam:coders:stream:0.1': lambda x, parser: map(parser, x) } - # We must prepend an underscore to this name so that the open-source unittest - # runner does not execute this method directly as a test. - @classmethod - def _create_test(cls, spec): - counter = 0 - name = spec.get('name', spec['coder']['urn'].split(':')[-2]) - unique_name = 'test_' + name - while hasattr(cls, unique_name): - counter += 1 - unique_name = 'test_%s_%d' % (name, counter) - setattr(cls, unique_name, lambda self: self._run_coder_test(spec)) - - # We must prepend an underscore to this name so that the open-source unittest - # runner does not execute this method directly as a test. - @classmethod - def _create_tests(cls, coder_test_specs): - if not os.path.exists(STANDARD_CODERS_YAML): - raise ValueError( - "Could not find the test spec: %s" % STANDARD_CODERS_YAML) - for ix, spec in enumerate(yaml.load_all(open(coder_test_specs))): - spec['index'] = ix - cls._create_test(spec) - - def _run_coder_test(self, spec): + @parameterized.expand(data(STANDARD_CODERS_YAML)) + def test_standard_coder(self, name, spec): coder = self.parse_coder(spec['coder']) parse_value = self.json_value_parser(spec['coder']) nested_list = [spec['nested']] if 'nested' in spec else [True, False] @@ -124,11 +116,6 @@ def quote(s): open(STANDARD_CODERS_YAML, 'w').write(doc_sep.join(docs)) -STANDARD_CODERS_YAML = os.path.join( - os.path.dirname(__file__), '..', 'tests', 'data', 'standard_coders.yaml') -StandardCodersTest._create_tests(STANDARD_CODERS_YAML) - - def encode_nested(coder, value, nested=True): out = coder_impl.create_OutputStream() coder.get_impl().encode_to_stream(value, out, nested) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 59c0994542ac..9fc970d90db5 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -100,6 +100,7 @@ def get_version(): REQUIRED_TEST_PACKAGES = [ + 'nose-parameterized>=0.5.0', 'pyhamcrest>=1.9,<2.0', ] From c9f0b312f8f547b022ecb05943c8775a6f9075a5 Mon Sep 17 00:00:00 2001 From: Vikas Kedigehalli Date: Thu, 16 Feb 2017 18:40:02 -0800 Subject: [PATCH 2/4] Disable RAT plugin for python .egg files --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 0688b733d346..76fad21195c3 100644 --- a/pom.xml +++ b/pom.xml @@ -1125,6 +1125,7 @@ **/test/**/.placeholder .repository/**/* **/nose-*.egg/**/* + **/.eggs/**/* **/.tox/**/* From 6457dbe6f7f418f3ac2eb448d2b506ef0a853f36 Mon Sep 17 00:00:00 2001 From: Vikas Kedigehalli Date: Mon, 20 Feb 2017 22:19:55 -0800 Subject: [PATCH 3/4] Address comments --- sdks/python/apache_beam/coders/standard_coders_test.py | 8 ++++++-- sdks/python/setup.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/coders/standard_coders_test.py b/sdks/python/apache_beam/coders/standard_coders_test.py index 299c1d68ea11..1fbf1a19557e 100644 --- a/sdks/python/apache_beam/coders/standard_coders_test.py +++ b/sdks/python/apache_beam/coders/standard_coders_test.py @@ -36,7 +36,11 @@ os.path.dirname(__file__), '..', 'tests', 'data', 'standard_coders.yaml') -def data(test_yaml): +def _load_test_cases(test_yaml): + """Load test data from yaml file and return an iterable of test cases. + + See ``standard_coders.yaml`` for more details. + """ if not os.path.exists(test_yaml): raise ValueError('Could not find the test spec: %s' % test_yaml) for ix, spec in enumerate(yaml.load_all(open(test_yaml))): @@ -68,7 +72,7 @@ class StandardCodersTest(unittest.TestCase): 'urn:beam:coders:stream:0.1': lambda x, parser: map(parser, x) } - @parameterized.expand(data(STANDARD_CODERS_YAML)) + @parameterized.expand(_load_test_cases(STANDARD_CODERS_YAML)) def test_standard_coder(self, name, spec): coder = self.parse_coder(spec['coder']) parse_value = self.json_value_parser(spec['coder']) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 9fc970d90db5..70f938b08420 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -100,7 +100,7 @@ def get_version(): REQUIRED_TEST_PACKAGES = [ - 'nose-parameterized>=0.5.0', + 'nose-parameterized>=0.5.0, <0.6.0', 'pyhamcrest>=1.9,<2.0', ] From 2b70f9d7232707f9d0d300b0160a48911cfc10d1 Mon Sep 17 00:00:00 2001 From: Vikas Kedigehalli Date: Tue, 21 Feb 2017 11:06:53 -0800 Subject: [PATCH 4/4] Clean up --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 70f938b08420..935cb90a1897 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -100,7 +100,7 @@ def get_version(): REQUIRED_TEST_PACKAGES = [ - 'nose-parameterized>=0.5.0, <0.6.0', + 'nose-parameterized>=0.5.0,<0.6.0', 'pyhamcrest>=1.9,<2.0', ]