From 15d0341f847a708843a233623ec4195ba69f65d2 Mon Sep 17 00:00:00 2001 From: Juta Date: Fri, 31 May 2019 23:10:23 +0200 Subject: [PATCH] [BEAM-5878] add tests for kwonly args in python 3 (#8505) * [BEAM-5878] add tests for kwonly args in python 3 * [BEAM-5878] skip python3 sytntax tests in pylint and tox on python 2 --- ...nsforms_keyword_only_args_py3_only_test.py | 106 ++++++++++++++++++ sdks/python/scripts/run_pylint.sh | 16 ++- sdks/python/tox.ini | 6 +- 3 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 sdks/python/apache_beam/transforms/transforms_keyword_only_args_py3_only_test.py diff --git a/sdks/python/apache_beam/transforms/transforms_keyword_only_args_py3_only_test.py b/sdks/python/apache_beam/transforms/transforms_keyword_only_args_py3_only_test.py new file mode 100644 index 0000000000000..a27a6f1ac8c49 --- /dev/null +++ b/sdks/python/apache_beam/transforms/transforms_keyword_only_args_py3_only_test.py @@ -0,0 +1,106 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""Unit tests for side inputs.""" + +from __future__ import absolute_import + +import logging +import unittest + +import apache_beam as beam +from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.util import assert_that +from apache_beam.testing.util import equal_to + + +@unittest.skip('TODO BEAM-5878: support kwonly args in python 3') +class KeywordOnlyArgsTests(unittest.TestCase): + # Enable nose tests running in parallel + _multiprocess_can_split_ = True + + def test_side_input_keyword_only_args(self): + pipeline = TestPipeline() + + def sort_with_side_inputs(x, *s, reverse=False): + for y in s: + yield sorted([x] + y, reverse=reverse) + + pcol = pipeline | 'start' >> beam.Create([1, 2]) + side = pipeline | 'side' >> beam.Create([3, 4]) # 2 values in side input. + result1 = pcol | 'compute1' >> beam.FlatMap( + sort_with_side_inputs, + beam.pvalue.AsList(side), reverse=True) + assert_that(result1, equal_to([[4,3,1], [4,3,2]]), label='assert1') + + result2 = pcol | 'compute2' >> beam.FlatMap( + sort_with_side_inputs, + beam.pvalue.AsIter(side)) + assert_that(result2, equal_to([[1,3,4], [2,3,4]]), label='assert2') + + result3 = pcol | 'compute3' >> beam.FlatMap( + sort_with_side_inputs) + assert_that(result3, equal_to([]), label='assert3') + + result4 = pcol | 'compute4' >> beam.FlatMap( + sort_with_side_inputs, reverse=True) + assert_that(result4, equal_to([]), label='assert4') + + pipeline.run() + + def test_combine_keyword_only_args(self): + pipeline = TestPipeline() + + def bounded_sum(values, *s, bound=500): + return min(sum(values) + sum(s), bound) + + pcoll = pipeline | 'start' >> beam.Create([6, 3, 1]) + result1 = pcoll | 'sum1' >> beam.CombineGlobally(bounded_sum, 5, 8, bound=20) + result2 = pcoll | 'sum2' >> beam.CombineGlobally(bounded_sum, 5, 8) + result3 = pcoll | 'sum3' >> beam.CombineGlobally(bounded_sum) + result4 = pcoll | 'sum4' >> beam.CombineGlobally(bounded_sum, bound=5) + + assert_that(result1, equal_to([20]), label='assert1') + assert_that(result2, equal_to([23]), label='assert2') + assert_that(result3, equal_to([10]), label='assert3') + assert_that(result4, equal_to([5]), label='assert4') + + pipeline.run() + + def test_do_fn_keyword_only_args(self): + pipeline = TestPipeline() + + class MyDoFn(beam.DoFn): + def process(self, element, *s, bound=500): + return [min(sum(s) + element, bound)] + + pcoll = pipeline | 'start' >> beam.Create([6, 3, 1]) + result1 = pcoll | 'sum1' >> beam.ParDo(MyDoFn(), 5, 8, bound=15) + result2 = pcoll | 'sum2' >> beam.ParDo(MyDoFn(), 5, 8) + result3 = pcoll | 'sum3' >> beam.ParDo(MyDoFn()) + result4 = pcoll | 'sum4' >> beam.ParDo(MyDoFn(), bound=5) + + assert_that(result1, equal_to([15,15,14]), label='assert1') + assert_that(result2, equal_to([19,16,14]), label='assert2') + assert_that(result3, equal_to([6,3,1]), label='assert3') + assert_that(result4, equal_to([5,3,1]), label='assert4') + pipeline.run() + + +if __name__ == '__main__': + logging.getLogger().setLevel(logging.DEBUG) + unittest.main() diff --git a/sdks/python/scripts/run_pylint.sh b/sdks/python/scripts/run_pylint.sh index 2814b4f5c5d0d..7056e069e2f0a 100755 --- a/sdks/python/scripts/run_pylint.sh +++ b/sdks/python/scripts/run_pylint.sh @@ -60,14 +60,22 @@ EXCLUDED_GENERATED_FILES=( apache_beam/portability/api/*pb2*.py ) +# TODO(BEAM-7372): Remove this list once Python 2 is no longer supported and drop _py3_only_test suffix. +# Following files contain python 3 syntax and are excluded from pylint in python 2 +EXCLUDED_PYTHON3_FILES=( +.*_py3_only_test.py +) + +EXCLUDED_FILES=( "${EXCLUDED_GENERATED_FILES[@]}" "${EXCLUDED_PYTHON3_FILES[@]}" ) + FILES_TO_IGNORE="" -for file in "${EXCLUDED_GENERATED_FILES[@]}"; do +for file in "${EXCLUDED_FILES[@]}"; do if test -z "$FILES_TO_IGNORE" then FILES_TO_IGNORE="$(basename $file)" else FILES_TO_IGNORE="$FILES_TO_IGNORE, $(basename $file)" fi done -echo "Skipping lint for generated files: $FILES_TO_IGNORE" +echo "Skipping lint for generated and python 3 files: $FILES_TO_IGNORE" echo "Running pylint for module $MODULE:" pylint -j8 ${MODULE} --ignore-patterns="$FILES_TO_IGNORE" @@ -75,7 +83,9 @@ echo "Running pycodestyle for module $MODULE:" pycodestyle ${MODULE} --exclude="$FILES_TO_IGNORE" echo "Running flake8 for module $MODULE:" # TODO(BEAM-3959): Add F821 (undefined names) as soon as that test passes -flake8 ${MODULE} --count --select=E9,F822,F823 --show-source --statistics +# flake8 exclusions are not the same syntax as pylint +FILES_TO_IGNORE=${FILES_TO_IGNORE//\.\*/*} +flake8 ${MODULE} --count --select=E9,F822,F823 --show-source --statistics --exclude="$FILES_TO_IGNORE" echo "Running isort for module $MODULE:" # Skip files where isort is behaving weirdly diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 2a221b0c5efee..1a34fcb40db40 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -54,7 +54,7 @@ commands_post = [testenv:py27] commands = python apache_beam/examples/complete/autocomplete_test.py - python setup.py nosetests + python setup.py nosetests --ignore-files=.*_py3_only_test.py [testenv:py35] setenv = @@ -85,7 +85,7 @@ commands = platform = linux2 commands = python apache_beam/examples/complete/autocomplete_test.py - python setup.py nosetests + python setup.py nosetests --ignore-files=.*_py3_only_test.py [testenv:py35-cython] # cython tests are only expected to work in linux (2.x and 3.x) @@ -127,7 +127,7 @@ commands = extras = test,gcp commands = python apache_beam/examples/complete/autocomplete_test.py - python setup.py nosetests + python setup.py nosetests --ignore-files=.*_py3_only_test.py # Old and new Datastore client unit tests cannot be run in the same process # due to conflicting protobuf modules. # TODO(BEAM-4543): Remove these separate nosetests invocations once the