[BEAM-5878] add tests for kwonly args in python 3 (#8505)

* [BEAM-5878] add tests for kwonly args in python 3
* [BEAM-5878] skip python3 sytntax tests in pylint and tox on python 2

sdks/python/apache_beam/transforms/transforms_keyword_only_args_py3_only_test.py

@@ -0,0 +1,106 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unit tests for side inputs."""
+
+from __future__ import absolute_import
+
+import logging
+import unittest
+
+import apache_beam as beam
+from apache_beam.testing.test_pipeline import TestPipeline
+from apache_beam.testing.util import assert_that
+from apache_beam.testing.util import equal_to
+
+
+@unittest.skip('TODO BEAM-5878: support kwonly args in python 3')
+class KeywordOnlyArgsTests(unittest.TestCase):
+  # Enable nose tests running in parallel
+  _multiprocess_can_split_ = True
+
+  def test_side_input_keyword_only_args(self):
+    pipeline = TestPipeline()
+
+    def sort_with_side_inputs(x, *s, reverse=False):
+      for y in s:
+        yield sorted([x] + y, reverse=reverse)
+
+    pcol = pipeline | 'start' >> beam.Create([1, 2])
+    side = pipeline | 'side' >> beam.Create([3, 4])  # 2 values in side input.
+    result1 = pcol | 'compute1' >> beam.FlatMap(
+        sort_with_side_inputs,
+        beam.pvalue.AsList(side), reverse=True)
+    assert_that(result1, equal_to([[4,3,1], [4,3,2]]), label='assert1')
+
+    result2 = pcol | 'compute2' >> beam.FlatMap(
+      sort_with_side_inputs,
+      beam.pvalue.AsIter(side))
+    assert_that(result2, equal_to([[1,3,4], [2,3,4]]), label='assert2')
+
+    result3 = pcol | 'compute3' >> beam.FlatMap(
+      sort_with_side_inputs)
+    assert_that(result3, equal_to([]), label='assert3')
+
+    result4 = pcol | 'compute4' >> beam.FlatMap(
+      sort_with_side_inputs, reverse=True)
+    assert_that(result4, equal_to([]), label='assert4')
+
+    pipeline.run()
+
+  def test_combine_keyword_only_args(self):
+    pipeline = TestPipeline()
+
+    def bounded_sum(values, *s, bound=500):
+      return min(sum(values) + sum(s), bound)
+
+    pcoll = pipeline | 'start' >> beam.Create([6, 3, 1])
+    result1 = pcoll | 'sum1' >> beam.CombineGlobally(bounded_sum, 5, 8, bound=20)
+    result2 = pcoll | 'sum2' >> beam.CombineGlobally(bounded_sum, 5, 8)
+    result3 = pcoll | 'sum3' >> beam.CombineGlobally(bounded_sum)
+    result4 = pcoll | 'sum4' >> beam.CombineGlobally(bounded_sum, bound=5)
+
+    assert_that(result1, equal_to([20]), label='assert1')
+    assert_that(result2, equal_to([23]), label='assert2')
+    assert_that(result3, equal_to([10]), label='assert3')
+    assert_that(result4, equal_to([5]), label='assert4')
+
+    pipeline.run()
+
+  def test_do_fn_keyword_only_args(self):
+    pipeline = TestPipeline()
+
+    class MyDoFn(beam.DoFn):
+      def process(self, element, *s, bound=500):
+        return [min(sum(s) + element, bound)]
+
+    pcoll = pipeline | 'start' >> beam.Create([6, 3, 1])
+    result1 = pcoll | 'sum1' >> beam.ParDo(MyDoFn(), 5, 8, bound=15)
+    result2 = pcoll | 'sum2' >> beam.ParDo(MyDoFn(), 5, 8)
+    result3 = pcoll | 'sum3' >> beam.ParDo(MyDoFn())
+    result4 = pcoll | 'sum4' >> beam.ParDo(MyDoFn(), bound=5)
+
+    assert_that(result1, equal_to([15,15,14]), label='assert1')
+    assert_that(result2, equal_to([19,16,14]), label='assert2')
+    assert_that(result3, equal_to([6,3,1]), label='assert3')
+    assert_that(result4, equal_to([5,3,1]), label='assert4')
+    pipeline.run()
+
+
+if __name__ == '__main__':
+  logging.getLogger().setLevel(logging.DEBUG)
+  unittest.main()

sdks/python/scripts/run_pylint.sh

@@ -60,22 +60,32 @@ EXCLUDED_GENERATED_FILES=(
 apache_beam/portability/api/*pb2*.py
 )
 
+# TODO(BEAM-7372): Remove this list once Python 2 is no longer supported and drop _py3_only_test suffix.
+# Following files contain python 3 syntax and are excluded from pylint in python 2
+EXCLUDED_PYTHON3_FILES=(
+.*_py3_only_test.py
+)
+
+EXCLUDED_FILES=( "${EXCLUDED_GENERATED_FILES[@]}" "${EXCLUDED_PYTHON3_FILES[@]}" )
+
 FILES_TO_IGNORE=""
-for file in "${EXCLUDED_GENERATED_FILES[@]}"; do
+for file in "${EXCLUDED_FILES[@]}"; do
   if test -z "$FILES_TO_IGNORE"
     then FILES_TO_IGNORE="$(basename $file)"
     else FILES_TO_IGNORE="$FILES_TO_IGNORE, $(basename $file)"
   fi
 done
-echo "Skipping lint for generated files: $FILES_TO_IGNORE"
+echo "Skipping lint for generated and python 3 files: $FILES_TO_IGNORE"
 
 echo "Running pylint for module $MODULE:"
 pylint -j8 ${MODULE} --ignore-patterns="$FILES_TO_IGNORE"
 echo "Running pycodestyle for module $MODULE:"
 pycodestyle ${MODULE} --exclude="$FILES_TO_IGNORE"
 echo "Running flake8 for module $MODULE:"
 # TODO(BEAM-3959): Add F821 (undefined names) as soon as that test passes
-flake8 ${MODULE} --count --select=E9,F822,F823 --show-source --statistics
+# flake8 exclusions are not the same syntax as pylint
+FILES_TO_IGNORE=${FILES_TO_IGNORE//\.\*/*}
+flake8 ${MODULE} --count --select=E9,F822,F823 --show-source --statistics --exclude="$FILES_TO_IGNORE"
 
 echo "Running isort for module $MODULE:"
 # Skip files where isort is behaving weirdly

sdks/python/tox.ini

@@ -54,7 +54,7 @@ commands_post =
 [testenv:py27]
 commands =
   python apache_beam/examples/complete/autocomplete_test.py
-  python setup.py nosetests
+  python setup.py nosetests --ignore-files=.*_py3_only_test.py
 
 [testenv:py35]
 setenv =
@@ -85,7 +85,7 @@ commands =
 platform = linux2
 commands =
   python apache_beam/examples/complete/autocomplete_test.py
-  python setup.py nosetests
+  python setup.py nosetests --ignore-files=.*_py3_only_test.py
 
 [testenv:py35-cython]
 # cython tests are only expected to work in linux (2.x and 3.x)
@@ -127,7 +127,7 @@ commands =
 extras = test,gcp
 commands =
   python apache_beam/examples/complete/autocomplete_test.py
-  python setup.py nosetests
+  python setup.py nosetests --ignore-files=.*_py3_only_test.py
   # Old and new Datastore client unit tests cannot be run in the same process
   # due to conflicting protobuf modules.
   # TODO(BEAM-4543): Remove these separate nosetests invocations once the