From 7e4e51fda633d208c4cc5e88182e5db16156f2cb Mon Sep 17 00:00:00 2001 From: Charles Chen Date: Tue, 13 Jun 2017 15:03:15 -0700 Subject: [PATCH] Move Runner API protos to portability/api --- .gitignore | 2 +- sdks/python/apache_beam/coders/coders.py | 2 +- sdks/python/apache_beam/pipeline.py | 4 ++-- .../portability/{runners => }/api/__init__.py | 0 .../portability/runners/__init__.py | 18 ------------------ sdks/python/apache_beam/pvalue.py | 2 +- .../runners/dataflow/dataflow_runner.py | 4 ++-- .../apache_beam/runners/pipeline_context.py | 2 +- .../runners/portability/fn_api_runner.py | 2 +- .../apache_beam/runners/worker/data_plane.py | 2 +- .../runners/worker/data_plane_test.py | 2 +- .../apache_beam/runners/worker/log_handler.py | 2 +- .../runners/worker/log_handler_test.py | 2 +- .../apache_beam/runners/worker/sdk_worker.py | 2 +- .../runners/worker/sdk_worker_main.py | 2 +- .../runners/worker/sdk_worker_test.py | 2 +- sdks/python/apache_beam/transforms/core.py | 2 +- .../apache_beam/transforms/ptransform.py | 2 +- sdks/python/apache_beam/transforms/trigger.py | 2 +- sdks/python/apache_beam/transforms/window.py | 4 ++-- sdks/python/apache_beam/utils/urns.py | 2 +- sdks/python/gen_protos.py | 2 +- sdks/python/run_pylint.sh | 2 +- 23 files changed, 24 insertions(+), 42 deletions(-) rename sdks/python/apache_beam/portability/{runners => }/api/__init__.py (100%) delete mode 100644 sdks/python/apache_beam/portability/runners/__init__.py diff --git a/.gitignore b/.gitignore index 631d7f32cb96..36c5cc8774ea 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,7 @@ sdks/python/**/*.egg sdks/python/LICENSE sdks/python/NOTICE sdks/python/README.md -sdks/python/apache_beam/portability/runners/api/*pb2*.* +sdks/python/apache_beam/portability/api/*pb2*.* # Ignore IntelliJ files. .idea/ diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py index 1be1f3c7a477..c56ef52301b7 100644 --- a/sdks/python/apache_beam/coders/coders.py +++ b/sdks/python/apache_beam/coders/coders.py @@ -25,7 +25,7 @@ import google.protobuf from apache_beam.coders import coder_impl -from apache_beam.portability.runners.api import beam_runner_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.utils import urns from apache_beam.utils import proto_utils diff --git a/sdks/python/apache_beam/pipeline.py b/sdks/python/apache_beam/pipeline.py index 05715d715704..ab77956a0c1a 100644 --- a/sdks/python/apache_beam/pipeline.py +++ b/sdks/python/apache_beam/pipeline.py @@ -492,7 +492,7 @@ def visit_value(self, value, _): def to_runner_api(self): """For internal use only; no backwards-compatibility guarantees.""" from apache_beam.runners import pipeline_context - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 context = pipeline_context.PipelineContext() # Mutates context; placing inline would force dependence on # argument evaluation order. @@ -692,7 +692,7 @@ def named_outputs(self): if isinstance(output, pvalue.PCollection)} def to_runner_api(self, context): - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 def transform_to_runner_api(transform, context): if transform is None: diff --git a/sdks/python/apache_beam/portability/runners/api/__init__.py b/sdks/python/apache_beam/portability/api/__init__.py similarity index 100% rename from sdks/python/apache_beam/portability/runners/api/__init__.py rename to sdks/python/apache_beam/portability/api/__init__.py diff --git a/sdks/python/apache_beam/portability/runners/__init__.py b/sdks/python/apache_beam/portability/runners/__init__.py deleted file mode 100644 index 0bce5d68f724..000000000000 --- a/sdks/python/apache_beam/portability/runners/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""For internal use only; no backwards-compatibility guarantees.""" diff --git a/sdks/python/apache_beam/pvalue.py b/sdks/python/apache_beam/pvalue.py index 8a774c4c5bf4..34a483e7bb9c 100644 --- a/sdks/python/apache_beam/pvalue.py +++ b/sdks/python/apache_beam/pvalue.py @@ -128,7 +128,7 @@ def __reduce_ex__(self, unused_version): return _InvalidUnpickledPCollection, () def to_runner_api(self, context): - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.internal import pickler return beam_runner_api_pb2.PCollection( unique_name='%d%s.%s' % ( diff --git a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py index a6cc25d71512..d6944b280264 100644 --- a/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py +++ b/sdks/python/apache_beam/runners/dataflow/dataflow_runner.py @@ -732,7 +732,7 @@ def run__NativeWrite(self, transform_node): @classmethod def serialize_windowing_strategy(cls, windowing): from apache_beam.runners import pipeline_context - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 context = pipeline_context.PipelineContext() windowing_proto = windowing.to_runner_api(context) return cls.byte_array_to_json_string( @@ -745,7 +745,7 @@ def deserialize_windowing_strategy(cls, serialized_data): # Imported here to avoid circular dependencies. # pylint: disable=wrong-import-order, wrong-import-position from apache_beam.runners import pipeline_context - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms.core import Windowing proto = beam_runner_api_pb2.MessageWithComponents() proto.ParseFromString(cls.json_string_to_byte_array(serialized_data)) diff --git a/sdks/python/apache_beam/runners/pipeline_context.py b/sdks/python/apache_beam/runners/pipeline_context.py index 1330c3904edf..e212abf8d9fc 100644 --- a/sdks/python/apache_beam/runners/pipeline_context.py +++ b/sdks/python/apache_beam/runners/pipeline_context.py @@ -24,7 +24,7 @@ from apache_beam import pipeline from apache_beam import pvalue from apache_beam import coders -from apache_beam.portability.runners.api import beam_runner_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms import core diff --git a/sdks/python/apache_beam/runners/portability/fn_api_runner.py b/sdks/python/apache_beam/runners/portability/fn_api_runner.py index 8c213ad08f38..90764f4dfac0 100644 --- a/sdks/python/apache_beam/runners/portability/fn_api_runner.py +++ b/sdks/python/apache_beam/runners/portability/fn_api_runner.py @@ -33,7 +33,7 @@ from apache_beam.internal import pickler from apache_beam.io import iobase from apache_beam.transforms.window import GlobalWindows -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.portability import maptask_executor_runner from apache_beam.runners.worker import data_plane from apache_beam.runners.worker import operation_specs diff --git a/sdks/python/apache_beam/runners/worker/data_plane.py b/sdks/python/apache_beam/runners/worker/data_plane.py index 734ee9cda36a..bc981a8d30ed 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane.py +++ b/sdks/python/apache_beam/runners/worker/data_plane.py @@ -28,7 +28,7 @@ import threading from apache_beam.coders import coder_impl -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 import grpc # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/worker/data_plane_test.py b/sdks/python/apache_beam/runners/worker/data_plane_test.py index a2b31e8eb72a..360468a86874 100644 --- a/sdks/python/apache_beam/runners/worker/data_plane_test.py +++ b/sdks/python/apache_beam/runners/worker/data_plane_test.py @@ -29,7 +29,7 @@ from concurrent import futures import grpc -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker import data_plane diff --git a/sdks/python/apache_beam/runners/worker/log_handler.py b/sdks/python/apache_beam/runners/worker/log_handler.py index dca0e4bd11b4..b8f635210d2e 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler.py +++ b/sdks/python/apache_beam/runners/worker/log_handler.py @@ -21,7 +21,7 @@ import Queue as queue import threading -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 import grpc # This module is experimental. No backwards-compatibility guarantees. diff --git a/sdks/python/apache_beam/runners/worker/log_handler_test.py b/sdks/python/apache_beam/runners/worker/log_handler_test.py index 6dd018f6ad45..2256bb5556f0 100644 --- a/sdks/python/apache_beam/runners/worker/log_handler_test.py +++ b/sdks/python/apache_beam/runners/worker/log_handler_test.py @@ -22,7 +22,7 @@ from concurrent import futures import grpc -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker import log_handler diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker.py b/sdks/python/apache_beam/runners/worker/sdk_worker.py index dc4f5c2ffab0..f662538e981d 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker.py @@ -38,7 +38,7 @@ from apache_beam.io import iobase from apache_beam.runners.dataflow.native_io import iobase as native_iobase from apache_beam.utils import counters -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker import operation_specs from apache_beam.runners.worker import operations diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py index 9c11068a972e..f3f1e023e0fc 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py @@ -24,7 +24,7 @@ import grpc from google.protobuf import text_format -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker.log_handler import FnApiLogRecordHandler from apache_beam.runners.worker.sdk_worker import SdkHarness diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py index 93f60d3e7424..c431bcdf2457 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_test.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_test.py @@ -29,7 +29,7 @@ from apache_beam.io.concat_source_test import RangeSource from apache_beam.io.iobase import SourceBundle -from apache_beam.portability.runners.api import beam_fn_api_pb2 +from apache_beam.portability.api import beam_fn_api_pb2 from apache_beam.runners.worker import data_plane from apache_beam.runners.worker import sdk_worker diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py index d7fa770af3be..a137a1357ad3 100644 --- a/sdks/python/apache_beam/transforms/core.py +++ b/sdks/python/apache_beam/transforms/core.py @@ -27,7 +27,7 @@ from apache_beam import typehints from apache_beam.coders import typecoders from apache_beam.internal import util -from apache_beam.portability.runners.api import beam_runner_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.transforms import ptransform from apache_beam.transforms.display import DisplayDataItem from apache_beam.transforms.display import HasDisplayData diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py index 79fe3add26dd..60413535f65a 100644 --- a/sdks/python/apache_beam/transforms/ptransform.py +++ b/sdks/python/apache_beam/transforms/ptransform.py @@ -430,7 +430,7 @@ def register_urn(cls, urn, parameter_type, constructor): cls._known_urns[urn] = parameter_type, constructor def to_runner_api(self, context): - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 urn, typed_param = self.to_runner_api_parameter(context) return beam_runner_api_pb2.FunctionSpec( urn=urn, diff --git a/sdks/python/apache_beam/transforms/trigger.py b/sdks/python/apache_beam/transforms/trigger.py index 41516070e8e3..89c6ec535db9 100644 --- a/sdks/python/apache_beam/transforms/trigger.py +++ b/sdks/python/apache_beam/transforms/trigger.py @@ -33,7 +33,7 @@ from apache_beam.transforms.window import TimestampCombiner from apache_beam.transforms.window import WindowedValue from apache_beam.transforms.window import WindowFn -from apache_beam.portability.runners.api import beam_runner_api_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 from apache_beam.utils.timestamp import MAX_TIMESTAMP from apache_beam.utils.timestamp import MIN_TIMESTAMP diff --git a/sdks/python/apache_beam/transforms/window.py b/sdks/python/apache_beam/transforms/window.py index 08c7a2d132f7..458fb747eb22 100644 --- a/sdks/python/apache_beam/transforms/window.py +++ b/sdks/python/apache_beam/transforms/window.py @@ -55,8 +55,8 @@ from google.protobuf import timestamp_pb2 from apache_beam.coders import coders -from apache_beam.portability.runners.api import beam_runner_api_pb2 -from apache_beam.portability.runners.api import standard_window_fns_pb2 +from apache_beam.portability.api import beam_runner_api_pb2 +from apache_beam.portability.api import standard_window_fns_pb2 from apache_beam.transforms import timeutil from apache_beam.utils import proto_utils from apache_beam.utils import urns diff --git a/sdks/python/apache_beam/utils/urns.py b/sdks/python/apache_beam/utils/urns.py index b925bcc9fbcd..e553eea95f6a 100644 --- a/sdks/python/apache_beam/utils/urns.py +++ b/sdks/python/apache_beam/utils/urns.py @@ -102,7 +102,7 @@ def to_runner_api(self, context): Prefer overriding self.to_runner_api_parameter. """ - from apache_beam.portability.runners.api import beam_runner_api_pb2 + from apache_beam.portability.api import beam_runner_api_pb2 urn, typed_param = self.to_runner_api_parameter(context) return beam_runner_api_pb2.SdkFunctionSpec( spec=beam_runner_api_pb2.FunctionSpec( diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py index a33c74b9cd92..a3d963d18d78 100644 --- a/sdks/python/gen_protos.py +++ b/sdks/python/gen_protos.py @@ -35,7 +35,7 @@ os.path.join('..', 'common', 'fn-api', 'src', 'main', 'proto') ] -PYTHON_OUTPUT_PATH = os.path.join('apache_beam', 'portability', 'runners', 'api') +PYTHON_OUTPUT_PATH = os.path.join('apache_beam', 'portability', 'api') def generate_proto_files(): diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh index 7434516bfdeb..2691be4ea42c 100755 --- a/sdks/python/run_pylint.sh +++ b/sdks/python/run_pylint.sh @@ -46,7 +46,7 @@ EXCLUDED_GENERATED_FILES=( "apache_beam/io/gcp/internal/clients/storage/storage_v1_client.py" "apache_beam/io/gcp/internal/clients/storage/storage_v1_messages.py" "apache_beam/coders/proto2_coder_test_messages_pb2.py" -apache_beam/portability/runners/api/*pb2*.py +apache_beam/portability/api/*pb2*.py ) FILES_TO_IGNORE=""