From f7b9450a8bee91f4626fdbfff94dd0ed21a9b6e0 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Tue, 7 Dec 2021 02:23:22 +0800 Subject: [PATCH] Remove default value Signed-off-by: Kevin Su --- flytekit/configuration/aws.py | 6 +++--- flytekit/remote/remote.py | 6 ++++-- flytekit/types/pickle/pickle.py | 6 +++++- plugins/flytekit-data-fsspec/tests/test_persist.py | 8 ++++---- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/flytekit/configuration/aws.py b/flytekit/configuration/aws.py index 785ee961de..a6af62bf6f 100644 --- a/flytekit/configuration/aws.py +++ b/flytekit/configuration/aws.py @@ -4,11 +4,11 @@ S3_SHARD_STRING_LENGTH = _config_common.FlyteIntegerConfigurationEntry("aws", "s3_shard_string_length", default=2) -S3_ENDPOINT = _config_common.FlyteStringConfigurationEntry("aws", "endpoint", default="http://localhost:30084") +S3_ENDPOINT = _config_common.FlyteStringConfigurationEntry("aws", "endpoint", default=None) -S3_ACCESS_KEY_ID = _config_common.FlyteStringConfigurationEntry("aws", "access_key_id", default="minio") +S3_ACCESS_KEY_ID = _config_common.FlyteStringConfigurationEntry("aws", "access_key_id", default=None) -S3_SECRET_ACCESS_KEY = _config_common.FlyteStringConfigurationEntry("aws", "secret_access_key", default="miniostorage") +S3_SECRET_ACCESS_KEY = _config_common.FlyteStringConfigurationEntry("aws", "secret_access_key", default=None) S3_ACCESS_KEY_ID_ENV_NAME = "AWS_ACCESS_KEY_ID" diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index 2c3e8ab41d..60ae7bbf3e 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -235,8 +235,6 @@ def __init__( # Not exposing this as a property for now. self._entrypoint_settings = entrypoint_settings - # Save the file access object locally, but also make it available for use from the context. - FlyteContextManager.with_context(FlyteContextManager.current_context().with_file_access(file_access).build()) raw_output_data_prefix = auth_config.RAW_OUTPUT_DATA_PREFIX.get() or os.path.join( sdk_config.LOCAL_SANDBOX.get(), "control_plane_raw" ) @@ -244,6 +242,10 @@ def __init__( local_sandbox_dir=os.path.join(sdk_config.LOCAL_SANDBOX.get(), "control_plane_metadata"), raw_output_prefix=raw_output_data_prefix, ) + # Save the file access object locally, but also make it available for use from the context. + FlyteContextManager.with_context( + FlyteContextManager.current_context().with_file_access(self._file_access).build() + ) # TODO: Reconsider whether we want this. Probably best to not cache. self._serialized_entity_cache = OrderedDict() diff --git a/flytekit/types/pickle/pickle.py b/flytekit/types/pickle/pickle.py index 57ead7ea09..3472dec7e6 100644 --- a/flytekit/types/pickle/pickle.py +++ b/flytekit/types/pickle/pickle.py @@ -79,7 +79,11 @@ def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], exp return Literal(scalar=Scalar(blob=Blob(metadata=meta, uri=remote_path))) def guess_python_type(self, literal_type: LiteralType) -> typing.Type[FlytePickle[typing.Any]]: - if literal_type.blob is not None and literal_type.blob.format == FlytePickleTransformer.PYTHON_PICKLE_FORMAT: + if ( + literal_type.blob is not None + and literal_type.blob.dimensionality == _core_types.BlobType.BlobDimensionality.SINGLE + and literal_type.blob.format == FlytePickleTransformer.PYTHON_PICKLE_FORMAT + ): return FlytePickle raise ValueError(f"Transformer {self} cannot reverse {literal_type}") diff --git a/plugins/flytekit-data-fsspec/tests/test_persist.py b/plugins/flytekit-data-fsspec/tests/test_persist.py index ce3a4d7e41..d2ac50a7f8 100644 --- a/plugins/flytekit-data-fsspec/tests/test_persist.py +++ b/plugins/flytekit-data-fsspec/tests/test_persist.py @@ -10,15 +10,15 @@ def test_s3_setup_args(): kwargs = s3_setup_args() - assert kwargs == {"client_kwargs": {"endpoint_url": "http://localhost:30084"}} + assert kwargs == {} - with aws.S3_ENDPOINT.get_patcher("http://flyte:30084"): + with aws.S3_ENDPOINT.get_patcher("http://localhost:30084"): kwargs = s3_setup_args() - assert kwargs == {"client_kwargs": {"endpoint_url": "http://flyte:30084"}} + assert kwargs == {"client_kwargs": {"endpoint_url": "http://localhost:30084"}} with aws.S3_ACCESS_KEY_ID.get_patcher("access"): kwargs = s3_setup_args() - assert kwargs == {"client_kwargs": {"endpoint_url": "http://localhost:30084"}} + assert kwargs == {} assert os.environ[aws.S3_ACCESS_KEY_ID_ENV_NAME] == "access"