From 617b2ea1f6cde778464a15688549a09acddda991 Mon Sep 17 00:00:00 2001 From: Arul Ajmani Date: Tue, 30 Jul 2019 19:58:50 -0700 Subject: [PATCH 1/2] add support for wheels --- databricks_cli/pipelines/api.py | 5 +++-- tests/pipelines/test_api.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/databricks_cli/pipelines/api.py b/databricks_cli/pipelines/api.py index a0292e1a..b269ab9c 100644 --- a/databricks_cli/pipelines/api.py +++ b/databricks_cli/pipelines/api.py @@ -38,6 +38,7 @@ BUFFER_SIZE = 1024 * 64 base_pipelines_dir = 'dbfs:/pipelines/code' +supported_lib_types = {'jar', 'whl'} class PipelinesApi(object): @@ -73,9 +74,9 @@ def _identify_local_libraries(lib_objects): local_lib_objects, external_lib_objects = [], [] for lib_object in lib_objects: parsed_uri = urllib.parse.urlparse(lib_object.path) - if lib_object.lib_type == 'jar' and parsed_uri.scheme == '': + if lib_object.lib_type in supported_lib_types and parsed_uri.scheme == '': local_lib_objects.append(lib_object) - elif lib_object.lib_type == 'jar' and parsed_uri.scheme.lower() == 'file': + elif lib_object.lib_type in supported_lib_types and parsed_uri.scheme.lower() == 'file': # exactly 1 or 3 if parsed_uri.path.startswith('//') or parsed_uri.netloc != '': raise RuntimeError('invalid file uri scheme, ' diff --git a/tests/pipelines/test_api.py b/tests/pipelines/test_api.py index 7b7a28ce..2007c62c 100644 --- a/tests/pipelines/test_api.py +++ b/tests/pipelines/test_api.py @@ -151,14 +151,14 @@ def test_partition_local_remote(pipelines_api): LibraryObject('jar', 'FILE:/all/caps.ext'), LibraryObject('jar', 'FiLe:/weird/case.ext'), LibraryObject('jar', 'file.ext'), + LibraryObject('whl', 'rel/path.ext'), # shouldn't be uploaded LibraryObject('jar', 'dbfs:/absolute/path/abc.ext'), LibraryObject('jar', 's3:file:/file/scheme/abs/path.ext'), LibraryObject('jar', 'scheme:file.ext'), LibraryObject('jar', 'scheme:/abs/path.ext'), LibraryObject('jar', 'scheme://abs/path.ext'), - LibraryObject('egg', 'file:/abs/path.ext'), - LibraryObject('whl', 'rel/path.ext') + LibraryObject('egg', 'file:/abs/path.ext') ] expected_llo = [ LibraryObject('jar', '/absolute/path/abc.ext'), @@ -166,7 +166,8 @@ def test_partition_local_remote(pipelines_api): LibraryObject('jar', '/file/scheme/abs/path.ext'), LibraryObject('jar', '/all/caps.ext'), LibraryObject('jar', '/weird/case.ext'), - LibraryObject('jar', 'file.ext') + LibraryObject('jar', 'file.ext'), + LibraryObject('whl', 'rel/path.ext') ] expected_external = [ LibraryObject('jar', 'dbfs:/absolute/path/abc.ext'), @@ -174,8 +175,7 @@ def test_partition_local_remote(pipelines_api): LibraryObject('jar', 'scheme:file.ext'), LibraryObject('jar', 'scheme:/abs/path.ext'), LibraryObject('jar', 'scheme://abs/path.ext'), - LibraryObject('egg', 'file:/abs/path.ext'), - LibraryObject('whl', 'rel/path.ext') + LibraryObject('egg', 'file:/abs/path.ext') ] llo, external = pipelines_api._identify_local_libraries(libraries) assert llo == expected_llo From c163a6ee18e694ffc6edae9008a559059e7479c3 Mon Sep 17 00:00:00 2001 From: Arul Ajmani Date: Tue, 30 Jul 2019 20:10:21 -0700 Subject: [PATCH 2/2] linter --- tests/pipelines/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/test_api.py b/tests/pipelines/test_api.py index 2007c62c..6594459a 100644 --- a/tests/pipelines/test_api.py +++ b/tests/pipelines/test_api.py @@ -207,4 +207,4 @@ def test_library_object_serialization_deserialization(): assert llo == library_objects libs = LibraryObject.to_json(library_objects) - assert libs == libraries \ No newline at end of file + assert libs == libraries