From 5c99e37e6d38853f9bbec0a4ab2b36ea757f6b6b Mon Sep 17 00:00:00 2001 From: Raphael Guzman Date: Fri, 4 Oct 2019 12:37:58 -0500 Subject: [PATCH 1/7] Fix WIN10 filepath to store as posix and fetch as user's platform. --- .travis.yml | 1 - datajoint/external.py | 4 ++-- tests/__init__.py | 10 +++++----- tests/test_filepath.py | 30 ++++++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7c664424f..012bb06f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,4 +70,3 @@ jobs: script: - sudo pip install python-coveralls - coveralls ->>>>>>> bff58baac8da58b43be95ba1c58174cb7721411f diff --git a/datajoint/external.py b/datajoint/external.py index 9d43372cf..8351f04b8 100644 --- a/datajoint/external.py +++ b/datajoint/external.py @@ -74,7 +74,7 @@ def s3(self): def _make_external_filepath(self, relative_filepath): """resolve the complete external path based on the relative path""" - return PurePosixPath(self.spec['location'], relative_filepath) + return PurePosixPath(Path(self.spec['location']), relative_filepath) def _make_uuid_path(self, uuid, suffix=''): """create external path based on the uuid hash""" @@ -211,7 +211,7 @@ def upload_filepath(self, local_filepath): """ local_filepath = Path(local_filepath) try: - relative_filepath = str(local_filepath.relative_to(self.spec['stage'])) + relative_filepath = str(local_filepath.relative_to(self.spec['stage']).as_posix()) except ValueError: raise DataJointError('The path {path} is not in stage {stage}'.format( path=local_filepath.parent, **self.spec)) from None diff --git a/tests/__init__.py b/tests/__init__.py index f5520b931..a8c2d1c5f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -113,7 +113,7 @@ def setup_package(): # Add old MySQL source = Path( - os.path.dirname(os.path.realpath(__file__)), + Path(__file__).resolve().parent, 'external-legacy-data') db_name = "djtest_blob_migrate" db_file = "v0_11.sql" @@ -127,7 +127,7 @@ def setup_package(): # Add old S3 source = Path( - os.path.dirname(os.path.realpath(__file__)), + Path(__file__).resolve().parent, 'external-legacy-data','s3') bucket = "migrate-test" region = "us-east-1" @@ -152,9 +152,9 @@ def setup_package(): # Add old File Content try: shutil.copytree( - Path(os.path.dirname(os.path.realpath(__file__)), - 'external-legacy-data','file','temp'), - Path(os.path.expanduser('~'),'temp')) + Path(Path(__file__).resolve().parent, + 'external-legacy-data','file','temp'), + Path(os.path.expanduser('~'),'temp')) except FileExistsError: pass diff --git a/tests/test_filepath.py b/tests/test_filepath.py index 610b3a5d5..4c81d0f9b 100644 --- a/tests/test_filepath.py +++ b/tests/test_filepath.py @@ -10,6 +10,36 @@ def setUp(self): dj.config['stores'] = stores_config +def test_path_match(store="repo"): + """ test file path matches and empty file""" + ext = schema.external[store] + stage_path = dj.config['stores'][store]['stage'] + + # create a mock file + relpath = 'path/to/films' + managed_file = Path(stage_path, relpath, 'vid.mov') + managed_file.parent.mkdir(parents=True, exist_ok=True) + open(managed_file, 'a').close() + + # put the file + uuid = ext.upload_filepath(managed_file) + + #remove + managed_file.unlink() + assert_false(managed_file.exists()) + + #check filepath + assert_equal( + (ext & {'hash': uuid}).fetch1('filepath'), + str(managed_file.relative_to(stage_path).as_posix())) + + # # Download the file and check its contents. + restored_path, checksum = ext.download_filepath(uuid) + assert_equal(restored_path, managed_file) + assert_equal(checksum, dj.hash.uuid_from_file(managed_file)) + + # cleanup + ext.delete(delete_external_files=True) def test_filepath(store="repo"): """ test file management """ From e2c3f2349040bc781a58e52313f80267147de364 Mon Sep 17 00:00:00 2001 From: Raphael Guzman Date: Fri, 4 Oct 2019 12:56:27 -0500 Subject: [PATCH 2/7] Fix relpath for Python3.5. --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index a8c2d1c5f..faa26a254 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -141,7 +141,7 @@ def setup_package(): if os.path.isfile(str(path)) and ".sql" not in str(path): minioClient.fput_object( bucket, str(Path( - os.path.relpath(path,Path(source,bucket))) + os.path.relpath(str(path),str(Path(source,bucket)))) .as_posix()), str(path)) # Add S3 try: From 93aeefc3b9198b28777774f1cd9058d3f3aee05d Mon Sep 17 00:00:00 2001 From: Raphael Guzman Date: Fri, 4 Oct 2019 13:31:27 -0500 Subject: [PATCH 3/7] Fix copytree for Python3.5. --- tests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index faa26a254..0384d240b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -152,9 +152,9 @@ def setup_package(): # Add old File Content try: shutil.copytree( - Path(Path(__file__).resolve().parent, + str(Path(Path(__file__).resolve().parent, 'external-legacy-data','file','temp'), - Path(os.path.expanduser('~'),'temp')) + Path(os.path.expanduser('~'),'temp'))) except FileExistsError: pass From 20719aebbd4543262634ad803d5ee206ce82b5f1 Mon Sep 17 00:00:00 2001 From: Raphael Guzman Date: Fri, 4 Oct 2019 13:40:42 -0500 Subject: [PATCH 4/7] Fix typo. --- tests/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 0384d240b..8869a4709 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -153,8 +153,8 @@ def setup_package(): try: shutil.copytree( str(Path(Path(__file__).resolve().parent, - 'external-legacy-data','file','temp'), - Path(os.path.expanduser('~'),'temp'))) + 'external-legacy-data','file','temp')), + str(Path(os.path.expanduser('~'),'temp'))) except FileExistsError: pass From 7ca00996234acf948f10e78afdb397a19fd1c1e3 Mon Sep 17 00:00:00 2001 From: Raphael Guzman Date: Fri, 4 Oct 2019 13:57:44 -0500 Subject: [PATCH 5/7] Fix for Python3.5. --- tests/__init__.py | 2 +- tests/test_filepath.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 8869a4709..ebc2e4a40 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -191,4 +191,4 @@ def teardown_package(): minioClient.remove_bucket(bucket) # Remove old File Content - shutil.rmtree(Path(os.path.expanduser('~'),'temp')) + shutil.rmtree(str(Path(os.path.expanduser('~'),'temp'))) diff --git a/tests/test_filepath.py b/tests/test_filepath.py index 4c81d0f9b..ca0c2b20f 100644 --- a/tests/test_filepath.py +++ b/tests/test_filepath.py @@ -19,7 +19,7 @@ def test_path_match(store="repo"): relpath = 'path/to/films' managed_file = Path(stage_path, relpath, 'vid.mov') managed_file.parent.mkdir(parents=True, exist_ok=True) - open(managed_file, 'a').close() + open(str(managed_file), 'a').close() # put the file uuid = ext.upload_filepath(managed_file) From f3ffd63d4a20c7f95985b766c8647a567bcbbf3b Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Fri, 4 Oct 2019 14:35:00 -0500 Subject: [PATCH 6/7] Update coveralls. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 012bb06f4..c25fe003f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,4 +69,5 @@ jobs: language: shell script: - sudo pip install python-coveralls + - coverage run --source=datajoint setup.py tests - coveralls From bb1b40f52dd2d1fc006a91536ee3ad204da430ac Mon Sep 17 00:00:00 2001 From: guzman-raphael Date: Fri, 4 Oct 2019 16:59:13 -0500 Subject: [PATCH 7/7] Update coverall env vars. --- .coveralls.yml | 1 - .travis.yml | 9 ++------- LNX-docker-compose.yml | 5 ++++- 3 files changed, 6 insertions(+), 9 deletions(-) delete mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml deleted file mode 100644 index 2eeede133..000000000 --- a/.coveralls.yml +++ /dev/null @@ -1 +0,0 @@ -repo_token: fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index c25fe003f..2a3c16a3b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ env: - COMPOSE_HTTP_TIMEOUT="300" - UID="2000" - GID="2000" + - COVERALLS_SERVICE_NAME="travis-ci" + - COVERALLS_REPO_TOKEN="fd0BoXG46TPReEem0uMy7BJO5j0w1MQiY" services: - docker main: &main @@ -64,10 +66,3 @@ jobs: env: - PY_VER: "3.5" - MYSQL_VER: "5.6" - - stage: Coverage - os: linux - language: shell - script: - - sudo pip install python-coveralls - - coverage run --source=datajoint setup.py tests - - coveralls diff --git a/LNX-docker-compose.yml b/LNX-docker-compose.yml index cf2c9902a..f5dead068 100644 --- a/LNX-docker-compose.yml +++ b/LNX-docker-compose.yml @@ -25,13 +25,16 @@ services: - PYTHON_USER=dja - JUPYTER_PASSWORD=datajoint - DISPLAY + - COVERALLS_SERVICE_NAME + - COVERALLS_REPO_TOKEN working_dir: /src command: > /bin/sh -c " - pip install --user nose nose-cov .; + pip install --user nose nose-cov coveralls .; pip freeze | grep datajoint; nosetests -vsw tests --with-coverage --cover-package=datajoint; + coveralls; # jupyter notebook; " # ports: