From 0ac717f1ab34403d81841576a86aa93ab4ce329b Mon Sep 17 00:00:00 2001 From: Diego Davila Date: Mon, 16 Sep 2019 21:20:44 +0200 Subject: [PATCH 1/5] making stashcp a packge not a module and changing the order for looking the caches.json file (software-3799) --- setup.py | 8 +++++--- stashcp.py => stashcp/__init__.py | 16 ++++++++-------- caches.json => stashcp/caches.json | 0 3 files changed, 13 insertions(+), 11 deletions(-) rename stashcp.py => stashcp/__init__.py (98%) rename caches.json => stashcp/caches.json (100%) diff --git a/setup.py b/setup.py index 12b1e90..6a78465 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ # For a discussion on single-sourcing the version across setup.py and the # project code, see # https://packaging.python.org/en/latest/single_source_version.html - version='5.4.1', # Required + version='5.5.0', # Required # This is a one-line description or tagline of what your project does. This # corresponds to the "Summary" metadata field: @@ -121,7 +121,8 @@ # the `py_modules` argument instead as follows, which will expect a file # called `my_module.py` to exist: # - py_modules=["stashcp"], + #py_modules=["stashcp"], + packages=["stashcp"], #packages=find_packages(exclude=['contrib', 'docs', 'tests']), # Required @@ -151,6 +152,7 @@ # # If using Python 2.6 or earlier, then these have to be included in # MANIFEST.in as well. + package_data={'stashcp': ['caches.json'],}, #package_data={ # Optional # '': ['bin/caches.json'], #}, @@ -160,7 +162,7 @@ # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # # In this case, 'data_file' will be installed into '/my_data' - data_files=[('', ['caches.json'])], # Optional + data_files=[('share/stashcache/', ['stashcp/caches.json'])], # To provide executable scripts, use entry points in preference to the # "scripts" keyword. Entry points provide cross-platform support and allow diff --git a/stashcp.py b/stashcp/__init__.py similarity index 98% rename from stashcp.py rename to stashcp/__init__.py index 1113a76..db13b4f 100755 --- a/stashcp.py +++ b/stashcp/__init__.py @@ -15,16 +15,16 @@ from urlparse import urlparse try: - from pkg_resources import resource_string + from pkg_resources import resource_filename except ImportError as e: - resource_string = None + resource_filename = None import logging from urlparse import urlparse # Version information for user-agent -VERSION = "5.4.1" +VERSION = "5.5.0" main_redirector = "root://redirector.osgstorage.org" stash_origin = "root://stash.osgconnect.net" @@ -554,12 +554,12 @@ def get_best_stashcache(): cache_files = [ caches_json_location ] else: prefix = os.environ.get("OSG_LOCATION", "/") - cache_files = [os.path.join(os.path.dirname(os.path.realpath(__file__)), "caches.json"), - os.path.join(prefix, "etc/stashcache/caches.json"), - os.path.join(prefix, "usr/share/stashcache/caches.json")] - if resource_string: + cache_files = [os.path.join(prefix, "etc/stashcache/caches.json"), + os.path.join(prefix, "usr/share/stashcache/caches.json"), + os.path.join(prefix, "usr/local/share/stashcache/caches.json")] + if resource_filename: try: - cache_files.insert(0, resource_string(__name__, 'caches.json')) + cache_files.append(resource_filename(__name__, 'caches.json')) except IOError as ioe: logging.debug("Unable to retrieve caches.json using resource string, trying other locations") diff --git a/caches.json b/stashcp/caches.json similarity index 100% rename from caches.json rename to stashcp/caches.json From 6168861b70e0eda9a98db0e3d127c62039bc12dc Mon Sep 17 00:00:00 2001 From: Diego Davila Date: Tue, 1 Oct 2019 01:39:08 +0200 Subject: [PATCH 2/5] fixing tests (software-3799 --- bin/stashcp2/tests/test_inside_docker.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/bin/stashcp2/tests/test_inside_docker.sh b/bin/stashcp2/tests/test_inside_docker.sh index c79f1cd..bf15d20 100755 --- a/bin/stashcp2/tests/test_inside_docker.sh +++ b/bin/stashcp2/tests/test_inside_docker.sh @@ -42,7 +42,7 @@ cp /StashCache/bin/stashcp2/tests/job.ad ./.job.ad # Test against a file that is known to not exist set +e -/StashCache/stashcp.py --cache=$XRD_CACHE /blah/does/not/exist ./ +/StashCache/stashcp/__init__.py --cache=$XRD_CACHE /blah/does/not/exist ./ if [ $? -eq 0 ]; then echo "Failed to exit with non-zero exit status when it should have" exit 1 @@ -50,14 +50,14 @@ fi set -e # Try copying with no forward slash -/StashCache/stashcp.py --cache=$XRD_CACHE user/dweitzel/public/blast/queries/query1 ./ +/StashCache/stashcp/__init__.py --cache=$XRD_CACHE user/dweitzel/public/blast/queries/query1 ./ result=`md5sum query1 | awk '{print $1;}'` rm query1 # Try copying with different destintion filename -/StashCache/stashcp.py --cache=$XRD_CACHE -d /user/dweitzel/public/blast/queries/query1 query.test +/StashCache/stashcp/__init__.py --cache=$XRD_CACHE -d /user/dweitzel/public/blast/queries/query1 query.test result=`md5sum query.test | awk '{print $1;}'` @@ -68,7 +68,7 @@ fi rm -f query.test # Perform tests -/StashCache/stashcp.py --cache=$XRD_CACHE -d /user/dweitzel/public/blast/queries/query1 ./ +/StashCache/stashcp/__init__.py --cache=$XRD_CACHE -d /user/dweitzel/public/blast/queries/query1 ./ result=`md5sum query1 | awk '{print $1;}'` @@ -78,7 +78,7 @@ fi rm -f query.test # Perform methods test -/StashCache/stashcp.py --cache=$XRD_CACHE --method=cvmfs,xrootd -d /user/dweitzel/public/blast/queries/query1 ./ +/StashCache/stashcp/__init__.py --cache=$XRD_CACHE --method=cvmfs,xrootd -d /user/dweitzel/public/blast/queries/query1 ./ result=`md5sum query1 | awk '{print $1;}'` @@ -88,7 +88,7 @@ fi rm -f query.test # Perform methods test -/StashCache/stashcp.py --cache=$XRD_CACHE --method=xrootd -d /user/dweitzel/public/blast/queries/query1 ./ +/StashCache/stashcp/__init__.py --cache=$XRD_CACHE --method=xrootd -d /user/dweitzel/public/blast/queries/query1 ./ result=`md5sum query1 | awk '{print $1;}'` @@ -98,7 +98,7 @@ fi rm -f query.test # Perform methods test -/StashCache/stashcp.py --cache=$XRD_CACHE --method=http,xrootd -d /user/dweitzel/public/blast/queries/query1 ./ +/StashCache/stashcp/__init__.py --cache=$XRD_CACHE --method=http,xrootd -d /user/dweitzel/public/blast/queries/query1 ./ result=`md5sum query1 | awk '{print $1;}'` @@ -108,7 +108,7 @@ fi rm -f query.test # Scheme test -/StashCache/stashcp.py --cache=$XRD_CACHE --method=http,xrootd -d stash:///user/dweitzel/public/blast/queries/query1 file:///tmp +/StashCache/stashcp/__init__.py --cache=$XRD_CACHE --method=http,xrootd -d stash:///user/dweitzel/public/blast/queries/query1 file:///tmp result=`md5sum /tmp/query1 | awk '{print $1;}'` @@ -117,7 +117,7 @@ if [ "$result" != "12bdb9a96cd5e8ca469b727a81593201" ]; then fi rm -f query.test -/StashCache/stashcp.py --cache=$XRD_CACHE -d -r /user/dweitzel/public/blast/queries ./ +/StashCache/stashcp/__init__.py --cache=$XRD_CACHE -d -r /user/dweitzel/public/blast/queries ./ ls -lah rm -rf queries From d2b00b324803080d930df2b07052c037288857c0 Mon Sep 17 00:00:00 2001 From: Diego Davila Date: Tue, 1 Oct 2019 01:41:18 +0200 Subject: [PATCH 3/5] fixing identation --- stashcp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stashcp/__init__.py b/stashcp/__init__.py index db13b4f..6af131e 100755 --- a/stashcp/__init__.py +++ b/stashcp/__init__.py @@ -559,7 +559,7 @@ def get_best_stashcache(): os.path.join(prefix, "usr/local/share/stashcache/caches.json")] if resource_filename: try: - cache_files.append(resource_filename(__name__, 'caches.json')) + cache_files.append(resource_filename(__name__, 'caches.json')) except IOError as ioe: logging.debug("Unable to retrieve caches.json using resource string, trying other locations") From 260224f9776647a58a41b61ff3a54720fa3fc3d6 Mon Sep 17 00:00:00 2001 From: Diego Davila Date: Tue, 1 Oct 2019 20:51:18 +0200 Subject: [PATCH 4/5] Install the package before testing it (software-3799) --- bin/stashcp2/tests/test_inside_docker.sh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bin/stashcp2/tests/test_inside_docker.sh b/bin/stashcp2/tests/test_inside_docker.sh index bf15d20..7aa9601 100755 --- a/bin/stashcp2/tests/test_inside_docker.sh +++ b/bin/stashcp2/tests/test_inside_docker.sh @@ -37,12 +37,15 @@ module load xrootd # For now, disable pylint failures #pylint /StashCache/bin/stashcp || /bin/true +# Install stashcp +pip install StashCache/ + # Copy in the .job.ad file: cp /StashCache/bin/stashcp2/tests/job.ad ./.job.ad # Test against a file that is known to not exist set +e -/StashCache/stashcp/__init__.py --cache=$XRD_CACHE /blah/does/not/exist ./ +stashcp --cache=$XRD_CACHE /blah/does/not/exist ./ if [ $? -eq 0 ]; then echo "Failed to exit with non-zero exit status when it should have" exit 1 @@ -50,14 +53,14 @@ fi set -e # Try copying with no forward slash -/StashCache/stashcp/__init__.py --cache=$XRD_CACHE user/dweitzel/public/blast/queries/query1 ./ +stashcp --cache=$XRD_CACHE user/dweitzel/public/blast/queries/query1 ./ result=`md5sum query1 | awk '{print $1;}'` rm query1 # Try copying with different destintion filename -/StashCache/stashcp/__init__.py --cache=$XRD_CACHE -d /user/dweitzel/public/blast/queries/query1 query.test +stashcp --cache=$XRD_CACHE -d /user/dweitzel/public/blast/queries/query1 query.test result=`md5sum query.test | awk '{print $1;}'` @@ -68,7 +71,7 @@ fi rm -f query.test # Perform tests -/StashCache/stashcp/__init__.py --cache=$XRD_CACHE -d /user/dweitzel/public/blast/queries/query1 ./ +stashcp --cache=$XRD_CACHE -d /user/dweitzel/public/blast/queries/query1 ./ result=`md5sum query1 | awk '{print $1;}'` @@ -78,7 +81,7 @@ fi rm -f query.test # Perform methods test -/StashCache/stashcp/__init__.py --cache=$XRD_CACHE --method=cvmfs,xrootd -d /user/dweitzel/public/blast/queries/query1 ./ +stashcp --cache=$XRD_CACHE --method=cvmfs,xrootd -d /user/dweitzel/public/blast/queries/query1 ./ result=`md5sum query1 | awk '{print $1;}'` @@ -88,7 +91,7 @@ fi rm -f query.test # Perform methods test -/StashCache/stashcp/__init__.py --cache=$XRD_CACHE --method=xrootd -d /user/dweitzel/public/blast/queries/query1 ./ +stashcp --cache=$XRD_CACHE --method=xrootd -d /user/dweitzel/public/blast/queries/query1 ./ result=`md5sum query1 | awk '{print $1;}'` @@ -98,7 +101,7 @@ fi rm -f query.test # Perform methods test -/StashCache/stashcp/__init__.py --cache=$XRD_CACHE --method=http,xrootd -d /user/dweitzel/public/blast/queries/query1 ./ +stashcp --cache=$XRD_CACHE --method=http,xrootd -d /user/dweitzel/public/blast/queries/query1 ./ result=`md5sum query1 | awk '{print $1;}'` @@ -108,7 +111,7 @@ fi rm -f query.test # Scheme test -/StashCache/stashcp/__init__.py --cache=$XRD_CACHE --method=http,xrootd -d stash:///user/dweitzel/public/blast/queries/query1 file:///tmp +stashcp --cache=$XRD_CACHE --method=http,xrootd -d stash:///user/dweitzel/public/blast/queries/query1 file:///tmp result=`md5sum /tmp/query1 | awk '{print $1;}'` @@ -117,7 +120,7 @@ if [ "$result" != "12bdb9a96cd5e8ca469b727a81593201" ]; then fi rm -f query.test -/StashCache/stashcp/__init__.py --cache=$XRD_CACHE -d -r /user/dweitzel/public/blast/queries ./ +stashcp --cache=$XRD_CACHE -d -r /user/dweitzel/public/blast/queries ./ ls -lah rm -rf queries From 992f9adb8a59a666bce37b8ca787b7131c496152 Mon Sep 17 00:00:00 2001 From: Diego Davila Date: Wed, 2 Oct 2019 00:04:46 +0200 Subject: [PATCH 5/5] install pip (software-3799) --- bin/stashcp2/tests/test_inside_docker.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/stashcp2/tests/test_inside_docker.sh b/bin/stashcp2/tests/test_inside_docker.sh index 7aa9601..9eed5e9 100755 --- a/bin/stashcp2/tests/test_inside_docker.sh +++ b/bin/stashcp2/tests/test_inside_docker.sh @@ -17,6 +17,8 @@ rpm -Uvh https://repo.opensciencegrid.org/osg/3.3/osg-3.3-el${OS_VERSION}-releas yum -y install osg-oasis +yum -y install python-pip + echo "user_allow_other" >> /etc/fuse.conf echo "CVMFS_HTTP_PROXY=DIRECT" >> /etc/cvmfs/default.local