Merge pull request #17 from NLeSC-GO-common-infrastructure/development

Development

.github/workflows/build.yml

@@ -22,15 +22,11 @@ jobs:
         run: |
           which python
           python --version
-      - name: Install dependencies
+      - name: Install package (with test dependencies)
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
-      - name: Build
-        shell: bash -l {0}
-        run: |
-          python setup.py build
+          python -m pip install -e .[test]
       - name: Test
         shell: bash -l {0}
-        run: |
-          python setup.py test
+        run: pytest --cov --cov-report xml --cov-report term --cov-report html
+

CHANGELOG.rst

@@ -5,6 +5,21 @@ Change Log
 All notable changes to this project will be documented in this file.
 This project adheres to `Semantic Versioning <http://semver.org/>`_.
 
+
+[0.2.1]
+*******
+
+Fixed
+-----
+
+* Process pool in copy_asset always spawn processes, avoiding issues with async filesystems
+
+Added
+-----
+
+* Global configure function
+
+
 [0.2.0]
 *******
 

CITATION.cff

@@ -9,8 +9,8 @@ authors:
     city: Amsterdam
     country: NL
     email:
-date-released: 2021-11-17
-version: "0.2.0"
+date-released: 2021-11-26
+version: "0.2.1"
 repository-code: "https://github.com/NLeSC-GO-common-infrastructure/stac2dcache"
 keywords:
   - STAC

setup.cfg

@@ -1,18 +1,6 @@
 [metadata]
 description-file = README.rst
 
-[aliases]
-# Define `python setup.py test`
-test=pytest
-
-[coverage:run]
-branch = True
-source = stac2dcache
-
-[tool:pytest]
-testpaths = tests
-addopts = --cov --cov-report xml --cov-report term --cov-report html
-
 # Define `python setup.py build_sphinx`
 [build_sphinx]
 source-dir = docs

setup.py

@@ -16,6 +16,14 @@
 with open('requirements.txt') as requirements_file:
     requirements = requirements_file.read().split()
 
+extras_require = {
+    'test': [
+        'pytest',
+        'pytest-cov',
+        'pycodestyle',
+    ],
+}
+
 setup(
     name='stac2dcache',
     version=version['__version__'],
@@ -40,16 +48,7 @@
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
     ],
-    test_suite='tests',
     python_requires=">=3.7",
     install_requires=requirements,
-    setup_requires=[
-        # dependency for `python setup.py test`
-        'pytest-runner',
-    ],
-    tests_require=[
-        'pytest',
-        'pytest-cov',
-        'pycodestyle',
-    ]
+    extras_require=extras_require,
 )

stac2dcache/__init__.py

@@ -5,7 +5,11 @@
 
 import logging
 
+from .configure import configure
 from .stac_io import configure_stac_io
 from .filesystem import configure_filesystem
 
 logging.getLogger(__name__).addHandler(logging.NullHandler())
+
+fs = None
+stac_io = None

stac2dcache/__version__.py

@@ -1 +1 @@
-__version__ = '0.2.0'
+__version__ = '0.2.1'

stac2dcache/configure.py

@@ -0,0 +1,21 @@
+import stac2dcache
+
+from .filesystem import configure_filesystem
+from .stac_io import configure_stac_io
+
+
+def configure(username=None, password=None, token_filename=None):
+    """
+    Configure authentication to dCache with either username/passwd or token.
+
+    :param username: (optional, str)
+    :param password: (optional, str)
+    :param token_filename: (optional, str) path to file with the token
+    """
+    stac2dcache.fs = configure_filesystem(
+        filesystem="dcache",
+        username=username,
+        password=password,
+        token_filename=token_filename,
+    )
+    stac2dcache.stac_io = configure_stac_io(stac2dcache.fs)

stac2dcache/utils.py

@@ -1,4 +1,5 @@
 import geopandas
+import multiprocessing
 import urlpath
 
 from concurrent.futures import ProcessPoolExecutor, as_completed
@@ -7,6 +8,9 @@
 from .filesystem import copy
 
 
+mp_context = multiprocessing.get_context("spawn")
+
+
 def catalog2geopandas(catalog, crs=None):
     """
     Create a geopandas data-frame with the catalog items
@@ -59,7 +63,8 @@ def copy_asset(catalog, asset_key, update_catalog=False, item_id=None,
     else:
         items = catalog.get_all_items()
 
-    with ProcessPoolExecutor(max_workers=max_workers) as executor:
+    with ProcessPoolExecutor(max_workers=max_workers, mp_context=mp_context) \
+            as executor:
 
         future_to_asset = {}
         for item in items: