small bump for build (#35)

* restructure to src layout and use importlib resources * fixes for new isis data server * rclone added to isis yml * remove ldd test from asap environment test, shouldn't really be there * patch cspice so * adding kernel db files again in a more explicit way * ugly hacks for download script issues
AndrewAnnex · Feb 8, 2023 · 9757e0b · 9757e0b
1 parent f2d216d
commit 9757e0b
Show file tree

Hide file tree

Showing 15 changed files with 183 additions and 128 deletions.
diff --git a/.github/workflows/ci-test-ctx.yml b/.github/workflows/ci-test-ctx.yml
@@ -2,13 +2,13 @@
 on: [pull_request]
 env:
   # Increase this value to reset cache
-  ISIS_CACHE_NUMBER: 20
-  ASAP_CACHE_NUMBER: 4
-  ASP_CACHE_NUMBER: 2
+  ISIS_CACHE_NUMBER: 34
+  ASAP_CACHE_NUMBER: 6
+  ASP_CACHE_NUMBER: 3
 jobs:
   cache_asp:
     name: Cache ASP distribution
-    runs-on: ubuntu-latest
+    runs-on: "ubuntu-latest"
     defaults:
       run:
         shell: bash -l {0}
@@ -22,11 +22,11 @@ jobs:
           key: asp-env-${{ runner.os }}-${{ env.ASP_CACHE_NUMBER }}-${{ hashFiles('asap.yml') }}
       - name: Download ASP tarball to tmp
         run: |
-          wget -c https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2023-01-01-daily-build/StereoPipeline-3.2.0-2023-01-01-x86_64-Linux.tar.bz2 -O /tmp/sp.tar.bz2 --progress=dot:giga
+          wget -c https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/3.2.0/StereoPipeline-3.2.0-2023-01-01-x86_64-Linux.tar.bz2 -O /tmp/sp.tar.bz2 --progress=dot:giga
           ls /tmp/sp.tar.bz2
   cache_isis_env:
     name: Cache ISIS conda environment
-    runs-on: ubuntu-latest
+    runs-on: "ubuntu-latest"
     defaults:
       run:
         shell: bash -l {0}
@@ -40,26 +40,35 @@ jobs:
           key: isis-env-${{ runner.os }}-${{ env.ISIS_CACHE_NUMBER }}-${{ hashFiles('isis.yml') }}
       - name: setup isis conda env
         if: steps.cache-isis.outputs.cache-hit != 'true'
-        uses: conda-incubator/setup-miniconda@v2.1.1
+        uses: conda-incubator/setup-miniconda@v2.2.0
         with:
-          use-mamba: false
+          miniforge-variant: Mambaforge
+          channels: usgs-astrogeology,conda-forge,defaults
           activate-environment: isis_for_asp
           environment-file: isis.yml
           auto-update-conda: true
           use-only-tar-bz2: true
-          clean-patched-environment-file: false
+      - name: patch cspice so file
+        if: steps.cache-isis.outputs.cache-hit != 'true'
+        run: |
+          ls $CONDA/envs/isis_for_asp/lib/
+          ln -s $CONDA/envs/isis_for_asp/lib/libcspice.so $CONDA/envs/isis_for_asp/lib/libcspice.so.66
       - name: conda info for isis
         if: steps.cache-isis.outputs.cache-hit != 'true'
         run: |
           echo $PATH
           conda env config vars list
           conda info
           conda list
+      - name: synce base kernels 1
+        if: steps.cache-isis.outputs.cache-hit != 'true'
+        run: downloadIsisData base $CONDA/envs/isis_for_asp/data/ --include='*/' --include="kernels.*.db" 
+      - name: synce base kernels 2
+        if: steps.cache-isis.outputs.cache-hit != 'true'
+        run: downloadIsisData base $CONDA/envs/isis_for_asp/data/ --exclude='kernels/' --exclude="dems/" --exclude="testData/" 
       - name: rsync base kernels
         if: steps.cache-isis.outputs.cache-hit != 'true'
         run: |
-          rsync -azv --exclude='kernels' --exclude "dems" --exclude "testData" --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/base $CONDA/envs/isis_for_asp/data/
-          rsync -azv --include '*/' --include "*.db"  --exclude "*"  --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/base/ $CONDA/envs/isis_for_asp/data/base
           mkdir -p $CONDA/envs/isis_for_asp/data/base/kernels/spk/
           mkdir -p $CONDA/envs/isis_for_asp/data/base/kernels/lsk/
           mkdir -p $CONDA/envs/isis_for_asp/data/base/kernels/pck/
@@ -69,14 +78,26 @@ jobs:
           wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/naif0012.tls -O $CONDA/envs/isis_for_asp/data/base/kernels/lsk/naif0012.tls
           wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/pck00009.tpc -O $CONDA/envs/isis_for_asp/data/base/kernels/pck/pck00009.tpc
           wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/molaMarsPlanetaryRadius0005_cropped.cub -O $CONDA/envs/isis_for_asp/data/base/dems/molaMarsPlanetaryRadius0005.cub
+          ls $CONDA/envs/isis_for_asp/data/base/kernels/spk/ -lahtr
+          ls $CONDA/envs/isis_for_asp/data/base/kernels/lsk/ -lahtr
+          ls $CONDA/envs/isis_for_asp/data/base/kernels/pck/ -lahtr
+          ls $CONDA/envs/isis_for_asp/data/base/dems/ -lahtr
+      - name: synce mro kernels 1
+        if: steps.cache-isis.outputs.cache-hit != 'true'
+        run: downloadIsisData mro $CONDA/envs/isis_for_asp/data/ --exclude='spk/' --exclude="sclk/" --exclude="ck/" --exclude="fk/" --exclude="iak/" --exclude="testData"
+      - name: synce mro kernels 2
+        if: steps.cache-isis.outputs.cache-hit != 'true'
+        run: downloadIsisData mro $CONDA/envs/isis_for_asp/data/ --include='*/' --include="kernels.*.db" --include="*.db" --exclude="*"
+      - name: synce mro kernels 3
+        if: steps.cache-isis.outputs.cache-hit != 'true'
+        run: downloadIsisData mro $CONDA/envs/isis_for_asp/data/ --include="*pck/*" 
       - name: rsync mro data
         if: steps.cache-isis.outputs.cache-hit != 'true'
         run: |
-          rsync -azv --exclude='kernels' --exclude "testData" --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/mro $CONDA/envs/isis_for_asp/data/
-          rsync -azv --include '*/' --include "*.db"  --exclude "*"  --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/mro/ $CONDA/envs/isis_for_asp/data/mro
           mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/spk/
           mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/sclk/
           mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/ck/
+          mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/pck/
           mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/iak/
           mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/fk/
           wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_v15.tf -O $CONDA/envs/isis_for_asp/data/mro/kernels/fk/mro_v15.tf
@@ -86,6 +107,16 @@ jobs:
           wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_sc_psp_081028_081103_subset.bc -O $CONDA/envs/isis_for_asp/data/mro/kernels/ck/mro_sc_psp_081028_081103.bc
           wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_psp1_ssd_mro110c_subset.bsp -O $CONDA/envs/isis_for_asp/data/mro/kernels/spk/mro_psp1_ssd_mro110c.bsp
           wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_psp9_ssd_mro110c_subset.bsp -O $CONDA/envs/isis_for_asp/data/mro/kernels/spk/mro_psp9_ssd_mro110c.bsp
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/pck00009.tpc -O $CONDA/envs/isis_for_asp/data/mro/kernels/pck/pck00009.tpc
+          ls $CONDA/envs/isis_for_asp/data/mro/kernels/spk/ -lahtr
+          ls $CONDA/envs/isis_for_asp/data/mro/kernels/sclk/ -lahtr
+          ls $CONDA/envs/isis_for_asp/data/mro/kernels/ck/ -lahtr
+          ls $CONDA/envs/isis_for_asp/data/mro/kernels/pck/ -lahtr
+          ls $CONDA/envs/isis_for_asp/data/mro/kernels/iak/ -lahtr
+          ls $CONDA/envs/isis_for_asp/data/mro/kernels/fk/ -lahtr
+      - name: show contents of data
+        if: steps.cache-isis.outputs.cache-hit != 'true'
+        run: ls -R $CONDA/envs/isis_for_asp/data/
       - name: pack isis conda env
         if: steps.cache-isis.outputs.cache-hit != 'true'
         run: |
@@ -109,14 +140,13 @@ jobs:
           key: asap-env-${{ runner.os }}-${{ env.ASAP_CACHE_NUMBER }}-${{ hashFiles('asap.yml') }}
       - name: setup asap conda env
         if: steps.cache-asap.outputs.cache-hit != 'true'
-        uses: conda-incubator/setup-miniconda@v2.1.1
+        uses: conda-incubator/setup-miniconda@v2.2.0
         with:
-          mamba-version: "*"
-          use-mamba: true
           activate-environment: asap
+          miniforge-variant: Mambaforge
+          channels: conda-forge
           environment-file: asap.yml
-          python-version: 3.9
-          channels: conda-forge,defaults
+          python-version: '3.11'
           use-only-tar-bz2: true
       - name: add ASP and ISIS to PATH of asap conda
         if: steps.cache-asap.outputs.cache-hit != 'true'
@@ -132,7 +162,6 @@ jobs:
           conda env config vars list
           conda info
           conda list
-          ldd `which qview`
       - name: pack asap conda env
         if: steps.cache-asap.outputs.cache-hit != 'true'
         run: |
@@ -151,7 +180,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup default miniconda
-        uses: conda-incubator/setup-miniconda@v2.1.1
+        uses: conda-incubator/setup-miniconda@v2.2.0
+        with:
+          miniforge-variant: Mambaforge
       - name: Get ASP from cache
         uses: actions/cache@v3
         with:
@@ -183,6 +214,8 @@ jobs:
           mkdir -p $CONDA/envs/asap
           tar -xvf asap-env.tar.gz -C /
           rm asap-env.tar.gz
+      - name: show contents of data
+        run: ls -R $CONDA/envs/isis_for_asp/data/
       - name: Check contents of isis env
         run: |
           pwd
@@ -230,7 +263,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup default miniconda
-        uses: conda-incubator/setup-miniconda@v2.1.1
+        uses: conda-incubator/setup-miniconda@v2.2.0
+        with:
+          miniforge-variant: Mambaforge
       - name: Get ASP from cache
         uses: actions/cache@v3
         with:
@@ -262,6 +297,8 @@ jobs:
           mkdir -p $CONDA/envs/asap
           tar -xvf asap-env.tar.gz -C /
           rm asap-env.tar.gz
+      - name: show contents of data
+        run: ls -R $CONDA/envs/isis_for_asp/data/
       - name: Check contents of isis env
         run: |
           pwd
@@ -420,7 +457,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Setup default miniconda
-        uses: conda-incubator/setup-miniconda@v2.1.1
+        uses: conda-incubator/setup-miniconda@v2.2.0
+        with:
+          miniforge-variant: Mambaforge
       - name: Get ASP from cache
         uses: actions/cache@v3
         with:
@@ -452,6 +491,8 @@ jobs:
           mkdir -p $CONDA/envs/asap
           tar -xvf asap-env.tar.gz -C /
           rm asap-env.tar.gz
+      - name: show contents of data
+        run: ls -R $CONDA/envs/isis_for_asp/data/
       - name: Check contents of isis env
         run: |
           pwd
@@ -460,6 +501,7 @@ jobs:
           ls $CONDA/
           ls $CONDA/envs/isis_for_asp -lahtr
           ls $CONDA/envs/isis_for_asp/bin -lahtr
+          ls $CONDA/envs/isis_for_asp/data/base/kernels/lsk/ -lahtr
       - name: Install asap
         run: |
           conda activate asap

diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
@@ -13,13 +13,13 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
-        python-version: '3.x'
+        python-version: '3.11'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine readme-renderer
+        pip install setuptools wheel twine readme-renderer build
     - name: Test readme rendering
       run: |
         python -m readme_renderer README.rst
@@ -28,5 +28,5 @@ jobs:
         TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
         TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
       run: |
-        python setup.py sdist bdist_wheel
+        python -m build
         twine upload dist/*
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,6 @@
 
 .idea/
-
+.fleet/
 
 ### Python template
 # Byte-compiled / optimized / DLL files

diff --git a/asap_stereo/__init__.py b/asap_stereo/__init__.py
diff --git a/asap_stereo/_version.py b/asap_stereo/_version.py
diff --git a/isis.yml b/isis.yml
@@ -5,3 +5,5 @@ channels:
   - defaults
 dependencies:
   - isis==7.1.0
+  - cspice
+  - rclone
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,55 @@
+[build-system]
+requires = ["setuptools >=65.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name     = 'asap_stereo'
+dynamic = ["version", "readme"]
+requires-python = ">=3.9"
+license  = {text = "BSD 3-Clause License"}
+description = 'A high level CLI and reproducible workflow for the Ames Stereo Pipeline'
+keywords = ['mars', 'nasa', 'asp', 'ames', 'stereo', 'pipeline', 'cli', 'tool', 'workflow']
+# Author details
+authors = [
+    {name = 'Andrew M. Annex', email='ama6fy@virginia.edu'}
+]
+dependencies = [
+    'requests',
+    'fire',
+    'moody>=0.2.0',
+    'sh',
+    'papermill',
+    'rasterio',
+    'pyproj'
+]
+classifiers=[
+    'Natural Language :: English',
+    'License :: OSI Approved :: BSD License',
+    'Development Status :: 3 - Alpha',
+    'Environment :: Console',
+    'Intended Audience :: Developers',
+    'Intended Audience :: Science/Research',
+    'Programming Language :: Python :: 3',
+    'Programming Language :: Python :: 3.9',
+    'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11',
+    'Topic :: Scientific/Engineering :: Astronomy',
+    'Topic :: Scientific/Engineering :: GIS'
+]
+
+[project.urls]
+repository = 'https://github.com/AndrewAnnex/asap_stereo/'
+
+[project.scripts]
+asap = 'asap_stereo.asap:main'
+
+[tool.setuptools.packages.find]
+where = ["src"]
+namespaces = false
+
+[tool.setuptools.package-data]
+"*" = ["*.ipynb"]
+
+[tool.setuptools.dynamic]
+version = {attr = "asap_stereo.__version__"}
+readme = {file = "README.rst"}
diff --git a/setup.py b/setup.py
@@ -1,48 +1,3 @@
 from setuptools import setup
-import io
 
-# some influences here came from https://github.com/audreyr/cookiecutter/blob/master/setup.py
-
-version = '0.2.1'
-
-
-with io.open('README.rst', 'r', encoding='utf-8') as readme_file:
-    readme = readme_file.read()
-
-
-setup(
-    name     = 'asap_stereo',
-    version  = version,
-    packages = ['asap_stereo'],
-    include_package_data=True,
-    license  = 'BSD-3-Clause',
-    description = 'A high level CLI and reproducible workflow for the Ames Stereo Pipeline',
-    long_description = readme,
-    # Author details
-    author='Andrew M. Annex',
-    author_email='ama6fy@virginia.edu',
-    url='https://github.com/AndrewAnnex/asap_stereo/',
-
-    install_requires=['requests', 'fire', 'moody>=0.2.0', 'sh', 'papermill', 'rasterio', 'pyproj'],
-
-    entry_points={
-        'console_scripts': [
-            'asap = asap_stereo.asap:main'
-        ]
-    },
-
-    keywords=['mars', 'nasa', 'asp', 'ames', 'stereo', 'pipeline', 'cli', 'tool', 'workflow'],
-
-    classifiers=[
-        'Natural Language :: English',
-        'License :: OSI Approved :: BSD License',
-        'Development Status :: 3 - Alpha',
-        'Environment :: Console',
-        'Intended Audience :: Developers',
-        'Intended Audience :: Science/Research',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.9',
-        'Topic :: Scientific/Engineering :: Astronomy',
-        'Topic :: Scientific/Engineering :: GIS'
-    ]
-)
+setup()
diff --git a/src/asap_stereo/__init__.py b/src/asap_stereo/__init__.py
@@ -0,0 +1 @@
+__version__ = "0.2.1"