migrating github actions to use new 2 step isis conda and asp install (…

…#23) * migrating github actions to use new 2 step isis conda and asp install method * consider a conda lock for isis * should also look into making a common workflow/job for the setup of isis/asp/asap between both workflows
AndrewAnnex · May 31, 2021 · aaed4ca · aaed4ca
1 parent 0b18500
commit aaed4ca
Show file tree

Hide file tree

Showing 5 changed files with 112 additions and 52 deletions.
diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
@@ -16,53 +16,75 @@ jobs:
           CACHE_NUMBER: 0
         with:
           path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-${{ hashFiles('asp.yml') }}
-      - name: setup miniconda
+          key: ${{ runner.os }}-${{ hashFiles('isis.yml') }}-${{ hashFiles('asap.yml') }}
+      - name: setup isis conda env
         uses: conda-incubator/setup-miniconda@v2.1.0
         with:
           mamba-version: "*"
-          activate-environment: asp
-          environment-file: asp.yml
-          python-version: 3.6
-          channels: nasa-ames-stereo-pipeline,usgs-astrogeology,conda-forge,defaults
+          use-mamba: true
+          activate-environment: isis_for_asp
+          environment-file: isis.yml
           channel-priority: strict
           auto-update-conda: true
+          python-version: 3.6
           use-only-tar-bz2: true
-      - name: isis3 path inits
-        run: python $CONDA_PREFIX/scripts/isis3VarInit.py
       - name: rsync base kernels
         run: |
-          rsync -azv --exclude='kernels' --exclude "dems" --exclude "testData" --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/base $CONDA_PREFIX/data/
-          rsync -azv --include '*/' --include "*.db"  --exclude "*"  --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/base/ $CONDA_PREFIX/data/base
-          mkdir -p $CONDA_PREFIX/data/base/kernels/spk/
-          mkdir -p $CONDA_PREFIX/data/base/kernels/lsk/
-          mkdir -p $CONDA_PREFIX/data/base/kernels/pck/
-          mkdir -p $CONDA_PREFIX/data/base/dems/
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/de430.bsp -O $CONDA_PREFIX/data/base/kernels/spk/de430.bsp
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mar097.bsp -O $CONDA_PREFIX/data/base/kernels/spk/mar097.bsp
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/naif0012.tls -O $CONDA_PREFIX/data/base/kernels/lsk/naif0012.tls
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/pck00009.tpc -O $CONDA_PREFIX/data/base/kernels/pck/pck00009.tpc
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/molaMarsPlanetaryRadius0005_cropped.cub -O $CONDA_PREFIX/data/base/dems/molaMarsPlanetaryRadius0005.cub
+          rsync -azv --exclude='kernels' --exclude "dems" --exclude "testData" --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/base $CONDA/envs/isis_for_asp/data/
+          rsync -azv --include '*/' --include "*.db"  --exclude "*"  --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/base/ $CONDA/envs/isis_for_asp/data/base
+          mkdir -p $CONDA/envs/isis_for_asp/data/base/kernels/spk/
+          mkdir -p $CONDA/envs/isis_for_asp/data/base/kernels/lsk/
+          mkdir -p $CONDA/envs/isis_for_asp/data/base/kernels/pck/
+          mkdir -p $CONDA/envs/isis_for_asp/data/base/dems/
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/de430.bsp -O $CONDA/envs/isis_for_asp/data/base/kernels/spk/de430.bsp
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mar097.bsp -O $CONDA/envs/isis_for_asp/data/base/kernels/spk/mar097.bsp
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/naif0012.tls -O $CONDA/envs/isis_for_asp/data/base/kernels/lsk/naif0012.tls
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/pck00009.tpc -O $CONDA/envs/isis_for_asp/data/base/kernels/pck/pck00009.tpc
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/molaMarsPlanetaryRadius0005_cropped.cub -O $CONDA/envs/isis_for_asp/data/base/dems/molaMarsPlanetaryRadius0005.cub
       - name: rsync mro data
         run: |
-          rsync -azv --exclude='kernels' --exclude "testData" --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/mro $CONDA_PREFIX/data/
-          rsync -azv --include '*/' --include "*.db"  --exclude "*"  --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/mro/ $CONDA_PREFIX/data/mro
-          mkdir -p $CONDA_PREFIX/data/mro/kernels/spk/
-          mkdir -p $CONDA_PREFIX/data/mro/kernels/sclk/
-          mkdir -p $CONDA_PREFIX/data/mro/kernels/ck/
-          mkdir -p $CONDA_PREFIX/data/mro/kernels/iak/
-          mkdir -p $CONDA_PREFIX/data/mro/kernels/fk/
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_v15.tf -O $CONDA_PREFIX/data/mro/kernels/fk/mro_v15.tf
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/MRO_SCLKSCET.00090.65536.tsc -O $CONDA_PREFIX/data/mro/kernels/sclk/MRO_SCLKSCET.00090.65536.tsc
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mroctxAddendum005.ti -O $CONDA_PREFIX/data/mro/kernels/iak/mroctxAddendum005.ti
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_sc_psp_061219_061225_subset.bc -O $CONDA_PREFIX/data/mro/kernels/ck/mro_sc_psp_061219_061225.bc
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_sc_psp_081028_081103_subset.bc -O $CONDA_PREFIX/data/mro/kernels/ck/mro_sc_psp_081028_081103.bc
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_psp1_ssd_mro110c_subset.bsp -O $CONDA_PREFIX/data/mro/kernels/spk/mro_psp1_ssd_mro110c.bsp
-          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_psp9_ssd_mro110c_subset.bsp -O $CONDA_PREFIX/data/mro/kernels/spk/mro_psp9_ssd_mro110c.bsp
+          rsync -azv --exclude='kernels' --exclude "testData" --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/mro $CONDA/envs/isis_for_asp/data/
+          rsync -azv --include '*/' --include "*.db"  --exclude "*"  --delete --partial isisdist.astrogeology.usgs.gov::isis3data/data/mro/ $CONDA/envs/isis_for_asp/data/mro
+          mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/spk/
+          mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/sclk/
+          mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/ck/
+          mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/iak/
+          mkdir -p $CONDA/envs/isis_for_asp/data/mro/kernels/fk/
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_v15.tf -O $CONDA/envs/isis_for_asp/data/mro/kernels/fk/mro_v15.tf
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/MRO_SCLKSCET.00090.65536.tsc -O $CONDA/envs/isis_for_asp/data/mro/kernels/sclk/MRO_SCLKSCET.00090.65536.tsc
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mroctxAddendum005.ti -O $CONDA/envs/isis_for_asp/data/mro/kernels/iak/mroctxAddendum005.ti
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_sc_psp_061219_061225_subset.bc -O $CONDA/envs/isis_for_asp/data/mro/kernels/ck/mro_sc_psp_061219_061225.bc
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_sc_psp_081028_081103_subset.bc -O $CONDA/envs/isis_for_asp/data/mro/kernels/ck/mro_sc_psp_081028_081103.bc
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_psp1_ssd_mro110c_subset.bsp -O $CONDA/envs/isis_for_asp/data/mro/kernels/spk/mro_psp1_ssd_mro110c.bsp
+          wget https://github.com/AndrewAnnex/asap_stereo_test_data/raw/master/mro_psp9_ssd_mro110c_subset.bsp -O $CONDA/envs/isis_for_asp/data/mro/kernels/spk/mro_psp9_ssd_mro110c.bsp
+      - name: download and extract ASP tarball to tmp
+        run: |
+          mkdir /tmp/sp
+          wget -c https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2.7.0/StereoPipeline-2.7.0-2020-07-29-x86_64-Linux.tar.bz2 -O /tmp/sp.tar.bz2 --progress=dot:giga
+          tar -xf /tmp/sp.tar.bz2 -C /tmp/sp --strip-components 1
+          ls /tmp/sp
+      - name: setup asap conda env
+        uses: conda-incubator/setup-miniconda@v2.1.0
+        with:
+          mamba-version: "*"
+          use-mamba: true
+          activate-environment: asap
+          environment-file: asap.yml
+          python-version: 3.8
+          channels: conda-forge,defaults
+          use-only-tar-bz2: true
+      - name: add ASP and ISIS to PATH of asap conda
+        run: |
+          conda env config vars set ISISROOT=$CONDA/envs/isis_for_asp
+          conda env config vars set ISISDATA=$CONDA/envs/isis_for_asp/data
+          conda env config vars set ISISTESTDATA=$CONDA/envs/isis_for_asp/testdata
+          conda env config vars set PATH=$PATH:/tmp/sp/bin:$CONDA/envs/isis_for_asp/bin
       - name: install asap
         run: pip install -e .
       - name: conda info
         run: |
+          echo $PATH
+          conda env config vars list
           conda info
           conda list
       - name: check isis shared libraries

diff --git a/.github/workflows/ci-workflow.yml b/.github/workflows/ci-workflow.yml
@@ -16,22 +16,46 @@ jobs:
           CACHE_NUMBER: 0
         with:
           path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-${{ hashFiles('asp.yml') }}
-      - name: setup miniconda
+          key: ${{ runner.os }}-${{ hashFiles('isis.yml') }}-${{ hashFiles('asap.yml') }}
+      - name: setup isis conda env
         uses: conda-incubator/setup-miniconda@v2.1.0
         with:
           mamba-version: "*"
-          activate-environment: asp
-          environment-file: asp.yml
-          python-version: 3.6
-          channels: nasa-ames-stereo-pipeline,usgs-astrogeology,conda-forge,defaults
+          use-mamba: true
+          activate-environment: isis_for_asp
+          environment-file: isis.yml
           channel-priority: strict
           auto-update-conda: true
+          python-version: 3.6
+          use-only-tar-bz2: true
+      - name: download and extract ASP tarball to tmp
+        run: |
+          mkdir /tmp/sp
+          wget -c https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2.7.0/StereoPipeline-2.7.0-2020-07-29-x86_64-Linux.tar.bz2 -O /tmp/sp.tar.bz2 --progress=dot:giga
+          tar -xf /tmp/sp.tar.bz2 -C /tmp/sp --strip-components 1
+          ls /tmp/sp
+      - name: setup asap conda env
+        uses: conda-incubator/setup-miniconda@v2.1.0
+        with:
+          mamba-version: "*"
+          use-mamba: true
+          activate-environment: asap
+          environment-file: asap.yml
+          python-version: 3.8
+          channels: conda-forge,defaults
           use-only-tar-bz2: true
+      - name: add ASP and ISIS to PATH of asap conda
+        run: |
+          conda env config vars set ISISROOT=$CONDA/envs/isis_for_asp
+          conda env config vars set ISISDATA=$CONDA/envs/isis_for_asp/data
+          conda env config vars set ISISTESTDATA=$CONDA/envs/isis_for_asp/testdata
+          conda env config vars set PATH=$PATH:/tmp/sp/bin:$CONDA/envs/isis_for_asp/bin
       - name: install asap
         run: pip install -e .
       - name: conda info
         run: |
+          echo $PATH
+          conda env config vars list
           conda info
           conda list
       - name: check isis shared libraries

diff --git a/asap.yml b/asap.yml
@@ -0,0 +1,20 @@
+name: asap
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=>3.8
+  - pvl
+  - nb_conda_kernels
+  - jupyterlab
+  - requests
+  - fire
+  - tqdm
+  - papermill
+  - rasterio
+  - pyproj
+  - shapely
+  - sh
+  - pip
+  - pip:
+      - moody>=0.0.4
diff --git a/asp.yml b/asp.yml
diff --git a/isis.yml b/isis.yml
@@ -0,0 +1,7 @@
+name: isis_for_asap
+channels:
+  - usgs-astrogeology
+  - conda-forge
+  - defaults
+dependencies:
+  - isis=4.4.0