attempting to update asap to run against asp 3.1.0 and isis 7.0.0 (#33)

* attempting to update asap to run against asp 3.1.0 and isis 7.0.0 * fix for parallel_bundle_adjust * added cache for asp toolkit in ci tests
AndrewAnnex · Jul 7, 2022 · 77c2ba3 · 77c2ba3
1 parent f24c4ec
commit 77c2ba3
Show file tree

Hide file tree

Showing 9 changed files with 128 additions and 59 deletions.
diff --git a/.github/workflows/ci-test-ctx.yml b/.github/workflows/ci-test-ctx.yml
@@ -2,9 +2,28 @@
 on: [workflow_dispatch]
 env:
   # Increase this value to reset cache
-  ISIS_CACHE_NUMBER: 2
-  ASAP_CACHE_NUMBER: 1
+  ISIS_CACHE_NUMBER: 3
+  ASAP_CACHE_NUMBER: 2
+  ASP_CACHE_NUMBER: 1
 jobs:
+  cache_asp:
+    name: Cache ASP distribution
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache ASP tarball
+        id: cache-asap
+        uses: actions/cache@v2
+        with:
+          path: "/tmp/sp.tar.bz2"
+          key: asp-env-${{ runner.os }}-${{ env.ASP_CACHE_NUMBER }}-${{ hashFiles('asap.yml') }}
+      - name: Download ASP tarball to tmp
+        run: |
+          wget -c https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/3.1.0/StereoPipeline-3.1.0-2022-05-18-x86_64-Linux.tar.bz2 -O /tmp/sp.tar.bz2 --progress=dot:giga
+          ls /tmp/sp.tar.bz2
   cache_isis_env:
     name: Cache ISIS conda environment
     runs-on: ubuntu-latest
@@ -23,14 +42,12 @@ jobs:
         if: steps.cache-isis.outputs.cache-hit != 'true'
         uses: conda-incubator/setup-miniconda@v2.1.1
         with:
-          mamba-version: "*"
-          use-mamba: true
+          use-mamba: false
           activate-environment: isis_for_asp
           environment-file: isis.yml
-          channel-priority: strict
           auto-update-conda: true
-          python-version: 3.6
           use-only-tar-bz2: true
+          clean-patched-environment-file: false
       - name: conda info for isis
         if: steps.cache-isis.outputs.cache-hit != 'true'
         run: |
@@ -98,7 +115,7 @@ jobs:
           use-mamba: true
           activate-environment: asap
           environment-file: asap.yml
-          python-version: 3.8
+          python-version: 3.9
           channels: conda-forge,defaults
           use-only-tar-bz2: true
       - name: add ASP and ISIS to PATH of asap conda
@@ -125,7 +142,7 @@ jobs:
   
   test_runner:
     name: ASAP test CTX pipeline
-    needs: [cache_isis_env, cache_asap_env]
+    needs: [cache_asp, cache_isis_env, cache_asap_env]
     runs-on: "ubuntu-latest"
     defaults:
       run:
@@ -134,10 +151,14 @@ jobs:
       - uses: actions/checkout@v2
       - name: Setup default miniconda
         uses: conda-incubator/setup-miniconda@v2.1.0
-      - name: Download and extract ASP tarball to tmp
+      - name: Get ASP from cache
+        uses: actions/cache@v2
+        with:
+          path: "/tmp/sp.tar.bz2"
+          key: asp-env-${{ runner.os }}-${{ env.ASP_CACHE_NUMBER }}-${{ hashFiles('asap.yml') }}
+      - name: Extract ASP tarball to tmp
         run: |
           mkdir /tmp/sp
-          wget -c https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2.7.0/StereoPipeline-2.7.0-2020-07-29-x86_64-Linux.tar.bz2 -O /tmp/sp.tar.bz2 --progress=dot:giga
           tar -xf /tmp/sp.tar.bz2 -C /tmp/sp --strip-components 1
           rm /tmp/sp.tar.bz2
           ls /tmp/sp

diff --git a/.github/workflows/ci-test-notebook.yml b/.github/workflows/ci-test-notebook.yml
@@ -2,9 +2,28 @@
 on: [workflow_dispatch]
 env:
   # Increase this value to reset cache
-  ISIS_CACHE_NUMBER: 2
-  ASAP_CACHE_NUMBER: 1
+  ISIS_CACHE_NUMBER: 3
+  ASAP_CACHE_NUMBER: 2
+  ASP_CACHE_NUMBER: 1
 jobs:
+  cache_asp:
+    name: Cache ASP distribution
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache ASP tarball
+        id: cache-asap
+        uses: actions/cache@v2
+        with:
+          path: "/tmp/sp.tar.bz2"
+          key: asp-env-${{ runner.os }}-${{ env.ASP_CACHE_NUMBER }}-${{ hashFiles('asap.yml') }}
+      - name: Download ASP tarball to tmp
+        run: |
+          wget -c https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/3.1.0/StereoPipeline-3.1.0-2022-05-18-x86_64-Linux.tar.bz2 -O /tmp/sp.tar.bz2 --progress=dot:giga
+          ls /tmp/sp.tar.bz2
   cache_isis_env:
     name: Cache ISIS conda environment
     runs-on: ubuntu-latest
@@ -23,14 +42,12 @@ jobs:
         if: steps.cache-isis.outputs.cache-hit != 'true'
         uses: conda-incubator/setup-miniconda@v2.1.1
         with:
-          mamba-version: "*"
-          use-mamba: true
+          use-mamba: false
           activate-environment: isis_for_asp
           environment-file: isis.yml
-          channel-priority: strict
           auto-update-conda: true
-          python-version: 3.6
           use-only-tar-bz2: true
+          clean-patched-environment-file: false
       - name: conda info for isis
         if: steps.cache-isis.outputs.cache-hit != 'true'
         run: |
@@ -99,7 +116,7 @@ jobs:
           use-mamba: true
           activate-environment: asap
           environment-file: asap.yml
-          python-version: 3.8
+          python-version: 3.9
           channels: conda-forge,defaults
           use-only-tar-bz2: true
       - name: add ASP and ISIS to PATH of asap conda
@@ -125,7 +142,7 @@ jobs:
   
   test_runner:
     name: ASAP test Notebook CTX pipeline
-    needs: [ cache_isis_env,  cache_asap_env ]
+    needs: [cache_asp, cache_isis_env, cache_asap_env]
     runs-on: "ubuntu-latest"
     defaults:
       run:
@@ -134,10 +151,14 @@ jobs:
       - uses: actions/checkout@v2
       - name: Setup default miniconda
         uses: conda-incubator/setup-miniconda@v2.1.0
-      - name: Download and extract ASP tarball to tmp
+      - name: Get ASP from cache
+        uses: actions/cache@v2
+        with:
+          path: "/tmp/sp.tar.bz2"
+          key: asp-env-${{ runner.os }}-${{ env.ASP_CACHE_NUMBER }}-${{ hashFiles('asap.yml') }}
+      - name: Extract ASP tarball to tmp
         run: |
           mkdir /tmp/sp
-          wget -c https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2.7.0/StereoPipeline-2.7.0-2020-07-29-x86_64-Linux.tar.bz2 -O /tmp/sp.tar.bz2 --progress=dot:giga
           tar -xf /tmp/sp.tar.bz2 -C /tmp/sp --strip-components 1
           rm /tmp/sp.tar.bz2
           ls /tmp/sp

diff --git a/.github/workflows/test_basic.yml b/.github/workflows/test_basic.yml
@@ -2,9 +2,28 @@
 on: [push, pull_request]
 env:
   # Increase this value to reset cache
-  ISIS_CACHE_NUMBER: 2
-  ASAP_CACHE_NUMBER: 1
+  ISIS_CACHE_NUMBER: 3
+  ASAP_CACHE_NUMBER: 2
+  ASP_CACHE_NUMBER: 1
 jobs:
+  cache_asp:
+    name: Cache ASP distribution
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Cache ASP tarball
+        id: cache-asap
+        uses: actions/cache@v2
+        with:
+          path: "/tmp/sp.tar.bz2"
+          key: asp-env-${{ runner.os }}-${{ env.ASP_CACHE_NUMBER }}-${{ hashFiles('asap.yml') }}
+      - name: Download ASP tarball to tmp
+        run: |
+          wget -c https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/3.1.0/StereoPipeline-3.1.0-2022-05-18-x86_64-Linux.tar.bz2 -O /tmp/sp.tar.bz2 --progress=dot:giga
+          ls /tmp/sp.tar.bz2
   cache_isis_env:
     name: Cache ISIS conda environment
     runs-on: ubuntu-latest
@@ -23,14 +42,12 @@ jobs:
         if: steps.cache-isis.outputs.cache-hit != 'true'
         uses: conda-incubator/setup-miniconda@v2.1.1
         with:
-          mamba-version: "*"
-          use-mamba: true
+          use-mamba: false
           activate-environment: isis_for_asp
           environment-file: isis.yml
-          channel-priority: strict
           auto-update-conda: true
-          python-version: 3.6
           use-only-tar-bz2: true
+          clean-patched-environment-file: false
       - name: conda info for isis
         if: steps.cache-isis.outputs.cache-hit != 'true'
         run: |
@@ -99,7 +116,7 @@ jobs:
           use-mamba: true
           activate-environment: asap
           environment-file: asap.yml
-          python-version: 3.8
+          python-version: 3.9
           channels: conda-forge,defaults
           use-only-tar-bz2: true
       - name: add ASP and ISIS to PATH of asap conda
@@ -125,7 +142,7 @@ jobs:
 
   test_basic:
     name: Basic ASAP Tests
-    needs: [ cache_isis_env, cache_asap_env ]
+    needs: [cache_asp, cache_isis_env, cache_asap_env]
     runs-on: "ubuntu-latest"
     defaults:
       run:
@@ -134,10 +151,14 @@ jobs:
       - uses: actions/checkout@v2
       - name: Setup default miniconda
         uses: conda-incubator/setup-miniconda@v2.1.0
-      - name: Download and extract ASP tarball to tmp
+      - name: Get ASP from cache
+        uses: actions/cache@v2
+        with:
+          path: "/tmp/sp.tar.bz2"
+          key: asp-env-${{ runner.os }}-${{ env.ASP_CACHE_NUMBER }}-${{ hashFiles('asap.yml') }}
+      - name: Extract ASP tarball to tmp
         run: |
           mkdir /tmp/sp
-          wget -c https://github.com/NeoGeographyToolkit/StereoPipeline/releases/download/2.7.0/StereoPipeline-2.7.0-2020-07-29-x86_64-Linux.tar.bz2 -O /tmp/sp.tar.bz2 --progress=dot:giga
           tar -xf /tmp/sp.tar.bz2 -C /tmp/sp --strip-components 1
           rm /tmp/sp.tar.bz2
           ls /tmp/sp

diff --git a/asap.yml b/asap.yml
@@ -1,9 +1,8 @@
 name: asap
 channels:
   - conda-forge
-  - defaults
 dependencies:
-  - python=>3.8
+  - python=>3.9
   - black=19.10b0
   - pvl
   - nb_conda_kernels

diff --git a/asap_36.yml b/asap_36.yml
diff --git a/asap_stereo/asap.py b/asap_stereo/asap.py
@@ -368,8 +368,7 @@ def __init__(self):
         self.ipmatch     = Command('ipmatch').bake(_out=sys.stdout, _err=sys.stderr)
         self.gdaltranslate = Command('gdal_translate').bake(_out=sys.stdout, _err=sys.stderr)
         self.ba = Command('parallel_bundle_adjust').bake(
-                '--threads-singleprocess', _threads_singleprocess,
-                '--threads-multiprocess', _threads_multiprocess,
+                '--threads', _threads_singleprocess,
                 _out=sys.stdout, _err=sys.stderr
             )
 
@@ -881,6 +880,36 @@ def estimate_median_disparity(self, ref_dem, src_dem=None):
         vals = self.get_geo_diff(ref_dem, src_dem)
         med_d = float(vals['Median difference'])       
         return med_d, abs(med_d)
+
+
+    def compute_footprints(self, *imgs):
+        """
+        for each footprint generate a vector footprint
+        :param imgs: gdal rasters with nodata defined
+        :return: 
+        """
+        poly = sh.Command('gdal_polygonize.py')
+        for img in tqdm.tqdm(imgs):
+            md = json.loads(str(sh.gdalinfo(img, '-json')))
+            if not 'noDataValue' in md['bands'][0]:
+                print('no noDataValue in image: ', img)
+                continue
+            # downsample to 10% size
+            ds_out_name = Path(img).stem + f'_ds.vrt'
+            _ = sh.gdal_translate(img, ds_out_name, '-of', 'vrt', '-outsize', '10%', '10%', '-r', 'cubic')
+            # scale to binary
+            eb_out_name = Path(img).stem + f'_eb.vrt'
+            _ = sh.gdal_translate(ds_out_name, eb_out_name, '-of', 'vrt', '-scale', '-ot', 'byte')
+            # scale to mask
+            vp_out_name = Path(img).stem + f'_vp.vrt'
+            _ = sh.gdal_translate(eb_out_name, vp_out_name, '-of', 'vrt', '-scale', '1', '255', '100', '100')
+            # make polygon
+            g_out_name = Path(img).stem + f'_footprint.geojson'
+            _ = poly('-of', 'geojson', '-8', vp_out_name, g_out_name)
+            # cleanup intermediate products
+            Path(ds_out_name).unlink(missing_ok=True)
+            Path(eb_out_name).unlink(missing_ok=True)
+            Path(vp_out_name).unlink(missing_ok=True)
 
 
 class CTX(object):

diff --git a/docs/asap_docs.yml b/docs/asap_docs.yml
@@ -3,7 +3,7 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - python=3.6
+  - python=3.9
   - numpy
   - pvl
   - sphinx==3.1.2

diff --git a/isis.yml b/isis.yml
@@ -4,4 +4,4 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - isis=>6.0.0
+  - isis==7.0.0
diff --git a/setup.py b/setup.py
@@ -41,7 +41,7 @@
         'Intended Audience :: Developers',
         'Intended Audience :: Science/Research',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Scientific/Engineering :: Astronomy',
         'Topic :: Scientific/Engineering :: GIS'
     ]