Added step_kwargs for CTX and fix for armadillo 9

* try this fix to ensure armadillo 9 is available * more fixes and always log out notebook and folders/files * added step_kwargs for step parameter injection, may not work/be robust
AndrewAnnex · Sep 12, 2021 · f598cf7 · f598cf7
1 parent 3a84587
commit f598cf7
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 18 deletions.
diff --git a/.github/workflows/ci-test-notebook.yml b/.github/workflows/ci-test-notebook.yml
@@ -102,11 +102,19 @@ jobs:
         run:  |
           wget https://raw.githubusercontent.com/NeoGeographyToolkit/StereoPipeline/master/examples/CTX/stereo.nonmap
           wget https://raw.githubusercontent.com/NeoGeographyToolkit/StereoPipeline/master/examples/CTX/stereo.map
-      - name: asap step 1
-        timeout-minutes: 5
-        run: asap ctx notebook_pipeline_make_dem B03_010644_1889_XN_08N001W P02_001902_1889_XI_08N001W ./stereo.nonmap --config2 ./stereo.map --downsample 8 --img_gsd 48 --dem_gsd 144
+      - name: asap notebook run
+        timeout-minutes: 30
+        run: |
+          asap ctx notebook_pipeline_make_dem B03_010644_1889_XN_08N001W P02_001902_1889_XI_08N001W ./stereo.nonmap --config2 ./stereo.map --downsample 8 --img_gsd 48 --dem_gsd 144 --step_kwargs '{"step_13": "--highest_accuracy False --max-num-reference-points 100000 --max-num-source-points 10000"}'
+      - name: ls directories to see what was made
+        continue-on-error: true
+        if: always()
+        run:
+          ls ./ -lahtr   
+          ls ./*/*/* -lahtr  
       - name: Archive Final Notebook from Papermill
         uses: actions/upload-artifact@v2
+        if: always()
         with:
           name: final
           path: |

diff --git a/asap_stereo/asap.py b/asap_stereo/asap.py
@@ -155,6 +155,13 @@ def isis3_to_dict(instr: str)-> Dict:
     return out
 
 
+def kwarg_parse(kwargs: Dict, key: str)-> str:
+    key_args = kwargs.get(key, {})
+    if isinstance(key_args, str):
+        return key_args
+    return ' '.join(map(str, kwargs_to_args(clean_kwargs(key_args))))
+
+
 def get_affine_from_file(file):
     md = json.loads(str(sh.gdalinfo(file, '-json')))
     gt = md['geoTransform']
@@ -886,7 +893,8 @@ def generate_ctx_pair_list(self, one, two):
 
     @staticmethod
     def notebook_pipeline_make_dem(left: str, right: str, config1: str, pedr_list: str = None, downsample: int = None, working_dir ='./', 
-                                   config2: Optional[str] = None, dem_gsd = 24.0, img_gsd = 6.0, maxdisp = None, out_notebook=None, **kwargs):
+                                   config2: Optional[str] = None, dem_gsd = 24.0, img_gsd = 6.0, maxdisp = None, step_kwargs = None, 
+                                   out_notebook = None, **kwargs):
         """
         First step in CTX DEM pipeline that uses papermill to persist log
 
@@ -901,9 +909,11 @@ def notebook_pipeline_make_dem(left: str, right: str, config1: str, pedr_list: s
         :param left: First image id
         :param right: Second image id
         :param maxdisp: Maximum expected displacement in meters, use None to determine it automatically 
+        :param step_kwargs: Arbitrary dict of kwargs for steps following {'step_#' : {'key': 'value}}
         :param downsample: Factor to downsample images for faster production
         :param dem_gsd: desired GSD of output DEMs (4x image GSD)
         :param img_gsd: desired GSD of output ortho images
+        :param kwargs: kwargs for papermill
         """
         if not out_notebook:
             out_notebook = f'{working_dir}/log_asap_notebook_pipeline_make_dem.ipynb'
@@ -921,6 +931,7 @@ def notebook_pipeline_make_dem(left: str, right: str, config1: str, pedr_list: s
                 'dem_gsd' : dem_gsd,
                 'img_gsd' : img_gsd,
                 'downsample' : downsample,
+                'step_kwargs' : step_kwargs
             },
             request_save_on_cell_execute=True,
             **kwargs

diff --git a/asap_stereo/asap_ctx_workflow.ipynb b/asap_stereo/asap_ctx_workflow.ipynb
@@ -14,6 +14,7 @@
     "maxdisp = None\n",
     "downsample = None\n",
     "pedr_list = None\n",
+    "step_kwargs = {}\n",
     "# todo: add reference_dem and use to conditional pedr things"
    ],
    "outputs": [],
@@ -144,7 +145,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_2  2>&1 | tee -i -a ./2_ctxedr2lev1eo.log ./full_log.log"
+    "!asap ctx step_2 {asap.kwarg_parse(step_kwargs, 'step_2')} 2>&1 | tee -i -a ./2_ctxedr2lev1eo.log ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -244,7 +245,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_4 2>&1 | tee -i -a ./2_bundle_adjust.log ./full_log.log"
+    "!asap ctx step_4 {asap.kwarg_parse(step_kwargs, 'step_4')} 2>&1 | tee -i -a ./2_bundle_adjust.log ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -258,7 +259,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_5 {config1}  2>&1 | tee -i -a ./3_lev1eo2dem.log ./full_log.log"
+    "!asap ctx step_5 {config1} {asap.kwarg_parse(step_kwargs, 'step_5')} 2>&1 | tee -i -a ./3_lev1eo2dem.log ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -272,7 +273,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_6 {config1}  2>&1 | tee -i -a ./3_lev1eo2dem.log ./full_log.log"
+    "!asap ctx step_6 {config1} {asap.kwarg_parse(step_kwargs, 'step_6')}  2>&1 | tee -i -a ./3_lev1eo2dem.log ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -286,7 +287,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_7 --mpp 100 --just_ortho False --dem_hole_fill_len 50 2>&1 | tee -i -a ./4_make_100m_dem.log ./full_log.log"
+    "!asap ctx step_7 --mpp 100 --just_ortho False --dem_hole_fill_len 50 {asap.kwarg_parse(step_kwargs, 'step_7')} 2>&1 | tee -i -a ./4_make_100m_dem.log ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -429,7 +430,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_9 --mpp {img_gsd} 2>&1 | tee -i -a ./5_mapproject_to_100m_dem.log ./full_log.log"
+    "!asap ctx step_9 --mpp {img_gsd} {asap.kwarg_parse(step_kwargs, 'step_9')} 2>&1 | tee -i -a ./5_mapproject_to_100m_dem.log ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -452,7 +453,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_10 {config2} 2>&1 | tee -i -a ./6_next_level_dem.log ./full_log.log"
+    "!asap ctx step_10 {config2} {asap.kwarg_parse(step_kwargs, 'step_10')} 2>&1 | tee -i -a ./6_next_level_dem.log ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -466,7 +467,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_11 {config2} 2>&1 | tee -i -a ./6_next_level_dem.log ./full_log.log"
+    "!asap ctx step_11 {config2} {asap.kwarg_parse(step_kwargs, 'step_11')} 2>&1 | tee -i -a ./6_next_level_dem.log ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -480,7 +481,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_7 --mpp {dem_gsd} --run results_map_ba"
+    "!asap ctx step_7 --mpp {dem_gsd} --run results_map_ba {asap.kwarg_parse(step_kwargs, 'step_7_2')}"
    ],
    "outputs": [],
    "metadata": {
@@ -645,7 +646,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_13 --maxd {maxdisp} 2>&1 | tee -i -a ./8_pc_align.log ./full_log.log"
+    "!asap ctx step_13 --maxd {maxdisp} {asap.kwarg_parse(step_kwargs, 'step_13')} 2>&1 | tee -i -a ./8_pc_align.log ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -659,7 +660,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_14 --mpp {dem_gsd}  2>&1 | tee -i -a ./9_dems_orthos.log ./full_log.log"
+    "!asap ctx step_14 --mpp {dem_gsd} {asap.kwarg_parse(step_kwargs, 'step_14')} 2>&1 | tee -i -a ./9_dems_orthos.log ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -673,7 +674,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "!asap ctx step_15 2>&1 | tee -i -a ./10_geoid_adjustment.log  ./full_log.log"
+    "!asap ctx step_15 {asap.kwarg_parse(step_kwargs, 'step_15')} 2>&1 | tee -i -a ./10_geoid_adjustment.log  ./full_log.log"
    ],
    "outputs": [],
    "metadata": {
@@ -701,7 +702,7 @@
    "cell_type": "code",
    "execution_count": null,
    "source": [
-    "img = './' + str(next(Path('./').glob('./*/results_map_ba/dem_align/*_ba_align_*-DEM-hillshade.tif')))\n",
+    "img = './' + str(next(Path('./').glob('./*/results_map_ba/dem_align/*-hillshade.tif')))\n",
     "out = img.replace('.tif', '.png')"
    ],
    "outputs": [],

diff --git a/isis.yml b/isis.yml
@@ -4,4 +4,5 @@ channels:
   - conda-forge
   - defaults
 dependencies:
-  - isis=4.4.0
+  - isis=4.4.0
+  - armadillo=9