Merge pull request #25 from nsidc/workflow-dmrpp-config

check workflow config for 'dmrpp' key if collection doesn't have it
ghrcdaac · Jan 5, 2022 · 0574f0a · 0574f0a
2 parents 50efe42 + dbd7b9c
commit 0574f0a
Show file tree

Hide file tree

Showing 2 changed files with 79 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -2,44 +2,99 @@
 Docker image to generate dmrpp files from netCDF and HDF files
 # Supported get_dmrpp configuration
 
-## Via Cumulus configuration
+## Via Cumulus Collection configuration
 ```code
 {
-    "config": {
-        "meta": {
+  "config": {
+    "meta": {
+      "dmrpp": {
+        "dmrpp_regex" : "^.*.H6",
+        "options": [
+          {
+            "flag": "-M"
+          },
+          {
+            "flag": "-s",
+            "opt": "s3://ghrcsbxw-public/dmrpp_config/file.config",
+            "download": "true"
+          },
+          {
+            "flag": "-c",
+            "opt": "s3://ghrcsbxw-public/aces1cont__1/aces1cont_2002.212_v2.50.tar.cmr.json",
+            "download": "false"
+          }
+        ]
+      }
+    }
+  }
+}
+```
+
+`opt` is the value that will come after the flag, if provided with `"download": "true"` the value will be ignored and the file provided will be downloaded and used with the `flag`.
+If the `opt` is provided with `"download": "false"` or without `download` the value of `opt` will be used as a letteral string in `get_dmrpp` executable.
+We are supporting HTTP and s3 protocols.
+
+## Via Cumulus Workflow configuration
+
+If you want the same `dmrpp` config to apply to multiple collections that use
+the same workflow, the configuration can be placed in the workflow at
+`${StepName}.Parameters.cma.task_config.dmrpp` instead of in each collection at
+`config.meta.dmrpp`:
+
+```
+   "HyraxProcessing": {
+      "Parameters": {
+        "cma": {
+          "event.$": "$",
+          "task_config": {
+            ...
             "dmrpp": {
-            "dmrpp_regex" : "^.*.H6",
-          "options": [
-            {
-              "flag": "-M"
-            },
-            {
-              "flag": "-s",
-              "opt": "s3://ghrcsbxw-public/dmrpp_config/file.config",
-              "download": "true"
-            },
-            {
-              "flag": "-c",
-              "opt": "s3://ghrcsbxw-public/aces1cont__1/aces1cont_2002.212_v2.50.tar.cmr.json",
-              "download": "false"
+              "dmrpp_regex" : "^.*.H6",
+              "options": [
+                {
+                  "flag": "-M"
+                },
+                {
+                  "flag": "-s",
+                  "opt": "s3://ghrcsbxw-public/dmrpp_config/file.config",
+                  "download": "true"
+                },
+                {
+                  "flag": "-c",
+                  "opt": "s3://ghrcsbxw-public/aces1cont__1/aces1cont_2002.212_v2.50.tar.cmr.json",
+                  "download": "false"
+                }
+              ]
             }
-          ]
+          }
         }
+      },
+
+    ...
     }
+```
+
+As with other variables in the JSON template file, the `dmrpp` config can be set in a terraform variable:
+
+```
+"task_config": {
+  ...
+  "dmrpp": ${jsonencode(dmrpp_config)}
 }
 ```
 
-`opt` is the value that will come after the flag, if provided with `"download": "true"` the value will be ignored and the file provided will be downloaded and used with the `flag`. 
-If the `opt` is provided with `"download": "false"` or without `download` the value of `opt` will be used as a letteral string in `get_dmrpp` executable.
-We are supporting HTTP and s3 protocols.
+If `dmrpp` configuration is set on the collection and in the workflow, the
+collection's configuration will override the workflow's. In other words, the
+workflow's `dmrpp` configuration acts as a default which can be overridden by
+any collection.
 
 # Supported get_dmrpp configuration
 ## Via env vars
 Create a PAYLOAD environment variable holding dmrpp options
 ```
 PAYLOAD='{"dmrpp_regex": "^.*.nc4", "options":[{"flag": "-M"}, {"flag": "-s", "opt": "s3://ghrcsbxw-public/dmrpp_config/file.config","download": "true"}]}'
 ```
-`dmrpp_regex` is optional to override the DMRPP-Generator regex 
+`dmrpp_regex` is optional to override the DMRPP-Generator regex
 # Generate DMRpp files locally without Hyrax server
 ```
 $./generate_and_validate_dmrpp --help
@@ -84,7 +139,7 @@ Now you can validate the result in localhost:8889
 ```code
 ./generate_and_validate_dmrpp -p <path/to/nc/hdf/files> -pyld $PAYLOAD
 ```
-or 
+or
 ```code
 docker run --rm -it --env-file ./env.list -v <path/to/nc/hdf/files>:/workstation ghrcdaac/dmrpp-generator
 ```

diff --git a/dmrpp_generator/main.py b/dmrpp_generator/main.py
@@ -81,7 +81,7 @@ def process(self):
         collection = self.config.get('collection')
         collection_files = collection.get('files', [])
         collection_meta = collection.get('meta', {})
-        dmrpp_meta = collection_meta.get('dmrpp', {})
+        dmrpp_meta = collection_meta.get('dmrpp', self.config.get('dmrpp', {}))
         buckets = self.config.get('buckets')
         granules = self.input['granules']
         self.processing_regex = dmrpp_meta.get('dmrpp_regex', self.processing_regex)