cli: add options step attribute (#881)

Add ability of specifying options for a step, initially supported for the docker runner.
The YAML spec is extended to support arbitrary options. This also contains tests and
additions to to documentation.

Co-authored-by: Ivo Jimenez <ivo.jimenez@gmail.com>

docs/sections/cli_features.md

@@ -1,4 +1,4 @@
-# CLI feautures
+# CLI features
 
 ## New workflow initialization
 

docs/sections/cn_workflows.md

@@ -45,6 +45,7 @@ attribute.
 | `secrets`   | **optional** A list of strings representing the names of secret variables to define<br>in the environment of the container for the step. For example,<br>`secrets: ["SECRET1", "SECRET2"]`. |
 | `skip_pull` | **optional** A boolean value that determines whether to pull the image before<br>executing the step. By default this is `false`. If the given container<br>image already exist (e.g. because it was built by a previous step in<br>the same workflow), assigning `true` skips downloading the image from<br>the registry. |
 | `dir`       | **optional** A string representing an absolute path inside the container to use as the<br>working directory. By default, this is `/workspace`. |
+| `options`   | **optional** Container configuration options. For instance:<br>`options: {ports: {8888:8888}, interactive: True, tty: True}`. Currently only<br> supported for the docker runtime. See the parameters of `client.containers.runs()`<br> in the [Docker Python SDK](https://docker-py.readthedocs.io/en/stable/containers.html?highlight=inspect) for the full list of options |
 
 ### Referencing images in a step
 
@@ -400,6 +401,11 @@ question (see [here][engconf] for more).
 
 [engconf]: ./cli_features#customizing-container-engine-behavior
 
+
+Alternatively, to restrict a configuration to a specific step in a workflow, set the desired parameters in the step's `options`
+**Note**: this is currently only supported for the Docker runtime 
+
+
 ## Resource Managers
 
 Popper can execute steps in a workflow through other resource managers

src/popper/parser.py

@@ -35,6 +35,7 @@ class WorkflowParser(object):
                                 "matching-rule": "any",
                                 "mapping": {"regex;(.+)": {"type": "str"}},
                             },
+                            "options": {"type": "map", "allowempty": True},
                         },
                     }
                 ],

src/popper/runner_host.py

@@ -257,7 +257,7 @@ def _get_container_kwargs(self, step, img, name):
         }
 
         self._update_with_engine_config(args)
-
+        args.update(step.options)
         log.debug(f"container args: {pu.prettystr(args)}\n")
 
         return args

src/test/test_parser.py

@@ -37,10 +37,11 @@ def test_new_workflow(self):
                     "env": {"EN": "EE"},
                     "secrets": ["S"],
                     "dir": "/path/to/",
+                    "options": {"name": "spam"},
                 },
                 {"uses": "bar", "runs": ["a", "b"], "args": ["c"], "skip_pull": True},
             ],
-            "options": {"env": {"FOO": "bar"}, "secrets": ["Z"]},
+            "options": {"env": {"FOO": "bar"}, "secrets": ["Z"],},
         }
         wf = WorkflowParser.parse(wf_data=wf_data)
 
@@ -50,6 +51,7 @@ def test_new_workflow(self):
         self.assertEqual(("Z", "S"), step.secrets)
         self.assertEqual({"EN": "EE", "FOO": "bar"}, step.env)
         self.assertEqual("/path/to/", step.dir)
+        self.assertEqual("spam", step.options.name)
         self.assertTrue(not step.runs)
         self.assertTrue(not step.args)
         self.assertFalse(step.skip_pull)

src/test/test_runner_host.py

@@ -159,6 +159,7 @@ def test_get_container_kwargs(self):
                 "args": ["ls"],
                 "id": "one",
                 "dir": "/tmp/",
+                "options": {"ports": {"8888/tcp": 8888}},
             },
             default_box=True,
         )
@@ -203,6 +204,7 @@ def test_get_container_kwargs(self):
                     "privileged": True,
                     "hostname": "popper.local",
                     "domainname": "www.example.org",
+                    "ports": {"8888/tcp": 8888},
                 },
             )
 
@@ -234,6 +236,7 @@ def test_get_container_kwargs(self):
                     "privileged": True,
                     "hostname": "popper.local",
                     "domainname": "www.example.org",
+                    "ports": {"8888/tcp": 8888},
                 },
             )