#72 allow multiple -a options to overrride e.g. defaults

bin/stetl

@@ -8,6 +8,7 @@
 from stetl.main import parse_args
 from stetl.etl import ETL
 from stetl.util import Util
+import sys
 
 log = Util.get_log('main')
 
@@ -18,12 +19,12 @@ def main():
     Args:
        -c  --config <config_file>  the Stetl config file.
        -s  --section <section_name> the section in the Stetl config (ini) file to execute (default is [etl]).
-       -a  --args <arglist> substitutable args for symbolic, {arg}, values in Stetl config file, in format "arg1=foo arg2=bar" etc.
+       -a  --args <arglist> zero or more substitutable args for symbolic, {arg}, values in Stetl config file, in format -a arg1=foo -a arg2=bar etc.
        -h  --help <subject> Get component documentation like its configuration parameters, e.g. stetl doc stetl.inputs.fileinput.FileInput
 
     """
 
-    args = parse_args()
+    args = parse_args(sys.argv[1:])
 
     if args.config_file:
         # Do the ETL

docs/using.rst

@@ -234,11 +234,13 @@ or even OGC WPS servers (planned).
 
 Reusable Stetl Configs
 ----------------------
-What we saw in the last example is that it is hard to reuse this `etl.cfg` when we have for example a different input file
-or want to map to different output files. For this Stetl supports `config parameter substitution`.
+What we saw in the last example is that it is hard to reuse this `etl.cfg`
+when we have for example a different input file
+or want to map to different output files.
+For this Stetl supports `config parameter substitution`.
 
 Dynamic or secret (e.g. database credentials) parameters in `etl.cfg` are declared
-symbolically and substituted at runtime via the commandline or the OS environment (new in 2018).
+symbolically and substituted at runtime via the commandline or the OS environment.
 
 A variable is declared between curly brackets like `{out_xml}`. See
 example `6_cmdargs <https://github.com/geopython/stetl/tree/master/examples/basics/6_cmdargs>`_. ::
@@ -258,7 +260,8 @@ example `6_cmdargs <https://github.com/geopython/stetl/tree/master/examples/basi
 	class = outputs.fileoutput.FileOutput
 	file_path = {out_xml}
 
-Note the symbolic input, xsl and output files. We can now perform this ETL using the `stetl -a option` in two ways.
+Note the symbolic input, xsl and output files. We can now perform
+the ETL using the `stetl -a option` in two basic ways.
 One, passing the arguments on the commandline, like ::
 
 	stetl -c etl.cfg -a "in_xml=input/cities.xml in_xsl=cities2gml.xsl out_xml=output/gmlcities.gml"
@@ -274,11 +277,21 @@ Where the content of the `etl.args` properties file is: ::
 	in_xsl=cities2gml.xsl
 	out_xml=output/gmlcities.gml
 
-A third way is to pass these key/value pairs (partly) as OS Environment variables.
+It is also possible to specify **multiple -a arguments**. This provides for situations
+where a `default.args` contains all default arguments and a `my.args` or explicit `-a` settings
+that override the default values in `default.args`. Overriding is determined by the order of
+the `-a` arguments. Examples: ::
+
+	stetl -c etl.cfg -a default.args -a my.args
+	stetl -c etl.cfg -a default.args -a "db_user=docker db_password=pass"
+	stetl -c etl.cfg -a default.args -a db_user=docker -a db_password=pass
+
+It is also possible to pass these key/value pairs via OS Environment variables.
 This is especially handy in Docker-based deployments like Docker Compose and Kubernetes.
 In this case the variable names need to be prepended with `STETL_` or `stetl_` as
 to not mix-up with other non-related OS-env vars. A mixture of commandline args (file)
-and environment vars is possible. The rule is that *OS Environment variables always take prevalence*.
+and environment vars is possible. The rule is that
+*OS Environment variables always override/overrule arguments specified with -a option(s)*.
 
 For example, the above args could also be passed as follows: ::
 
@@ -287,11 +300,16 @@ For example, the above args could also be passed as follows: ::
 	export stetl_out_xml="output/gmlcities.gml"
 	stetl -c etl.cfg
 
-or only override the input file name from `etl.args`: ::
+or only override the input file name `in_xml` from `etl.args`: ::
 
 	export stetl_in_xml="input/cities2.xml"
 	stetl -c etl.cfg -a etl.args
 
+or even with multiple `-a args`: ::
+
+	export stetl_in_xml="input/cities2.xml"
+	stetl -c etl.cfg -a etl.args -a my.args
+
 This makes an ETL chain highly reusable.
 A very elaborate Stetl config with parameter substitution can be seen in the
 `Top10NL ETL <https://github.com/geopython/stetl/blob/master/examples/top10nl/etl-top10nl.cfg>`_.

examples/basics/6_cmdargs/etl.sh

@@ -14,3 +14,6 @@ stetl  -c etl.cfg -a "in_xml=input/cities.xml in_xsl=cities2gml.xsl out_xml=outp
 
 # Option 2: using a properties file
 stetl  -c etl.cfg -a etl.args
+
+# Option 3: multiple -a options e.g. overriding one or more default args (file)
+stetl  -c etl.cfg -a etl.args -a "in_xml=input/amsterdam.xml"

examples/basics/6_cmdargs/input/amsterdam.xml

@@ -0,0 +1,8 @@
+<?xml version='1.0' encoding='utf-8'?>
+<cities>
+    <city>
+        <name>Amsterdam</name>
+        <lat>52.4</lat>
+        <lon>4.9</lon>
+    </city>
+</cities>

examples/basics/6_cmdargs/output/gmlcities.gml

@@ -22,24 +22,4 @@
       </ogr:geometry>
     </ogr:City>
   </gml:featureMember>
-  <gml:featureMember>
-    <ogr:City>
-      <ogr:name>Bonn</ogr:name>
-      <ogr:geometry>
-        <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
-          <gml:coordinates>50.7,7.1</gml:coordinates>
-        </gml:Point>
-      </ogr:geometry>
-    </ogr:City>
-  </gml:featureMember>
-  <gml:featureMember>
-    <ogr:City>
-      <ogr:name>Rome</ogr:name>
-      <ogr:geometry>
-        <gml:Point srsName="urn:ogc:def:crs:EPSG:4326">
-          <gml:coordinates>41.9,12.5</gml:coordinates>
-        </gml:Point>
-      </ogr:geometry>
-    </ogr:City>
-  </gml:featureMember>
 </ogr:FeatureCollection>