Issue #48: new option --debug

README.md

@@ -54,3 +54,4 @@
  - http://nose.readthedocs.io/en/latest/
  - https://pypi.python.org/pypi/bandit
  - https://www.contributor-covenant.org/
+ - http://ddt.readthedocs.io/en/latest/index.html

docs/options.rst

@@ -48,3 +48,34 @@ following output:
     spline.components.tasks -  | # give back result
     spline.components.tasks -  | exit ${docker_error}
     spline.components.tasks - Processing Bash code: finished
+
+Debug
+-----
+The option `--debug` adjust the Bash option `set -x` which activates the debug
+mode in Bash. The primes example in the spline repository gives you a good example.
+I'm just printing the first 20 lines:
+
+::
+
+    $ spline --definition=examples/primes.yaml --debug 2>&1 | head -20
+    2018-01-05 19:31:12,023 - spline.application - Running with Python 2.7.13 (default, Jan 19 2017, 14:48:08) [GCC 6.3.0 20170118]
+    2018-01-05 19:31:12,028 - spline.application - Running on platform Linux-4.9.0-3-amd64-x86_64-with-debian-9.1
+    2018-01-05 19:31:12,028 - spline.application - Current cpu count is 4
+    2018-01-05 19:31:12,029 - spline.application - Processing pipeline definition 'examples/primes.yaml'
+    2018-01-05 19:31:12,032 - spline.application - Schema validation for 'examples/primes.yaml' succeeded
+    2018-01-05 19:31:12,032 - spline.components.stage - Processing pipeline stage 'Calculate Primes'
+    2018-01-05 19:31:12,033 - spline.components.tasks - Processing group of tasks (parallel=no)
+    2018-01-05 19:31:12,033 - spline.components.tasks - Processing Bash code: start
+    2018-01-05 19:31:12,043 - spline.components.bash - Running script /tmp/pipeline-script-5iRCsz.sh
+    2018-01-05 19:31:12,050 - spline.components.tasks -  | ++ seq 0 100
+    2018-01-05 19:31:12,051 - spline.components.tasks -  | + for n in $(seq 0 100)
+    2018-01-05 19:31:12,052 - spline.components.tasks -  | ++ is_prime 0
+    2018-01-05 19:31:12,052 - spline.components.tasks -  | ++ n=0
+    2018-01-05 19:31:12,052 - spline.components.tasks -  | ++ '[' 0 -lt 2 ']'
+    2018-01-05 19:31:12,052 - spline.components.tasks -  | ++ return
+    2018-01-05 19:31:12,053 - spline.components.tasks -  | + '[' '' == yes ']'
+    2018-01-05 19:31:12,053 - spline.components.tasks -  | + for n in $(seq 0 100)
+    2018-01-05 19:31:12,053 - spline.components.tasks -  | ++ is_prime 1
+    2018-01-05 19:31:12,053 - spline.components.tasks -  | ++ n=1
+    2018-01-05 19:31:12,053 - spline.components.tasks -  | ++ '[' 1 -lt 2 ']'
+

requirements.txt

@@ -18,4 +18,5 @@ sphinx_rtd_theme == 0.2.4
 schema == 0.6.6
 pycontracts == 1.8.0
 bandit == 1.0.1
-epydoc
+epydoc
+ddt == 1.1.1

spline/application.py

@@ -173,6 +173,8 @@ def run(self, definition):
               help="When enabled then it does log event details")
 @click.option('--dry-run', is_flag=True, default=False,
               help="When enabled then no Bash script is executed but shown")
+@click.option('--debug', is_flag=True, default=False,
+              help="When enabled then using 'set -x' for debugging Bash scripts")
 def main(**kwargs):
     """The Pipeline tool."""
     options = ApplicationOptions(**kwargs)

spline/components/bash.py

@@ -45,13 +45,10 @@ def __init__(self, config):
         """
         self.event = Event.create(__name__)
         self.logger = Logger.get_logger(__name__)
+        self.config = config
         self.success = True
-        self.script = config.script
-        self.model = config.model
-        self.dry_run = config.dry_run
         self.env = os.environ.copy()
         self.env.update(config.env)
-        self.item = config.item
 
         if len(config.title) > 0:
             self.logger.info(config.title)
@@ -94,7 +91,7 @@ def create_file_for(self, script):
             prefix="pipeline-script-", mode='w+t', suffix=".sh", delete=False)
 
         self.update_script_filename(temp.name)
-        rendered_script = render(script, model=self.model, env=self.env, item=self.item)
+        rendered_script = render(script, model=self.config.model, env=self.env, item=self.config.item)
         if rendered_script is None:
             self.success = False
             temp.close()
@@ -107,7 +104,7 @@ def create_file_for(self, script):
             content = str(open(rendered_script).read())
             temp.writelines(content)
         else:
-            temp.write(u'#!/bin/bash\n')
+            temp.write(u"#!/bin/bash\n%s" % ("set -x\n" if self.config.debug else ''))
             temp.write(to_file_map[sys.version_info.major](rendered_script))
         temp.close()
         # make Bash script executable
@@ -139,9 +136,9 @@ def process_script(self, filename):
 
     def process(self):
         """Running the Bash code."""
-        temp_filename = self.create_file_for(self.script)
+        temp_filename = self.create_file_for(self.config.script)
         if temp_filename is not None:
-            if self.dry_run:
+            if self.config.dry_run:
                 self.logger.info("Dry run mode for script %s", temp_filename)
                 for line in open(temp_filename):
                     yield line[0:-1] if line[-1] == '\n' else line

spline/components/config.py

@@ -20,6 +20,7 @@
     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
     OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 """
+# pylint: disable=too-many-instance-attributes
 import logging
 from schema import Schema, SchemaError, And, Or, Optional
 from ..tools.adapter import Adapter
@@ -34,7 +35,8 @@ class ShellConfig(object):
         Optional('model', default={}): {Optional(And(str, len)): object},
         Optional('env', default={}): {Optional(And(str, len)): And(str, len)},
         Optional('item', default=None): object,
-        Optional('dry_run', default=False): bool
+        Optional('dry_run', default=False): bool,
+        Optional('debug', default=False): bool
     }
 
     def __init__(self, **kwargs):
@@ -47,6 +49,7 @@ def __init__(self, **kwargs):
             self.env = arguments.env.data
             self.item = arguments.item
             self.dry_run = arguments.dry_run
+            self.debug = arguments.debug
         except SchemaError as exception:
             logging.getLogger(__name__).error(exception)
             raise RuntimeError(str(exception))
@@ -62,11 +65,17 @@ class ApplicationOptions(object):
         Optional('validate_only', default=False): bool,
         Optional('dry_run', default=False): bool,
         Optional('event_logging', default=False): bool,
-        Optional('logging_config', default=''): Or(type(''), type(u''))
+        Optional('logging_config', default=''): Or(type(''), type(u'')),
+        Optional('debug', default=False): bool
     }
 
     def __init__(self, **kwargs):
-        """Initializing and validating fields."""
+        """
+        Initializing and validating fields.
+
+        @type kwargs: dict
+        @param kwargs: application command line options.
+        """
         try:
             arguments = Adapter(Schema(ApplicationOptions.SCHEMA).validate(kwargs))
             self.definition = arguments.definition
@@ -76,6 +85,7 @@ def __init__(self, **kwargs):
             self.dry_run = arguments.dry_run
             self.event_logging = arguments.event_logging
             self.logging_config = arguments.logging_config
+            self.debug = arguments.debug
         except SchemaError as exception:
             logging.getLogger(__name__).error(exception)
             raise RuntimeError(str(exception))

spline/components/tasks.py

@@ -46,7 +46,7 @@ def worker(data):
                     ShellConfig(script=data['entry']['script'],
                                 title=data['entry']['title'] if 'title' in data['entry'] else '',
                                 model=data['model'], env=data['env'], item=data['item'],
-                                dry_run=data['dry_run']))
+                                dry_run=data['dry_run'], debug=data['debug']))
     output = []
     for line in shell.process():
         output.append(line)
@@ -84,7 +84,8 @@ def prepare_shell_data(self, shells, key, entry):
                     'model': self.pipeline.model,
                     'env': self.get_merged_env(),
                     'item': item,
-                    'dry_run': self.pipeline.options.dry_run})
+                    'dry_run': self.pipeline.options.dry_run,
+                    'debug': self.pipeline.options.debug})
 
     def get_parallel_mode(self):
         """Logging helper for visualizing parallel state."""
@@ -146,7 +147,7 @@ def process_shells_ordered(self, shells):
             entry = shell['entry']
             config = ShellConfig(script=entry['script'], title=entry['title'] if 'title' in entry else '',
                                  model=shell['model'], env=shell['env'], item=shell['item'],
-                                 dry_run=shell['dry_run'])
+                                 dry_run=shell['dry_run'], debug=shell['debug'])
             result = Adapter(self.process_shell(get_creator_by_name(shell['creator']), entry, config))
             output += result.output
             if not result.success:
@@ -204,7 +205,8 @@ def run_cleanup(self, env, exit_code):
             env.update({'PIPELINE_SHELL_EXIT_CODE': str(exit_code)})
             config = ShellConfig(script=self.pipeline.data.hooks.cleanup,
                                  model=self.pipeline.model, env=env,
-                                 dry_run=self.pipeline.options.dry_run)
+                                 dry_run=self.pipeline.options.dry_run,
+                                 debug=self.pipeline.options.debug)
             cleanup_shell = Bash(config)
             for line in cleanup_shell.process():
                 output.append(line)

tests/test_application_options.py

@@ -18,6 +18,7 @@ def test_minimal_valid(self):
         assert_that(options.event_logging, equal_to(False))
         assert_that(options.validate_only, equal_to(False))
         assert_that(options.dry_run, equal_to(False))
+        assert_that(options.debug, equal_to(False))
 
     def test_missing_mandatory(self):
         """Testing missing mandatory parameter."""

tests/test_shell_config.py

@@ -2,9 +2,11 @@
 # pylint: disable=no-self-use, invalid-name, redundant-unittest-assert
 import unittest
 from hamcrest import assert_that, equal_to, contains_string
+from ddt import ddt, data
 from spline.components.config import ShellConfig
 
 
+@ddt
 class TestShellConfig(unittest.TestCase):
     """Testing of class ShellConfig."""
 
@@ -17,17 +19,21 @@ def test_minimal_valid(self):
         assert_that(config.env, equal_to({}))
         assert_that(config.item, equal_to(None))
         assert_that(config.dry_run, equal_to(False))
+        assert_that(config.debug, equal_to(False))
 
-    def test_complete_valid(self):
+    @data({'dry_run': True}, {'debug': True}, {'dry_run': True}, {'item': 'hello'},
+          {'env': {'message': 'hello'}}, {'model': {'foo': 123}}, {'title': 'test'})
+    def test_individual_valid(self, kwargs):
         """Testing to provide mandatory and all optional parameters."""
-        config = ShellConfig(script='echo "hello world"', title='test', model={'foo': 123},
-                             env={'bar': "xyz"}, item='hello', dry_run=True)
-        assert_that(config.script, equal_to('echo "hello world"'))
-        assert_that(config.title, equal_to('test'))
-        assert_that(config.model, equal_to({'foo': 123}))
-        assert_that(config.env, equal_to({'bar': "xyz"}))
-        assert_that(config.item, equal_to('hello'))
-        assert_that(config.dry_run, equal_to(True))
+        # defaults
+        final_kwargs = {'script': 'echo "hello world"', 'title': '', 'debug': False,
+                        'dry_run': False, 'item': None, 'env': {}, 'model': {}}
+        final_kwargs.update(kwargs)
+
+        config = ShellConfig(**final_kwargs)
+        for key, value in final_kwargs.items():
+            assert_that(key in config.__dict__, equal_to(True))
+            assert_that(config.__dict__[key], equal_to(value))
 
     def test_missing_mandatory(self):
         """Testing invalid parameter."""

tests/test_tasks.py

@@ -147,7 +147,7 @@ def test_worker(self):
         data = {'creator': 'shell',
                 'entry': {'script': '''echo "{{model.mode}}:{{env.message}}"'''},
                 'env': {'message': 'hello'}, 'model': {'mode': 'test'}, 'item': None,
-                'dry_run': False}
+                'dry_run': False, 'debug': False}
         result = worker(data)
         output = [line for line in result['output'] if line.find("hello") >= 0]
         assert_that(result['success'], equal_to(True))