add a step queue for post processing

Signed-off-by: Adam Stokes <adam.stokes@ubuntu.com>

conjure-up-rsyslog.conf

@@ -7,6 +7,8 @@ template(name="ConjureUpLogFileName" type="list") {
 }
 if $programname == 'conjure-up' then \
   /var/log/conjure-up/combined.log
+if $programname == 'conjure-craft' then \
+  /var/log/conjure-up/craft.log
 if $programname == 'conjure-up' then \
   if $syslogtag contains 'conjure-up/' then \
     ?ConjureUpLogFileName

conjure/controllers/finish/tui.py

@@ -7,6 +7,8 @@
 import json
 import os
 import sys
+import time
+from collections import deque
 from glob import glob
 
 
@@ -35,6 +37,10 @@ def render():
         utils.info("Setting up prerequisites")
         try:
             sh = common.run_script(pre_exec_sh)
+            if sh.returncode > 0:
+                raise Exception(
+                    "Cannot execute pre-processing script: {}".format(
+                        sh.stderr.decode('utf8')))
             result = json.loads(sh.stdout.decode('utf8'))
             app.log.debug("pre execution done: {}".format(result))
         except CalledProcessError as e:
@@ -52,20 +58,36 @@ def render():
 
     # post step processing
     steps = sorted(glob(os.path.join(bundle_scripts, '*.sh')))
+    steps_queue = deque()
     for step in steps:
         if "00_pre.sh" in step or "00_post-bootstrap.sh" in step:
             app.log.debug("Skipping pre and post-bootstrap steps.")
             continue
 
         if os.access(step, os.X_OK):
+            steps_queue.append(step)
+
+    is_requeued = False
+    while steps_queue:
+        step = steps_queue.popleft()
+        if not is_requeued:
             utils.info(
                 "Running: {}".format(common.parse_description(step)))
-        try:
-            sh = common.run_script(step)
-            result = json.loads(sh.stdout.decode('utf8'))
-            app.log.debug("post execution done: {}".format(result))
-        except CalledProcessError as e:
-            utils.warning(
-                "Failure in step: {}".format(e))
+        sh = common.run_script(step)
+        result = json.loads(sh.stdout.decode('utf8'))
+        if result['returnCode'] > 0:
+            utils.error(
+                "Failure in step: {}".format(result['message']))
             sys.exit(1)
+        if not result['isComplete']:
+            time.sleep(5)
+            if not is_requeued:
+                utils.warning("{}, please wait".format(
+                    result['message']))
+            steps_queue.appendleft(step)
+            is_requeued = True
+            continue
+        utils.info(result['message'])
+        is_requeued = False
+        app.log.debug("post execution done: {}".format(result))
     finish()

share/hooklib/common.sh

@@ -23,6 +23,12 @@ info() {
     logger -t "conjure-up/$name" "[INFO] $@"
 }
 
+log() {
+    if [ $CONJURE_UP_HEADLESS ]; then
+        echo -e "\e[32m\e[1m[ info ]\e[0m $@]"
+    fi
+}
+
 # Gets current juju state for machine
 #
 # Arguments: