Merge pull request #31 from johandahlberg/compatiblity_st2_1.4

WIP: Compatibility with st2 1.4
arteria-project · May 24, 2016 · 1385c4d · 1385c4d
2 parents f27daf8 + 4b171af
commit 1385c4d
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 51 deletions.
diff --git a/actions/create_sis_style_checksums.yaml b/actions/create_sis_style_checksums.yaml
@@ -42,7 +42,6 @@
         cmd:
             immutable: true
             default: 'ssh -v {{host}} "bash -s" < /opt/stackstorm/packs/arteria-packs/actions/lib/create_sis_style_checksums.sh {{ source }} {{ include_file }} {{ output_file }}'
-            position: 0
         connect_timeout:
             type: 'integer'
             description: 'SSH connect timeout in seconds'

diff --git a/actions/lib/poll_status.py b/actions/lib/poll_status.py
@@ -47,12 +47,13 @@ def query(self, url):
             self.log("{0} -- {1} - an error was encountered: {2}".format(current_time, url, err))
             return None, None
 
-    def run(self, url, sleep, log, max_retries = 3):
+    def run(self, url, sleep, log, ignore_result, max_retries = 3):
         """
         Query the url end-point. Can be called directly from StackStorm, or via the script cli
         :param url: to call
         :param sleep: minutes to sleep between attempts
         :param log: file name to write log to
+        :param ignore_result: return 0 exit status even if polling failed (for known errors).
         :param max_retries: maximum number of retries
         :return: None
         """
@@ -81,14 +82,18 @@ def run(self, url, sleep, log, max_retries = 3):
                                                                                                    url,
                                                                                                    state))
                 self.log(resp.json())
-                self.shutdown(1)
+                if ignore_result:
+                    self.shutdown(0)
+                else:
+                    self.shutdown(1)
+
             elif not state and retry_attempts < max_retries:
                 retry_attempts += 1
                 self.log("{0} -- {1} did not report state. "
                          "Probably due to a connection error, will retry. Attempt {3} of {4}.".format(current_time,
                                                                                                       url,
                                                                                                       retry_attempts,
-                                                                                                      retries))
+                                                                                                      max_retries))
                 time.sleep(sleep * 60)
             else:
                 self.log("{0} -- {1} returned state unknown state {2}. "
@@ -103,15 +108,18 @@ def run(self, url, sleep, log, max_retries = 3):
 @click.option("--log", required = False,
               default = "/var/log/arteria/poll_status.log",
               help = "Path to log file (default /var/log/arteria/poll_status.log)")
-def start(url, sleep, log):
+@click.option("--ignore_result", required = False,
+              default = False,
+              help = "Return 0 exit status even if polling failed.")
+def start(url, sleep, log, ignore_result):
     """ Accepts an URL to poll (e.g. http://testarteria1:10900/api/1.0/qc/status/4224)
         and sleeps a number of minutes between every poll (default 1 minute).
 
         Will continue to poll as long as a returned JSON field called state contains 'started'.
         Exits with an error if 'error' or 'none' is received, and with success if 'done'
         is received.
     """
-    PollStatus().run(url, sleep, log)
+    PollStatus().run(url, sleep, log, ignore_result)
 
 if __name__ == "__main__":
     start()
diff --git a/actions/poll_status.yaml b/actions/poll_status.yaml
@@ -20,6 +20,12 @@ parameters:
         required: false
         position: 2
         default: 1
+    ignore_result:
+        type: boolean
+        description: Allows manually overriding the result, e.g. even script will return 0 exit status even if polling failed. Useful for e.g. overriding qc result checking.
+        required: false
+        position: 2
+        default: false
     log:
         type: string
         description: Path to log file (default /var/log/arteria/poll_status.log)

diff --git a/actions/rsync.yaml b/actions/rsync.yaml
@@ -46,5 +46,4 @@
         cmd:
             immutable: true
             default: 'ssh -v {{ source_host_user }}@{{ source_host }} rsync {{args}} {% if include_file %} --include-from {{ include_file }}  {% endif %}{{source}} {{dest_user}}@{{dest_server}}:{{destination}}/'
-            position: 0
 
diff --git a/actions/workflows/ngi_uu_workflow.yaml b/actions/workflows/ngi_uu_workflow.yaml
@@ -34,23 +34,23 @@ workflows:
             get_config:
               action: arteria-packs.get_pack_config
               publish:
-                hermes_base_url: <% $.get_config.result.hermes_base_url %>
-                hermes_user: <% $.get_config.result.hermes_user %>
-                hermes_token: <% $.get_config.result.hermes_token %>
-                remote_host: <% $.get_config.result.remote_host %>
-                remote_user: <% $.get_config.result.remote_user%>
-                remote_destination: <% $.get_config.result.remote_destination %>
-                summary_host: <% $.get_config.result.summary_host %>
-                summary_user: <% $.get_config.result.summary_user %>
-                summary_destination: <% $.get_config.result.summary_destination %>
-                siswrap_service_port: <% $.get_config.result.siswrap_service_port %>
-                bcl2fastq_service_port: <% $.get_config.result.bcl2fastq_service_port %>
-                runfolder_service_port: <% $.get_config.result.runfolder_service_port %>
-                send_mail_to: <% $.get_config.result.send_mail_to %>
-                remote_sisyphus_location: <% $.get_config.result.remote_sisyphus_location %>
-                nestor_remote_path: <% $.get_config.result.nestor_remote_path %>
-                irma_remote_path: <% $.get_config.result.irma_remote_path %>
-                ngi_pipeline_url: <% $.get_config.result.ngi_pipeline_url %>
+                hermes_base_url: <% task(get_config).result.result.hermes_base_url %>
+                hermes_user: <% task(get_config).result.result.hermes_user %>
+                hermes_token: <% task(get_config).result.result.hermes_token %>
+                remote_host: <% task(get_config).result.result.remote_host %>
+                remote_user: <% task(get_config).result.result.remote_user%>
+                remote_destination: <% task(get_config).result.result.remote_destination %>
+                summary_host: <% task(get_config).result.result.summary_host %>
+                summary_user: <% task(get_config).result.result.summary_user %>
+                summary_destination: <% task(get_config).result.result.summary_destination %>
+                siswrap_service_port: <% task(get_config).result.result.siswrap_service_port %>
+                bcl2fastq_service_port: <% task(get_config).result.result.bcl2fastq_service_port %>
+                runfolder_service_port: <% task(get_config).result.result.runfolder_service_port %>
+                send_mail_to: <% task(get_config).result.result.send_mail_to %>
+                remote_sisyphus_location: <% task(get_config).result.result.remote_sisyphus_location %>
+                nestor_remote_path: <% task(get_config).result.result.nestor_remote_path %>
+                irma_remote_path: <% task(get_config).result.result.irma_remote_path %>
+                ngi_pipeline_url: <% task(get_config).result.result.ngi_pipeline_url %>
               on-success:
                  - mark_as_started
 
@@ -68,7 +68,7 @@ workflows:
               input:
                 cmd: python -c 'import sys; sys.stdout.write("<% $.runfolder %>".split("/")[-1])'
               publish:
-                runfolder_name: "<% $.get_runfolder_name.stdout %>"
+                runfolder_name: "<% task(get_runfolder_name).result.stdout %>"
               on-success:
                 - get_flowcell_name
 
@@ -77,7 +77,7 @@ workflows:
                 input:
                     runfolder_name: <% $.runfolder_name %>
                 publish:
-                    flowcell_name: <% $.get_flowcell_name.result %>
+                    flowcell_name: <% task(get_flowcell_name).result.result %>
                 on-success:
                     - download_samplesheet
 
@@ -87,9 +87,11 @@ workflows:
                 action: core.http
                 input:
                     url: <% $.hermes_base_url %>/<% $.flowcell_name %>/samplesheetfile
-                    headers: USER=<% $.hermes_user %>&X-XSRF-TOKEN=<% $.hermes_token %>
+                    headers: 
+                        USER: <% $.hermes_user %>
+                        X-XSRF-TOKEN: <% $.hermes_token %>
                 publish:
-                    samplesheet_string: <% $.download_samplesheet.body.samplesheet %>
+                    samplesheet_string: <% task(download_samplesheet).result.body.samplesheet %>
                 on-success:
                     - construct_bcl2fastq_body
 
@@ -106,7 +108,7 @@ workflows:
                     use_base_mask: "<% $.use_base_mask %>"
                     additional_arguments: "<% $.bcl2fastq_additional_arguments %>"
                 publish:
-                    bcl2fastq_body: <% $.construct_bcl2fastq_body.stdout %>
+                    bcl2fastq_body: <% task(construct_bcl2fastq_body).result.stdout %>
                 on-success:
                     - run_bcl2fastq
 
@@ -115,10 +117,11 @@ workflows:
                 input:
                     url: "http://<% $.host %>:<% $.bcl2fastq_service_port %>/api/1.0/start/<% $.runfolder_name %>"
                     method: "POST"
-                    headers: "Content-Type=application/json"
+                    headers:
+                        Content-Type: application/json
                     body: <% $.bcl2fastq_body %>
                 publish:
-                    bcl2fastq_status_url: <% $.run_bcl2fastq.body.link %>
+                    bcl2fastq_status_url: <% task(run_bcl2fastq).result.body.link %>
                 on-success:
                     - poll_demultiplex_status
 
@@ -135,9 +138,11 @@ workflows:
                 action: core.http
                 input:
                     url: <% $.hermes_base_url %>/<% $.flowcell_name %>/configfile
-                    headers: USER=<% $.hermes_user %>&X-XSRF-TOKEN=<% $.hermes_token %>
+                    headers:
+                      USER: <% $.hermes_user %>
+                      X-XSRF-TOKEN: <% $.hermes_token %>
                 publish:
-                    sisyphus_conf_string: <% $.download_sisyphus_config.body.config %>
+                    sisyphus_conf_string: <% task(download_sisyphus_config).result.body.config %>
                 on-success:
                     - construct_report_body
 
@@ -147,7 +152,7 @@ workflows:
                     runfolder: "<% $.runfolder_name %>"
                     sisyphus_config: "<% $.sisyphus_conf_string %>"
                 publish:
-                    report_body: <% $.construct_report_body.stdout %>
+                    report_body: <% task(construct_report_body).result.stdout %>
                 on-success:
                     - run_sisyphus_quick_report
 
@@ -156,10 +161,11 @@ workflows:
                 input:
                     url: "http://<% $.host %>:<% $.siswrap_service_port %>/api/1.0/report/run/<% $.runfolder_name %>"
                     method: "POST"
-                    headers: "Content-Type=application/json"
+                    headers:
+                      Content-Type: application/json
                     body: <% $.report_body %>
                 publish:
-                    report_status_url: "<% $.run_sisyphus_quick_report.body.link %>"
+                    report_status_url: <% task(run_sisyphus_quick_report).result.body.link %>
                 on-success:
                     - poll_report_status
 
@@ -176,9 +182,11 @@ workflows:
                 action: core.http
                 input:
                     url: <% $.hermes_base_url %>/<% $.flowcell_name %>/qcfile
-                    headers: USER=<% $.hermes_user %>&X-XSRF-TOKEN=<% $.hermes_token %>
+                    headers:
+                      USER: <% $.hermes_user %>
+                      X-XSRF-TOKEN: <% $.hermes_token %>
                 publish:
-                    qc_conf_string: <% $.download_qc_config.body.qc %>
+                    qc_conf_string: <% task(download_qc_config).result.body.qc %>
                 on-success:
                     - construct_qc_body
 
@@ -188,7 +196,7 @@ workflows:
                     runfolder: "<% $.runfolder_name %>"
                     qc_config: "<% $.qc_conf_string %>"
                 publish:
-                    qc_body: <% $.construct_qc_body.stdout %>
+                    qc_body: <% task(construct_qc_body).result.stdout %>
                 on-success:
                     - run_sisyphus_qc
 
@@ -197,25 +205,21 @@ workflows:
                 input:
                     url: "http://<% $.host %>:<% $.siswrap_service_port %>/api/1.0/qc/run/<% $.runfolder_name %>"
                     method: "POST"
-                    headers: "Content-Type=application/json"
+                    headers:
+                      Content-Type: application/json
                     body: <% $.qc_body %>
                 publish:
-                    qc_status_url: "<% $.run_sisyphus_qc.body.link %>"
+                    qc_status_url: "<% task(run_sisyphus_qc).result.body.link %>"
                 on-success:
                     - poll_qc_status
 
             poll_qc_status:
                 action: arteria-packs.poll_status
                 input:
                     url: <% $.qc_status_url %>
+                    ignore_result: <% $.ignore_sisyphus_qc_result %>
                 on-success:
                     - rsync_to_summary_host
-                on-error:
-                    # When we choose to ignore the sisphus qc rsults, just move on
-                    # otherwise mark as failed and notify.
-                    - rsync_to_summary_host: <% $.ignore_sisyphus_qc_result %>
-                    - mark_as_failed: <% not $.ignore_sisyphus_qc_result %>
-                    - oh_shit_error: <% not $.ignore_sisyphus_qc_result %>
             ### QUALITY CONTROL END ###
 
             ### TRANSFER FILES TO UPPMAX START ###
@@ -236,10 +240,12 @@ workflows:
                action: core.http
                input:
                     url: <% $.hermes_base_url %>/<% $.flowcell_name %>/flowcell/analysishostinfo
-                    headers: USER=<% $.hermes_user %>&X-XSRF-TOKEN=<% $.hermes_token %>
+                    headers: 
+                      USER: <% $.hermes_user %>
+                      X-XSRF-TOKEN: <% $.hermes_token %>
                on-success:
-                    - rsync_to_milou: <% 'milou-b.uppmax.uu.se' in $.check_hosts_to_rsync_to.body.remote_hosts %>
-                    - rsync_to_irma: <% 'irma.uppmax.uu.se' in $.check_hosts_to_rsync_to.body.remote_hosts %>
+                    - rsync_to_milou: <% 'milou-b.uppmax.uu.se' in task(check_hosts_to_rsync_to).result.body.remote_hosts %>
+                    - rsync_to_irma: <% 'irma.uppmax.uu.se' in task(check_hosts_to_rsync_to).result.body.remote_hosts %>
 
             rsync_to_milou:
                 action: arteria-packs.sync_workflow
@@ -329,7 +335,7 @@ workflows:
                 input:
                     to: <% $.send_mail_to %>
                     subject: "'[ARTERIA] - Finished processing runfolder <% $.runfolder_name %>'"
-                    body: "Finished processing on <% $.host %> for <% $.runfolder_name %>: <% $.demultiplex_message %> <% $.report_message %> <% $.qc_message %>"
+                    body: "Finished processing on <% $.host %> for <% $.runfolder_name %>"
 
             oh_shit_error:
                 action: core.sendmail