Merge pull request #2824 from StackStorm/pluggable-runners

Adds pluggable runner support for ST2.

Makefile

@@ -7,6 +7,7 @@ BINARIES := bin
 
 # All components are prefixed by st2
 COMPONENTS := $(wildcard st2*)
+COMPONENTS += $(wildcard contrib/runners/*)
 
 # Components that implement a component-controlled test-runner. These components provide an
 # in-component Makefile. (Temporary fix until I can generalize the pecan unittest setup. -mar)
@@ -150,7 +151,7 @@ compile:
 .st2common-circular-dependencies-check:
 	@echo "Checking st2common for circular dependencies"
 	find ${ROOT_DIR}/st2common/st2common/ -name \*.py -type f -print0 | xargs -0 cat | grep st2reactor ; test $$? -eq 1
-	find ${ROOT_DIR}/st2common/st2common/ \( -name \*.py ! -name runnersregistrar\.py \) -type f -print0 | xargs -0 cat | grep st2actions ; test $$? -eq 1
+	find ${ROOT_DIR}/st2common/st2common/ \( -name \*.py ! -name runnersregistrar\.py ! -wholename "*runners/__init__.py" ! -name python_action_wrapper.py ! -wholename "*/query/base.py" \) -type f -print0 | xargs -0 cat | grep st2actions ; test $$? -eq 1
 	find ${ROOT_DIR}/st2common/st2common/ -name \*.py -type f -print0 | xargs -0 cat | grep st2api ; test $$? -eq 1
 	find ${ROOT_DIR}/st2common/st2common/ -name \*.py -type f -print0 | xargs -0 cat | grep st2auth ; test $$? -eq 1
 	find ${ROOT_DIR}/st2common/st2common/ -name \*.py -type f -print0 | xargs -0 cat | grep st2debug; test $$? -eq 1

contrib/chatops/actions/format_execution_result.py

@@ -1,7 +1,7 @@
 import six
 import os
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 from st2client.client import Client
 from st2common.util import jinja as jinja_utils
 

contrib/chatops/tests/fixtures/http_execution.json

@@ -44,7 +44,7 @@
         "enabled": true,
         "id": "54c6bb640640fd5211edef0b",
         "name": "http-request",
-        "runner_module": "st2actions.runners.httprunner",
+        "runner_module": "httprunner",
         "runner_parameters": {
             "url": {
                 "type": "string"

contrib/chatops/tests/fixtures/local_cmd_execution.json

@@ -37,7 +37,7 @@
         "enabled": true,
         "id": "54c6bb640640fd5211edef0b",
         "name": "local-shell-cmd",
-        "runner_module": "st2actions.runners.localrunner",
+        "runner_module": "local_runner",
         "runner_parameters": {
             "cmd": {
                 "type": "string"

contrib/chatops/tests/fixtures/python_action_execution.json

@@ -36,7 +36,7 @@
         "enabled": true,
         "id": "54c6bb640640fd5211edef0b",
         "name": "python-script",
-        "runner_module": "st2actions.runners.pythonrunner",
+        "runner_module": "pythonrunner",
         "runner_parameters": {
 
         }

contrib/chatops/tests/fixtures/remote_cmd_execution.json

@@ -53,7 +53,7 @@
         "enabled": true,
         "id": "54c6bb640640fd5211edef0b",
         "name": "remote-shell-cmd",
-        "runner_module": "st2actions.runners.remote_command_runner",
+        "runner_module": "remote_command_runner",
         "runner_parameters": {
             "cmd": {
                 "type": "string"

contrib/examples/actions/print_config.py

@@ -1,6 +1,6 @@
 from pprint import pprint
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 
 
 class PrintConfigAction(Action):

contrib/examples/actions/pythonactions/isprime.py

@@ -1,6 +1,6 @@
 import math
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 
 
 class PrimeCheckerAction(Action):

contrib/examples/actions/pythonactions/json_string_to_object.py

@@ -1,6 +1,6 @@
 import json
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 
 
 class JsonStringToObject(Action):

contrib/examples/actions/pythonactions/object_return.py

@@ -1,4 +1,4 @@
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 
 
 class ObjectReturnAction(Action):

contrib/examples/actions/pythonactions/yaml_string_to_object.py

@@ -1,6 +1,6 @@
 import yaml
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 
 
 class YamlStringToObject(Action):

contrib/linux/actions/dig.py

@@ -3,7 +3,7 @@
 import subprocess
 import random
 import re
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 
 
 class DigAction(Action):

contrib/linux/actions/wait_for_ssh.py

@@ -4,8 +4,8 @@
 
 from oslo_config import cfg
 
-from st2actions.runners.pythonrunner import Action
-from st2actions.runners.ssh.paramiko_ssh import ParamikoSSHClient
+from st2common.runners.base_action import Action
+from st2common.runners.paramiko_ssh import ParamikoSSHClient
 
 
 class BaseAction(Action):

contrib/packs/actions/check_auto_deploy_repo.py

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 
 
 class CheckAutoDeployRepo(Action):

contrib/packs/actions/expand_repo_name.py

@@ -15,7 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 
 
 class ExpandRepoName(Action):

contrib/packs/actions/pack_mgmt/delete.py

@@ -18,7 +18,7 @@
 
 from oslo_config import cfg
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 from st2common.constants.pack import SYSTEM_PACK_NAMES
 from st2common.util.shell import quote_unix
 

contrib/packs/actions/pack_mgmt/download.py

@@ -23,7 +23,7 @@
 from git.repo import Repo
 from lockfile import LockFile
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 from st2common.util.green import shell
 
 ALL_PACKS = '*'

contrib/packs/actions/pack_mgmt/info.py

@@ -16,7 +16,7 @@
 import os
 import json
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 from st2common.content.utils import get_packs_base_paths
 
 GITINFO_FILE = '.gitinfo'

contrib/packs/actions/pack_mgmt/setup_virtualenv.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 from st2common.util.virtualenvs import setup_pack_virtualenv
 
 __all__ = [

contrib/packs/actions/pack_mgmt/unload.py

@@ -16,7 +16,7 @@
 from oslo_config import cfg
 
 from st2common.models.db import db_setup
-from st2actions.runners.pythonrunner import Action as BaseAction
+from st2common.runners.base_action import Action as BaseAction
 from st2common.persistence.pack import Pack
 from st2common.persistence.pack import ConfigSchema
 from st2common.persistence.reactor import SensorType

contrib/packs/actions/pack_mgmt/virtualenv_setup_prerun.py

@@ -15,7 +15,7 @@
 
 import six
 
-from st2actions.runners.pythonrunner import Action
+from st2common.runners.base_action import Action
 
 
 class PacksTransformationAction(Action):

st2actions/st2actions/runners/actionchainrunner.py → contrib/runners/action_chain_runner/action_chain_runner.py

@@ -20,7 +20,7 @@
 
 from jsonschema import exceptions as json_schema_exceptions
 
-from st2actions.runners import ActionRunner
+from st2common.runners import ActionRunner
 from st2common import log as logging
 from st2common.constants.action import ACTION_CONTEXT_KV_PREFIX
 from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED

contrib/runners/action_chain_runner/runner.yaml

@@ -0,0 +1,14 @@
+- aliases: []
+  description: A runner for launching linear action chains.
+  enabled: true
+  name: action-chain
+  runner_module: action_chain_runner
+  runner_parameters:
+    display_published:
+      default: false
+      description: Intermediate published variables will be stored and displayed.
+      type: boolean
+    skip_notify:
+      default: []
+      description: List of tasks to skip notifications for.
+      type: array

st2actions/tests/unit/test_actionchain.py → contrib/runners/action_chain_runner/tests/unit/test_actionchain.py

@@ -14,8 +14,7 @@
 # limitations under the License.
 
 import mock
-
-from st2actions.runners import actionchainrunner as acr
+import action_chain_runner as acr
 from st2actions.container.service import RunnerContainerService
 from st2common.constants.action import LIVEACTION_STATUS_RUNNING
 from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED

st2actions/tests/unit/test_actionchain_notifications.py → contrib/runners/action_chain_runner/tests/unit/test_actionchain_notifications.py

@@ -15,13 +15,13 @@
 
 import mock
 
-from st2actions.runners import actionchainrunner as acr
 from st2actions.container.service import RunnerContainerService
 from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED
 from st2common.services import action as action_service
 from st2common.util import action_db as action_db_util
 from st2tests import DbTestCase
 from st2tests.fixturesloader import FixturesLoader
+import action_chain_runner as acr
 
 
 class DummyActionExecution(object):

st2actions/tests/unit/test_actionchain_params_rendering.py → contrib/runners/action_chain_runner/tests/unit/test_actionchain_params_rendering.py

@@ -15,7 +15,7 @@
 
 import unittest2
 
-from st2actions.runners import actionchainrunner as acr
+import action_chain_runner as acr
 from st2common.exceptions.action import ParameterRenderingFailedException
 from st2common.models.system.actionchain import Node
 

st2actions/st2actions/runners/announcementrunner.py → contrib/runners/announcement_runner/announcement_runner.py

@@ -15,7 +15,7 @@
 
 import uuid
 
-from st2actions.runners import ActionRunner
+from st2common.runners import ActionRunner
 from st2common import log as logging
 from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED
 from st2common.exceptions import actionrunner as runnerexceptions

contrib/runners/announcement_runner/runner.yaml

@@ -0,0 +1,18 @@
+- aliases: []
+  description: A runner for emitting an announcement event on the stream.
+  enabled: true
+  name: announcement
+  runner_module: announcement_runner
+  runner_parameters:
+    experimental:
+      default: false
+      description: Flag to indicate acknowledment of using experimental runner
+      required: true
+      type: boolean
+    route:
+      default: general
+      description: The routing_key used to route the message to consumers. Might be
+        a list of words, delimited by dots.
+      maxLength: 255
+      minLength: 1
+      type: string

st2actions/tests/unit/test_announcementrunner.py → contrib/runners/announcement_runner/tests/test_announcementrunner.py

@@ -15,11 +15,11 @@
 
 import mock
 
-from st2actions.runners import announcementrunner
 from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED
 from st2common.models.api.trace import TraceContext
 from base import RunnerTestCase
 import st2tests.config as tests_config
+import announcementrunner
 
 
 mock_dispatcher = mock.Mock()

st2actions/st2actions/runners/cloudslang/cloudslang_runner.py → contrib/runners/cloudslang_runner/cloudslang_runner.py

@@ -25,7 +25,7 @@
 from st2common.util.shell import kill_process
 from st2common.util.shell import quote_unix
 from st2common import log as logging
-from st2actions.runners import ActionRunner
+from st2common.runners import ActionRunner
 from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED
 from st2common.constants.action import LIVEACTION_STATUS_FAILED
 from st2common.constants.runners import LOCAL_RUNNER_DEFAULT_ACTION_TIMEOUT

contrib/runners/cloudslang_runner/runner.yaml

@@ -0,0 +1,16 @@
+- aliases: []
+  description: A runner to execute cloudslang flows.
+  enabled: true
+  name: cloudslang
+  runner_module: cloudslang_runner
+  runner_parameters:
+    inputs:
+      default: {}
+      description: Inputs which will be available to CLoudSlang flow execution(e.g.
+        input1=val1,input2=val2)
+      type: object
+    timeout:
+      default: 60
+      description: Action timeout in seconds. Action will get killed if it doesn't
+        finish in timeout seconds.
+      type: integer

st2actions/tests/unit/test_cloudslang.py → contrib/runners/cloudslang_runner/tests/test_cloudslang.py

@@ -18,9 +18,9 @@
 
 from unittest2 import TestCase
 
-import st2actions.runners.cloudslang.cloudslang_runner as csr
 from st2common.constants.action import LIVEACTION_STATUS_SUCCEEDED
 from st2common.constants.action import LIVEACTION_STATUS_FAILED
+from cloudslang import cloudslang_runner as csr
 
 import st2tests.config as tests_config
 tests_config.parse_args()
@@ -49,8 +49,8 @@ def test_pre_run_sets_attributes(self):
         self.assertEqual(runner._inputs, inputs)
         self.assertEqual(runner._timeout, timeout)
 
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.quote_unix')
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.run_command')
+    @mock.patch('cloudslang.cloudslang_runner.quote_unix')
+    @mock.patch('cloudslang.cloudslang_runner.run_command')
     def test_run_calls_a_new_process_success(self, mock_run_command, mock_quote_unix):
         entry_point = 'path'
         timeout = 1
@@ -69,8 +69,8 @@ def test_run_calls_a_new_process_success(self, mock_run_command, mock_quote_unix
         self.assertTrue(mock_run_command.called)
         self.assertEqual(LIVEACTION_STATUS_SUCCEEDED, result[0])
 
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.quote_unix')
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.run_command')
+    @mock.patch('cloudslang.cloudslang_runner.quote_unix')
+    @mock.patch('cloudslang.cloudslang_runner.run_command')
     def test_run_calls_a_new_process_failure(self, mock_run_command, mock_quote_unix):
         timeout = 1
         runner = csr.get_runner()
@@ -86,7 +86,7 @@ def test_run_calls_a_new_process_failure(self, mock_run_command, mock_quote_unix
         self.assertTrue(mock_run_command.called)
         self.assertEqual(LIVEACTION_STATUS_FAILED, result[0])
 
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.run_command')
+    @mock.patch('cloudslang.cloudslang_runner.run_command')
     def test_run_calls_a_new_process_timeout(self, mock_run_command):
         entry_point = 'path'
         timeout = 1
@@ -102,8 +102,8 @@ def test_run_calls_a_new_process_timeout(self, mock_run_command):
         self.assertTrue(mock_run_command.called)
         self.assertEqual(LIVEACTION_STATUS_FAILED, result[0])
 
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.run_command')
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.yaml.safe_dump')
+    @mock.patch('cloudslang.cloudslang_runner.run_command')
+    @mock.patch('cloudslang.cloudslang_runner.yaml.safe_dump')
     def test_inputs_are_save_to_file_properly(self, mock_yaml_dump, mock_run_command):
         entry_point = 'path'
         inputs = {'a': 1}
@@ -122,8 +122,8 @@ def test_inputs_are_save_to_file_properly(self, mock_yaml_dump, mock_run_command
         mock_yaml_dump.assert_called_with(inputs, default_flow_style=False)
         self.assertEqual(LIVEACTION_STATUS_FAILED, result[0])
 
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.run_command')
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.os.remove')
+    @mock.patch('cloudslang.cloudslang_runner.run_command')
+    @mock.patch('cloudslang.cloudslang_runner.os.remove')
     def test_temp_file_deletes_when_exception_occurs(self, mock_os_remove, mock_run_command):
         entry_point = 'path'
         inputs = {'a': 1}
@@ -144,7 +144,7 @@ def test_temp_file_deletes_when_exception_occurs(self, mock_os_remove, mock_run_
         # lets really remove it now
         os.remove(mock_os_remove.call_args[0][0])
 
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.run_command')
+    @mock.patch('cloudslang.cloudslang_runner.run_command')
     def test_inputs_provided_via_inputs_runner_parameter(self, mock_run_command):
         entry_point = 'path'
         inputs = {'a': 1}
@@ -164,7 +164,7 @@ def test_inputs_provided_via_inputs_runner_parameter(self, mock_run_command):
         runner.run({})
         runner._write_inputs_to_a_temp_file.assert_called_with(inputs=inputs)
 
-    @mock.patch('st2actions.runners.cloudslang.cloudslang_runner.run_command')
+    @mock.patch('cloudslang.cloudslang_runner.run_command')
     def test_inputs_provided_via_action_parameters(self, mock_run_command):
         entry_point = 'path'
         inputs = None