Add extra_vars parsing directives due to Jinja

CHANGES.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## v0.6.0
+
+* Add extra_vars parsing directives to get around difficulties with Jinja (which casts all
+  extra_vars as strings). When passing in an object via Jinja, all values become strings. To get
+  around this, add "!AST", "!JSON", or "!INT" directives in your action-chain yaml:
+
+```yaml
+chain:
+  name: 'example'
+  ref: 'ansible.command_local'
+  extra_vars:
+    -
+      keyA: "!AST{{ jinja_variable_a }}"
+      keyB: "!JSON{{ jinja_variable_b | tojson }}"
+      keyC: "!INT{{ jinja_variable_c | int }}"
+```
+
 ## v0.5.2
 
 * Added pywinrm to requirements so connection to windows hosts is possible.

README.md

@@ -83,6 +83,53 @@ sample_task:
           key8: value8
 ```
 
+##### Structured input and Jinja
+Note that `value3`, in the previous example, was passed in as a variable in a Jinja expression.
+StackStorm normally casts variables to the types like int, array, and object.
+However, it can't do that for extra_vars because we cannot know ahead of time,
+for all Ansible playbooks, what schema and data types each entry will be. As such,
+within extra_vars, all Jinja expressions result in strings. To get around this, we've
+added some extra_vars parsing diretives: `!INT`, `!JSON`, and `!AST`.
+
+Use `!INT` if you are using a playbook that expects a value to be an integer:
+
+```yaml
+extra_vars:
+  port: '!INT{{ port_number }}'
+```
+
+Use `!JSON` when you have something that is already a JSON string or when it makes sense
+to convert it into a json string using Jinja filters:
+
+```yaml
+extra_vars:
+  special_config: '!JSON{{ config | tojson }}'
+```
+
+Use `!AST` (referring to python AST) when you have an object that you want to send straight over
+without any additional jinja filters:
+
+```yaml
+extra_vars:
+  data: '!AST{{ data }}'
+```
+
+These directives are supported recursively. So, you can embed directives in other directives.
+Consider this contrived example (though mostly you'll only hit embedding directives or objects
+if you're chaining workflows together to build up a data object):
+
+```yaml
+vars:
+  answer: "!INT{{ life_universe_everything }}"
+chain:
+  -
+    name: '...'
+    ref: 'ansible.playbook'
+    ...
+    extra_vars:
+      earth: '!JSON{"question": "unknown", "answer": {{ answer }} }'
+```
+
 ##### Structured output
 ```sh
 # get structured JSON output from a playbook

actions/lib/ansible_base.py

@@ -1,10 +1,10 @@
-import os
-import sys
-import subprocess
-import shell
 import ast
 import json
+import os
+import shell
 import six
+import subprocess
+import sys
 
 __all__ = [
     'AnsibleBaseRunner'
@@ -27,6 +27,24 @@ def __init__(self, args):
         self._parse_extra_vars()  # handle multiple entries in --extra_vars arg
         self._prepend_venv_path()
 
+    def _parse_extra_vars_directives(self, value):
+        if isinstance(value, six.string_types):
+            if value.strip().startswith("!AST"):
+                a = ast.literal_eval(value.strip()[4:])
+                return self._parse_extra_vars_directives(a)
+            elif value.strip().startswith("!JSON"):
+                j = json.loads(value.strip()[5:])
+                return self._parse_extra_vars_directives(j)
+            elif value.strip().startswith("!INT"):
+                return int(value.strip()[4:])
+            else:
+                return value
+        elif isinstance(value, dict):
+            return {k: self._parse_extra_vars_directives(v) for k, v in six.iteritems(value)}
+        elif isinstance(value, list):
+            return [self._parse_extra_vars_directives(item) for item in value]
+        return value
+
     def _parse_extra_vars(self):
         """
         This method turns the string list ("--extra_vars=[...]") passed in from the args
@@ -50,10 +68,14 @@ def _parse_extra_vars(self):
                     if isinstance(n, six.string_types):
                         if n.strip().startswith("@"):
                             var_list.append(('file', n.strip()))
+                        elif (n.strip().startswith("!AST") or
+                              n.strip().startswith("!JSON") or
+                              '=' not in n):
+                            var_list.append(('json', self._parse_extra_vars_directives(n)))
                         else:
                             var_list.append(('kwarg', n.strip()))
                     elif isinstance(n, dict):
-                        var_list.append(('json', n))
+                        var_list.append(('json', self._parse_extra_vars_directives(n)))
 
                 last = ''
                 kv_param = ''

pack.yaml

@@ -6,6 +6,6 @@ keywords:
   - ansible
   - cfg management
   - configuration management
-version : 0.5.2
+version : 0.6.0
 author : StackStorm, Inc.
 email : info@stackstorm.com

tests/fixtures/extra_vars_complex.yaml

@@ -25,4 +25,70 @@
       key6:
         key7: 'value7'
     - 'key8=value8'
+-
+  name: ast_directive
+  test:
+    - "!AST{u'key1': u'12345', u'key2': u'value2'}"
+  expected:
+    -
+      key1: '12345'
+      key2: 'value2'
+-
+  name: json_directive
+  test:
+    - '!JSON{"key1": "12345", "key2": "value2"}'
+  expected:
+    -
+      key1: '12345'
+      key2: 'value2'
+-
+  name: sub_ast_directive
+  test:
+    -
+      astkey: "!AST[{u'key1': u'12345', u'key2': u'value2'}]"
+  expected:
+    -
+      astkey:
+        -
+          key1: '12345'
+          key2: 'value2'
+-
+  name: sub_json_directive
+  test:
+    -
+      jsonkey: '!JSON[{"key1": "12345", "key2": "value2"}]'
+  expected:
+    -
+      jsonkey:
+        -
+          key1: '12345'
+          key2: 'value2'
+-
+  name: int_directive
+  test:
+    -
+      key1: "!INT12345"
+  expected:
+    -
+      key1: 12345
+-
+  name: multi_directives_ast_int
+  test:
+    -
+      astkey: "!AST[{u'intkey': u'!INT12345'}]"
+  expected:
+    -
+      astkey:
+        -
+          intkey: 12345
+-
+  name: multi_directives_json_int
+  test:
+    -
+      jsonkey: '!JSON[{"intkey": "!INT12345"}]'
+  expected:
+    -
+      jsonkey:
+        -
+          intkey: 12345
 

tests/test_actions_lib_ansiblebaserunner.py

@@ -121,3 +121,24 @@ def extra_vars_complex_yaml_fixture(self, test_name):
 
     def test_parse_extra_vars_complex_yaml_arbitrarily_complex(self):
         self.extra_vars_complex_yaml_fixture('arbitrarily_complex')
+
+    def test_parse_extra_vars_complex_yaml_ast_directive(self):
+        self.extra_vars_complex_yaml_fixture('ast_directive')
+
+    def test_parse_extra_vars_complex_yaml_json_directive(self):
+        self.extra_vars_complex_yaml_fixture('json_directive')
+
+    def test_parse_extra_vars_complex_yaml_sub_ast_directive(self):
+        self.extra_vars_complex_yaml_fixture('sub_ast_directive')
+
+    def test_parse_extra_vars_complex_yaml_sub_json_directive(self):
+        self.extra_vars_complex_yaml_fixture('sub_json_directive')
+
+    def test_parse_extra_vars_complex_yaml_int_directive(self):
+        self.extra_vars_complex_yaml_fixture('int_directive')
+
+    def test_parse_extra_vars_complex_yaml_multi_directives_ast_int(self):
+        self.extra_vars_complex_yaml_fixture('multi_directives_ast_int')
+
+    def test_parse_extra_vars_complex_yaml_multi_directives_json_int(self):
+        self.extra_vars_complex_yaml_fixture('multi_directives_json_int')