Permalink
Browse files

Change from automatic to explicit template declarations

Eliminate support for splitting scripts by line
  • Loading branch information...
1 parent b0e3b1a commit 5877547b1c430e09671ce107a57402eb01cb7367 @jaydoane jaydoane committed Aug 11, 2011
Showing with 157 additions and 37 deletions.
  1. +21 −4 README.rst
  2. +9 −1 provision/config.py
  3. +25 −22 provision/nodelib.py
  4. +5 −10 test/test_deploy.py
  5. +97 −0 test/test_script_templates.py
View
@@ -174,8 +174,27 @@ memory, and are mapped into bundles in __init__.py, which can then be
specified in the command line using -b bundle-name, or added to
DEFAULT_BUNDLES, to get installed for every deploy.
-The __init__.py can also be used to override default settings in the
-provision.config module, which gets passed into init() as a parameter.
+It is sometimes useful to be able to substitute variables into scripts
+at runtime. This can be done by using the --subvars command line
+option with script templating.
+
+Embed one of the following lines in a script to activate variable
+substitution::
+
+ # provision-template-type: format-string
+ or
+ # provision-template-type: template-string
+
+See `format string documentation
+<http://docs.python.org/library/string.html#format-string-syntax>`_
+and `template strings documentation
+<http://docs.python.org/library/string.html#template-strings>`_ for
+the respective syntaxes. Also see test cases in
+test_script_templates.py.
+
+The __init__.py file can also be used to override default settings in
+the provision.config module, which gets passed into init() as a
+parameter.
This is an example of an __init__.py file::
@@ -206,5 +225,3 @@ Default Configuration Directory Locations
When provision.config is first imported, it will try to load
configuration directory in ~/.provision/secrets. If it cannot locate
one, it will then try $VIRTUAL_ENV/provision_secrets.
-
-
View
@@ -60,7 +60,15 @@
# Set to None or '' to ignore metadata
NODE_METADATA_CONTAINER_NAME = 'node_meta'
-SPLIT_RE = re.compile('split-lines:\W*true', re.IGNORECASE)
+#SPLIT_RE = re.compile('split-lines:\W*true', re.IGNORECASE)
+TEMPLATE_RE = re.compile('#.+provision-template-type:\W*(?P<type>[\w-]+)')
+
+TEMPLATE_TYPEMAP = {
+ # http://docs.python.org/library/string.html#format-string-syntax
+ 'format-string': lambda text, submap: text.format(**submap),
+ # http://docs.python.org/library/string.html#template-strings
+ 'template-string': lambda text, submap: string.Template(text).safe_substitute(submap),
+ }
CODEPATH = os.path.dirname(__file__)
View
@@ -7,6 +7,7 @@
import json
import os
import re
+import string
import libcloud.providers
import libcloud.deployment
@@ -32,10 +33,6 @@ def list_nodes(driver):
return driver.list_nodes()
-def flatten(lst):
- return list(itertools.chain(*lst))
-
-
class NodeProxy(object):
"""Wrap a libcloud.base.Node object and adds some functionality"""
@@ -88,25 +85,32 @@ def sum_exit_status(self):
return sum([sd.exit_status for sd in self.node.script_deployments])
-def named_script_deployments(path, script, submap=None):
+def substitute(script, submap):
+
+ """Check for presence of template indicator and if found, perform
+ variable substition on script based on template type, returning
+ script."""
+
+ match = config.TEMPLATE_RE.search(script)
+ if match:
+ template_type = match.groupdict()['type']
+ try:
+ return config.TEMPLATE_TYPEMAP[template_type](script, submap)
+ except KeyError:
+ logger.error('Unsupported template type: %s' % template_type)
+ raise
+ return script
- """Perform variable substition on script, possibly breaking into
- separate scripts per line, and return a list of ScriptDeployments."""
+
+def script_deployment(path, script, submap=None):
+
+ """Return a ScriptDeployment from script with possible template
+ substitutions."""
if submap is None:
submap = {}
-
- if config.SPLIT_RE.search(script):
- lines = script.split('\n')
- deployments = []
- base, ext = os.path.splitext(path)
- for number, line in enumerate(lines):
- linepath = '%s_%02d%s' % (base, number, ext) #TODO: smarter or configurable
- deployments.append(
- libcloud.deployment.ScriptDeployment(line.format(**submap), linepath))
- return deployments
- else:
- return [libcloud.deployment.ScriptDeployment(script.format(**submap), path)]
+ script = substitute(script, submap)
+ return libcloud.deployment.ScriptDeployment(script, path)
def merge_load(items, amap):
@@ -184,9 +188,8 @@ def __init__(self, name=None, bundles=[], pubkey=config.DEFAULT_PUBKEY,
logger.debug('files {0}'.format(self.filemap.keys()))
logger.debug('scripts {0}'.format(scriptmap.keys()))
- self.script_deployments = flatten(
- [named_script_deployments(path, script, config.SUBMAP)
- for path, script in scriptmap.items()])
+ self.script_deployments = [script_deployment(path, script, config.SUBMAP)
+ for path, script in scriptmap.items()]
logger.debug('len(script_deployments) = {0}'.format(len(self.script_deployments)))
steps = [libcloud.deployment.SSHKeyDeployment(''.join(self.pubkeys))]
View
@@ -6,20 +6,15 @@
import provision.nodelib as nodelib
class TestDeploy(unittest.TestCase):
+
def test_random_str(self):
assert len(config.random_str()) == 6
- def test_flatten(self):
- assert [1,2,3,4] == nodelib.flatten([[1],[2,3],[4]])
+
def test_unsplit_named_script_deployments(self):
- names = [s.name for s in
- nodelib.named_script_deployments('foo.sh', 'l1\nl2\nl3\n')]
- assert ['foo.sh'] == names
- def test_split_named_script_deployments(self):
- names = [s.name for s in nodelib.named_script_deployments(
- 'foo.sh', '# -*- split-lines: true; -*-\ns2\ns3')]
- assert ['foo_00.sh', 'foo_01.sh', 'foo_02.sh'] == names
+ assert 'foo.sh' == \
+ nodelib.script_deployment('foo.sh', 'l1\nl2\nl3\n').name
+
def test_node_deployment(self):
nd = nodelib.Deployment(bundles=['mta'])
assert nd.name.startswith(config.DEFAULT_NAME_PREFIX)
assert libcloud.deployment.SSHKeyDeployment == type(nd.deployment.steps[0])
-
@@ -0,0 +1,97 @@
+import unittest
+
+import provision.config as config
+import provision.nodelib as nodelib
+
+class Test(unittest.TestCase):
+
+ def setUp(self):
+
+ self.submap = {'ci_user_host': 'testuser@testhost',}
+
+ self.format_string_script = '''# provision-template-type: format-string
+scp -pr -o StrictHostKeyChecking=no {ci_user_host}:pkg/libcloud .
+'''
+ self.template_string_script = '''# provision-template-type: template-string
+scp -pr -o StrictHostKeyChecking=no ${ci_user_host}:pkg/libcloud .
+'''
+ self.template_string_script_simple = ''' # provision-template-type: template-string
+scp -pr -o StrictHostKeyChecking=no $ci_user_host:pkg/libcloud .'''
+
+ self.template_script_unsupported_type = '''# provision-template-type: mako
+scp -pr -o StrictHostKeyChecking=no ${ci_user_host}:pkg/libcloud .
+'''
+ self.non_template_script = '''apt-get -y install python-setuptools python-dev gcc
+'''
+ # don't do this
+ self.multiple_specifiers_one_line = '''
+# provision-template-type: format-string provision-template-type: template-string
+apt-get -y install python-setuptools python-dev gcc
+'''
+ # or this
+ self.multiple_specifiers_multiple_lines = '''
+# provision-template-type: format-string
+# provision-template-type: template-string
+apt-get -y install python-setuptools python-dev gcc
+'''
+ self.bash_function = '''# provision-template-type: template-string
+download_file() {
+ echo wget -q -O $DOWNLOAD_DIR/$1 $2
+ wget -q -O $DOWNLOAD_DIR/$1 $2
+ http_rc=$?
+ if [[ "$http_rc" != "0" ]]; then
+ echo "Download of $1 failed, return code was $http_rc"
+ exit 1
+ fi
+ if [ ! -f $DOWNLOAD_DIR/$1 ]; then
+ echo "Downloaded file $1 not found"
+ exit 1
+ fi
+}
+'''
+ self.awk_function = '''# provision-template-type: template-string
+eth1=`ifconfig eth1 | grep -o -E 'inet addr:[0-9.]+' | awk -F: '{print $2}'`'''
+
+
+ def test_format_strings_detection(self):
+ assert 'format-string' == \
+ config.TEMPLATE_RE.search(self.format_string_script).groupdict()['type']
+
+ def test_template_strings_detection(self):
+ assert 'template-string' == \
+ config.TEMPLATE_RE.search(self.template_string_script).groupdict()['type']
+
+ def test_multiple_specifiers_one_line(self):
+ assert 'template-string' == \
+ config.TEMPLATE_RE.search(self.multiple_specifiers_one_line).groupdict()['type']
+
+ def test_multiple_specifiers_multiple_lines(self):
+ assert 'format-string' == \
+ config.TEMPLATE_RE.search(self.multiple_specifiers_multiple_lines).groupdict()['type']
+
+ def test_bash_function_unchanged(self):
+ assert self.bash_function == nodelib.substitute(
+ self.bash_function, self.submap)
+
+ def test_awk_function_unchanged(self):
+ assert self.awk_function == nodelib.substitute(
+ self.awk_function, self.submap)
+
+ def test_format_string_substitution(self):
+ assert nodelib.substitute(self.format_string_script, self.submap).find(
+ self.submap['ci_user_host']) > 0
+
+ def test_template_string_substitution(self):
+ assert nodelib.substitute(self.template_string_script, self.submap).find(
+ self.submap['ci_user_host']) > 0
+
+ def test_template_string_simple_substitution(self):
+ assert nodelib.substitute(self.template_string_script_simple, self.submap).find(
+ self.submap['ci_user_host']) > 0
+
+ def test_template_script_unsupported_type(self):
+ try:
+ nodelib.substitute(self.template_script_unsupported_type, self.submap)
+ self.fail()
+ except KeyError:
+ pass

0 comments on commit 5877547

Please sign in to comment.