Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Change from automatic to explicit template declarations

Eliminate support for splitting scripts by line
  • Loading branch information...
commit 5877547b1c430e09671ce107a57402eb01cb7367 1 parent b0e3b1a
Jay Doane jaydoane authored
25 README.rst
View
@@ -174,8 +174,27 @@ memory, and are mapped into bundles in __init__.py, which can then be
specified in the command line using -b bundle-name, or added to
DEFAULT_BUNDLES, to get installed for every deploy.
-The __init__.py can also be used to override default settings in the
-provision.config module, which gets passed into init() as a parameter.
+It is sometimes useful to be able to substitute variables into scripts
+at runtime. This can be done by using the --subvars command line
+option with script templating.
+
+Embed one of the following lines in a script to activate variable
+substitution::
+
+ # provision-template-type: format-string
+ or
+ # provision-template-type: template-string
+
+See `format string documentation
+<http://docs.python.org/library/string.html#format-string-syntax>`_
+and `template strings documentation
+<http://docs.python.org/library/string.html#template-strings>`_ for
+the respective syntaxes. Also see test cases in
+test_script_templates.py.
+
+The __init__.py file can also be used to override default settings in
+the provision.config module, which gets passed into init() as a
+parameter.
This is an example of an __init__.py file::
@@ -206,5 +225,3 @@ Default Configuration Directory Locations
When provision.config is first imported, it will try to load
configuration directory in ~/.provision/secrets. If it cannot locate
one, it will then try $VIRTUAL_ENV/provision_secrets.
-
-
10 provision/config.py
View
@@ -60,7 +60,15 @@
# Set to None or '' to ignore metadata
NODE_METADATA_CONTAINER_NAME = 'node_meta'
-SPLIT_RE = re.compile('split-lines:\W*true', re.IGNORECASE)
+#SPLIT_RE = re.compile('split-lines:\W*true', re.IGNORECASE)
+TEMPLATE_RE = re.compile('#.+provision-template-type:\W*(?P<type>[\w-]+)')
+
+TEMPLATE_TYPEMAP = {
+ # http://docs.python.org/library/string.html#format-string-syntax
+ 'format-string': lambda text, submap: text.format(**submap),
+ # http://docs.python.org/library/string.html#template-strings
+ 'template-string': lambda text, submap: string.Template(text).safe_substitute(submap),
+ }
CODEPATH = os.path.dirname(__file__)
47 provision/nodelib.py
View
@@ -7,6 +7,7 @@
import json
import os
import re
+import string
import libcloud.providers
import libcloud.deployment
@@ -32,10 +33,6 @@ def list_nodes(driver):
return driver.list_nodes()
-def flatten(lst):
- return list(itertools.chain(*lst))
-
-
class NodeProxy(object):
"""Wrap a libcloud.base.Node object and adds some functionality"""
@@ -88,25 +85,32 @@ def sum_exit_status(self):
return sum([sd.exit_status for sd in self.node.script_deployments])
-def named_script_deployments(path, script, submap=None):
+def substitute(script, submap):
+
+ """Check for presence of template indicator and if found, perform
+ variable substition on script based on template type, returning
+ script."""
+
+ match = config.TEMPLATE_RE.search(script)
+ if match:
+ template_type = match.groupdict()['type']
+ try:
+ return config.TEMPLATE_TYPEMAP[template_type](script, submap)
+ except KeyError:
+ logger.error('Unsupported template type: %s' % template_type)
+ raise
+ return script
- """Perform variable substition on script, possibly breaking into
- separate scripts per line, and return a list of ScriptDeployments."""
+
+def script_deployment(path, script, submap=None):
+
+ """Return a ScriptDeployment from script with possible template
+ substitutions."""
if submap is None:
submap = {}
-
- if config.SPLIT_RE.search(script):
- lines = script.split('\n')
- deployments = []
- base, ext = os.path.splitext(path)
- for number, line in enumerate(lines):
- linepath = '%s_%02d%s' % (base, number, ext) #TODO: smarter or configurable
- deployments.append(
- libcloud.deployment.ScriptDeployment(line.format(**submap), linepath))
- return deployments
- else:
- return [libcloud.deployment.ScriptDeployment(script.format(**submap), path)]
+ script = substitute(script, submap)
+ return libcloud.deployment.ScriptDeployment(script, path)
def merge_load(items, amap):
@@ -184,9 +188,8 @@ def __init__(self, name=None, bundles=[], pubkey=config.DEFAULT_PUBKEY,
logger.debug('files {0}'.format(self.filemap.keys()))
logger.debug('scripts {0}'.format(scriptmap.keys()))
- self.script_deployments = flatten(
- [named_script_deployments(path, script, config.SUBMAP)
- for path, script in scriptmap.items()])
+ self.script_deployments = [script_deployment(path, script, config.SUBMAP)
+ for path, script in scriptmap.items()]
logger.debug('len(script_deployments) = {0}'.format(len(self.script_deployments)))
steps = [libcloud.deployment.SSHKeyDeployment(''.join(self.pubkeys))]
15 test/test_deploy.py
View
@@ -6,20 +6,15 @@
import provision.nodelib as nodelib
class TestDeploy(unittest.TestCase):
+
def test_random_str(self):
assert len(config.random_str()) == 6
- def test_flatten(self):
- assert [1,2,3,4] == nodelib.flatten([[1],[2,3],[4]])
+
def test_unsplit_named_script_deployments(self):
- names = [s.name for s in
- nodelib.named_script_deployments('foo.sh', 'l1\nl2\nl3\n')]
- assert ['foo.sh'] == names
- def test_split_named_script_deployments(self):
- names = [s.name for s in nodelib.named_script_deployments(
- 'foo.sh', '# -*- split-lines: true; -*-\ns2\ns3')]
- assert ['foo_00.sh', 'foo_01.sh', 'foo_02.sh'] == names
+ assert 'foo.sh' == \
+ nodelib.script_deployment('foo.sh', 'l1\nl2\nl3\n').name
+
def test_node_deployment(self):
nd = nodelib.Deployment(bundles=['mta'])
assert nd.name.startswith(config.DEFAULT_NAME_PREFIX)
assert libcloud.deployment.SSHKeyDeployment == type(nd.deployment.steps[0])
-
97 test/test_script_templates.py
View
@@ -0,0 +1,97 @@
+import unittest
+
+import provision.config as config
+import provision.nodelib as nodelib
+
+class Test(unittest.TestCase):
+
+ def setUp(self):
+
+ self.submap = {'ci_user_host': 'testuser@testhost',}
+
+ self.format_string_script = '''# provision-template-type: format-string
+scp -pr -o StrictHostKeyChecking=no {ci_user_host}:pkg/libcloud .
+'''
+ self.template_string_script = '''# provision-template-type: template-string
+scp -pr -o StrictHostKeyChecking=no ${ci_user_host}:pkg/libcloud .
+'''
+ self.template_string_script_simple = ''' # provision-template-type: template-string
+scp -pr -o StrictHostKeyChecking=no $ci_user_host:pkg/libcloud .'''
+
+ self.template_script_unsupported_type = '''# provision-template-type: mako
+scp -pr -o StrictHostKeyChecking=no ${ci_user_host}:pkg/libcloud .
+'''
+ self.non_template_script = '''apt-get -y install python-setuptools python-dev gcc
+'''
+ # don't do this
+ self.multiple_specifiers_one_line = '''
+# provision-template-type: format-string provision-template-type: template-string
+apt-get -y install python-setuptools python-dev gcc
+'''
+ # or this
+ self.multiple_specifiers_multiple_lines = '''
+# provision-template-type: format-string
+# provision-template-type: template-string
+apt-get -y install python-setuptools python-dev gcc
+'''
+ self.bash_function = '''# provision-template-type: template-string
+download_file() {
+ echo wget -q -O $DOWNLOAD_DIR/$1 $2
+ wget -q -O $DOWNLOAD_DIR/$1 $2
+ http_rc=$?
+ if [[ "$http_rc" != "0" ]]; then
+ echo "Download of $1 failed, return code was $http_rc"
+ exit 1
+ fi
+ if [ ! -f $DOWNLOAD_DIR/$1 ]; then
+ echo "Downloaded file $1 not found"
+ exit 1
+ fi
+}
+'''
+ self.awk_function = '''# provision-template-type: template-string
+eth1=`ifconfig eth1 | grep -o -E 'inet addr:[0-9.]+' | awk -F: '{print $2}'`'''
+
+
+ def test_format_strings_detection(self):
+ assert 'format-string' == \
+ config.TEMPLATE_RE.search(self.format_string_script).groupdict()['type']
+
+ def test_template_strings_detection(self):
+ assert 'template-string' == \
+ config.TEMPLATE_RE.search(self.template_string_script).groupdict()['type']
+
+ def test_multiple_specifiers_one_line(self):
+ assert 'template-string' == \
+ config.TEMPLATE_RE.search(self.multiple_specifiers_one_line).groupdict()['type']
+
+ def test_multiple_specifiers_multiple_lines(self):
+ assert 'format-string' == \
+ config.TEMPLATE_RE.search(self.multiple_specifiers_multiple_lines).groupdict()['type']
+
+ def test_bash_function_unchanged(self):
+ assert self.bash_function == nodelib.substitute(
+ self.bash_function, self.submap)
+
+ def test_awk_function_unchanged(self):
+ assert self.awk_function == nodelib.substitute(
+ self.awk_function, self.submap)
+
+ def test_format_string_substitution(self):
+ assert nodelib.substitute(self.format_string_script, self.submap).find(
+ self.submap['ci_user_host']) > 0
+
+ def test_template_string_substitution(self):
+ assert nodelib.substitute(self.template_string_script, self.submap).find(
+ self.submap['ci_user_host']) > 0
+
+ def test_template_string_simple_substitution(self):
+ assert nodelib.substitute(self.template_string_script_simple, self.submap).find(
+ self.submap['ci_user_host']) > 0
+
+ def test_template_script_unsupported_type(self):
+ try:
+ nodelib.substitute(self.template_script_unsupported_type, self.submap)
+ self.fail()
+ except KeyError:
+ pass
Please sign in to comment.
Something went wrong with that request. Please try again.