diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4034f6b06..ab52a973b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,7 @@ History Unreleased ========== +* Add the `--parallel` flag to experimentally allow molecule to be run in parallel. * `dependency` step is now run by default before any playbook sequence step, including `create` and `destroy`. This allows the use of roles in all sequence step playbooks. * Removed validation regex for docker registry passwords, all ``string`` values are now valid. diff --git a/docs/examples.rst b/docs/examples.rst index fe5b97a01..3b82584bb 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -279,3 +279,41 @@ lives in a shared location and ``molecule.yml`` is points to the shared tests. directory: ../resources/tests/ lint: name: flake8 + +.. _parallel-usage-example: + +Running Molecule processes in parallel mode +=========================================== + +.. important:: + + This functionality should be considered experimental. It is part of ongoing + work towards enabling parallelizable functionality across all moving parts + in the execution of the Molecule feature set. + +.. note:: + + Only the following sequences support parallelizable functionality: + + * ``check_sequence``: ``molecule check --parallel`` + * ``destroy_sequence``: ``molecule destroy --parallel`` + * ``test_sequence``: ``molecule test --parallel`` + + It is currently only available for use with the Docker driver. + +It is possible to run Molecule processes in parallel using another tool to +orchestrate the parallelization (such as `GNU Parallel`_ or `Pytest`_). + +When Molecule receives the ``--parallel`` flag it will generate a `UUID`_ for +the duration of the testing sequence and will use that unique identifier to +cache the run-time state for that process. The parallel Molecule processes +cached state and created instances will therefore not interfere with each +other. + +Molecule uses a new and separate caching folder for this in the +``$HOME/.cache/molecule_parallel`` location. Molecule exposes a new environment +variable ``MOLECULE_PARALLEL`` which can enable this functionality. + +.. _GNU Parallel: https://www.gnu.org/software/parallel/ +.. _Pytest: https://docs.pytest.org/en/latest/ +.. _UUID: https://en.wikipedia.org/wiki/Universally_unique_identifier diff --git a/docs/faq.rst b/docs/faq.rst index 148a4b8cf..326400ef2 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -78,3 +78,9 @@ Are there similar tools to Molecule? .. _`ansible-test`: https://github.com/nylas/ansible-test .. _`abandoned`: https://github.com/nylas/ansible-test/issues/14 .. _`RoleSpec`: https://github.com/nickjj/rolespec + + +Can I run Molecule processes in parallel? +========================================= + +Please see :ref:`parallel-usage-example` for usage. diff --git a/molecule/command/base.py b/molecule/command/base.py index 58d5bdbed..d4721de9a 100644 --- a/molecule/command/base.py +++ b/molecule/command/base.py @@ -110,6 +110,8 @@ def execute_cmdline_scenarios(scenario_name, execute_subcommand(scenario.config, 'destroy') # always prune ephemeral dir if destroying on failure scenario.prune() + if scenario.config.is_parallel: + scenario._remove_scenario_state_directory() util.sysexit() else: raise @@ -144,6 +146,9 @@ def execute_scenario(scenario): if 'destroy' in scenario.sequence: scenario.prune() + if scenario.config.is_parallel: + scenario._remove_scenario_state_directory() + def get_configs(args, command_args, ansible_args=()): """ diff --git a/molecule/command/check.py b/molecule/command/check.py index 00288f8f6..b239f70fc 100644 --- a/molecule/command/check.py +++ b/molecule/command/check.py @@ -18,12 +18,15 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import os import click from molecule import logger from molecule.command import base +from molecule import util LOG = logger.get_logger(__name__) +MOLECULE_PARALLEL = os.environ.get('MOLECULE_PARALLEL', False) class Check(base.Base): @@ -58,6 +61,12 @@ class Check(base.Base): Load an env file to read variables from when rendering molecule.yml. + + .. program:: molecule --parallel check + + .. option:: molecule --parallel check + + Run in parallelizable mode. """ def execute(self): @@ -79,7 +88,11 @@ def execute(self): default=base.MOLECULE_DEFAULT_SCENARIO_NAME, help='Name of the scenario to target. ({})'.format( base.MOLECULE_DEFAULT_SCENARIO_NAME)) -def check(ctx, scenario_name): # pragma: no cover +@click.option( + '--parallel/--no-parallel', + default=MOLECULE_PARALLEL, + help='Enable or disable parallel mode. Default is disabled.') +def check(ctx, scenario_name, parallel): # pragma: no cover """ Use the provisioner to perform a Dry-Run (destroy, dependency, create, prepare, converge). @@ -87,7 +100,11 @@ def check(ctx, scenario_name): # pragma: no cover args = ctx.obj.get('args') subcommand = base._get_subcommand(__name__) command_args = { + 'parallel': parallel, 'subcommand': subcommand, } + if parallel: + util.validate_parallel_cmd_args(command_args) + base.execute_cmdline_scenarios(scenario_name, args, command_args) diff --git a/molecule/command/destroy.py b/molecule/command/destroy.py index 7492848c7..fe7deb34a 100644 --- a/molecule/command/destroy.py +++ b/molecule/command/destroy.py @@ -18,13 +18,16 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import os import click from molecule import config from molecule import logger from molecule.command import base +from molecule import util LOG = logger.get_logger(__name__) +MOLECULE_PARALLEL = os.environ.get('MOLECULE_PARALLEL', False) class Destroy(base.Base): @@ -71,6 +74,12 @@ class Destroy(base.Base): Load an env file to read variables from when rendering molecule.yml. + + .. program:: molecule --parallel destroy + + .. option:: molecule --parallel destroy + + Run in parallelizable mode. """ def execute(self): @@ -112,13 +121,19 @@ def execute(self): @click.option( '--all/--no-all', '__all', - default=False, + default=MOLECULE_PARALLEL, help='Destroy all scenarios. Default is False.') -def destroy(ctx, scenario_name, driver_name, __all): # pragma: no cover +@click.option( + '--parallel/--no-parallel', + default=False, + help='Enable or disable parallel mode. Default is disabled.') +def destroy(ctx, scenario_name, driver_name, __all, + parallel): # pragma: no cover """ Use the provisioner to destroy the instances. """ args = ctx.obj.get('args') subcommand = base._get_subcommand(__name__) command_args = { + 'parallel': parallel, 'subcommand': subcommand, 'driver_name': driver_name, } @@ -126,4 +141,7 @@ def destroy(ctx, scenario_name, driver_name, __all): # pragma: no cover if __all: scenario_name = None + if parallel: + util.validate_parallel_cmd_args(command_args) + base.execute_cmdline_scenarios(scenario_name, args, command_args) diff --git a/molecule/command/test.py b/molecule/command/test.py index 7db0cd879..6ea891b84 100644 --- a/molecule/command/test.py +++ b/molecule/command/test.py @@ -18,13 +18,16 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import os import click from molecule import config from molecule import logger from molecule.command import base +from molecule import util LOG = logger.get_logger(__name__) +MOLECULE_PARALLEL = os.environ.get('MOLECULE_PARALLEL', False) class Test(base.Base): @@ -71,6 +74,12 @@ class Test(base.Base): Load an env file to read variables from when rendering molecule.yml. + + .. program:: molecule --parallel test + + .. option:: molecule --parallel test + + Run in parallelizable mode. """ def execute(self): @@ -106,7 +115,12 @@ def execute(self): default='always', help=('The destroy strategy used at the conclusion of a ' 'Molecule run (always).')) -def test(ctx, scenario_name, driver_name, __all, destroy): # pragma: no cover +@click.option( + '--parallel/--no-parallel', + default=MOLECULE_PARALLEL, + help='Enable or disable parallel mode. Default is disabled.') +def test(ctx, scenario_name, driver_name, __all, destroy, + parallel): # pragma: no cover """ Test (lint, cleanup, destroy, dependency, syntax, create, prepare, converge, idempotence, side_effect, verify, cleanup, destroy). @@ -115,6 +129,7 @@ def test(ctx, scenario_name, driver_name, __all, destroy): # pragma: no cover args = ctx.obj.get('args') subcommand = base._get_subcommand(__name__) command_args = { + 'parallel': parallel, 'destroy': destroy, 'subcommand': subcommand, 'driver_name': driver_name, @@ -123,4 +138,7 @@ def test(ctx, scenario_name, driver_name, __all, destroy): # pragma: no cover if __all: scenario_name = None + if parallel: + util.validate_parallel_cmd_args(command_args) + base.execute_cmdline_scenarios(scenario_name, args, command_args) diff --git a/molecule/config.py b/molecule/config.py index baf0fd465..ed57b8625 100644 --- a/molecule/config.py +++ b/molecule/config.py @@ -18,6 +18,7 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +from uuid import uuid4 import os import anyconfig @@ -109,11 +110,16 @@ def __init__(self, self.ansible_args = ansible_args self.config = self._get_config() self._action = None + self._run_uuid = str(uuid4()) def after_init(self): self.config = self._reget_config() self._validate() + @property + def is_parallel(self): + return self.command_args.get('parallel', False) + @property def debug(self): return self.args.get('debug', MOLECULE_DEBUG) @@ -138,6 +144,10 @@ def action(self, value): def project_directory(self): return os.getcwd() + @property + def cache_directory(self): + return 'molecule_parallel' if self.is_parallel else 'molecule' + @property def molecule_directory(self): return molecule_directory(self.project_directory) @@ -198,6 +208,7 @@ def env(self): 'MOLECULE_DEBUG': str(self.debug), 'MOLECULE_FILE': self.molecule_file, 'MOLECULE_ENV_FILE': self.env_file, + 'MOLECULE_STATE_FILE': self.state.state_file, 'MOLECULE_INVENTORY_FILE': self.provisioner.inventory_file, 'MOLECULE_EPHEMERAL_DIRECTORY': self.scenario.ephemeral_directory, 'MOLECULE_SCENARIO_DIRECTORY': self.scenario.directory, @@ -224,7 +235,8 @@ def lint(self): @property @util.memoize def platforms(self): - return platforms.Platforms(self) + return platforms.Platforms( + self, parallelize_platforms=self.is_parallel) @property @util.memoize diff --git a/molecule/platforms.py b/molecule/platforms.py index ed3a98343..1cbfb823d 100644 --- a/molecule/platforms.py +++ b/molecule/platforms.py @@ -19,6 +19,7 @@ # DEALINGS IN THE SOFTWARE. from molecule import logger +from molecule import util LOG = logger.get_logger(__name__) @@ -65,13 +66,16 @@ class Platforms(object): - child_group1 """ - def __init__(self, config): + def __init__(self, config, parallelize_platforms=False): """ Initialize a new platform class and returns None. :param config: An instance of a Molecule config. :return: None """ + if parallelize_platforms: + config.config['platforms'] = util._parallelize_platforms( + config.config, config._run_uuid) self._config = config @property diff --git a/molecule/provisioner/ansible/plugins/filters/molecule_core.py b/molecule/provisioner/ansible/plugins/filters/molecule_core.py index c99559f62..a5e1b1fc1 100644 --- a/molecule/provisioner/ansible/plugins/filters/molecule_core.py +++ b/molecule/provisioner/ansible/plugins/filters/molecule_core.py @@ -43,7 +43,9 @@ def from_yaml(data): i = interpolation.Interpolator(interpolation.TemplateWithDefaults, env) interpolated_data = i.interpolate(data) - return util.safe_load(interpolated_data) + loaded_data = util.safe_load(interpolated_data) + loaded_data = _parallelize_config(loaded_data) + return loaded_data def to_yaml(data): @@ -65,6 +67,14 @@ def get_docker_networks(data): return network_list +def _parallelize_config(data): + state = util.safe_load_file(os.environ['MOLECULE_STATE_FILE']) + if state['is_parallel']: + data['platforms'] = util._parallelize_platforms( + data, state['run_uuid']) + return data + + class FilterModule(object): """ Core Molecule filter plugins. """ diff --git a/molecule/scenario.py b/molecule/scenario.py index ed17618c0..a1c4706aa 100644 --- a/molecule/scenario.py +++ b/molecule/scenario.py @@ -18,6 +18,7 @@ # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +import shutil import os import fnmatch try: @@ -98,6 +99,14 @@ def __init__(self, config): self.config = config self._setup() + def _remove_scenario_state_directory(self): + """Remove scenario cached disk stored state. + + :return: None + """ + LOG.info('Removing scenario state directory from cache') + shutil.rmtree(Path(self.ephemeral_directory).parent) + def prune(self): """ Prune the scenario ephemeral directory files and returns None. @@ -137,9 +146,14 @@ def directory(self): @property def ephemeral_directory(self): project_directory = os.path.basename(self.config.project_directory) - scenario_name = self.name - project_scenario_directory = os.path.join( - 'molecule', project_directory, scenario_name) + + if self.config.is_parallel: + project_directory = '{}-{}'.format(project_directory, + self.config._run_uuid) + + project_scenario_directory = os.path.join(self.config.cache_directory, + project_directory, self.name) + path = ephemeral_directory(project_scenario_directory) return ephemeral_directory(path) diff --git a/molecule/state.py b/molecule/state.py index 1dfa1592d..c5045f128 100644 --- a/molecule/state.py +++ b/molecule/state.py @@ -30,6 +30,8 @@ 'driver', 'prepared', 'sanity_checked', + 'run_uuid', + 'is_parallel', ] @@ -101,6 +103,14 @@ def prepared(self): def sanity_checked(self): return self._data.get('sanity_checked') + @property + def run_uuid(self): + return self._data.get('run_uuid') + + @property + def is_parallel(self): + return self._data.get('is_parallel') + @marshal def reset(self): self._data = self._default_data() @@ -133,6 +143,8 @@ def _default_data(self): 'driver': None, 'prepared': None, 'sanity_checked': False, + 'run_uuid': self._config._run_uuid, + 'is_parallel': self._config.is_parallel, } def _load_file(self): diff --git a/molecule/util.py b/molecule/util.py index 34246afd0..2a50ce95d 100644 --- a/molecule/util.py +++ b/molecule/util.py @@ -311,3 +311,17 @@ def wrapper(*args, **kwargs): return memo[args] return wrapper + + +def validate_parallel_cmd_args(cmd_args): + if cmd_args.get('parallel') and cmd_args.get('destroy') == 'never': + msg = 'Combining "--parallel" and "--destroy=never" is not supported' + sysexit_with_message(msg) + + +def _parallelize_platforms(config, run_uuid): + def parallelize(platform): + platform['name'] = '{}-{}'.format(platform['name'], run_uuid) + return platform + + return [parallelize(platform) for platform in config['platforms']] diff --git a/test/unit/test_config.py b/test/unit/test_config.py index 60a6d9be3..a9fb9a025 100644 --- a/test/unit/test_config.py +++ b/test/unit/test_config.py @@ -323,6 +323,7 @@ def test_env(config_instance): 'MOLECULE_PROVISIONER_NAME': 'ansible', 'MOLECULE_PROVISIONER_LINT_NAME': 'ansible-lint', 'MOLECULE_SCENARIO_NAME': 'default', + 'MOLECULE_STATE_FILE': config_instance.state.state_file, 'MOLECULE_VERIFIER_NAME': 'testinfra', 'MOLECULE_VERIFIER_LINT_NAME': 'flake8', 'MOLECULE_VERIFIER_TEST_DIRECTORY': config_instance.verifier.directory,