From 29ddf56aab31227468494baf6dc2973f03557b09 Mon Sep 17 00:00:00 2001 From: Nemanja Cimbaljevic Date: Tue, 2 Sep 2025 01:50:48 +0200 Subject: [PATCH 1/3] Python3 migration --- Makefile | 16 +- bin/command/docker_exec_command.py | 24 +- bin/command/generate_graph_command.py | 4 +- bin/command/generate_provision_command.py | 2 +- bin/command/test_testinfra_command.py | 2 +- bin/console | 20 +- bin/requirements.txt | 2 +- bin/webdevops/Command.py | 7 +- bin/webdevops/Configuration.py | 10 +- bin/webdevops/Dockerfile.py | 2 +- bin/webdevops/DockerfileUtility.py | 10 +- bin/webdevops/Provisioner.py | 4 +- bin/webdevops/__init__.py | 2 +- bin/webdevops/command/BaseCommand.py | 28 +- bin/webdevops/command/DoitCommand.py | 2 +- bin/webdevops/command/__init__.py | 4 +- bin/webdevops/docker/DockerBaseClient.py | 6 +- bin/webdevops/docker/DockerPyClient.py | 2 +- bin/webdevops/docker/__init__.py | 6 +- bin/webdevops/doit/DoitReporter.py | 6 +- bin/webdevops/doit/__init__.py | 2 +- .../taskloader/BaseDockerTaskLoader.py | 9 +- bin/webdevops/taskloader/BaseTaskLoader.py | 29 +- .../taskloader/DockerBuildTaskLoader.py | 55 ++- .../taskloader/DockerPullTaskLoader.py | 33 +- .../taskloader/DockerPushTaskLoader.py | 33 +- .../DockerTestServerspecTaskLoader.py | 69 ++-- .../DockerTestTestinfraTaskLoader.py | 30 +- bin/webdevops/taskloader/__init__.py | 14 +- .../testinfra/TestinfraDockerPlugin.py | 2 +- bin/webdevops/testinfra/__init__.py | 2 +- conftest.py | 17 +- .../resources/images/docker-image-layout.gv | 328 +++--------------- provisioning/base/general/bin/provision.py | 12 +- pytest.ini | 14 + tests/testinfra/conftest.py | 33 ++ tests/testinfra/test_liquibase.py | 4 +- 37 files changed, 403 insertions(+), 442 deletions(-) create mode 100644 pytest.ini create mode 100644 tests/testinfra/conftest.py diff --git a/Makefile b/Makefile index 8c239643d..dc3f5fa74 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ full: provision build all: build build: - python2 ./bin/console docker:build --threads=auto + python3 ./bin/console docker:build --threads=auto bootstrap: webdevops/bootstrap webdevops/ansible base: webdevops/base webdevops/base-app webdevops/storage @@ -36,27 +36,27 @@ misc: webdevops/mail-sandbox webdevops/sphinx webdevops/liquibase setup: requirements requirements: - pip install --upgrade -I -r ./bin/requirements.txt + pip3 install --upgrade -I -r ./bin/requirements.txt cd tests/serverspec && bundle install --path=vendor test: - python2 bin/console test:serverspec --threads=auto -v + python3 bin/console test:serverspec --threads=auto -v structure-test: cd tests/structure-test && ./run.sh provision: - python2 bin/console generate:dockerfile - python2 bin/console generate:provision + python3 bin/console generate:dockerfile + python3 bin/console generate:provision push: - python2 ./bin/console docker:push --threads=auto + python3 ./bin/console docker:push --threads=auto graph: - python2 ./bin/console generate:graph + python3 ./bin/console generate:graph graph-full: - python2 ./bin/console generate:graph --all\ + python3 ./bin/console generate:graph --all\ --filename docker-image-full-layout.gv documentation: diff --git a/bin/command/docker_exec_command.py b/bin/command/docker_exec_command.py index 0f84f7627..7ecbd6bb4 100644 --- a/bin/command/docker_exec_command.py +++ b/bin/command/docker_exec_command.py @@ -1,4 +1,4 @@ -#!/usr/bin/env/python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # (c) 2016 WebDevOps.io @@ -51,11 +51,11 @@ def run_task(self, configuration): for dockerfile in dockerfile_list: title = dockerfile['image']['fullname'] - print title - print '~' * len(title) + print(title) + print('~' * len(title)) if configuration['dryRun']: - print ' exec: %s' % (docker_command) + print(' exec: %s' % (docker_command)) else: cmd = [ @@ -71,19 +71,19 @@ def run_task(self, configuration): status = Command.execute(cmd) if status: - print colored(' -> successfull', 'green') + print(colored(' -> successfull', 'green')) else: - print colored(' -> failed', 'red') + print(colored(' -> failed', 'red')) image_fail_list.append(dockerfile['image']['fullname']) - print '' - print '' + print('') + print('') if len(image_fail_list) >= 1: - print '' - print colored(' => failed images (%s):' % (str(len(image_fail_list))), 'red') + print('') + print(colored(' => failed images (%s):' % (str(len(image_fail_list))), 'red')) for image in image_fail_list: - print ' - %s ' % image - print '' + print(' - %s ' % image) + print('') return False else: diff --git a/bin/command/generate_graph_command.py b/bin/command/generate_graph_command.py index 1a82ee378..f9b25235f 100644 --- a/bin/command/generate_graph_command.py +++ b/bin/command/generate_graph_command.py @@ -43,7 +43,7 @@ class GenerateGraphCommand(BaseCommand): '--format': Enum(['png', 'jpg', 'pdf', 'svg']) } - from_regex = re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P.+))?', re.MULTILINE) + from_regex = re.compile(r'FROM\s+(?P[^\s:]+)(:(?P.+))?', re.MULTILINE) containers = {} @@ -116,7 +116,7 @@ def __append_tag(self, docker_image, tag): :return: self """ - if not self.tags.has_key(docker_image): + if docker_image not in self.tags: self.tags[docker_image] = {} self.tags[docker_image][tag] = tag return self diff --git a/bin/command/generate_provision_command.py b/bin/command/generate_provision_command.py index cdf3cec44..707263238 100644 --- a/bin/command/generate_provision_command.py +++ b/bin/command/generate_provision_command.py @@ -22,7 +22,7 @@ import yaml import yamlordereddictloader import time -import Queue +import queue as Queue import shutil import grp from cleo import Output diff --git a/bin/command/test_testinfra_command.py b/bin/command/test_testinfra_command.py index 39508bd45..7204c6542 100644 --- a/bin/command/test_testinfra_command.py +++ b/bin/command/test_testinfra_command.py @@ -1,4 +1,4 @@ -#!/usr/bin/env/python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # (c) 2016 WebDevOps.io diff --git a/bin/console b/bin/console index 0ec2645a0..4e9d07b5c 100755 --- a/bin/console +++ b/bin/console @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os, sys @@ -6,9 +6,9 @@ import os, sys # prevent bytecode sys.dont_write_bytecode = True -# unbuffered stdout / stderr -sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) -sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0) +# unbuffered stdout / stderr - fixed for Python 3 +sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1) +sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', buffering=1) import re, yaml from cleo import Application @@ -43,19 +43,19 @@ if __name__ == '__main__': # Read console.yml for configuration with open(os.path.join(conf_path, 'console.yml'), 'r') as stream: try: - configuration = yaml.load(stream) + configuration = yaml.load(stream, Loader=yaml.SafeLoader) configuration['confPath'] = conf_path except yaml.YAMLError as e: configuration = None - print ' !!! Exception while loading configuration from %s:' % conf_path - print '' - print e - print '' + print(' !!! Exception while loading configuration from %s:' % conf_path) + print('') + print(e) + print('') sys.exit(1) # Check if configuration is valid if configuration is None: - print ' !!! Configuration not found' + print(' !!! Configuration not found') sys.exit(1) # generate full paths diff --git a/bin/requirements.txt b/bin/requirements.txt index 0a1be6eca..5963d19cf 100644 --- a/bin/requirements.txt +++ b/bin/requirements.txt @@ -3,7 +3,7 @@ graphviz>=0.4.10 cleo<0.7.0 yamlordereddictloader>=0.1.0 testinfra>=1.4.2 -doit==0.29.0 +doit>=0.36.0 termcolor>=1.1.0 pytest-timeout>=1.0.0 pytest-rerunfailures>=1.0.0 diff --git a/bin/webdevops/Command.py b/bin/webdevops/Command.py index c7979e8d3..3defbdb65 100644 --- a/bin/webdevops/Command.py +++ b/bin/webdevops/Command.py @@ -25,7 +25,7 @@ def execute(cmd, cwd=False, env=None): Execute cmd and output stdout/stderr """ - print 'Execute: %s' % ' '.join(cmd) + print('Execute: %s' % ' '.join(cmd)) if env is not None: env = copy.deepcopy(env) @@ -56,7 +56,6 @@ def execute(cmd, cwd=False, env=None): cmd, stdout=file_stdout, stderr=file_stdout, - bufsize=-1, env=env ) @@ -67,7 +66,7 @@ def execute(cmd, cwd=False, env=None): # output stdout with open(file_stdout.name, 'r') as f: for line in f: - print line.rstrip('\n') + print(line.rstrip('\n')) # restore current work directory os.chdir(path_current) @@ -75,5 +74,5 @@ def execute(cmd, cwd=False, env=None): if proc.returncode == 0: return True else: - print '>> failed command with return code %s' % proc.returncode + print(')>> failed command with return code %s' % proc.returncode) return False diff --git a/bin/webdevops/Configuration.py b/bin/webdevops/Configuration.py index 3d6cfad25..537fab7d0 100644 --- a/bin/webdevops/Configuration.py +++ b/bin/webdevops/Configuration.py @@ -44,7 +44,7 @@ 'docker': { 'imagePrefix': '', 'autoLatestTag': False, - 'fromRegExp': re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P.+))?', re.MULTILINE), + 'fromRegExp': re.compile(r'FROM\s+(?P[^\s:]+)(:(?P.+))?', re.MULTILINE), 'pathRegex': False, 'autoPull': False, 'autoPullWhitelist': False, @@ -83,7 +83,7 @@ def dictmerge(original, update): Recursively update a dict. Subdict's won't be overwritten but also updated. """ - for key, value in original.iteritems(): + for key, value in original.items(): if key not in update: update[key] = value elif isinstance(value, dict): @@ -101,7 +101,7 @@ def __init__(self, value=None): for key in value: self.__setitem_internal__(key, value[key]) else: - raise TypeError, 'expected dict' + raise TypeError('expected dict') def __setitem_internal__(self, key, value): """ @@ -116,7 +116,7 @@ def __setitem__(self, key, value): myKey, restOfKey = key.split('.', 1) target = self.setdefault(myKey, dotdictify()) if not isinstance(target, dotdictify): - raise KeyError, 'cannot set "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target)) + raise KeyError('cannot set "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target))) target[restOfKey] = value else: if isinstance(value, dict) and not isinstance(value, dotdictify): @@ -129,7 +129,7 @@ def __getitem__(self, key, raw=False): myKey, restOfKey = key.split('.', 1) target = dict.get(self, myKey, None) if not isinstance(target, dotdictify): - raise KeyError, 'cannot get "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target)) + raise KeyError('cannot get "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target))) return target[restOfKey] def __contains__(self, key): diff --git a/bin/webdevops/Dockerfile.py b/bin/webdevops/Dockerfile.py index 8955a0597..2f6b8a75d 100644 --- a/bin/webdevops/Dockerfile.py +++ b/bin/webdevops/Dockerfile.py @@ -40,7 +40,7 @@ def finder(dockerfile_path, filename="Dockerfile", filter=[]): :rtype: list """ dockerfile_stack = [] - filter_regex = re.compile(ur'.*(%s).*' % "|".join(filter), re.IGNORECASE) + filter_regex = re.compile(r'.*(%s).*' % "|".join(filter), re.IGNORECASE) # pprint(filter_regex.pattern) for root, dirs, files in os.walk(dockerfile_path): for file in files: diff --git a/bin/webdevops/DockerfileUtility.py b/bin/webdevops/DockerfileUtility.py index 32b32a966..c8c567e21 100644 --- a/bin/webdevops/DockerfileUtility.py +++ b/bin/webdevops/DockerfileUtility.py @@ -1,4 +1,4 @@ -#!/usr/bin/env/python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # (c) 2016 WebDevOps.io @@ -21,8 +21,8 @@ import os import re -DOCKERFILE_STATEMENT_FROM_RE = re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P[^\s:]+))?(?!.*\s+AS)', re.MULTILINE) -DOCKERFILE_STATEMENT_FROM_MULTISTAGE_RE = re.compile(ur'FROM\s+(?P[^\s:]+)(:(?P[^\s:]+))?(\s+AS)', re.MULTILINE) +DOCKERFILE_STATEMENT_FROM_RE = re.compile(r'FROM\s+(?P[^\s:]+)(:(?P[^\s:]+))?(?!.*\s+AS)', re.MULTILINE) +DOCKERFILE_STATEMENT_FROM_MULTISTAGE_RE = re.compile(r'FROM\s+(?P[^\s:]+)(:(?P[^\s:]+))?(\s+AS)', re.MULTILINE) def find_file_in_path(dockerfile_path, filename="Dockerfile", whitelist=False, blacklist=False): """ @@ -64,7 +64,7 @@ def find_file_in_path(dockerfile_path, filename="Dockerfile", whitelist=False, b if blacklist: for term in blacklist: - file_list = filter(lambda x: term not in x, file_list) + file_list = list(filter(lambda x: term not in x, file_list)) return file_list @@ -143,7 +143,7 @@ def filter_dockerfile(dockerfile_list, whitelist=False, blacklist=False): if blacklist: for term in blacklist: - dockerfile_list = filter(lambda x: term not in x['image']['fullname'], dockerfile_list) + dockerfile_list = list(filter(lambda x: term not in x['image']['fullname'], dockerfile_list)) return dockerfile_list diff --git a/bin/webdevops/Provisioner.py b/bin/webdevops/Provisioner.py index 08dc1ad7a..f24354567 100644 --- a/bin/webdevops/Provisioner.py +++ b/bin/webdevops/Provisioner.py @@ -24,7 +24,7 @@ import os from distutils.dir_util import copy_tree, remove_tree from threading import Thread -import Queue +import queue as Queue import shutil @@ -114,7 +114,7 @@ def __deploy_configuration(self): """ Deploy the configuration to the container """ - for src, tag in self.image_config['configuration'].iteritems(): + for src, tag in self.image_config['configuration'].items(): if Output.VERBOSITY_NORMAL <= self.output.get_verbosity(): self.line("%s => %s:%s" % (src, self.image_name, tag)) if isinstance(tag, list): diff --git a/bin/webdevops/__init__.py b/bin/webdevops/__init__.py index 0688bc71a..40a6fb278 100644 --- a/bin/webdevops/__init__.py +++ b/bin/webdevops/__init__.py @@ -18,7 +18,7 @@ # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from Provisioner import Provisioner +from .Provisioner import Provisioner __all__ = [ 'Provisioner', diff --git a/bin/webdevops/command/BaseCommand.py b/bin/webdevops/command/BaseCommand.py index 6734f5917..85fb0ddd1 100644 --- a/bin/webdevops/command/BaseCommand.py +++ b/bin/webdevops/command/BaseCommand.py @@ -49,11 +49,11 @@ def handle(self): try: exitcode = self.run_task(configuration=self.configuration) except KeyboardInterrupt as e: - print ' !!! Execution aborted by user' + print(' !!! Execution aborted by user') exitcode = 1 except SystemExit as e: - print ' !!! Execution aborted by SystemExit' - print '' + print(' !!! Execution aborted by SystemExit') + print('') traceback.print_exc(file=sys.stdout) exitcode = 1 @@ -90,26 +90,26 @@ def startup(self): DoitReporter.simulation_mode = True - print 'Executing %s (%s)' % (self.name, ', '.join(options)) - print '' + print('Executing %s (%s)' % (self.name, ', '.join(options))) + print('') try: whitelist = self.get_whitelist() if whitelist: - print 'WHITELIST active:' + print('WHITELIST active:') for item in whitelist: - print ' - %s' % item - print '' + print(' - %s' % item) + print('') except: pass try: blacklist = self.get_blacklist() if blacklist: - print 'BLACKLIST active:' + print('BLACKLIST active:') for item in blacklist: - print ' - %s' % item - print '' + print(' - %s' % item) + print('') except: pass @@ -128,11 +128,11 @@ def shutdown(self, exitcode=0): self.teardown(exitcode) - print '' + print('') if exitcode == 0: - print '> finished execution in %s successfully' % (duration) + print('> finished execution in %s successfully' % (duration)) else: - print '> finished execution in %s with errors (exitcode %s)' % (duration, exitcode) + print('> finished execution in %s with errors (exitcode %s)' % (duration, exitcode)) def build_configuration(self): """ diff --git a/bin/webdevops/command/DoitCommand.py b/bin/webdevops/command/DoitCommand.py index c7d1dc3ed..6af842c1d 100644 --- a/bin/webdevops/command/DoitCommand.py +++ b/bin/webdevops/command/DoitCommand.py @@ -28,7 +28,7 @@ def run_doit(self, task_loader, configuration): extra_configuration = {} if 'threads' in configuration and configuration.get('threads') > 1: - arguments.extend(['-n', str(configuration.get('threads')), '--parallel-type', 'process']) + arguments.extend(['-n', str(configuration.get('threads')), '--parallel-type', 'thread']) if 'doitConfig' in configuration: extra_configuration = configuration.get('doitConfig') diff --git a/bin/webdevops/command/__init__.py b/bin/webdevops/command/__init__.py index 9be4e15cb..9bf0d0486 100644 --- a/bin/webdevops/command/__init__.py +++ b/bin/webdevops/command/__init__.py @@ -18,8 +18,8 @@ # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from BaseCommand import BaseCommand -from DoitCommand import DoitCommand +from .BaseCommand import BaseCommand +from .DoitCommand import DoitCommand __all__ = [ 'BaseCommand', diff --git a/bin/webdevops/docker/DockerBaseClient.py b/bin/webdevops/docker/DockerBaseClient.py index b8b22cd9c..3874b8690 100644 --- a/bin/webdevops/docker/DockerBaseClient.py +++ b/bin/webdevops/docker/DockerBaseClient.py @@ -29,19 +29,19 @@ def pull_image(self, name, tag): """ Build dockerfile """ - print 'Pull image %s:%s' % (name, tag) + print('Pull image %s:%s' % (name, tag)) return True def build_dockerfile(self, path, name, nocache=False): """ Build dockerfile """ - print 'Build Dockerfile %s with name %s' % (path, name) + print('Build Dockerfile %s with name %s' % (path, name)) return True def push_image(self, name): """ Push one Docker image to registry """ - print 'Build image %s' % (name) + print('Build image %s' % (name)) return True diff --git a/bin/webdevops/docker/DockerPyClient.py b/bin/webdevops/docker/DockerPyClient.py index 507a190f0..4f685b672 100644 --- a/bin/webdevops/docker/DockerPyClient.py +++ b/bin/webdevops/docker/DockerPyClient.py @@ -110,5 +110,5 @@ def output_message(message, prevent_repeat=False): if 'id' in line: message += ' ' + line['id'] output_message(message) - print '' + print('') return ret diff --git a/bin/webdevops/docker/__init__.py b/bin/webdevops/docker/__init__.py index 9124d6a99..5b6708643 100644 --- a/bin/webdevops/docker/__init__.py +++ b/bin/webdevops/docker/__init__.py @@ -18,9 +18,9 @@ # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from DockerBaseClient import DockerBaseClient -from DockerPyClient import DockerPyClient -from DockerCliClient import DockerCliClient +from .DockerBaseClient import DockerBaseClient +from .DockerPyClient import DockerPyClient +from .DockerCliClient import DockerCliClient __all__ = [ 'DockerBaseClient', diff --git a/bin/webdevops/doit/DoitReporter.py b/bin/webdevops/doit/DoitReporter.py index 84c85ee53..645b3f83c 100644 --- a/bin/webdevops/doit/DoitReporter.py +++ b/bin/webdevops/doit/DoitReporter.py @@ -18,7 +18,7 @@ # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import os, sys, time, datetime, StringIO +import os, sys, time, datetime, io import termcolor from termcolor import colored from ..taskloader.BaseTaskLoader import BaseTaskLoader @@ -121,9 +121,9 @@ def __init__(self, outstream, options=None): #pylint: disable=W0613 # than the data. so anything that is sent to stdout/err needs to # be captured. self._old_out = sys.stdout - sys.stdout = StringIO.StringIO() + sys.stdout = io.StringIO() self._old_err = sys.stderr - sys.stderr = StringIO.StringIO() + sys.stderr = io.StringIO() self.outstream = outstream # runtime and cleanup errors self.errors = [] diff --git a/bin/webdevops/doit/__init__.py b/bin/webdevops/doit/__init__.py index 29fb79a7c..65496fdce 100644 --- a/bin/webdevops/doit/__init__.py +++ b/bin/webdevops/doit/__init__.py @@ -18,7 +18,7 @@ # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from DoitReporter import DoitReporter +from .DoitReporter import DoitReporter __all__ = [ 'DoitReporter', diff --git a/bin/webdevops/taskloader/BaseDockerTaskLoader.py b/bin/webdevops/taskloader/BaseDockerTaskLoader.py index 42404c5a6..82abb3915 100644 --- a/bin/webdevops/taskloader/BaseDockerTaskLoader.py +++ b/bin/webdevops/taskloader/BaseDockerTaskLoader.py @@ -27,6 +27,7 @@ class BaseDockerTaskLoader(BaseTaskLoader): docker_client = False + cmd_options = () def __init__(self, configuration): """ @@ -37,12 +38,10 @@ def __init__(self, configuration): # Init docker client self.docker_client = configuration.get('dockerClient') - def load_tasks(self, cmd, opt_values, pos_args): + def load_tasks(self, cmd, pos_args): """ DOIT task list generator """ - config = {'verbosity': self.configuration.get('verbosity')} - dockerfile_list = DockerfileUtility.find_dockerfiles_in_path( base_path=self.configuration.get('dockerPath'), path_regex=self.configuration.get('docker.pathRegex'), @@ -52,7 +51,7 @@ def load_tasks(self, cmd, opt_values, pos_args): ) dockerfile_list = self.process_dockerfile_list(dockerfile_list) - #import json,sys;print json.dumps(dockerfile_list, sort_keys=True, indent = 4, separators = (',', ': '));sys.exit(0); + #import json,sys;print(j)son.dumps(dockerfile_list, sort_keys=True, indent = 4, separators = (',', ': '));sys.exit(0); tasklist = self.generate_task_list(dockerfile_list) @@ -61,7 +60,7 @@ def load_tasks(self, cmd, opt_values, pos_args): tasklist = self.process_tasklist(tasklist) - return tasklist, config + return tasklist def process_dockerfile_list(self, dockerfile_list): """ diff --git a/bin/webdevops/taskloader/BaseTaskLoader.py b/bin/webdevops/taskloader/BaseTaskLoader.py index a919fb934..b3d233127 100644 --- a/bin/webdevops/taskloader/BaseTaskLoader.py +++ b/bin/webdevops/taskloader/BaseTaskLoader.py @@ -18,7 +18,7 @@ # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import sys, re, time, StringIO, tempfile, json, base64, os +import sys, re, time, io, tempfile, json, base64, os from webdevops import DockerfileUtility from doit.cmd_base import TaskLoader from doit.task import dict_to_task @@ -29,6 +29,7 @@ class BaseTaskLoader(TaskLoader): reporter = False TASK_COUNT = 0 TASK_RESULTS = {} + cmd_options = () def __init__(self, configuration): """ @@ -37,6 +38,22 @@ def __init__(self, configuration): # Build configuration as namespace object self.configuration = configuration + def setup(self, opt_values): + """ + Delayed initialization. + + :param opt_values: (dict) with values for cmd_options + """ + pass + + def load_doit_config(self): + """ + Load doit configuration. + + :return: (dict) Dictionary of doit configuration values. + """ + return {} + def process_tasklist(self, tasklist): """ Process task list and create task objects @@ -45,7 +62,7 @@ def process_tasklist(self, tasklist): for task in tasklist: ret.append(dict_to_task(task)) - print 'Starting execution of %s tasks...' % (len(ret)) + print('Starting execution of %s tasks...' % (len(ret))) BaseTaskLoader.TASK_COUNT = len(ret) @@ -117,7 +134,7 @@ def task_runner(func, args, task): Will return the stdout if task fails as exception """ backup = sys.stdout - sys.stdout = StringIO.StringIO() + sys.stdout = io.StringIO() status = func(task=task, *args) output = sys.stdout.getvalue().strip() sys.stdout.close() @@ -126,7 +143,7 @@ def task_runner(func, args, task): if not status: raise Exception(output) else: - print output + print(output) return status @@ -134,7 +151,7 @@ def task_runner(func, args, task): @staticmethod def task_statusfile(task): - return '%s/%s' % (tempfile.gettempdir(), base64.b64encode(task.name)) + return '%s/%s' % (tempfile.gettempdir(), base64.b64encode(task.name.encode()).decode()) @staticmethod def task_write_statusfile(task, data): @@ -164,5 +181,5 @@ def task_get_statusfile(task, remove=True): @staticmethod def task_remove_statusfile(task): - filename = '%s/%s' % (tempfile.gettempdir(), base64.b64encode(task.name)) + filename = '%s/%s' % (tempfile.gettempdir(), base64.b64encode(task.name.encode()).decode()) os.remove(filename) diff --git a/bin/webdevops/taskloader/DockerBuildTaskLoader.py b/bin/webdevops/taskloader/DockerBuildTaskLoader.py index 761e7cbe3..897c9bae2 100644 --- a/bin/webdevops/taskloader/DockerBuildTaskLoader.py +++ b/bin/webdevops/taskloader/DockerBuildTaskLoader.py @@ -23,7 +23,34 @@ from .BaseDockerTaskLoader import BaseDockerTaskLoader from webdevops import DockerfileUtility +def run_dependency_puller_task(dockerfileList, configuration_dict, task): + """ + Standalone function for dependency puller task + """ + from webdevops.docker.DockerCliClient import DockerCliClient + from webdevops.Configuration import dotdictify + + # Recreate objects in worker process + docker_client = DockerCliClient() + configuration = dotdictify(configuration_dict) + + return DockerBuildTaskLoader.task_dependency_puller(docker_client, dockerfileList, configuration, task) + +def run_build_task(dockerfile, configuration_dict, task): + """ + Standalone function for docker build task + """ + from webdevops.docker.DockerCliClient import DockerCliClient + from webdevops.Configuration import dotdictify + + # Recreate objects in worker process + docker_client = DockerCliClient() + configuration = dotdictify(configuration_dict) + + return DockerBuildTaskLoader.task_run(docker_client, dockerfile, configuration, task) + class DockerBuildTaskLoader(BaseDockerTaskLoader): + cmd_options = () def generate_task_list(self, dockerfileList): """ @@ -31,12 +58,14 @@ def generate_task_list(self, dockerfileList): """ tasklist = [] + # Convert configuration to dict for serialization + configuration_dict = self.configuration.to_dict() + # TASK: dependency puller task = { 'name': 'DockerBuild|DependencyPuller', 'title': DockerBuildTaskLoader.task_title_dependency_puller, - 'actions': [(BaseTaskLoader.task_runner, - [DockerBuildTaskLoader.task_dependency_puller, [self.docker_client, dockerfileList, self.configuration]])], + 'actions': [(run_dependency_puller_task, [dockerfileList, configuration_dict])], 'task_dep': [] } tasklist.append(task) @@ -46,7 +75,7 @@ def generate_task_list(self, dockerfileList): task = { 'name': 'DockerBuild|%s' % dockerfile['image']['fullname'], 'title': DockerBuildTaskLoader.task_title, - 'actions': [(BaseTaskLoader.task_runner, [DockerBuildTaskLoader.task_run, [self.docker_client, dockerfile, self.configuration]])], + 'actions': [(run_build_task, [dockerfile, configuration_dict])], 'task_dep': ['DockerBuild|DependencyPuller'] } @@ -73,7 +102,7 @@ def task_dependency_puller(docker_client, dockerfileList, configuration, task): Pulls dependency images before building """ def pull_image(image): - print ' -> Pull base image %s ' % image + print(') -> Pull base image %s ' % image) if configuration.get('dryRun'): return True @@ -91,9 +120,9 @@ def pull_image(image): if pull_status: break elif retry_count < (configuration.get('retry') - 1): - print ' failed, retrying... (try %s)' % (retry_count + 1) + print(') failed, retrying... (try %s)' % (retry_count + 1)) else: - print ' failed, giving up' + print(') failed, giving up') if not pull_status: return False @@ -130,17 +159,17 @@ def task_run(docker_client, dockerfile, configuration, task): # check if dockerfile is symlink, skipping tests if just a duplicate image # image is using the same hashes if dockerfile['image']['duplicate'] and not task.task_dep: - print ' Docker image %s is build from symlink but not included in build chain, please include %s' % (dockerfile['image']['fullname'], dockerfile['image']['from']) - print ' -> failing build' + print(') Docker image %s is build from symlink but not included in build chain, please include %s' % (dockerfile['image']['fullname'], dockerfile['image']['from'])) + print(') -> failing build') return False if configuration.get('dryRun'): - print ' path: %s' % dockerfile['path'] - print ' dep: %s' % (DockerBuildTaskLoader.human_task_name_list(task.task_dep) if task.task_dep else 'none') + print(') path: %s' % dockerfile['path']) + print(') dep: %s' % (DockerBuildTaskLoader.human_task_name_list(task.task_dep) if task.task_dep else 'none')) return True ## Build image - print ' -> Building image %s ' % dockerfile['image']['fullname'] + print(') -> Building image %s ' % dockerfile['image']['fullname']) build_status = False for retry_count in range(0, configuration.get('retry')): build_status = docker_client.build_dockerfile( @@ -152,9 +181,9 @@ def task_run(docker_client, dockerfile, configuration, task): if build_status: break elif retry_count < (configuration.get('retry')-1): - print ' failed, retrying... (try %s)' % (retry_count+1) + print(') failed, retrying... (try %s)' % (retry_count+1)) else: - print ' failed, giving up' + print(') failed, giving up') if build_status and dockerfile['image']['duplicate']: BaseTaskLoader.set_task_status(task, 'finished (duplicate)', 'success2') diff --git a/bin/webdevops/taskloader/DockerPullTaskLoader.py b/bin/webdevops/taskloader/DockerPullTaskLoader.py index 4bed18a51..93f38c12c 100644 --- a/bin/webdevops/taskloader/DockerPullTaskLoader.py +++ b/bin/webdevops/taskloader/DockerPullTaskLoader.py @@ -23,7 +23,28 @@ from .BaseDockerTaskLoader import BaseDockerTaskLoader from webdevops import DockerfileUtility +# Define standalone functions for better multiprocessing compatibility +def task_run_wrapper(dockerfile, configuration_dict, task): + """ + Wrapper for docker pull task that recreates objects in worker process + """ + from webdevops.docker.DockerCliClient import DockerCliClient + from webdevops.Configuration import dotdictify + + # Recreate objects in worker process + docker_client = DockerCliClient() + configuration = dotdictify(configuration_dict) + + return DockerPullTaskLoader.task_run(docker_client, dockerfile, configuration, task) + +def task_runner_wrapper(func, args, task): + """ + Wrapper for task runner + """ + return BaseTaskLoader.task_runner(func, args, task) + class DockerPullTaskLoader(BaseDockerTaskLoader): + cmd_options = () def generate_task_list(self, dockerfile_list): """ @@ -31,11 +52,14 @@ def generate_task_list(self, dockerfile_list): """ tasklist = [] + # Convert configuration to dict for serialization + configuration_dict = self.configuration.to_dict() + for dockerfile in dockerfile_list: task = { 'name': 'DockerPull|%s' % dockerfile['image']['fullname'], 'title': DockerPullTaskLoader.task_title, - 'actions': [(BaseTaskLoader.task_runner, [DockerPullTaskLoader.task_run, [self.docker_client, dockerfile, self.configuration]])], + 'actions': [(task_runner_wrapper, [task_run_wrapper, [dockerfile, configuration_dict]])], 'task_dep': [] } tasklist.append(task) @@ -56,7 +80,7 @@ def task_run(docker_client, dockerfile, configuration, task): Pull one Docker image from registry """ if configuration.get('dryRun'): - print ' pull: %s' % (dockerfile['image']['fullname']) + print(') pull: %s' % (dockerfile['image']['fullname'])) return True pull_status = False @@ -69,9 +93,9 @@ def task_run(docker_client, dockerfile, configuration, task): if pull_status: break elif retry_count < (configuration.get('retry') - 1): - print ' failed, retrying... (try %s)' % (retry_count+1) + print(') failed, retrying... (try %s)' % (retry_count+1)) else: - print ' failed, giving up' + print(') failed, giving up') return pull_status @@ -81,4 +105,3 @@ def task_title(task): Pull task title function """ return "Docker pull %s" % (BaseTaskLoader.human_task_name(task.name)) -0 diff --git a/bin/webdevops/taskloader/DockerPushTaskLoader.py b/bin/webdevops/taskloader/DockerPushTaskLoader.py index 974bdea7f..ab42e5f3e 100644 --- a/bin/webdevops/taskloader/DockerPushTaskLoader.py +++ b/bin/webdevops/taskloader/DockerPushTaskLoader.py @@ -24,7 +24,28 @@ from .BaseDockerTaskLoader import BaseDockerTaskLoader from webdevops import DockerfileUtility +# Define standalone functions for better multiprocessing compatibility +def task_run_wrapper(dockerfile, configuration_dict, task): + """ + Wrapper for docker push task that recreates objects in worker process + """ + from webdevops.docker.DockerCliClient import DockerCliClient + from webdevops.Configuration import dotdictify + + # Recreate objects in worker process + docker_client = DockerCliClient() + configuration = dotdictify(configuration_dict) + + return DockerPushTaskLoader.task_run(docker_client, dockerfile, configuration, task) + +def task_runner_wrapper(func, args, task): + """ + Wrapper for task runner + """ + return BaseTaskLoader.task_runner(func, args, task) + class DockerPushTaskLoader(BaseDockerTaskLoader): + cmd_options = () def generate_task_list(self, dockerfile_list): """ @@ -32,11 +53,14 @@ def generate_task_list(self, dockerfile_list): """ tasklist = [] + # Convert configuration to dict for serialization + configuration_dict = self.configuration.to_dict() + for dockerfile in dockerfile_list: task = { 'name': 'DockerPush|%s' % dockerfile['image']['fullname'], 'title': DockerPushTaskLoader.task_title, - 'actions': [(BaseTaskLoader.task_runner, [DockerPushTaskLoader.task_run, [self.docker_client, dockerfile, self.configuration]])], + 'actions': [(task_runner_wrapper, [task_run_wrapper, [dockerfile, configuration_dict]])], 'task_dep': [] } @@ -61,7 +85,7 @@ def task_run(docker_client, dockerfile, configuration, task): Push one Docker image to registry """ if configuration.get('dryRun'): - print ' push: %s' % (dockerfile['image']['fullname']) + print(') push: %s' % (dockerfile['image']['fullname'])) return True push_status = False @@ -73,10 +97,10 @@ def task_run(docker_client, dockerfile, configuration, task): if push_status: break elif retry_count < (configuration.get('retry') - 1): - print ' failed, retrying... (try %s)' % (retry_count+1) + print(') failed, retrying... (try %s)' % (retry_count+1)) time.sleep(randint(10, 30)) else: - print ' failed, giving up' + print(') failed, giving up') return push_status @@ -86,4 +110,3 @@ def task_title(task): Push task title function """ return "Docker push %s" % (BaseTaskLoader.human_task_name(task.name)) -0 diff --git a/bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py b/bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py index 0c11aff10..b5729fcca 100644 --- a/bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py +++ b/bin/webdevops/taskloader/DockerTestServerspecTaskLoader.py @@ -25,7 +25,26 @@ from doit.task import dict_to_task import pytest +# Define standalone functions for better multiprocessing compatibility +def task_run_wrapper(dockerfile, configuration_dict, task): + """ + Wrapper for docker test serverspec task that recreates objects in worker process + """ + from webdevops.Configuration import dotdictify + + # Recreate objects in worker process + configuration = dotdictify(configuration_dict) + + return DockerTestServerspecTaskLoader.task_run(dockerfile, configuration, task) + +def task_runner_wrapper(func, args, task): + """ + Wrapper for task runner + """ + return BaseTaskLoader.task_runner(func, args, task) + class DockerTestServerspecTaskLoader(BaseDockerTaskLoader): + cmd_options = () def generate_task_list(self, dockerfile_list): """ @@ -33,11 +52,14 @@ def generate_task_list(self, dockerfile_list): """ tasklist = [] + # Convert configuration to dict for serialization + configuration_dict = self.configuration.to_dict() + for dockerfile in dockerfile_list: task = { 'name': 'DockerTestServerspec|%s' % dockerfile['image']['fullname'], 'title': DockerTestServerspecTaskLoader.task_title, - 'actions': [(BaseTaskLoader.task_runner, [DockerTestServerspecTaskLoader.task_run, [dockerfile, self.configuration]])], + 'actions': [(task_runner_wrapper, [task_run_wrapper, [dockerfile, configuration_dict]])], 'task_dep': [] } @@ -65,8 +87,8 @@ def task_run(dockerfile, configuration, task): # check if dockerfile is symlink, skipping tests if just a duplicate image # image is using the same hashes if dockerfile['image']['duplicate']: - print ' Docker image %s is build from symlink and duplicate of %s' % (dockerfile['image']['fullname'], dockerfile['image']['from']) - print ' -> skipping tests' + print(') Docker image %s is build from symlink and duplicate of %s' % (dockerfile['image']['fullname'], dockerfile['image']['from'])) + print(') -> skipping tests') BaseTaskLoader.set_task_status(task, 'skipped (symlink)', 'skipped') return True @@ -83,7 +105,7 @@ def task_run(dockerfile, configuration, task): # create dockerfile tmp_suffix = '.%s_%s_%s.tmp' % (dockerfile['image']['repository'], dockerfile['image']['imageName'], dockerfile['image']['tag']) tmp_suffix = tmp_suffix.replace('/', '_') - test_dockerfile = tempfile.NamedTemporaryFile(prefix='Dockerfile.', suffix=tmp_suffix, dir=configuration.get('serverspecPath'), bufsize=0, delete=False) + test_dockerfile = tempfile.NamedTemporaryFile(prefix='Dockerfile.', suffix=tmp_suffix, dir=configuration.get('serverspecPath'), delete=False) # serverspec conf serverspec_conf = DockerTestServerspecTaskLoader.generate_serverspec_configuration( @@ -95,7 +117,7 @@ def task_run(dockerfile, configuration, task): # serverspec options serverspec_opts = [] - serverspec_opts.extend([spec_path, dockerfile['image']['fullname'], base64.b64encode(json.dumps(serverspec_conf)), os.path.basename(test_dockerfile.name)]) + serverspec_opts.extend([spec_path, dockerfile['image']['fullname'], base64.b64encode(json.dumps(serverspec_conf).encode('utf-8')).decode('utf-8'), os.path.basename(test_dockerfile.name)]) # dockerfile content dockerfile_content = DockerTestServerspecTaskLoader.generate_dockerfile( @@ -107,24 +129,24 @@ def task_run(dockerfile, configuration, task): # DryRun if configuration.get('dryRun'): if not os.path.isfile(spec_abs_path): - print ' no tests found' - - print ' image: %s' % (dockerfile['image']['fullname']) - print ' path: %s' % (spec_path) - print ' args: %s' % (' '.join(serverspec_opts)) - print '' - print 'spec configuration:' - print '-------------------' - print json.dumps(serverspec_conf, indent=4, sort_keys=True) - print '' - print 'Dockerfile:' - print '-----------' - print dockerfile_content + print(') no tests found') + + print(') image: %s' % (dockerfile['image']['fullname'])) + print(') path: %s' % (spec_path)) + print(') args: %s' % (' '.join(serverspec_opts))) + print(')') + print(')spec configuration:') + print(')-------------------') + print(json.dumps(serverspec_conf, indent=4, sort_keys=True)) + print(')') + print(')Dockerfile:') + print(')-----------') + print(dockerfile_content) return True # check if we have any tests if not os.path.isfile(spec_abs_path): - print ' no tests defined (%s)' % (spec_path) + print(') no tests defined (%s)' % (spec_path)) BaseTaskLoader.set_task_status(task, 'skipped (no test)', 'skipped') return True @@ -133,26 +155,25 @@ def task_run(dockerfile, configuration, task): cmd.extend(serverspec_opts) # create Dockerfile - with open(test_dockerfile.name, mode='w', buffering=0) as f: + with open(test_dockerfile.name, mode='w') as f: f.write(dockerfile_content) f.flush() os.fsync(f.fileno()) - f.close() test_status = False for retry_count in range(0, configuration.get('retry')): try: test_status = Command.execute(cmd, cwd=configuration.get('serverspecPath')) except Exception as e: - print e + print(e) pass if test_status: break elif retry_count < (configuration.get('retry') - 1): - print ' failed, retrying... (try %s)' % (retry_count + 1) + print(') failed, retrying... (try %s)' % (retry_count + 1)) else: - print ' failed, giving up' + print(') failed, giving up') return test_status diff --git a/bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py b/bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py index a9bfe37db..49d34553d 100644 --- a/bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py +++ b/bin/webdevops/taskloader/DockerTestTestinfraTaskLoader.py @@ -1,4 +1,4 @@ -#!/usr/bin/env/python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # (c) 2016 WebDevOps.io @@ -25,7 +25,26 @@ from doit.task import dict_to_task import pytest +# Define standalone functions for better multiprocessing compatibility +def task_run_wrapper(dockerfile, configuration_dict, task): + """ + Wrapper for docker test testinfra task that recreates objects in worker process + """ + from webdevops.Configuration import dotdictify + + # Recreate objects in worker process + configuration = dotdictify(configuration_dict) + + return DockerTestTestinfraTaskLoader.task_run(dockerfile, configuration, task) + +def task_runner_wrapper(func, args, task): + """ + Wrapper for task runner + """ + return BaseTaskLoader.task_runner(func, args, task) + class DockerTestTestinfraTaskLoader(BaseDockerTaskLoader): + cmd_options = () def generate_task_list(self, dockerfile_list): """ @@ -33,11 +52,14 @@ def generate_task_list(self, dockerfile_list): """ tasklist = [] + # Convert configuration to dict for serialization + configuration_dict = self.configuration.to_dict() + for dockerfile in dockerfile_list: task = { 'name': 'DockerTestTestinfra|%s' % dockerfile['image']['fullname'], 'title': DockerTestTestinfraTaskLoader.task_title, - 'actions': [(BaseTaskLoader.task_runner, [DockerTestTestinfraTaskLoader.task_run, [dockerfile, self.configuration]])], + 'actions': [(task_runner_wrapper, [task_run_wrapper, [dockerfile, configuration_dict]])], 'task_dep': [] } @@ -70,8 +92,8 @@ def task_run(dockerfile, configuration, task): test_opts.extend(['-v']) if configuration.get('dryRun'): - print ' image: %s' % (dockerfile['image']['fullname']) - print ' args: %s' % (' '.join(test_opts)) + print(') image: %s' % (dockerfile['image']['fullname'])) + print(') args: %s' % (' '.join(test_opts))) return True exitcode = pytest.main(test_opts, plugins=[TestinfraDockerPlugin(configuration=configuration, docker_image=dockerfile['image']['fullname'])]) diff --git a/bin/webdevops/taskloader/__init__.py b/bin/webdevops/taskloader/__init__.py index e333eaa0a..cf82b5628 100644 --- a/bin/webdevops/taskloader/__init__.py +++ b/bin/webdevops/taskloader/__init__.py @@ -18,13 +18,13 @@ # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from BaseTaskLoader import BaseTaskLoader -from BaseDockerTaskLoader import BaseDockerTaskLoader -from DockerBuildTaskLoader import DockerBuildTaskLoader -from DockerPushTaskLoader import DockerPushTaskLoader -from DockerPullTaskLoader import DockerPullTaskLoader -from DockerTestTestinfraTaskLoader import DockerTestTestinfraTaskLoader -from DockerTestServerspecTaskLoader import DockerTestServerspecTaskLoader +from .BaseTaskLoader import BaseTaskLoader +from .BaseDockerTaskLoader import BaseDockerTaskLoader +from .DockerBuildTaskLoader import DockerBuildTaskLoader +from .DockerPushTaskLoader import DockerPushTaskLoader +from .DockerPullTaskLoader import DockerPullTaskLoader +from .DockerTestTestinfraTaskLoader import DockerTestTestinfraTaskLoader +from .DockerTestServerspecTaskLoader import DockerTestServerspecTaskLoader __all__ = [ 'BaseTaskLoader', diff --git a/bin/webdevops/testinfra/TestinfraDockerPlugin.py b/bin/webdevops/testinfra/TestinfraDockerPlugin.py index ffe9ed322..7af0d1f45 100644 --- a/bin/webdevops/testinfra/TestinfraDockerPlugin.py +++ b/bin/webdevops/testinfra/TestinfraDockerPlugin.py @@ -1,4 +1,4 @@ -#!/usr/bin/env/python +#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # (c) 2016 WebDevOps.io diff --git a/bin/webdevops/testinfra/__init__.py b/bin/webdevops/testinfra/__init__.py index 7a919e2a8..0058f9b56 100644 --- a/bin/webdevops/testinfra/__init__.py +++ b/bin/webdevops/testinfra/__init__.py @@ -18,7 +18,7 @@ # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from TestinfraDockerPlugin import TestinfraDockerPlugin +from .TestinfraDockerPlugin import TestinfraDockerPlugin __all__ = [ 'TestinfraDockerPlugin', diff --git a/conftest.py b/conftest.py index b26fcf303..3aea564c9 100644 --- a/conftest.py +++ b/conftest.py @@ -7,9 +7,9 @@ test_conf_app_path = os.path.join(test_conf_path, 'app') # Use testinfra to get a handy function to run commands locally -check_output = testinfra.get_backend( +check_output = testinfra.get_host( "local://" -).get_module("Command").check_output +).run @pytest.fixture @@ -35,15 +35,14 @@ def TestinfraBackend(request): docker_image ) - def teardown(): - check_output("docker rm -f %s", docker_id) - - # Destroy the container at the end of the fixture life - request.addfinalizer(teardown) - # wait for getting the image up if docker_sleeptime: time.sleep(docker_sleeptime) # Return a dynamic created backend - return testinfra.get_backend("docker://" + docker_id) + backend = testinfra.get_host("docker://" + docker_id) + + yield backend + + # Cleanup after the test + check_output("docker rm -f %s", docker_id) diff --git a/documentation/docs/resources/images/docker-image-layout.gv b/documentation/docs/resources/images/docker-image-layout.gv index aa88dd565..a6e167892 100644 --- a/documentation/docs/resources/images/docker-image-layout.gv +++ b/documentation/docs/resources/images/docker-image-layout.gv @@ -2,277 +2,57 @@ digraph webdevops { graph [bgcolor=white fontcolor=black fontsize=16 rankdir=TP] node [color=white fillcolor="#E1E1E1" fontcolor=black fontname=Helvetica shape=box style=filled] edge [arrowhead=open color=black fontcolor=white fontname=Courier fontsize=12 style=dashed] - label = "\n\nWebdevops Images\n2017-02-18" - newrank=true; - subgraph cluster_php { - graph [fillcolor="#c0e5a8" style=filled] - node [color=black fillcolor="#78c445" fontcolor=black fontname=Helvetica shape=box style=filled] - label = "PHP images" - "webdevops/php-dev" -> "webdevops/php-nginx-dev" - "webdevops/php" -> "webdevops/php-nginx" - "webdevops/php-dev" -> "webdevops/php-apache-dev" - "webdevops/php" - "webdevops/php" -> "webdevops/php-apache" - "webdevops/php" -> "webdevops/php-dev" - } - subgraph cluster_service { - graph [fillcolor="#fbd3b5" style=filled] - node [color=black fillcolor="#ffa35f" fontcolor=black fontname=Helvetica shape=box style=filled] - label = "Service images" - "webdevops/varnish" - "webdevops/nginx" -> "webdevops/nginx-dev" - "webdevops/ssh" - "webdevops/mail-sandbox" - "webdevops/vsftp" - "webdevops/nginx" - "webdevops/apache" - "webdevops/samson-deployment" - "webdevops/apache" -> "webdevops/apache-dev" - "webdevops/postfix" - } - subgraph cluster_official { - graph [fillcolor=gray style=dashed] - node [color=black fillcolor="#e1e1e1" fontcolor=black fontname=Helvetica shape=box style=filled] - label = "Official images (Docker hub)" - busybox - centos - java - ubuntu - debian - alpine - solr - } - subgraph cluster_application { - graph [fillcolor="#f2e3b5" style=filled] - node [color=black fillcolor="#e5b931" fontcolor=black fontname=Helvetica shape=box style=filled] - label = "Application images" - "webdevops/typo3" - "webdevops/piwik" - "webdevops/typo3-solr" -> "webdevops/typo3-solr" - } - subgraph cluster_base { - graph [fillcolor=gray style=filled] - node [color=black fillcolor="#e1e1e1" fontcolor=black fontname=Helvetica shape=box style=filled] - label = "Base images" - "webdevops/bootstrap" -> "webdevops/ansible" - "webdevops/base" -> "webdevops/base-app" - "webdevops/storage" - "webdevops/bootstrap" - "webdevops/bootstrap" -> "webdevops/base" - } - subgraph cluster_upstream { - graph [fillcolor=gray style=dashed] - node [color=black fillcolor="#e1e1e1" fontcolor=black fontname=Helvetica shape=box style=filled] - label = "Upstream images (Docker hub)" - "zendesk/samson" - "guywithnose/solr" - } - subgraph cluster_hhvm { - graph [fillcolor="#c1c3f2" style=filled] - node [color=black fillcolor="#7f84f1" fontcolor=black fontname=Helvetica shape=box style=filled] - label = "HHVM images" - "webdevops/hhvm" -> "webdevops/hhvm-nginx" - "webdevops/hhvm" -> "webdevops/hhvm-apache" - "webdevops/hhvm" - } - subgraph cluster_tools { - graph [fillcolor="#c0e5a8" style=filled] - node [color=black fillcolor="#78c445" fontcolor=black fontname=Helvetica shape=box style=filled] - label = "Tools images" - "webdevops/sphinx" - "webdevops/liquibase" - "webdevops/certbot" - } - subgraph "cluster_php-dev" { - graph [fillcolor="#c0e5a8" style=filled] - node [color=black fillcolor="#78c445" fontcolor=black fontname=Helvetica shape=box style=filled] - label = "PHP development images" - } - "webdevops/base-app" -> "webdevops/php" - "webdevops/php-apache" -> "webdevops/typo3" - "webdevops/base-app" -> "webdevops/ssh" - "webdevops/php-nginx" -> "webdevops/mail-sandbox" - "webdevops/base-app" -> "webdevops/postfix" - "webdevops/base" -> "webdevops/apache" - busybox -> "webdevops/storage" - "webdevops/base" -> "webdevops/nginx" - "webdevops/base" -> "webdevops/vsftp" - java -> "webdevops/liquibase" - ubuntu -> "webdevops/bootstrap" - "webdevops/bootstrap" -> "webdevops/sphinx" - "webdevops/php-nginx" -> "webdevops/piwik" - "webdevops/base" -> "webdevops/varnish" - "zendesk/samson" -> "webdevops/samson-deployment" - "webdevops/bootstrap" -> "webdevops/certbot" - "webdevops/base-app" -> "webdevops/hhvm" - { "busybox" -> "webdevops/ansible" [style=invis] } - { "busybox" -> "webdevops/base-app" [style=invis] } - { "busybox" -> "webdevops/storage" [style=invis] } - { "busybox" -> "webdevops/bootstrap" [style=invis] } - { "busybox" -> "webdevops/base" [style=invis] } - { "centos" -> "webdevops/ansible" [style=invis] } - { "centos" -> "webdevops/base-app" [style=invis] } - { "centos" -> "webdevops/storage" [style=invis] } - { "centos" -> "webdevops/bootstrap" [style=invis] } - { "centos" -> "webdevops/base" [style=invis] } - { "zendesk/samson" -> "webdevops/ansible" [style=invis] } - { "zendesk/samson" -> "webdevops/base-app" [style=invis] } - { "zendesk/samson" -> "webdevops/storage" [style=invis] } - { "zendesk/samson" -> "webdevops/bootstrap" [style=invis] } - { "zendesk/samson" -> "webdevops/base" [style=invis] } - { "java" -> "webdevops/ansible" [style=invis] } - { "java" -> "webdevops/base-app" [style=invis] } - { "java" -> "webdevops/storage" [style=invis] } - { "java" -> "webdevops/bootstrap" [style=invis] } - { "java" -> "webdevops/base" [style=invis] } - { "ubuntu" -> "webdevops/ansible" [style=invis] } - { "ubuntu" -> "webdevops/base-app" [style=invis] } - { "ubuntu" -> "webdevops/storage" [style=invis] } - { "ubuntu" -> "webdevops/bootstrap" [style=invis] } - { "ubuntu" -> "webdevops/base" [style=invis] } - { "debian" -> "webdevops/ansible" [style=invis] } - { "debian" -> "webdevops/base-app" [style=invis] } - { "debian" -> "webdevops/storage" [style=invis] } - { "debian" -> "webdevops/bootstrap" [style=invis] } - { "debian" -> "webdevops/base" [style=invis] } - { "alpine" -> "webdevops/ansible" [style=invis] } - { "alpine" -> "webdevops/base-app" [style=invis] } - { "alpine" -> "webdevops/storage" [style=invis] } - { "alpine" -> "webdevops/bootstrap" [style=invis] } - { "alpine" -> "webdevops/base" [style=invis] } - { "solr" -> "webdevops/ansible" [style=invis] } - { "solr" -> "webdevops/base-app" [style=invis] } - { "solr" -> "webdevops/storage" [style=invis] } - { "solr" -> "webdevops/bootstrap" [style=invis] } - { "solr" -> "webdevops/base" [style=invis] } - { "guywithnose/solr" -> "webdevops/ansible" [style=invis] } - { "guywithnose/solr" -> "webdevops/base-app" [style=invis] } - { "guywithnose/solr" -> "webdevops/storage" [style=invis] } - { "guywithnose/solr" -> "webdevops/bootstrap" [style=invis] } - { "guywithnose/solr" -> "webdevops/base" [style=invis] } - { "webdevops/ansible" -> "webdevops/typo3" [style=invis] } - { "webdevops/ansible" -> "webdevops/php-nginx-dev" [style=invis] } - { "webdevops/ansible" -> "webdevops/piwik" [style=invis] } - { "webdevops/ansible" -> "webdevops/php-nginx" [style=invis] } - { "webdevops/ansible" -> "webdevops/sphinx" [style=invis] } - { "webdevops/ansible" -> "webdevops/varnish" [style=invis] } - { "webdevops/ansible" -> "webdevops/nginx-dev" [style=invis] } - { "webdevops/ansible" -> "webdevops/ssh" [style=invis] } - { "webdevops/ansible" -> "webdevops/mail-sandbox" [style=invis] } - { "webdevops/ansible" -> "webdevops/vsftp" [style=invis] } - { "webdevops/ansible" -> "webdevops/nginx" [style=invis] } - { "webdevops/ansible" -> "webdevops/apache" [style=invis] } - { "webdevops/ansible" -> "webdevops/hhvm-nginx" [style=invis] } - { "webdevops/ansible" -> "webdevops/php-apache-dev" [style=invis] } - { "webdevops/ansible" -> "webdevops/php" [style=invis] } - { "webdevops/ansible" -> "webdevops/php-apache" [style=invis] } - { "webdevops/ansible" -> "webdevops/php-dev" [style=invis] } - { "webdevops/ansible" -> "webdevops/samson-deployment" [style=invis] } - { "webdevops/ansible" -> "webdevops/apache-dev" [style=invis] } - { "webdevops/ansible" -> "webdevops/typo3-solr" [style=invis] } - { "webdevops/ansible" -> "webdevops/hhvm-apache" [style=invis] } - { "webdevops/ansible" -> "webdevops/postfix" [style=invis] } - { "webdevops/ansible" -> "webdevops/liquibase" [style=invis] } - { "webdevops/ansible" -> "webdevops/hhvm" [style=invis] } - { "webdevops/ansible" -> "webdevops/certbot" [style=invis] } - { "webdevops/base-app" -> "webdevops/typo3" [style=invis] } - { "webdevops/base-app" -> "webdevops/php-nginx-dev" [style=invis] } - { "webdevops/base-app" -> "webdevops/piwik" [style=invis] } - { "webdevops/base-app" -> "webdevops/php-nginx" [style=invis] } - { "webdevops/base-app" -> "webdevops/sphinx" [style=invis] } - { "webdevops/base-app" -> "webdevops/varnish" [style=invis] } - { "webdevops/base-app" -> "webdevops/nginx-dev" [style=invis] } - { "webdevops/base-app" -> "webdevops/ssh" [style=invis] } - { "webdevops/base-app" -> "webdevops/mail-sandbox" [style=invis] } - { "webdevops/base-app" -> "webdevops/vsftp" [style=invis] } - { "webdevops/base-app" -> "webdevops/nginx" [style=invis] } - { "webdevops/base-app" -> "webdevops/apache" [style=invis] } - { "webdevops/base-app" -> "webdevops/hhvm-nginx" [style=invis] } - { "webdevops/base-app" -> "webdevops/php-apache-dev" [style=invis] } - { "webdevops/base-app" -> "webdevops/php" [style=invis] } - { "webdevops/base-app" -> "webdevops/php-apache" [style=invis] } - { "webdevops/base-app" -> "webdevops/php-dev" [style=invis] } - { "webdevops/base-app" -> "webdevops/samson-deployment" [style=invis] } - { "webdevops/base-app" -> "webdevops/apache-dev" [style=invis] } - { "webdevops/base-app" -> "webdevops/typo3-solr" [style=invis] } - { "webdevops/base-app" -> "webdevops/hhvm-apache" [style=invis] } - { "webdevops/base-app" -> "webdevops/postfix" [style=invis] } - { "webdevops/base-app" -> "webdevops/liquibase" [style=invis] } - { "webdevops/base-app" -> "webdevops/hhvm" [style=invis] } - { "webdevops/base-app" -> "webdevops/certbot" [style=invis] } - { "webdevops/storage" -> "webdevops/typo3" [style=invis] } - { "webdevops/storage" -> "webdevops/php-nginx-dev" [style=invis] } - { "webdevops/storage" -> "webdevops/piwik" [style=invis] } - { "webdevops/storage" -> "webdevops/php-nginx" [style=invis] } - { "webdevops/storage" -> "webdevops/sphinx" [style=invis] } - { "webdevops/storage" -> "webdevops/varnish" [style=invis] } - { "webdevops/storage" -> "webdevops/nginx-dev" [style=invis] } - { "webdevops/storage" -> "webdevops/ssh" [style=invis] } - { "webdevops/storage" -> "webdevops/mail-sandbox" [style=invis] } - { "webdevops/storage" -> "webdevops/vsftp" [style=invis] } - { "webdevops/storage" -> "webdevops/nginx" [style=invis] } - { "webdevops/storage" -> "webdevops/apache" [style=invis] } - { "webdevops/storage" -> "webdevops/hhvm-nginx" [style=invis] } - { "webdevops/storage" -> "webdevops/php-apache-dev" [style=invis] } - { "webdevops/storage" -> "webdevops/php" [style=invis] } - { "webdevops/storage" -> "webdevops/php-apache" [style=invis] } - { "webdevops/storage" -> "webdevops/php-dev" [style=invis] } - { "webdevops/storage" -> "webdevops/samson-deployment" [style=invis] } - { "webdevops/storage" -> "webdevops/apache-dev" [style=invis] } - { "webdevops/storage" -> "webdevops/typo3-solr" [style=invis] } - { "webdevops/storage" -> "webdevops/hhvm-apache" [style=invis] } - { "webdevops/storage" -> "webdevops/postfix" [style=invis] } - { "webdevops/storage" -> "webdevops/liquibase" [style=invis] } - { "webdevops/storage" -> "webdevops/hhvm" [style=invis] } - { "webdevops/storage" -> "webdevops/certbot" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/typo3" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/php-nginx-dev" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/piwik" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/php-nginx" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/sphinx" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/varnish" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/nginx-dev" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/ssh" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/mail-sandbox" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/vsftp" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/nginx" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/apache" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/hhvm-nginx" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/php-apache-dev" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/php" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/php-apache" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/php-dev" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/samson-deployment" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/apache-dev" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/typo3-solr" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/hhvm-apache" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/postfix" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/liquibase" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/hhvm" [style=invis] } - { "webdevops/bootstrap" -> "webdevops/certbot" [style=invis] } - { "webdevops/base" -> "webdevops/typo3" [style=invis] } - { "webdevops/base" -> "webdevops/php-nginx-dev" [style=invis] } - { "webdevops/base" -> "webdevops/piwik" [style=invis] } - { "webdevops/base" -> "webdevops/php-nginx" [style=invis] } - { "webdevops/base" -> "webdevops/sphinx" [style=invis] } - { "webdevops/base" -> "webdevops/varnish" [style=invis] } - { "webdevops/base" -> "webdevops/nginx-dev" [style=invis] } - { "webdevops/base" -> "webdevops/ssh" [style=invis] } - { "webdevops/base" -> "webdevops/mail-sandbox" [style=invis] } - { "webdevops/base" -> "webdevops/vsftp" [style=invis] } - { "webdevops/base" -> "webdevops/nginx" [style=invis] } - { "webdevops/base" -> "webdevops/apache" [style=invis] } - { "webdevops/base" -> "webdevops/hhvm-nginx" [style=invis] } - { "webdevops/base" -> "webdevops/php-apache-dev" [style=invis] } - { "webdevops/base" -> "webdevops/php" [style=invis] } - { "webdevops/base" -> "webdevops/php-apache" [style=invis] } - { "webdevops/base" -> "webdevops/php-dev" [style=invis] } - { "webdevops/base" -> "webdevops/samson-deployment" [style=invis] } - { "webdevops/base" -> "webdevops/apache-dev" [style=invis] } - { "webdevops/base" -> "webdevops/typo3-solr" [style=invis] } - { "webdevops/base" -> "webdevops/hhvm-apache" [style=invis] } - { "webdevops/base" -> "webdevops/postfix" [style=invis] } - { "webdevops/base" -> "webdevops/liquibase" [style=invis] } - { "webdevops/base" -> "webdevops/hhvm" [style=invis] } - { "webdevops/base" -> "webdevops/certbot" [style=invis] } -} \ No newline at end of file +label = "\n\nWebdevops Images\n2025-09-02"newrank=true; "webdevops/toolbox" + "webdevops/dockerfile-build-env" + subgraph cluster_official { + graph [fillcolor=gray style=dashed] + node [color=black fillcolor="#e1e1e1" fontcolor=black fontname=Helvetica shape=box style=filled] + label = "Official images (Docker hub)" alpine + ubuntu + php + } + subgraph cluster_upstream { + graph [fillcolor=gray style=dashed] + node [color=black fillcolor="#e1e1e1" fontcolor=black fontname=Helvetica shape=box style=filled] + label = "Upstream images (Docker hub)" } + subgraph cluster_base { + graph [fillcolor=gray style=filled] + node [color=black fillcolor="#e1e1e1" fontcolor=black fontname=Helvetica shape=box style=filled] + label = "Base images" "webdevops/bootstrap" -> "webdevops/base" + "webdevops/base" -> "webdevops/base-app" + "webdevops/bootstrap" + "webdevops/storage" + } + subgraph cluster_php { + graph [fillcolor="#c0e5a8" style=filled] + node [color=black fillcolor="#78c445" fontcolor=black fontname=Helvetica shape=box style=filled] + label = "PHP images" "webdevops/php-official" -> "webdevops/php" + "webdevops/php" -> "webdevops/php-apache" + "webdevops/php-apache" -> "webdevops/php-apache-dev" + "webdevops/php-official" + "webdevops/php" -> "webdevops/php-nginx" + "webdevops/php-nginx" -> "webdevops/php-nginx-dev" + "webdevops/php" -> "webdevops/php-dev" + } + subgraph "cluster_php-dev" { + graph [fillcolor="#c0e5a8" style=filled] + node [color=black fillcolor="#78c445" fontcolor=black fontname=Helvetica shape=box style=filled] + label = "PHP development images" } + subgraph cluster_service { + graph [fillcolor="#fbd3b5" style=filled] + node [color=black fillcolor="#ffa35f" fontcolor=black fontname=Helvetica shape=box style=filled] + label = "Service images" "webdevops/ssh" + "webdevops/nginx" + "webdevops/vsftp" + "webdevops/apache" + } + ubuntu -> "webdevops/bootstrap" + php -> "webdevops/php-official" + alpine -> "webdevops/storage" + "webdevops/base-app" -> "webdevops/ssh" + "webdevops/base" -> "webdevops/nginx" + alpine -> "webdevops/toolbox" + "webdevops/base-app" -> "webdevops/dockerfile-build-env" + "webdevops/base" -> "webdevops/vsftp" + "webdevops/base" -> "webdevops/apache" +{ "webdevops/base" -> "webdevops/php" [style=invis] }{ "webdevops/base" -> "webdevops/php-apache" [style=invis] }{ "webdevops/base" -> "webdevops/php-apache-dev" [style=invis] }{ "webdevops/base" -> "webdevops/php-official" [style=invis] }{ "webdevops/base" -> "webdevops/php-nginx" [style=invis] }{ "webdevops/base" -> "webdevops/php-nginx-dev" [style=invis] }{ "webdevops/base" -> "webdevops/ssh" [style=invis] }{ "webdevops/base" -> "webdevops/nginx" [style=invis] }{ "webdevops/base" -> "webdevops/vsftp" [style=invis] }{ "webdevops/base" -> "webdevops/apache" [style=invis] }{ "webdevops/base" -> "webdevops/php-dev" [style=invis] }{ "webdevops/base-app" -> "webdevops/php" [style=invis] }{ "webdevops/base-app" -> "webdevops/php-apache" [style=invis] }{ "webdevops/base-app" -> "webdevops/php-apache-dev" [style=invis] }{ "webdevops/base-app" -> "webdevops/php-official" [style=invis] }{ "webdevops/base-app" -> "webdevops/php-nginx" [style=invis] }{ "webdevops/base-app" -> "webdevops/php-nginx-dev" [style=invis] }{ "webdevops/base-app" -> "webdevops/ssh" [style=invis] }{ "webdevops/base-app" -> "webdevops/nginx" [style=invis] }{ "webdevops/base-app" -> "webdevops/vsftp" [style=invis] }{ "webdevops/base-app" -> "webdevops/apache" [style=invis] }{ "webdevops/base-app" -> "webdevops/php-dev" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/php" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/php-apache" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/php-apache-dev" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/php-official" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/php-nginx" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/php-nginx-dev" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/ssh" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/nginx" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/vsftp" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/apache" [style=invis] }{ "webdevops/bootstrap" -> "webdevops/php-dev" [style=invis] }{ "webdevops/storage" -> "webdevops/php" [style=invis] }{ "webdevops/storage" -> "webdevops/php-apache" [style=invis] }{ "webdevops/storage" -> "webdevops/php-apache-dev" [style=invis] }{ "webdevops/storage" -> "webdevops/php-official" [style=invis] }{ "webdevops/storage" -> "webdevops/php-nginx" [style=invis] }{ "webdevops/storage" -> "webdevops/php-nginx-dev" [style=invis] }{ "webdevops/storage" -> "webdevops/ssh" [style=invis] }{ "webdevops/storage" -> "webdevops/nginx" [style=invis] }{ "webdevops/storage" -> "webdevops/vsftp" [style=invis] }{ "webdevops/storage" -> "webdevops/apache" [style=invis] }{ "webdevops/storage" -> "webdevops/php-dev" [style=invis] }{ "alpine" -> "webdevops/base" [style=invis] }{ "alpine" -> "webdevops/base-app" [style=invis] }{ "alpine" -> "webdevops/bootstrap" [style=invis] }{ "alpine" -> "webdevops/storage" [style=invis] }{ "ubuntu" -> "webdevops/base" [style=invis] }{ "ubuntu" -> "webdevops/base-app" [style=invis] }{ "ubuntu" -> "webdevops/bootstrap" [style=invis] }{ "ubuntu" -> "webdevops/storage" [style=invis] }{ "php" -> "webdevops/base" [style=invis] }{ "php" -> "webdevops/base-app" [style=invis] }{ "php" -> "webdevops/bootstrap" [style=invis] }{ "php" -> "webdevops/storage" [style=invis] }} diff --git a/provisioning/base/general/bin/provision.py b/provisioning/base/general/bin/provision.py index d367c2812..b50453aff 100755 --- a/provisioning/base/general/bin/provision.py +++ b/provisioning/base/general/bin/provision.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import os import argparse @@ -158,7 +158,7 @@ def actionPlaybook(args): playbook = buildPlaybookFromArgs(args) if playbook: - print playbook + print(playbook) else: sys.exit(1) @@ -171,7 +171,7 @@ def actionList(args): for tag in args.tags: if tag in json: for role in json[tag]: - print role + print(role) @@ -200,10 +200,10 @@ def actionSummary(args): if roleList: maxLength = len(max(roleList.keys(), key=len)) - print "Roles in " + tag + ":" + print("Roles in " + tag + ":") for role in roleList: - print ' - ' + role.ljust(maxLength, ' ') + ' [priority: ' + str(roleList[role]['priority']) + ']' - print '' + print(' - ' + role.ljust(maxLength, ' ') + ' [priority: ' + str(roleList[role]['priority']) + ']') + print('') diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..e986882a4 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,14 @@ +[tool:pytest] +testpaths = tests +python_files = test_*.py *_test.py +python_classes = Test* +python_functions = test_* +addopts = + --strict-markers + --disable-warnings + --tb=short +markers = + docker_images: mark test to run against specific docker images + docker_loop: mark test to run in loop mode + docker_images_blacklist: mark test to exclude specific docker images + destructive: mark test as destructive (requires function scope) diff --git a/tests/testinfra/conftest.py b/tests/testinfra/conftest.py new file mode 100644 index 000000000..b7d40644e --- /dev/null +++ b/tests/testinfra/conftest.py @@ -0,0 +1,33 @@ +import pytest +import sys +import os + +# Add the bin directory to the Python path +sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'bin')) +from webdevops.testinfra import TestinfraDockerPlugin + +def pytest_configure(config): + """ + Register custom pytest marks + """ + config.addinivalue_line( + "markers", "docker_images: mark test to run against specific docker images" + ) + config.addinivalue_line( + "markers", "docker_loop: mark test to run in loop mode" + ) + config.addinivalue_line( + "markers", "docker_images_blacklist: mark test to exclude specific docker images" + ) + config.addinivalue_line( + "markers", "destructive: mark test as destructive (requires function scope)" + ) + +def pytest_generate_tests(metafunc): + """ + Generate tests using TestinfraDockerPlugin + """ + if "TestinfraBackend" in metafunc.fixturenames: + # This will be handled by the TestinfraDockerPlugin when running through the console + # For direct pytest runs, we need to handle this differently + pass diff --git a/tests/testinfra/test_liquibase.py b/tests/testinfra/test_liquibase.py index 741d79a2d..3e8b94edd 100644 --- a/tests/testinfra/test_liquibase.py +++ b/tests/testinfra/test_liquibase.py @@ -4,7 +4,9 @@ @pytest.mark.docker_loop() @pytest.mark.docker_images('webdevops/liquibase') -def test_liquibase_cmd(Command, File): +def test_liquibase_cmd(TestinfraBackend): + Command = TestinfraBackend.run + File = TestinfraBackend.file assert File('/usr/local/bin/liquibase').exists assert File('/usr/local/bin/liquibase').mode == 0o777 From 34c99cab9c8fd1936740fc322cc8e3d75d6b1ec6 Mon Sep 17 00:00:00 2001 From: Nemanja Cimbaljevic Date: Tue, 2 Sep 2025 01:57:29 +0200 Subject: [PATCH 2/3] updated coding standards --- bin/webdevops/taskloader/DockerBuildTaskLoader.py | 14 +++++++------- bin/webdevops/testinfra/TestinfraDockerPlugin.py | 6 +++--- conftest.py | 4 ++-- tests/testinfra/conftest.py | 2 +- tests/testinfra/test_liquibase.py | 12 ++++++------ 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/bin/webdevops/taskloader/DockerBuildTaskLoader.py b/bin/webdevops/taskloader/DockerBuildTaskLoader.py index 897c9bae2..7170390eb 100644 --- a/bin/webdevops/taskloader/DockerBuildTaskLoader.py +++ b/bin/webdevops/taskloader/DockerBuildTaskLoader.py @@ -23,7 +23,7 @@ from .BaseDockerTaskLoader import BaseDockerTaskLoader from webdevops import DockerfileUtility -def run_dependency_puller_task(dockerfileList, configuration_dict, task): +def run_dependency_puller_task(dockerfile_list, configuration_dict, task): """ Standalone function for dependency puller task """ @@ -34,7 +34,7 @@ def run_dependency_puller_task(dockerfileList, configuration_dict, task): docker_client = DockerCliClient() configuration = dotdictify(configuration_dict) - return DockerBuildTaskLoader.task_dependency_puller(docker_client, dockerfileList, configuration, task) + return DockerBuildTaskLoader.task_dependency_puller(docker_client, dockerfile_list, configuration, task) def run_build_task(dockerfile, configuration_dict, task): """ @@ -52,7 +52,7 @@ def run_build_task(dockerfile, configuration_dict, task): class DockerBuildTaskLoader(BaseDockerTaskLoader): cmd_options = () - def generate_task_list(self, dockerfileList): + def generate_task_list(self, dockerfile_list): """ Generate task list for docker build """ @@ -65,13 +65,13 @@ def generate_task_list(self, dockerfileList): task = { 'name': 'DockerBuild|DependencyPuller', 'title': DockerBuildTaskLoader.task_title_dependency_puller, - 'actions': [(run_dependency_puller_task, [dockerfileList, configuration_dict])], + 'actions': [(run_dependency_puller_task, [dockerfile_list, configuration_dict])], 'task_dep': [] } tasklist.append(task) # TASK: dockerfile build - for dockerfile in dockerfileList: + for dockerfile in dockerfile_list: task = { 'name': 'DockerBuild|%s' % dockerfile['image']['fullname'], 'title': DockerBuildTaskLoader.task_title, @@ -97,7 +97,7 @@ def generate_task_list(self, dockerfileList): return tasklist @staticmethod - def task_dependency_puller(docker_client, dockerfileList, configuration, task): + def task_dependency_puller(docker_client, dockerfile_list, configuration, task): """ Pulls dependency images before building """ @@ -130,7 +130,7 @@ def pull_image(image): return True image_list = [] - for dockerfile in dockerfileList: + for dockerfile in dockerfile_list: # Pull base image (FROM: xxx) first if DockerfileUtility.check_if_base_image_needs_pull(dockerfile['image']['from'], configuration): image_list.append(dockerfile['image']['from']) diff --git a/bin/webdevops/testinfra/TestinfraDockerPlugin.py b/bin/webdevops/testinfra/TestinfraDockerPlugin.py index 7af0d1f45..ed521839c 100644 --- a/bin/webdevops/testinfra/TestinfraDockerPlugin.py +++ b/bin/webdevops/testinfra/TestinfraDockerPlugin.py @@ -80,7 +80,7 @@ def pytest_generate_tests(self, metafunc): """ Generate tests """ - if "TestinfraBackend" in metafunc.fixturenames: + if "testinfra_backend" in metafunc.fixturenames: images = [] # Lookup "docker_images" marker @@ -103,7 +103,7 @@ def pytest_generate_tests(self, metafunc): if marker is not None: images = ['{}#loop'.format(item) for item in images] - # If the test has a destructive marker, we scope TestinfraBackend + # If the test has a destructive marker, we scope testinfra_backend # at function level (i.e. executing for each test). If not we scope # at session level (i.e. all tests will share the same container) if getattr(metafunc.function, "destructive", None) is not None: @@ -112,5 +112,5 @@ def pytest_generate_tests(self, metafunc): scope = "session" metafunc.parametrize( - "TestinfraBackend", images, indirect=True, scope=scope + "testinfra_backend", images, indirect=True, scope=scope ) diff --git a/conftest.py b/conftest.py index 3aea564c9..1dac99312 100644 --- a/conftest.py +++ b/conftest.py @@ -13,8 +13,8 @@ @pytest.fixture -def TestinfraBackend(request): - # Override the TestinfraBackend fixture, +def testinfra_backend(request): + # Override the testinfra_backend fixture, # all testinfra fixtures (i.e. modules) depend on it. docker_command = '' diff --git a/tests/testinfra/conftest.py b/tests/testinfra/conftest.py index b7d40644e..09384702b 100644 --- a/tests/testinfra/conftest.py +++ b/tests/testinfra/conftest.py @@ -27,7 +27,7 @@ def pytest_generate_tests(metafunc): """ Generate tests using TestinfraDockerPlugin """ - if "TestinfraBackend" in metafunc.fixturenames: + if "testinfra_backend" in metafunc.fixturenames: # This will be handled by the TestinfraDockerPlugin when running through the console # For direct pytest runs, we need to handle this differently pass diff --git a/tests/testinfra/test_liquibase.py b/tests/testinfra/test_liquibase.py index 3e8b94edd..e46f87f03 100644 --- a/tests/testinfra/test_liquibase.py +++ b/tests/testinfra/test_liquibase.py @@ -4,13 +4,13 @@ @pytest.mark.docker_loop() @pytest.mark.docker_images('webdevops/liquibase') -def test_liquibase_cmd(TestinfraBackend): - Command = TestinfraBackend.run - File = TestinfraBackend.file - assert File('/usr/local/bin/liquibase').exists - assert File('/usr/local/bin/liquibase').mode == 0o777 +def test_liquibase_cmd(testinfra_backend): + command = testinfra_backend.run + file = testinfra_backend.file + assert file('/usr/local/bin/liquibase').exists + assert file('/usr/local/bin/liquibase').mode == 0o777 - cmd = Command("liquibase --version") + cmd = command("liquibase --version") assert cmd.rc == 0 assert 'Liquibase Version:' in cmd.stderr From d179977618cd92e25ef0c583dd8519f18c102e21 Mon Sep 17 00:00:00 2001 From: Nemanja Cimbaljevic Date: Tue, 2 Sep 2025 02:11:18 +0200 Subject: [PATCH 3/3] code optimization --- .../taskloader/DockerBuildTaskLoader.py | 85 +++++++++++-------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/bin/webdevops/taskloader/DockerBuildTaskLoader.py b/bin/webdevops/taskloader/DockerBuildTaskLoader.py index 7170390eb..6af81bc32 100644 --- a/bin/webdevops/taskloader/DockerBuildTaskLoader.py +++ b/bin/webdevops/taskloader/DockerBuildTaskLoader.py @@ -97,55 +97,68 @@ def generate_task_list(self, dockerfile_list): return tasklist @staticmethod - def task_dependency_puller(docker_client, dockerfile_list, configuration, task): + def _should_pull_image(image, configuration): """ - Pulls dependency images before building + Check if an image should be pulled based on configuration """ - def pull_image(image): - print(') -> Pull base image %s ' % image) - - if configuration.get('dryRun'): - return True - - pull_image_name = DockerfileUtility.image_basename(image) - pull_image_tag = DockerfileUtility.extract_image_name_tag(image) - - pull_status = False - for retry_count in range(0, configuration.get('retry')): - pull_status = docker_client.pull_image( - name=pull_image_name, - tag=pull_image_tag, - ) - - if pull_status: - break - elif retry_count < (configuration.get('retry') - 1): - print(') failed, retrying... (try %s)' % (retry_count + 1)) - else: - print(') failed, giving up') - - if not pull_status: - return False - - return True + return DockerfileUtility.check_if_base_image_needs_pull(image, configuration) + @staticmethod + def _extract_images_to_pull(dockerfile_list, configuration): + """ + Extract all images that need to be pulled from dockerfile list + """ image_list = [] for dockerfile in dockerfile_list: # Pull base image (FROM: xxx) first - if DockerfileUtility.check_if_base_image_needs_pull(dockerfile['image']['from'], configuration): + if DockerBuildTaskLoader._should_pull_image(dockerfile['image']['from'], configuration): image_list.append(dockerfile['image']['from']) - # Pull straged images (multi-stage dockerfiles) + # Pull staged images (multi-stage dockerfiles) for multiStageImage in dockerfile['image']['multiStageImages']: - if DockerfileUtility.check_if_base_image_needs_pull(multiStageImage, configuration): + if DockerBuildTaskLoader._should_pull_image(multiStageImage, configuration): image_list.append(multiStageImage) - # filter only unique image names - image_list = set(image_list) + return list(set(image_list)) # filter only unique image names + + @staticmethod + def _pull_single_image(image, configuration, docker_client): + """ + Pull a single image with retry logic + """ + print(') -> Pull base image %s ' % image) + + if configuration.get('dryRun'): + return True + + pull_image_name = DockerfileUtility.image_basename(image) + pull_image_tag = DockerfileUtility.extract_image_name_tag(image) + + for retry_count in range(0, configuration.get('retry')): + pull_status = docker_client.pull_image( + name=pull_image_name, + tag=pull_image_tag, + ) + + if pull_status: + return True + elif retry_count < (configuration.get('retry') - 1): + print(') failed, retrying... (try %s)' % (retry_count + 1)) + else: + print(') failed, giving up') + + return False + + @staticmethod + def task_dependency_puller(docker_client, dockerfile_list, configuration, task): + """ + Pulls dependency images before building + """ + image_list = DockerBuildTaskLoader._extract_images_to_pull(dockerfile_list, configuration) # pull images - for image in set(image_list): - if not pull_image(image): + for image in image_list: + if not DockerBuildTaskLoader._pull_single_image(image, configuration, docker_client): return False return True