Python3 migration

Makefile

@@ -17,7 +17,7 @@ full:      provision build
 all:       build
 
 build:
-	python2 ./bin/console docker:build --threads=auto
+	python3 ./bin/console docker:build --threads=auto
 
 bootstrap: webdevops/bootstrap webdevops/ansible
 base:      webdevops/base webdevops/base-app webdevops/storage
@@ -36,27 +36,27 @@ misc:      webdevops/mail-sandbox webdevops/sphinx webdevops/liquibase
 setup:     requirements
 
 requirements:
-	pip install --upgrade -I -r ./bin/requirements.txt
+	pip3 install --upgrade -I -r ./bin/requirements.txt
 	cd tests/serverspec && bundle install --path=vendor
 
 test:
-	python2 bin/console test:serverspec --threads=auto -v
+	python3 bin/console test:serverspec --threads=auto -v
 
 structure-test:
 	cd tests/structure-test && ./run.sh
 
 provision:
-	python2 bin/console generate:dockerfile
-	python2 bin/console generate:provision
+	python3 bin/console generate:dockerfile
+	python3 bin/console generate:provision
 
 push:
-	python2 ./bin/console docker:push --threads=auto
+	python3 ./bin/console docker:push --threads=auto
 
 graph:
-	python2 ./bin/console generate:graph
+	python3 ./bin/console generate:graph
 
 graph-full:
-	python2 ./bin/console generate:graph --all\
+	python3 ./bin/console generate:graph --all\
 		--filename docker-image-full-layout.gv
 
 documentation:

bin/command/docker_exec_command.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env/python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # (c) 2016 WebDevOps.io
@@ -51,11 +51,11 @@ def run_task(self, configuration):
         for dockerfile in dockerfile_list:
             title = dockerfile['image']['fullname']
 
-            print title
-            print '~' * len(title)
+            print(title)
+            print('~' * len(title))
 
             if configuration['dryRun']:
-                print '  exec: %s' % (docker_command)
+                print('  exec: %s' % (docker_command))
             else:
 
                 cmd = [
@@ -71,19 +71,19 @@ def run_task(self, configuration):
                 status = Command.execute(cmd)
 
                 if status:
-                    print colored(' -> successfull', 'green')
+                    print(colored(' -> successfull', 'green'))
                 else:
-                    print colored(' -> failed', 'red')
+                    print(colored(' -> failed', 'red'))
                     image_fail_list.append(dockerfile['image']['fullname'])
-            print ''
-            print ''
+            print('')
+            print('')
 
         if len(image_fail_list) >= 1:
-            print ''
-            print colored(' => failed images (%s):' % (str(len(image_fail_list))), 'red')
+            print('')
+            print(colored(' => failed images (%s):' % (str(len(image_fail_list))), 'red'))
             for image in image_fail_list:
-                print '   - %s ' % image
-            print ''
+                print('   - %s ' % image)
+            print('')
 
             return False
         else:

bin/command/generate_graph_command.py

@@ -43,7 +43,7 @@ class GenerateGraphCommand(BaseCommand):
         '--format': Enum(['png', 'jpg', 'pdf', 'svg'])
     }
 
-    from_regex = re.compile(ur'FROM\s+(?P<image>[^\s:]+)(:(?P<tag>.+))?', re.MULTILINE)
+    from_regex = re.compile(r'FROM\s+(?P<image>[^\s:]+)(:(?P<tag>.+))?', re.MULTILINE)
 
     containers = {}
 
@@ -116,7 +116,7 @@ def __append_tag(self, docker_image, tag):
 
         :return: self
         """
-        if not self.tags.has_key(docker_image):
+        if docker_image not in self.tags:
             self.tags[docker_image] = {}
         self.tags[docker_image][tag] = tag
         return self

bin/command/generate_provision_command.py

@@ -22,7 +22,7 @@
 import yaml
 import yamlordereddictloader
 import time
-import Queue
+import queue as Queue
 import shutil
 import grp
 from cleo import Output

bin/command/test_testinfra_command.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env/python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # (c) 2016 WebDevOps.io

bin/console

@@ -1,14 +1,14 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import os, sys
 
 # prevent bytecode
 sys.dont_write_bytecode = True
 
-# unbuffered stdout / stderr
-sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
-sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0)
+# unbuffered stdout / stderr - fixed for Python 3
+sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
+sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', buffering=1)
 
 import re, yaml
 from cleo import Application
@@ -43,19 +43,19 @@ if __name__ == '__main__':
     # Read console.yml for configuration
     with open(os.path.join(conf_path, 'console.yml'), 'r') as stream:
         try:
-            configuration = yaml.load(stream)
+            configuration = yaml.load(stream, Loader=yaml.SafeLoader)
             configuration['confPath'] = conf_path
         except yaml.YAMLError as e:
             configuration = None
-            print ' !!! Exception while loading configuration from %s:' % conf_path
-            print ''
-            print e
-            print ''
+            print(' !!! Exception while loading configuration from %s:' % conf_path)
+            print('')
+            print(e)
+            print('')
             sys.exit(1)
 
     # Check if configuration is valid
     if configuration is None:
-        print ' !!! Configuration not found'
+        print(' !!! Configuration not found')
         sys.exit(1)
 
     # generate full paths

bin/requirements.txt

@@ -3,7 +3,7 @@ graphviz>=0.4.10
 cleo<0.7.0
 yamlordereddictloader>=0.1.0
 testinfra>=1.4.2
-doit==0.29.0
+doit>=0.36.0
 termcolor>=1.1.0
 pytest-timeout>=1.0.0
 pytest-rerunfailures>=1.0.0

bin/webdevops/Command.py

@@ -25,7 +25,7 @@ def execute(cmd, cwd=False, env=None):
     Execute cmd and output stdout/stderr
     """
 
-    print 'Execute: %s' % ' '.join(cmd)
+    print('Execute: %s' % ' '.join(cmd))
 
     if env is not None:
         env = copy.deepcopy(env)
@@ -56,7 +56,6 @@ def execute(cmd, cwd=False, env=None):
         cmd,
         stdout=file_stdout,
         stderr=file_stdout,
-        bufsize=-1,
         env=env
     )
 
@@ -67,13 +66,13 @@ def execute(cmd, cwd=False, env=None):
     # output stdout
     with open(file_stdout.name, 'r') as f:
         for line in f:
-            print line.rstrip('\n')
+            print(line.rstrip('\n'))
 
     # restore current work directory
     os.chdir(path_current)
 
     if proc.returncode == 0:
         return True
     else:
-        print '>> failed command with return code %s' % proc.returncode
+        print(')>> failed command with return code %s' % proc.returncode)
         return False

bin/webdevops/Configuration.py

@@ -44,7 +44,7 @@
     'docker': {
         'imagePrefix': '',
         'autoLatestTag': False,
-        'fromRegExp': re.compile(ur'FROM\s+(?P<image>[^\s:]+)(:(?P<tag>.+))?', re.MULTILINE),
+        'fromRegExp': re.compile(r'FROM\s+(?P<image>[^\s:]+)(:(?P<tag>.+))?', re.MULTILINE),
         'pathRegex': False,
         'autoPull': False,
         'autoPullWhitelist': False,
@@ -83,7 +83,7 @@ def dictmerge(original, update):
         Recursively update a dict.
         Subdict's won't be overwritten but also updated.
         """
-        for key, value in original.iteritems():
+        for key, value in original.items():
             if key not in update:
                 update[key] = value
             elif isinstance(value, dict):
@@ -101,7 +101,7 @@ def __init__(self, value=None):
             for key in value:
                 self.__setitem_internal__(key, value[key])
         else:
-            raise TypeError, 'expected dict'
+            raise TypeError('expected dict')
 
     def __setitem_internal__(self, key, value):
         """
@@ -116,7 +116,7 @@ def __setitem__(self, key, value):
             myKey, restOfKey = key.split('.', 1)
             target = self.setdefault(myKey, dotdictify())
             if not isinstance(target, dotdictify):
-                raise KeyError, 'cannot set "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target))
+                raise KeyError('cannot set "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target)))
             target[restOfKey] = value
         else:
             if isinstance(value, dict) and not isinstance(value, dotdictify):
@@ -129,7 +129,7 @@ def __getitem__(self, key, raw=False):
         myKey, restOfKey = key.split('.', 1)
         target = dict.get(self, myKey, None)
         if not isinstance(target, dotdictify):
-            raise KeyError, 'cannot get "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target))
+            raise KeyError('cannot get "%s" in "%s" (%s)' % (restOfKey, myKey, repr(target)))
         return target[restOfKey]
 
     def __contains__(self, key):

bin/webdevops/Dockerfile.py

@@ -40,7 +40,7 @@ def finder(dockerfile_path, filename="Dockerfile", filter=[]):
     :rtype: list
     """
     dockerfile_stack = []
-    filter_regex = re.compile(ur'.*(%s).*' % "|".join(filter), re.IGNORECASE)
+    filter_regex = re.compile(r'.*(%s).*' % "|".join(filter), re.IGNORECASE)
     # pprint(filter_regex.pattern)
     for root, dirs, files in os.walk(dockerfile_path):
         for file in files:

bin/webdevops/DockerfileUtility.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env/python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # (c) 2016 WebDevOps.io
@@ -21,8 +21,8 @@
 import os
 import re
 
-DOCKERFILE_STATEMENT_FROM_RE = re.compile(ur'FROM\s+(?P<image>[^\s:]+)(:(?P<tag>[^\s:]+))?(?!.*\s+AS)', re.MULTILINE)
-DOCKERFILE_STATEMENT_FROM_MULTISTAGE_RE = re.compile(ur'FROM\s+(?P<image>[^\s:]+)(:(?P<tag>[^\s:]+))?(\s+AS)', re.MULTILINE)
+DOCKERFILE_STATEMENT_FROM_RE = re.compile(r'FROM\s+(?P<image>[^\s:]+)(:(?P<tag>[^\s:]+))?(?!.*\s+AS)', re.MULTILINE)
+DOCKERFILE_STATEMENT_FROM_MULTISTAGE_RE = re.compile(r'FROM\s+(?P<image>[^\s:]+)(:(?P<tag>[^\s:]+))?(\s+AS)', re.MULTILINE)
 
 def find_file_in_path(dockerfile_path, filename="Dockerfile", whitelist=False, blacklist=False):
     """
@@ -64,7 +64,7 @@ def find_file_in_path(dockerfile_path, filename="Dockerfile", whitelist=False, b
 
     if blacklist:
         for term in blacklist:
-            file_list = filter(lambda x: term not in x, file_list)
+            file_list = list(filter(lambda x: term not in x, file_list))
 
     return file_list
 
@@ -143,7 +143,7 @@ def filter_dockerfile(dockerfile_list, whitelist=False, blacklist=False):
 
     if blacklist:
         for term in blacklist:
-            dockerfile_list = filter(lambda x: term not in x['image']['fullname'], dockerfile_list)
+            dockerfile_list = list(filter(lambda x: term not in x['image']['fullname'], dockerfile_list))
 
     return dockerfile_list
 

bin/webdevops/Provisioner.py

@@ -24,7 +24,7 @@
 import os
 from distutils.dir_util import copy_tree, remove_tree
 from threading import Thread
-import Queue
+import queue as Queue
 import shutil
 
 
@@ -114,7 +114,7 @@ def __deploy_configuration(self):
         """
         Deploy the configuration to the container
         """
-        for src, tag in self.image_config['configuration'].iteritems():
+        for src, tag in self.image_config['configuration'].items():
             if Output.VERBOSITY_NORMAL <= self.output.get_verbosity():
                 self.line("<fg=cyan>%s</> <info>=></info> %s:%s" % (src, self.image_name, tag))
             if isinstance(tag, list):

bin/webdevops/__init__.py

@@ -18,7 +18,7 @@
 # OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-from Provisioner import Provisioner
+from .Provisioner import Provisioner
 
 __all__ = [
     'Provisioner',

bin/webdevops/command/BaseCommand.py

@@ -49,11 +49,11 @@ def handle(self):
         try:
             exitcode = self.run_task(configuration=self.configuration)
         except KeyboardInterrupt as e:
-            print ' !!! Execution aborted by user'
+            print(' !!! Execution aborted by user')
             exitcode = 1
         except SystemExit as e:
-            print ' !!! Execution aborted by SystemExit'
-            print ''
+            print(' !!! Execution aborted by SystemExit')
+            print('')
             traceback.print_exc(file=sys.stdout)
             exitcode = 1
 
@@ -90,26 +90,26 @@ def startup(self):
             DoitReporter.simulation_mode = True
 
 
-        print 'Executing %s (%s)' % (self.name, ', '.join(options))
-        print ''
+        print('Executing %s (%s)' % (self.name, ', '.join(options)))
+        print('')
 
         try:
             whitelist = self.get_whitelist()
             if whitelist:
-                print 'WHITELIST active:'
+                print('WHITELIST active:')
                 for item in whitelist:
-                    print ' - %s' % item
-                print ''
+                    print(' - %s' % item)
+                print('')
         except:
             pass
 
         try:
             blacklist = self.get_blacklist()
             if blacklist:
-                print 'BLACKLIST active:'
+                print('BLACKLIST active:')
                 for item in blacklist:
-                    print ' - %s' % item
-                print ''
+                    print(' - %s' % item)
+                print('')
         except:
             pass
 
@@ -128,11 +128,11 @@ def shutdown(self, exitcode=0):
 
         self.teardown(exitcode)
 
-        print ''
+        print('')
         if exitcode == 0:
-            print '> finished execution in %s successfully' % (duration)
+            print('> finished execution in %s successfully' % (duration))
         else:
-            print '> finished execution in %s with errors (exitcode %s)' % (duration, exitcode)
+            print('> finished execution in %s with errors (exitcode %s)' % (duration, exitcode))
 
     def build_configuration(self):
         """