Move Opstool to python 3

.bumpversion.cfg

@@ -1,12 +1,12 @@
 [bumpversion]
-current_version = 1.9
+current_version = 1.9.4
 commit = True
 tag = True
 tag_name = {new_version}
 message = [RELEASE] - Release version {new_version}
-parse = (?P<major>\d+)\.(?P<minor>\d+)
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
 serialize = 
-	{major}.{minor}
+	{major}.{minor}.{patch}
 
 [bumpversion:file:setup.py]
 

.travis.yml

@@ -5,20 +5,26 @@ services:
 branches:
   only:
   - master
-  - python3
   - /\d+\.\d+/
+  - /\d+\.\d+\.\d+/
+
 env:
   - BOTO_CONFIG=/dev/null
-python:
-- '2.7'
+
 script:
 # build
 - bash build_scripts/freeze_requirements.sh
 - bash build_scripts/build_package.sh
 # dry run
 - pip install --no-cache-dir dist/ops*.tar.gz && ops --verbose -h
+
+# Output something every 5 minutes or Travis kills the job
+- while sleep 5m; do echo "=====[ $SECONDS seconds still running ]====="; done &
 # build docker image
-- travis_wait 30 docker build -f build_scripts/Dockerfile -t ops .
+- docker build -f build_scripts/Dockerfile -t ops .
+# Killing background sleep loop
+- kill %1
+
 deploy:
   matrix:
   - provider: releases

README.md

@@ -76,10 +76,34 @@ Here is a link about how to install and use virtualenv:
 https://virtualenv.pypa.io/en/stable/
 
 ### Ops tool installation
+
+#### Python 3
 ```sh
-# Ops tool works on python2 only at the moment.
+# Make sure pip is up to date
+curl https://bootstrap.pypa.io/get-pip.py | python3
+
+# Install virtualenv
+pip install --upgrade virtualenv
+pip install --upgrade virtualenvwrapper
+
+echo 'export WORKON_HOME=$HOME/.virtualenvs' >> ~/.bash_profile
+echo 'source /usr/local/bin/virtualenvwrapper.sh' >> ~/.bash_profile
+source ~/.bash_profile
+
+# create virtualenv
+mkvirtualenv ops
+workon ops
+
+# uninstall previous `ops` version (if you have it)
+pip uninstall ops --yes
 
-# Make sure pip is up to date (min version: 9.0.3)
+# install ops-cli v1.9.4 stable release
+pip install --upgrade ops-cli
+```
+
+#### Python 2
+```sh
+# Make sure pip is up to date
 curl https://bootstrap.pypa.io/2.6/get-pip.py | python2
 
 # Install virtualenv
@@ -89,21 +113,26 @@ pip2 install -U virtualenv
 virtualenv ops
 source ops/bin/activate
 
-# install opswrapper v1.9 stable release
-pip2 install --upgrade https://github.com/adobe/ops-cli/releases/download/1.9/ops-1.9.tar.gz
+# uninstall previous `ops` version (if you have it)
+pip uninstall ops --yes
 
-# Optionally, install terraform to be able to access terraform plugin
-# See https://www.terraform.io/intro/getting-started/install.html
-# Also for pretty formatting of terraform plan output you can install https://github.com/coinbase/terraform-landscape (use gem install for MacOS)
+# install ops-cli v1.9.4 stable release
+pip2 install --upgrade ops-cli
 ```
 
+
+### Terraform
+Optionally, install terraform to be able to access terraform plugin. See https://www.terraform.io/intro/getting-started/install.html
+Also for pretty formatting of terraform plan output you can install https://github.com/coinbase/terraform-landscape (use gem install for MacOS)
+
+
 ## Using docker image
 
 You can try out `ops-cli`, by using docker. The docker image has all required prerequisites (python, terraform, helm, git, ops-cli etc).
 
 To start out a container, running the latest `ops-cli` docker image run:
 ```sh
-docker run -it adobe/ops-cli:1.9 bash
+docker run -it adobe/ops-cli:1.9.4 bash
 ```
 
 After the container has started, you can start using `ops-cli`:

build_scripts/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2.7.16-alpine3.9 AS compile-image
+FROM python:3.7-alpine3.10 AS compile-image
 ARG TERRAFORM_VERSION="0.12.6"
 ARG AZURE_CLI_VERSION="2.0.67"
 
@@ -22,7 +22,7 @@ RUN bash build_scripts/build_package.sh
 RUN apk del --purge build
 
 
-FROM python:2.7.16-alpine3.9
+FROM python:3.7-alpine3.10
 ARG TERRAFORM_VERSION="0.12.6"
 ARG VAULT_VERSION="1.1.3"
 ARG KUBECTL_VERSION="v1.13.7"
@@ -59,7 +59,7 @@ RUN adduser ops -Du 2342 -h /home/ops \
     && wget -q https://github.com/roboll/helmfile/releases/download/${HELM_FILE_VERSION}/helmfile_linux_amd64 -O /usr/local/bin/helmfile \
     && chmod +x /usr/local/bin/helmfile
 # Fix https://github.com/kubernetes-client/python-base/pull/126/files
-COPY build_scripts/patches/kube_config.py /usr/local/lib/python2.7/site-packages/kubernetes/config/kube_config.py
+COPY build_scripts/patches/kube_config.py /usr/local/lib/python3.7/site-packages/kubernetes/config/kube_config.py
 
 # install utils under `ops` user
 USER ops

build_scripts/docker_push.sh

@@ -2,5 +2,5 @@
 set -e
 
 echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
-docker tag ops adobe/ops-cli:1.9
-docker push adobe/ops-cli:1.9
+docker tag ops adobe/ops-cli:1.9.4
+docker push adobe/ops-cli:1.9.4

build_scripts/freeze_requirements.sh

@@ -3,6 +3,7 @@ set -e
 
 echo "Freezing requirements.txt"
 pip install pipenv
+
 rm -rf Pipfile* deps
-pipenv lock --clear --two --requirements 1>deps
+pipenv lock --clear --three --requirements 1>deps
 grep '==' deps | sed "s/;\\sextra.*//" > requirements.txt

requirements.txt

@@ -14,4 +14,5 @@ hvac==0.9.3
 passgen
 inflection==0.3.1
 kubernetes==9.0.0
-himl==0.1.18
+himl==0.2.1
+six

setup.py

@@ -23,7 +23,7 @@
 _requires = [ r for r in open(os.path.sep.join((_mydir,'requirements.txt')), "r").read().split('\n') if len(r)>1 ]
 setup(
     name='ops-cli',
-    version='1.9',
+    version='1.9.4',
     description='Ops - wrapper for Terraform, Ansible, and SSH for cloud automation',
     long_description=_readme + '\n\n',
     long_description_content_type='text/markdown',
@@ -38,9 +38,13 @@
         'Intended Audience :: Developers',
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: OS Independent',
-        'Programming Language :: Python',
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP :: Dynamic Content',

src/ops/__init__.py

@@ -12,11 +12,14 @@
 import re
 from distutils.version import StrictVersion
 from subprocess import call, Popen, PIPE
-from ops.cli import display
+
+from six import PY3
+
+from .cli import display
 
 
 def validate_ops_version(min_ops_version):
-    current_ops_version = [x.version for x in pkg_resources.working_set if x.project_name == "ops"][0]
+    current_ops_version = [x.version for x in pkg_resources.working_set if x.project_name == "ops-cli"][0]
     if StrictVersion(current_ops_version) < StrictVersion(min_ops_version):
         raise Exception("The current ops version {0} is lower than the minimum required version {1}. "
                         "Please upgrade by following the instructions seen here: "
@@ -30,7 +33,7 @@ def __call__(self, result, pass_trough=True, cwd=None):
         try:
             return self._execute(result, pass_trough, cwd)
         except Exception as ex:
-            display(ex.message, stderr=True, color='red')
+            display(str(ex) if PY3 else ex.message, stderr=True, color='red')
             display('------- TRACEBACK ----------', stderr=True, color='dark gray')
             import traceback
             traceback.print_exc()
@@ -48,7 +51,7 @@ def _execute(self, result, pass_trough=True, cwd=None):
             else:
                 p = Popen(shell_command, shell=True, stdout=PIPE, stderr=PIPE, cwd=cwd)
                 output, errors = p.communicate()
-                display(output)
+                display(str(output))
                 if errors:
                     display("%s" % self.shadow_credentials(errors), stderr=True, color='red')
                 exit_code = p.returncode

src/ops/ansible/filter_plugins/commonfilters.py

@@ -11,6 +11,8 @@
 from __future__ import absolute_import
 import os
 from ops.cli import display
+from six import iteritems
+
 
 def read_file(fname):
     if os.path.exists(fname):
@@ -98,7 +100,7 @@ def write_vault(
         namespace=None, mount_point=None, auto_prompt=auto_prompt)
     new_data = {}
     if isinstance(data, dict):
-        for k,v in data.iteritems():
+        for k,v in iteritems(data):
             new_data[k] = str(v)
     elif key:
         new_data[key] = str(data)

src/ops/cli/__init__.py

@@ -25,7 +25,7 @@ def display(msg, **kwargs):
         from ansible.playbook.play import display
         display.display(msg, **kwargs)
     except ImportError:
-        print msg
+        print(msg)
 
 def err(msg):
     display(str(msg), stderr=True, color='red')

src/ops/cli/inventory.py

@@ -12,8 +12,8 @@
 
 from ansible.parsing.yaml.dumper import AnsibleDumper
 from ansible.utils.color import stringc
-from ops.cli import display
-from parser import configure_common_arguments, SubParserConfig
+from . import display
+from .parser import configure_common_arguments, SubParserConfig
 
 
 class InventoryParserConfig(SubParserConfig):
@@ -63,7 +63,6 @@ def get_inventory_hosts(self, args):
 
     def get_host_facts(self, host, indent="\t"):
         vars = host.get_vars()
-        vars = {unicode(key): var for key, var in vars.items()} # the yaml dumper doesn't handle mixed keys encoding
         ret = yaml.dump(vars, indent=4, allow_unicode=True, default_flow_style=False, Dumper=AnsibleDumper)
         ret = "\n".join([indent + line for line in ret.split("\n")])
 

src/ops/cli/parser.py

@@ -12,6 +12,8 @@
 
 import sys
 
+from six import PY3
+
 
 class RootParser(object):
     def __init__(self, sub_parsers=None):
@@ -52,14 +54,14 @@ def _check_args_for_unicode(args):
 
         try:
             for value in args:
-                if isinstance(value, unicode):
+                if not PY3 and isinstance(value, unicode):
                     # Python3 or some Python3 compatibility mode can make arguments to be unicode, not str
                     value.encode('utf-8').encode('utf-8')
                 elif isinstance(value, str):  # Python 2 str, check if it can be represented in utf8
                     value.encode('utf-8')
         except UnicodeDecodeError as e:
-            print 'Invalid character in argument "{0}", most likely an "en dash", replace it with normal dash -'.format(
-                e.args[1])
+            print('Invalid character in argument "{0}", most likely an "en dash", replace it with normal dash -'.format(
+                e.args[1]))
             raise
 
     def parse_args(self, args=None):

src/ops/cli/playbook.py

@@ -8,8 +8,8 @@
 #OF ANY KIND, either express or implied. See the License for the specific language
 #governing permissions and limitations under the License.
 
-from ops.cli.parser import SubParserConfig
-from parser import configure_common_ansible_args, configure_common_arguments
+from .parser import SubParserConfig
+from .parser import configure_common_ansible_args, configure_common_arguments
 import getpass
 
 

src/ops/cli/run.py

@@ -8,9 +8,7 @@
 #OF ANY KIND, either express or implied. See the License for the specific language
 #governing permissions and limitations under the License.
 
-import subprocess
-from . import *
-from parser import configure_common_ansible_args, configure_common_arguments, SubParserConfig
+from .parser import configure_common_ansible_args, SubParserConfig
 
 
 class CommandParserConfig(SubParserConfig):

src/ops/cli/ssh.py

@@ -10,9 +10,9 @@
 
 from subprocess import call
 
-from ops.cli import display
-from ops.cli.parser import SubParserConfig
-from parser import configure_common_arguments
+from . import display
+from .parser import SubParserConfig
+from .parser import configure_common_arguments
 from ansible.inventory.host import Host
 
 from . import err
@@ -105,7 +105,7 @@ def run(self, args):
                 else:
                     #generate ssh keypair. The passphrase will be the name of the cluster
                     cmd = "ssh-keygen -t rsa -b 4096 -N {} -f {}".format(self.cluster_name,prv_key_file).split(' ')
-                    print cmd
+                    print(cmd)
                     call(cmd)
             return
 

src/ops/cli/sync.py

@@ -11,7 +11,7 @@
 import getpass
 import subprocess
 
-from parser import SubParserConfig
+from .parser import SubParserConfig
 from . import *
 
 

src/ops/cli/terraform.py

@@ -203,5 +203,6 @@ def run_v2_composition(self, args, terraform_path, composition):
         config['terraform'] = {}
         config['terraform']["path"] = "{}{}".format(terraform_path, composition)
         config['terraform']["variables_file"] = "variables.tfvars.json"
-        config['cluster'] = "auto_generated_" + hashlib.md5(self.cluster_config_path).hexdigest()[:6]
+        cluster_id = hashlib.md5(self.cluster_config_path.encode('utf-8')).hexdigest()[:6]
+        config['cluster'] = "auto_generated_{}".format(cluster_id)
         return self.run_composition(args, config)

src/ops/hierarchical/composition_config_generator.py

@@ -113,7 +113,7 @@ def __init__(self, composition_order):
         self.composition_order = composition_order
 
     def get_sorted_compositions(self, compositions, reverse=False):
-        result = filter(lambda x: x in compositions, self.composition_order)
+        result = list(filter(lambda x: x in compositions, self.composition_order))
         return tuple(reversed(result)) if reverse else result