diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 807cee39..ea8368ad 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,12 +1,12 @@ [bumpversion] -current_version = 1.9 +current_version = 1.9.4 commit = True tag = True tag_name = {new_version} message = [RELEASE] - Release version {new_version} -parse = (?P\d+)\.(?P\d+) +parse = (?P\d+)\.(?P\d+)\.(?P\d+) serialize = - {major}.{minor} + {major}.{minor}.{patch} [bumpversion:file:setup.py] diff --git a/.travis.yml b/.travis.yml index 848e3e07..d48596d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,20 +5,26 @@ services: branches: only: - master - - python3 - /\d+\.\d+/ + - /\d+\.\d+\.\d+/ + env: - BOTO_CONFIG=/dev/null -python: -- '2.7' + script: # build - bash build_scripts/freeze_requirements.sh - bash build_scripts/build_package.sh # dry run - pip install --no-cache-dir dist/ops*.tar.gz && ops --verbose -h + +# Output something every 5 minutes or Travis kills the job +- while sleep 5m; do echo "=====[ $SECONDS seconds still running ]====="; done & # build docker image -- travis_wait 30 docker build -f build_scripts/Dockerfile -t ops . +- docker build -f build_scripts/Dockerfile -t ops . +# Killing background sleep loop +- kill %1 + deploy: matrix: - provider: releases diff --git a/README.md b/README.md index 218f219b..641a5c72 100644 --- a/README.md +++ b/README.md @@ -76,10 +76,34 @@ Here is a link about how to install and use virtualenv: https://virtualenv.pypa.io/en/stable/ ### Ops tool installation + +#### Python 3 ```sh -# Ops tool works on python2 only at the moment. +# Make sure pip is up to date +curl https://bootstrap.pypa.io/get-pip.py | python3 + +# Install virtualenv +pip install --upgrade virtualenv +pip install --upgrade virtualenvwrapper + +echo 'export WORKON_HOME=$HOME/.virtualenvs' >> ~/.bash_profile +echo 'source /usr/local/bin/virtualenvwrapper.sh' >> ~/.bash_profile +source ~/.bash_profile + +# create virtualenv +mkvirtualenv ops +workon ops + +# uninstall previous `ops` version (if you have it) +pip uninstall ops --yes -# Make sure pip is up to date (min version: 9.0.3) +# install ops-cli v1.9.4 stable release +pip install --upgrade ops-cli +``` + +#### Python 2 +```sh +# Make sure pip is up to date curl https://bootstrap.pypa.io/2.6/get-pip.py | python2 # Install virtualenv @@ -89,21 +113,26 @@ pip2 install -U virtualenv virtualenv ops source ops/bin/activate -# install opswrapper v1.9 stable release -pip2 install --upgrade https://github.com/adobe/ops-cli/releases/download/1.9/ops-1.9.tar.gz +# uninstall previous `ops` version (if you have it) +pip uninstall ops --yes -# Optionally, install terraform to be able to access terraform plugin -# See https://www.terraform.io/intro/getting-started/install.html -# Also for pretty formatting of terraform plan output you can install https://github.com/coinbase/terraform-landscape (use gem install for MacOS) +# install ops-cli v1.9.4 stable release +pip2 install --upgrade ops-cli ``` + +### Terraform +Optionally, install terraform to be able to access terraform plugin. See https://www.terraform.io/intro/getting-started/install.html +Also for pretty formatting of terraform plan output you can install https://github.com/coinbase/terraform-landscape (use gem install for MacOS) + + ## Using docker image You can try out `ops-cli`, by using docker. The docker image has all required prerequisites (python, terraform, helm, git, ops-cli etc). To start out a container, running the latest `ops-cli` docker image run: ```sh -docker run -it adobe/ops-cli:1.9 bash +docker run -it adobe/ops-cli:1.9.4 bash ``` After the container has started, you can start using `ops-cli`: diff --git a/build_scripts/Dockerfile b/build_scripts/Dockerfile index 2de953ac..e0378e4b 100644 --- a/build_scripts/Dockerfile +++ b/build_scripts/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2.7.16-alpine3.9 AS compile-image +FROM python:3.7-alpine3.10 AS compile-image ARG TERRAFORM_VERSION="0.12.6" ARG AZURE_CLI_VERSION="2.0.67" @@ -22,7 +22,7 @@ RUN bash build_scripts/build_package.sh RUN apk del --purge build -FROM python:2.7.16-alpine3.9 +FROM python:3.7-alpine3.10 ARG TERRAFORM_VERSION="0.12.6" ARG VAULT_VERSION="1.1.3" ARG KUBECTL_VERSION="v1.13.7" @@ -59,7 +59,7 @@ RUN adduser ops -Du 2342 -h /home/ops \ && wget -q https://github.com/roboll/helmfile/releases/download/${HELM_FILE_VERSION}/helmfile_linux_amd64 -O /usr/local/bin/helmfile \ && chmod +x /usr/local/bin/helmfile # Fix https://github.com/kubernetes-client/python-base/pull/126/files -COPY build_scripts/patches/kube_config.py /usr/local/lib/python2.7/site-packages/kubernetes/config/kube_config.py +COPY build_scripts/patches/kube_config.py /usr/local/lib/python3.7/site-packages/kubernetes/config/kube_config.py # install utils under `ops` user USER ops diff --git a/build_scripts/docker_push.sh b/build_scripts/docker_push.sh index 4debd2cb..554c2faf 100644 --- a/build_scripts/docker_push.sh +++ b/build_scripts/docker_push.sh @@ -2,5 +2,5 @@ set -e echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin -docker tag ops adobe/ops-cli:1.9 -docker push adobe/ops-cli:1.9 +docker tag ops adobe/ops-cli:1.9.4 +docker push adobe/ops-cli:1.9.4 diff --git a/build_scripts/freeze_requirements.sh b/build_scripts/freeze_requirements.sh index c05e4415..4e6246bc 100644 --- a/build_scripts/freeze_requirements.sh +++ b/build_scripts/freeze_requirements.sh @@ -3,6 +3,7 @@ set -e echo "Freezing requirements.txt" pip install pipenv + rm -rf Pipfile* deps -pipenv lock --clear --two --requirements 1>deps +pipenv lock --clear --three --requirements 1>deps grep '==' deps | sed "s/;\\sextra.*//" > requirements.txt diff --git a/requirements.txt b/requirements.txt index 78b8ac17..4fd7b2c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,5 @@ hvac==0.9.3 passgen inflection==0.3.1 kubernetes==9.0.0 -himl==0.1.18 +himl==0.2.1 +six diff --git a/setup.py b/setup.py index 02028b63..f6b1adc8 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ _requires = [ r for r in open(os.path.sep.join((_mydir,'requirements.txt')), "r").read().split('\n') if len(r)>1 ] setup( name='ops-cli', - version='1.9', + version='1.9.4', description='Ops - wrapper for Terraform, Ansible, and SSH for cloud automation', long_description=_readme + '\n\n', long_description_content_type='text/markdown', @@ -38,9 +38,13 @@ 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', - 'Programming Language :: Python', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', diff --git a/src/ops/__init__.py b/src/ops/__init__.py index 84741560..dedccfcb 100644 --- a/src/ops/__init__.py +++ b/src/ops/__init__.py @@ -12,11 +12,14 @@ import re from distutils.version import StrictVersion from subprocess import call, Popen, PIPE -from ops.cli import display + +from six import PY3 + +from .cli import display def validate_ops_version(min_ops_version): - current_ops_version = [x.version for x in pkg_resources.working_set if x.project_name == "ops"][0] + current_ops_version = [x.version for x in pkg_resources.working_set if x.project_name == "ops-cli"][0] if StrictVersion(current_ops_version) < StrictVersion(min_ops_version): raise Exception("The current ops version {0} is lower than the minimum required version {1}. " "Please upgrade by following the instructions seen here: " @@ -30,7 +33,7 @@ def __call__(self, result, pass_trough=True, cwd=None): try: return self._execute(result, pass_trough, cwd) except Exception as ex: - display(ex.message, stderr=True, color='red') + display(str(ex) if PY3 else ex.message, stderr=True, color='red') display('------- TRACEBACK ----------', stderr=True, color='dark gray') import traceback traceback.print_exc() @@ -48,7 +51,7 @@ def _execute(self, result, pass_trough=True, cwd=None): else: p = Popen(shell_command, shell=True, stdout=PIPE, stderr=PIPE, cwd=cwd) output, errors = p.communicate() - display(output) + display(str(output)) if errors: display("%s" % self.shadow_credentials(errors), stderr=True, color='red') exit_code = p.returncode diff --git a/src/ops/ansible/filter_plugins/commonfilters.py b/src/ops/ansible/filter_plugins/commonfilters.py index 8a33ee5a..78bb93aa 100644 --- a/src/ops/ansible/filter_plugins/commonfilters.py +++ b/src/ops/ansible/filter_plugins/commonfilters.py @@ -11,6 +11,8 @@ from __future__ import absolute_import import os from ops.cli import display +from six import iteritems + def read_file(fname): if os.path.exists(fname): @@ -98,7 +100,7 @@ def write_vault( namespace=None, mount_point=None, auto_prompt=auto_prompt) new_data = {} if isinstance(data, dict): - for k,v in data.iteritems(): + for k,v in iteritems(data): new_data[k] = str(v) elif key: new_data[key] = str(data) diff --git a/src/ops/cli/__init__.py b/src/ops/cli/__init__.py index 89146ab6..b069f34a 100644 --- a/src/ops/cli/__init__.py +++ b/src/ops/cli/__init__.py @@ -25,7 +25,7 @@ def display(msg, **kwargs): from ansible.playbook.play import display display.display(msg, **kwargs) except ImportError: - print msg + print(msg) def err(msg): display(str(msg), stderr=True, color='red') diff --git a/src/ops/cli/inventory.py b/src/ops/cli/inventory.py index 254aff78..17eee157 100644 --- a/src/ops/cli/inventory.py +++ b/src/ops/cli/inventory.py @@ -12,8 +12,8 @@ from ansible.parsing.yaml.dumper import AnsibleDumper from ansible.utils.color import stringc -from ops.cli import display -from parser import configure_common_arguments, SubParserConfig +from . import display +from .parser import configure_common_arguments, SubParserConfig class InventoryParserConfig(SubParserConfig): @@ -63,7 +63,6 @@ def get_inventory_hosts(self, args): def get_host_facts(self, host, indent="\t"): vars = host.get_vars() - vars = {unicode(key): var for key, var in vars.items()} # the yaml dumper doesn't handle mixed keys encoding ret = yaml.dump(vars, indent=4, allow_unicode=True, default_flow_style=False, Dumper=AnsibleDumper) ret = "\n".join([indent + line for line in ret.split("\n")]) diff --git a/src/ops/cli/parser.py b/src/ops/cli/parser.py index af2a1a3c..45ccab82 100644 --- a/src/ops/cli/parser.py +++ b/src/ops/cli/parser.py @@ -12,6 +12,8 @@ import sys +from six import PY3 + class RootParser(object): def __init__(self, sub_parsers=None): @@ -52,14 +54,14 @@ def _check_args_for_unicode(args): try: for value in args: - if isinstance(value, unicode): + if not PY3 and isinstance(value, unicode): # Python3 or some Python3 compatibility mode can make arguments to be unicode, not str value.encode('utf-8').encode('utf-8') elif isinstance(value, str): # Python 2 str, check if it can be represented in utf8 value.encode('utf-8') except UnicodeDecodeError as e: - print 'Invalid character in argument "{0}", most likely an "en dash", replace it with normal dash -'.format( - e.args[1]) + print('Invalid character in argument "{0}", most likely an "en dash", replace it with normal dash -'.format( + e.args[1])) raise def parse_args(self, args=None): diff --git a/src/ops/cli/playbook.py b/src/ops/cli/playbook.py index 39784c9f..b97f6084 100644 --- a/src/ops/cli/playbook.py +++ b/src/ops/cli/playbook.py @@ -8,8 +8,8 @@ #OF ANY KIND, either express or implied. See the License for the specific language #governing permissions and limitations under the License. -from ops.cli.parser import SubParserConfig -from parser import configure_common_ansible_args, configure_common_arguments +from .parser import SubParserConfig +from .parser import configure_common_ansible_args, configure_common_arguments import getpass diff --git a/src/ops/cli/run.py b/src/ops/cli/run.py index 46823d1e..ea61bf83 100644 --- a/src/ops/cli/run.py +++ b/src/ops/cli/run.py @@ -8,9 +8,7 @@ #OF ANY KIND, either express or implied. See the License for the specific language #governing permissions and limitations under the License. -import subprocess -from . import * -from parser import configure_common_ansible_args, configure_common_arguments, SubParserConfig +from .parser import configure_common_ansible_args, SubParserConfig class CommandParserConfig(SubParserConfig): diff --git a/src/ops/cli/ssh.py b/src/ops/cli/ssh.py index 1df3e481..dce30dfd 100644 --- a/src/ops/cli/ssh.py +++ b/src/ops/cli/ssh.py @@ -10,9 +10,9 @@ from subprocess import call -from ops.cli import display -from ops.cli.parser import SubParserConfig -from parser import configure_common_arguments +from . import display +from .parser import SubParserConfig +from .parser import configure_common_arguments from ansible.inventory.host import Host from . import err @@ -105,7 +105,7 @@ def run(self, args): else: #generate ssh keypair. The passphrase will be the name of the cluster cmd = "ssh-keygen -t rsa -b 4096 -N {} -f {}".format(self.cluster_name,prv_key_file).split(' ') - print cmd + print(cmd) call(cmd) return diff --git a/src/ops/cli/sync.py b/src/ops/cli/sync.py index dce48187..a2306a60 100644 --- a/src/ops/cli/sync.py +++ b/src/ops/cli/sync.py @@ -11,7 +11,7 @@ import getpass import subprocess -from parser import SubParserConfig +from .parser import SubParserConfig from . import * diff --git a/src/ops/cli/terraform.py b/src/ops/cli/terraform.py index 67fb9c96..bea7f3a5 100644 --- a/src/ops/cli/terraform.py +++ b/src/ops/cli/terraform.py @@ -203,5 +203,6 @@ def run_v2_composition(self, args, terraform_path, composition): config['terraform'] = {} config['terraform']["path"] = "{}{}".format(terraform_path, composition) config['terraform']["variables_file"] = "variables.tfvars.json" - config['cluster'] = "auto_generated_" + hashlib.md5(self.cluster_config_path).hexdigest()[:6] + cluster_id = hashlib.md5(self.cluster_config_path.encode('utf-8')).hexdigest()[:6] + config['cluster'] = "auto_generated_{}".format(cluster_id) return self.run_composition(args, config) diff --git a/src/ops/hierarchical/composition_config_generator.py b/src/ops/hierarchical/composition_config_generator.py index a73ddf05..459928f8 100644 --- a/src/ops/hierarchical/composition_config_generator.py +++ b/src/ops/hierarchical/composition_config_generator.py @@ -113,7 +113,7 @@ def __init__(self, composition_order): self.composition_order = composition_order def get_sorted_compositions(self, compositions, reverse=False): - result = filter(lambda x: x in compositions, self.composition_order) + result = list(filter(lambda x: x in compositions, self.composition_order)) return tuple(reversed(result)) if reverse else result diff --git a/src/ops/inventory/SKMS.py b/src/ops/inventory/SKMS.py index 458c0cbd..2c12be03 100644 --- a/src/ops/inventory/SKMS.py +++ b/src/ops/inventory/SKMS.py @@ -34,7 +34,7 @@ class WebApiClient(object): """Class to allow easy access to the SKMS Web API""" # Version Constants - CLIENT_TYPE = "python2.6.requests" + CLIENT_TYPE = "python3.7.requests" CLIENT_VERSION = "1.10" # Properties @@ -102,7 +102,7 @@ def __init__(self, username, passkey, skms_domain=None, enable_session_optimizat if not os.path.exists(skms_dir): return try: - os.chmod(skms_dir, 0700) + os.chmod(skms_dir, 0o700) except OSError: return except OSError: @@ -310,7 +310,7 @@ def send_request(self, object_name, method_name, method_param_dict=None): self.skms_session_storage_file, "w" ) as skms_session_storage_file_ptr: skms_session_storage_file_ptr.write('') - os.chmod(self.skms_session_storage_file, 0600) + os.chmod(self.skms_session_storage_file, 0o600) except OSError: populate_session_file = False if populate_session_file is True: @@ -362,7 +362,7 @@ def send_request(self, object_name, method_name, method_param_dict=None): ) return False - except requests.RequestException, exc: + except requests.RequestException as exc: self.error_message = type(exc).__name__ self.status = False return self.status diff --git a/src/ops/inventory/__init__.py b/src/ops/inventory/__init__.py index 7a7261d3..abf46c50 100644 --- a/src/ops/inventory/__init__.py +++ b/src/ops/inventory/__init__.py @@ -8,4 +8,4 @@ #OF ANY KIND, either express or implied. See the License for the specific language #governing permissions and limitations under the License. -from ec2inventory import Ec2Inventory \ No newline at end of file +from .ec2inventory import Ec2Inventory diff --git a/src/ops/inventory/azurerm.py b/src/ops/inventory/azurerm.py index 550b7d61..a7d76e66 100644 --- a/src/ops/inventory/azurerm.py +++ b/src/ops/inventory/azurerm.py @@ -195,7 +195,7 @@ ''' import argparse -import ConfigParser +from six.moves import configparser import json import os import re @@ -205,6 +205,8 @@ from os.path import expanduser +from six import iteritems + HAS_AZURE = True HAS_AZURE_EXC = None @@ -291,7 +293,7 @@ def __init__(self, args): def log(self, msg): if self.debug: - print (msg + u'\n') + print(msg + u'\n') def fail(self, msg): raise Exception(msg) @@ -300,7 +302,7 @@ def _get_profile(self, profile="default"): path = expanduser("~") path += "/.azure/credentials" try: - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() config.read(path) except Exception as exc: self.fail("Failed to access {0}. Check that the file exists and you have read " @@ -319,7 +321,7 @@ def _get_profile(self, profile="default"): def _get_env_credentials(self): env_credentials = dict() - for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.iteritems(): + for attribute, env_variable in iteritems(AZURE_CREDENTIAL_ENV_MAPPING): env_credentials[attribute] = os.environ.get(env_variable, None) if env_credentials['profile'] is not None: @@ -338,7 +340,7 @@ def _get_credentials(self, params): self.log('Getting credentials') arg_credentials = dict() - for attribute, env_variable in AZURE_CREDENTIAL_ENV_MAPPING.iteritems(): + for attribute, env_variable in iteritems(AZURE_CREDENTIAL_ENV_MAPPING): arg_credentials[attribute] = getattr(params, attribute) # try module params @@ -446,7 +448,7 @@ def __init__(self): self.include_powerstate = False self.get_inventory() - print (self._json_format_dict(pretty=self._args.pretty)) + print(self._json_format_dict(pretty=self._args.pretty)) sys.exit(0) def _parse_cli_args(self): @@ -677,7 +679,7 @@ def _add_host(self, vars): self._inventory['azure'].append(host_name) if self.group_by_tag and vars.get('tags'): - for key, value in vars['tags'].iteritems(): + for key, value in iteritems(vars['tags']): safe_key = self._to_safe(key) safe_value = self._to_safe(value) if not self._inventory.get(safe_key): @@ -737,7 +739,7 @@ def _to_boolean(self, value): def _get_env_settings(self): env_settings = dict() - for attribute, env_variable in AZURE_CONFIG_SETTINGS.iteritems(): + for attribute, env_variable in iteritems(AZURE_CONFIG_SETTINGS): env_settings[attribute] = os.environ.get(env_variable, None) return env_settings diff --git a/src/ops/inventory/caching.py b/src/ops/inventory/caching.py index 9dbfe37d..a3605fe6 100644 --- a/src/ops/inventory/caching.py +++ b/src/ops/inventory/caching.py @@ -13,6 +13,9 @@ import os import time +from six import PY3 + + def cache_callback_result(directory, func, max_age, cache_key_args): directory = os.path.expanduser(directory) path = get_cache_path(directory, cache_key_args) @@ -24,7 +27,10 @@ def cache_callback_result(directory, func, max_age, cache_key_args): def get_cache_path(dir, args): m = hashlib.md5() - m.update(json.dumps(args)) + json_dump = json.dumps(args) + if PY3: + json_dump = json_dump.encode('utf-8') + m.update(json_dump) return os.path.join(dir, m.hexdigest()) diff --git a/src/ops/inventory/ec2inventory.py b/src/ops/inventory/ec2inventory.py index 8a1b1202..62403cbe 100644 --- a/src/ops/inventory/ec2inventory.py +++ b/src/ops/inventory/ec2inventory.py @@ -16,6 +16,8 @@ from boto import ec2 from boto.pyami.config import Config +from six import iteritems, string_types, integer_types + class Ec2Inventory(object): def _empty_inventory(self): @@ -86,7 +88,7 @@ def get_instances_by_region(self, region): # connect_to_region will fail "silently" by returning None if the region name is wrong or not supported if conn is None: - print("region name: %s likely not supported, or AWS is down. connection to region failed." % region) + print("region name: {} likely not supported, or AWS is down. connection to region failed.".format(region)) sys.exit(1) reservations = conn.get_all_instances(filters=self.filters) @@ -99,20 +101,17 @@ def get_instances_by_region(self, region): # sort the instance based on name and index, in this order def sort_key(instance): - components = instance.tags.get('Name', '').rsplit('-', 1) - if len(components) == 2: - return (components[0], int(components[1]) if components[1].isdigit() else 0) - else: - return components[0] + name = instance.tags.get('Name', '') + return "{}-{}".format(name, instance.id) for instance in sorted(instances, key=sort_key): self.add_instance(bastion_ip, instance, region) - except boto.provider.ProfileNotFoundError, e: + except boto.provider.ProfileNotFoundError as e: raise Exception("{}, configure it with 'aws configure --profile {}'".format(e.message, self.boto_profile)) - except boto.exception.BotoServerError, e: - print e + except boto.exception.BotoServerError as e: + print(e) sys.exit(1) def get_instance(self, region, instance_id): @@ -196,9 +195,9 @@ def get_host_info_dict_from_instance(self, instance): elif key == 'ec2__previous_state': instance_vars['ec2_previous_state'] = instance.previous_state or '' instance_vars['ec2_previous_state_code'] = instance.previous_state_code - elif type(value) in [int, bool]: + elif type(value) in integer_types or type(value) == bool: instance_vars[key] = value - elif type(value) in [str, unicode]: + elif type(value) in string_types: instance_vars[key] = value.strip() elif type(value) == type(None): instance_vars[key] = '' @@ -207,7 +206,7 @@ def get_host_info_dict_from_instance(self, instance): elif key == 'ec2__placement': instance_vars['ec2_placement'] = value.zone elif key == 'ec2_tags': - for k, v in value.iteritems(): + for k, v in iteritems(value): key = self.to_safe('ec2_tag_' + k) instance_vars[key] = v elif key == 'ec2_groups': diff --git a/src/ops/inventory/generator.py b/src/ops/inventory/generator.py index 26ac27f7..96987261 100644 --- a/src/ops/inventory/generator.py +++ b/src/ops/inventory/generator.py @@ -15,7 +15,7 @@ import ansible.inventory as ansible_inventory import ansible.vars as ansible_vars -import caching +from . import caching from ansible.inventory.manager import InventoryManager from ansible.parsing.dataloader import DataLoader from ansible.playbook.play import display @@ -139,8 +139,6 @@ def generate(self): except KeyError as e: error = 'Required key %s not found' % e errors.append(dict(entry=entry, error=error)) - except Exception as e: - errors.append(dict(entry=entry, error=e)) found_generator = True break @@ -178,9 +176,9 @@ class PluginInventoryGenerator(object): template = """#!/usr/bin/env python # CONFIG: {config} # PLUGIN PATH: {plugin_path} -print \"\"\" +print(\"\"\" {json_content} -\"\"\" +\"\"\") """ def __init__(self, cluster_name, inventory_plugins): @@ -205,7 +203,7 @@ def generate(self, dest, config): with open(script_dest, 'w+') as f: f.write(script_content) - os.fchmod(f.fileno(), 0500) + os.fchmod(f.fileno(), 0o500) class ShellInventoryGenerator(object): @@ -257,7 +255,7 @@ def generate(self, dest, config): with open(script_dest, 'w+') as f: f.write(script_content) - os.fchmod(f.fileno(), 0500) + os.fchmod(f.fileno(), 0o500) class AnsibleInventory(object): @@ -284,10 +282,10 @@ def get_hosts(self, limit): return self.inventory.get_hosts(limit) def get_host(self, host): - return self.inventory.get_host(unicode(host)) + return self.inventory.get_host(str(host)) def get_vars(self, host): - return self.inventory.get_vars(unicode(host)) + return self.inventory.get_vars(str(host)) def get_ssh_config(self): return self.ssh_config_path diff --git a/src/ops/inventory/plugin/__init__.py b/src/ops/inventory/plugin/__init__.py index af915d9e..6d8610e3 100644 --- a/src/ops/inventory/plugin/__init__.py +++ b/src/ops/inventory/plugin/__init__.py @@ -8,8 +8,8 @@ #OF ANY KIND, either express or implied. See the License for the specific language #governing permissions and limitations under the License. -from ec2 import ec2 -from cns import cns -from legacy_pcs import legacy_pcs -from azr import azr -from skms import skms +from .ec2 import ec2 +from .cns import cns +from .legacy_pcs import legacy_pcs +from .azr import azr +from .skms import skms diff --git a/src/ops/inventory/plugin/azr.py b/src/ops/inventory/plugin/azr.py index 359db4fa..ce75f5e5 100644 --- a/src/ops/inventory/plugin/azr.py +++ b/src/ops/inventory/plugin/azr.py @@ -8,17 +8,21 @@ #OF ANY KIND, either express or implied. See the License for the specific language #governing permissions and limitations under the License. -import json + from ops.inventory.azurerm import * from ansible.playbook.play import display +from six import iteritems + class DictGlue(object): def __init__(self,data={}): self.__dict__.update(data) + class EnvironmentMissingException(Exception): pass + class OpsAzureInventory(AzureInventory): """ We inherit from the original implementation and override what we need here @@ -89,7 +93,7 @@ def __init__(self,args={}): self.get_inventory() bastions = {} - for host, hostvars in self._inventory['_meta']['hostvars'].iteritems(): + for host, hostvars in iteritems(self._inventory['_meta']['hostvars']): if ('role' in hostvars['tags'] and hostvars['tags']['role'] == 'bastion') or \ ('Adobe:Class' in hostvars['tags'] and hostvars['tags']['Adobe:Class'] == 'bastion'): if hostvars['public_ip'] is not None: @@ -101,7 +105,7 @@ def __init__(self,args={}): display.display("Warning, bastion host found but has no public IP (is the host stopped?)", color='yellow') if bastions: - for host, hostvars in self._inventory['_meta']['hostvars'].iteritems(): + for host, hostvars in iteritems(self._inventory['_meta']['hostvars']): if ('role' in hostvars['tags'] and hostvars['tags']['role'] == 'bastion') or \ ('Adobe:Class' in hostvars['tags'] and hostvars['tags']['Adobe:Class'] == 'bastion'): pass diff --git a/src/ops/inventory/plugin/cns.py b/src/ops/inventory/plugin/cns.py index d06ab0a0..f52cab14 100644 --- a/src/ops/inventory/plugin/cns.py +++ b/src/ops/inventory/plugin/cns.py @@ -10,7 +10,7 @@ import json -from ec2 import ec2 +from .ec2 import ec2 def cns(args): diff --git a/src/ops/inventory/plugin/legacy_pcs.py b/src/ops/inventory/plugin/legacy_pcs.py index f22e9e60..436cb663 100644 --- a/src/ops/inventory/plugin/legacy_pcs.py +++ b/src/ops/inventory/plugin/legacy_pcs.py @@ -9,8 +9,8 @@ #governing permissions and limitations under the License. import json -from ec2 import ec2 -from cns import merge_inventories +from .ec2 import ec2 +from .cns import merge_inventories def legacy_pcs(args): diff --git a/src/ops/main.py b/src/ops/main.py index 56d25ffc..b6feacb9 100644 --- a/src/ops/main.py +++ b/src/ops/main.py @@ -12,27 +12,27 @@ import logging import os -from cli.config_generator import ConfigGeneratorParserConfig, ConfigGeneratorRunner +from .cli.config_generator import ConfigGeneratorParserConfig, ConfigGeneratorRunner from simpledi import Container, auto, cache, instance, ListInstanceProvider -from cli.config import ClusterConfigGenerator, ClusterConfig -from cli.inventory import InventoryParserConfig -from cli.inventory import InventoryRunner -from cli.parser import RootParser -from cli.playbook import PlaybookRunner, PlaybookParserConfig -from cli.run import CommandRunner, CommandParserConfig -from cli.ssh import SshParserConfig, SshRunner -from cli.sync import SyncParserConfig, SyncRunner -from cli.terraform import TerraformParserConfig, TerraformRunner -from cli.helmfile import HelmfileParserConfig, HelmfileRunner -from cli.packer import PackerParserConfig, PackerRunner -from inventory.generator import DirInventoryGenerator, ShellInventoryGenerator, AnsibleInventory, \ +from .cli.config import ClusterConfigGenerator, ClusterConfig +from .cli.inventory import InventoryParserConfig +from .cli.inventory import InventoryRunner +from .cli.parser import RootParser +from .cli.playbook import PlaybookRunner, PlaybookParserConfig +from .cli.run import CommandRunner, CommandParserConfig +from .cli.ssh import SshParserConfig, SshRunner +from .cli.sync import SyncParserConfig, SyncRunner +from .cli.terraform import TerraformParserConfig, TerraformRunner +from .cli.helmfile import HelmfileParserConfig, HelmfileRunner +from .cli.packer import PackerParserConfig, PackerRunner +from .inventory.generator import DirInventoryGenerator, ShellInventoryGenerator, AnsibleInventory, \ PluginInventoryGenerator, InventoryGenerator, CachedInventoryGenerator -from inventory.plugin import ec2, legacy_pcs, cns, azr, skms -from inventory.sshconfig import SshConfigGenerator -from ops import OpsException, Executor, validate_ops_version -from ops.jinja import Template -from opsconfig import OpsConfig +from .inventory.plugin import ec2, legacy_pcs, cns, azr, skms +from .inventory.sshconfig import SshConfigGenerator +from . import OpsException, Executor, validate_ops_version +from .jinja import Template +from .opsconfig import OpsConfig logger = logging.getLogger(__name__) diff --git a/src/ops/opsconfig.py b/src/ops/opsconfig.py index 139162c0..17d25a7c 100644 --- a/src/ops/opsconfig.py +++ b/src/ops/opsconfig.py @@ -120,7 +120,7 @@ def ansible_filter_plugins(self): # the default filters are in this package filters = [self.package_dir + '/ansible/filter_plugins'] - if self.config.has_key('ansible.filter_plugins'): + if 'ansible.filter_plugins' in self.config: filters.append(self.config['ansible.filter_plugins']) return os.path.pathsep.join(filters) @@ -138,7 +138,7 @@ def ansible_config_path(self): def ansible_vars_plugins(self): vars = [self.package_dir + '/ansible/vars_plugins'] - if self.config.has_key('ansible.vars_plugins'): + if 'ansible.vars_plugins' in self.config: vars.append(self.config['ansible.vars_plugins']) return os.path.pathsep.join(vars) @@ -147,7 +147,7 @@ def ansible_vars_plugins(self): def ansible_callback_plugins(self): vars = [self.package_dir + '/ansible/callback_plugins'] - if self.config.has_key('ansible.callback_plugins'): + if 'ansible.callback_plugins' in self.config: vars.append(self.config['ansible.callback_plugins']) return os.path.pathsep.join(vars) diff --git a/src/ops/simpleconsul.py b/src/ops/simpleconsul.py index f4870c90..22f0b9f9 100644 --- a/src/ops/simpleconsul.py +++ b/src/ops/simpleconsul.py @@ -18,6 +18,8 @@ import consul import hashmerge +from six import iteritems + DEFAULT_CONNECT = { 'host': '127.0.0.1', 'port': 8500, @@ -87,7 +89,7 @@ def get(self, key, recurse=False): index, keys_list = self.conn.kv.get(key+'/', recurse=recurse) if keys_list: keys_dict = {i['Key'] : i['Value'] for i in keys_list} - for k, v in keys_dict.iteritems(): + for k, v in iteritems(keys_dict): tmp = {} path_atoms = k.split('/') leaf = path_atoms.pop() @@ -109,5 +111,5 @@ def put(self, key, value): for item in value: self.conn.kv.put(key, item, "True") elif isinstance(value, dict): - for k, v in value.iteritems(): + for k, v in iteritems(value): self.put(key + '/' + k, v) diff --git a/src/ops/simplevault.py b/src/ops/simplevault.py index 48f40c1d..dbf0cf03 100644 --- a/src/ops/simplevault.py +++ b/src/ops/simplevault.py @@ -21,7 +21,8 @@ import os import hvac import getpass -from ops.cli import display +from .cli import display +from six import iteritems MAX_LDAP_ATTEMPTS = 3 class SimpleVault(object): @@ -118,7 +119,7 @@ def put(self, path, value, lease=None, wrap_ttl=None): if isinstance(value, (basestring, int, float, bool)): payload['value'] = str(value) elif isinstance(value, dict): - for k,v in value.iteritems(): + for k,v in iteritems(value): payload[k] = str(v) else: raise Exception('Unsupported data type for secret payload') diff --git a/src/ops/terraform/terraform_cmd_generator.py b/src/ops/terraform/terraform_cmd_generator.py index 48e48aba..1c361f04 100644 --- a/src/ops/terraform/terraform_cmd_generator.py +++ b/src/ops/terraform/terraform_cmd_generator.py @@ -11,6 +11,7 @@ import os import re import shutil +import logging from jinja2 import FileSystemLoader from subprocess import Popen, PIPE @@ -289,7 +290,7 @@ def generate(self, args): variables_file=variables_file ) elif args.subcommand is not None: - # Examples: + # Examples: # - command = "state push errored.tfstate" # - command = "force-unlock " generate_module_templates = True @@ -356,10 +357,10 @@ def check_terraform_version(self): try: execution = Popen(['terraform', '--version'], stdin=PIPE, stdout=PIPE, stderr=PIPE) except Exception as e: - err('Terraform does not appear to be installed, please ensure terraform is in your PATH') - raise e + logging.exception("Terraform does not appear to be installed, please ensure terraform is in your PATH") + exit(1) current_version, execution_error = execution.communicate() - current_version = current_version.replace('Terraform ', '').split('\n', 1)[0] + current_version = current_version.decode('utf-8').replace('Terraform ', '').split('\n', 1)[0] if expected_version == 'latest': return current_version @@ -380,7 +381,7 @@ def copy_static_files(self, path, terraform_path): def write_var_file(self, path, variables): fname = os.path.join(path, 'ops.auto.tfvars') - with open(fname, 'w') as f: + with open(fname, 'wb') as f: for key, val in variables.items(): if val[0] != '"': val = '"{}"'.format(val) @@ -393,7 +394,7 @@ def write_module_templates(self, path=''): folder = os.path.dirname(fname) if not os.path.exists(folder): os.makedirs(folder) - with open(fname, 'w') as f: + with open(fname, 'wb') as f: f.write(result.encode('utf8')) def remove_module_template(self): diff --git a/tests/e2e/test_inventory.py b/tests/e2e/test_inventory.py index 18ce0e53..03db39bb 100644 --- a/tests/e2e/test_inventory.py +++ b/tests/e2e/test_inventory.py @@ -12,6 +12,8 @@ # coding=utf-8 import os import pytest +from six import PY3 + from ops.main import AppContainer from simpledi import * @@ -51,7 +53,8 @@ def test_plugin_generator(capsys): # we should have the 3 hosts in the inventory output out, err = capsys.readouterr() - print out, err + print(out) + print(err) assert 'bastion.host' in out assert 'web1.host' in out assert 'web2.host' in out @@ -61,11 +64,13 @@ def test_inventory_limit(capsys): # when we run with limit, then we should have only one host run(current_dir + '/fixture/inventory/clusters/plugin_generator.yaml', 'inventory', '--limit', 'bastion') out, err = capsys.readouterr() - print out, err + print(out) + print(err) assert 'bastion.host' in out assert 'web1.host' not in out -def test_inventory_limit_unicode_dash(): - with pytest.raises(UnicodeDecodeError): - run(current_dir + '/fixture/inventory/clusters/plugin_generator.yaml', 'inventory', '––limit', 'bastion') +if not PY3: + def test_inventory_limit_unicode_dash(): + with pytest.raises(UnicodeDecodeError): + run(current_dir + '/fixture/inventory/clusters/plugin_generator.yaml', 'inventory', '––limit', 'bastion') diff --git a/tests/e2e/test_playbook.py b/tests/e2e/test_playbook.py index 9a8d0e76..136af6be 100644 --- a/tests/e2e/test_playbook.py +++ b/tests/e2e/test_playbook.py @@ -11,6 +11,10 @@ import os import pytest +from ops import display + +from six import PY3 + from ops.main import AppContainer from simpledi import * @@ -31,6 +35,8 @@ def test_loading_of_modules_and_extensions(capsys, app): command = container.run() code = container.execute(command, pass_trough=False) out, err = capsys.readouterr() + display(out, color='gray') + display(err, color='red') assert code is 0 # the filter plugins work @@ -42,10 +48,9 @@ def test_loading_of_modules_and_extensions(capsys, app): # cluster is present as a variable in the command line assert '-e cluster=test' in command['command'] - -def test_ssh_user_unicode_dash(capsys, app): - with pytest.raises(UnicodeDecodeError): - root_dir = current_dir + '/fixture/ansible' - app([u'–vv', '--root-dir', root_dir, 'clusters/test.yaml', 'play', - 'playbooks/play_module.yaml']).run() - +if not PY3: + def test_ssh_user_unicode_dash(capsys, app): + with pytest.raises(UnicodeDecodeError): + root_dir = current_dir + '/fixture/ansible' + app([u'–vv', '--root-dir', root_dir, 'clusters/test.yaml', 'play', + 'playbooks/play_module.yaml']).run() diff --git a/tests/e2e/test_ssh.py b/tests/e2e/test_ssh.py index 2fc8a226..d6204566 100644 --- a/tests/e2e/test_ssh.py +++ b/tests/e2e/test_ssh.py @@ -11,6 +11,9 @@ import os import re + +from six import PY3 + import test_inventory import pytest @@ -35,9 +38,10 @@ def test_ssh_user(): assert re.match('ssh -F .+/ssh.config bastion.host -l remote_user', command['command']) -def test_ssh_user_unicode_dash(): - with pytest.raises(UnicodeDecodeError): - run(current_dir + '/fixture/inventory/clusters/plugin_generator.yaml', 'ssh', 'bastion', '–l', 'remote_user') +if not PY3: + def test_ssh_user_unicode_dash(): + with pytest.raises(UnicodeDecodeError): + run(current_dir + '/fixture/inventory/clusters/plugin_generator.yaml', 'ssh', 'bastion', '–l', 'remote_user') def test_ssh_user_default(): diff --git a/tests/e2e/test_terraform.py b/tests/e2e/test_terraform.py index 40469679..80f0d60a 100644 --- a/tests/e2e/test_terraform.py +++ b/tests/e2e/test_terraform.py @@ -28,7 +28,8 @@ def test_terraform_templating_for_file_plugin(capsys, app): app(['--root-dir', current_dir + '/fixture/terraform', 'clusters/prod/test.yaml', 'terraform', 'template']).run() out, err = capsys.readouterr() - print out, err + print(out) + print(err) assert 'my_user_data' in out