From ec155b6fc36cb485e21412f3071f40410d950cfe Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 25 May 2020 13:38:38 +0300 Subject: [PATCH] Upgrade to Ansible 2.9.6 (#261) Ansible 2.9.1 is the first version that is compatible with macOS and Python >=3.8.0, so currently running Ansible via `make configure` fails on macOS. Unfortunately, so far, we've relied on installing requirements defined in `setup.py` by directly invoking `python3 setup.py ...` which uses `easy_install` to install the dependencies[1]. Apart from being a big anti-pattern, Ansible >= 2.9 fails to install with easy_install regardless of OS. Install Ansible 2.9.6 (matching the version in esbench) via pip and clean up Python version deps in setup.py and left-over Ansible configuration options. [1] http://cerfacs.fr/coop/python3_doc/pip_install/ --- install.sh | 4 +- night_rally/fixtures/ansible/ansible.cfg | 4 +- .../ansible/inventory/production/inventory.py | 39 +++++++++---------- setup.py | 20 +++++++--- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/install.sh b/install.sh index 1729140ae96a4..d5996d9061c60 100755 --- a/install.sh +++ b/install.sh @@ -6,7 +6,7 @@ if python3 -c 'import os, sys; sys.exit(0) if "VIRTUAL_ENV" in os.environ else sys.exit(1)' >/dev/null 2>&1 then # inside virtual env - python3 setup.py -q develop --upgrade + python3 -mpip install --upgrade -e . else - python3 setup.py -q develop --user --upgrade + python3 -mpip install --user --upgrade -e . fi diff --git a/night_rally/fixtures/ansible/ansible.cfg b/night_rally/fixtures/ansible/ansible.cfg index 92cfa0d77021c..12bbbabcb0016 100644 --- a/night_rally/fixtures/ansible/ansible.cfg +++ b/night_rally/fixtures/ansible/ansible.cfg @@ -2,13 +2,13 @@ callback_whitelist = timer display_skipped_hosts = false host_key_checking = false -inventory = inventory/local library = library +interpreter_python = "/usr/bin/env python3" +inventory = inventory/production lookup_plugins = plugins/lookup_plugins nocows = 1 retry_files_save_path = ~/.ansible roles_path = roles -ansible_python_interpreter = python3 # needed to prevent Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user allow_world_readable_tmpfiles = true # https://github.com/ansible/ansible/issues/10698 diff --git a/night_rally/fixtures/ansible/inventory/production/inventory.py b/night_rally/fixtures/ansible/inventory/production/inventory.py index a8cb6a951b902..7a59d604c55f0 100755 --- a/night_rally/fixtures/ansible/inventory/production/inventory.py +++ b/night_rally/fixtures/ansible/inventory/production/inventory.py @@ -1,20 +1,16 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -''' +""" Extremely simple dynamic inventory script for Ansible. Expects TARGET_HOSTS env var to create the target_hosts group and COORDINATING_NODES env var to create the coordinating_nodes group. -''' +""" import argparse +import json import os -try: - import json -except ImportError: - import simplejson as json - ANSIBLE_SSH_PRIVATE_KEY_FILE = "/var/lib/jenkins/.ssh/rally" ANSIBLE_USER = "rally" @@ -45,15 +41,15 @@ def build_inventory(self): inventory.update(self.dynamic_groups(group_name="coordinating_nodes", env_var="COORDINATING_NODES")) inventory.update( { - '_meta': { - 'hostvars': {} + "_meta": { + "hostvars": {} } } ) - return inventory - def dynamic_groups(self, group_name=None, env_var=None): + @staticmethod + def dynamic_groups(group_name=None, env_var=None): if group_name is None or env_var is None or env_var not in os.environ: return {} else: @@ -61,22 +57,23 @@ def dynamic_groups(self, group_name=None, env_var=None): return { group_name: { - 'hosts': group_hosts.split(","), - 'vars': { - 'ansible_ssh_user': ANSIBLE_USER, - 'ansible_ssh_private_key_file': ANSIBLE_SSH_PRIVATE_KEY_FILE + "hosts": group_hosts.split(","), + "vars": { + "ansible_ssh_user": ANSIBLE_USER, + "ansible_ssh_private_key_file": ANSIBLE_SSH_PRIVATE_KEY_FILE } } } - def empty_inventory(self): - return {'_meta': {'hostvars': {}}} + @staticmethod + def empty_inventory(): + return {"_meta": {"hostvars": {}}} def read_cli_args(self): parser = argparse.ArgumentParser() - parser.add_argument('--list', action = 'store_true') - parser.add_argument('--host', action = 'store') - parser.add_argument('--refresh', action = 'store_true') + parser.add_argument("--list", action="store_true") + parser.add_argument("--host", action="store") + parser.add_argument("--refresh", action="store_true") self.args = parser.parse_args() diff --git a/setup.py b/setup.py index 397bb21f35455..16dd6cce2a82b 100644 --- a/setup.py +++ b/setup.py @@ -18,9 +18,12 @@ def str_from_file(name): long_description = str_from_file("README.md") +# tuples of (major, minor) of supported Python versions ordered from lowest to highest +supported_python_versions = [(3, 8)] + install_requires = [ # required for night-rally fixtures and deploying ini files - "ansible==2.8.8", + "ansible==2.9.6", # License: Apache 2.0 # transitive dependency urllib3: MIT "elasticsearch==7.6.0", @@ -40,6 +43,13 @@ def str_from_file(name): "pytest-cov" ] +python_version_classifiers = ["Programming Language :: Python :: {}.{}".format(major, minor) + for major, minor in supported_python_versions] + +first_supported_version = "{}.{}".format(supported_python_versions[0][0], supported_python_versions[0][1]) +# next minor after the latest supported version +first_unsupported_version = "{}.{}".format(supported_python_versions[-1][0], supported_python_versions[-1][1] + 1) + setup(name="night_rally", version=__versionstr__, description="Nightly Benchmark Scripts for Elasticsearch Benchmarks based on Rally", @@ -50,6 +60,9 @@ def str_from_file(name): exclude=("tests*", "external*") ), include_package_data=True, + # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires + python_requires=">={},<{}".format(first_supported_version, first_unsupported_version), + package_data={"": ["*.json", "*.yml", "*.cfg", "*.j2"]}, install_requires=install_requires, test_suite="tests", @@ -72,8 +85,5 @@ def str_from_file(name): "Operating System :: POSIX", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6" - ], + ] + python_version_classifiers, zip_safe=False)