Upgrade to Ansible 2.9.6 (elastic#261)

Ansible 2.9.1 is the first version that is compatible with macOS and
Python >=3.8.0, so currently running Ansible via `make configure` fails
on macOS.

Unfortunately, so far, we've relied on installing requirements defined
in `setup.py` by directly invoking `python3 setup.py ...` which uses
`easy_install` to install the dependencies[1]. Apart from being a big
anti-pattern, Ansible >= 2.9 fails to install with
easy_install regardless of OS.

Install Ansible 2.9.6 (matching the version in esbench) via pip and
clean up Python version deps in setup.py and left-over Ansible
configuration options.

[1] http://cerfacs.fr/coop/python3_doc/pip_install/

install.sh

@@ -6,7 +6,7 @@
 if python3 -c 'import os, sys; sys.exit(0) if "VIRTUAL_ENV" in os.environ else sys.exit(1)' >/dev/null 2>&1
 then
     # inside virtual env
-    python3 setup.py -q develop --upgrade
+    python3 -mpip install --upgrade -e .
 else
-    python3 setup.py -q develop --user --upgrade
+    python3 -mpip install --user --upgrade -e .
 fi

night_rally/fixtures/ansible/ansible.cfg

@@ -2,13 +2,13 @@
 callback_whitelist = timer
 display_skipped_hosts = false
 host_key_checking = false
-inventory = inventory/local
 library = library
+interpreter_python = "/usr/bin/env python3"
+inventory = inventory/production
 lookup_plugins = plugins/lookup_plugins
 nocows = 1
 retry_files_save_path = ~/.ansible
 roles_path = roles
-ansible_python_interpreter = python3
 # needed to prevent Failed to set permissions on the temporary files Ansible needs to create when becoming an unprivileged user
 allow_world_readable_tmpfiles = true
 # https://github.com/ansible/ansible/issues/10698

night_rally/fixtures/ansible/inventory/production/inventory.py

@@ -1,20 +1,16 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
-'''
+"""
 Extremely simple dynamic inventory script for Ansible.
 
 Expects TARGET_HOSTS env var to create the target_hosts group
 and COORDINATING_NODES env var to create the coordinating_nodes group.
-'''
+"""
 
 import argparse
+import json
 import os
 
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
 ANSIBLE_SSH_PRIVATE_KEY_FILE = "/var/lib/jenkins/.ssh/rally"
 ANSIBLE_USER = "rally"
 
@@ -45,38 +41,39 @@ def build_inventory(self):
         inventory.update(self.dynamic_groups(group_name="coordinating_nodes", env_var="COORDINATING_NODES"))
         inventory.update(
             {
-                '_meta': {
-                    'hostvars': {}
+                "_meta": {
+                    "hostvars": {}
                 }
             }
         )
-
         return inventory
 
-    def dynamic_groups(self, group_name=None, env_var=None):
+    @staticmethod
+    def dynamic_groups(group_name=None, env_var=None):
         if group_name is None or env_var is None or env_var not in os.environ:
             return {}
         else:
             group_hosts = os.environ[env_var]
 
         return {
             group_name: {
-                'hosts': group_hosts.split(","),
-                'vars': {
-                    'ansible_ssh_user': ANSIBLE_USER,
-                    'ansible_ssh_private_key_file': ANSIBLE_SSH_PRIVATE_KEY_FILE
+                "hosts": group_hosts.split(","),
+                "vars": {
+                    "ansible_ssh_user": ANSIBLE_USER,
+                    "ansible_ssh_private_key_file": ANSIBLE_SSH_PRIVATE_KEY_FILE
                 }
             }
         }
 
-    def empty_inventory(self):
-        return {'_meta': {'hostvars': {}}}
+    @staticmethod
+    def empty_inventory():
+        return {"_meta": {"hostvars": {}}}
 
     def read_cli_args(self):
         parser = argparse.ArgumentParser()
-        parser.add_argument('--list', action = 'store_true')
-        parser.add_argument('--host', action = 'store')
-        parser.add_argument('--refresh', action = 'store_true')
+        parser.add_argument("--list", action="store_true")
+        parser.add_argument("--host", action="store")
+        parser.add_argument("--refresh", action="store_true")
         self.args = parser.parse_args()
 
 

setup.py

@@ -18,9 +18,12 @@ def str_from_file(name):
 
 long_description = str_from_file("README.md")
 
+# tuples of (major, minor) of supported Python versions ordered from lowest to highest
+supported_python_versions = [(3, 8)]
+
 install_requires = [
     # required for night-rally fixtures and deploying ini files
-    "ansible==2.8.8",
+    "ansible==2.9.6",
     # License: Apache 2.0
     # transitive dependency urllib3: MIT
     "elasticsearch==7.6.0",
@@ -40,6 +43,13 @@ def str_from_file(name):
     "pytest-cov"
 ]
 
+python_version_classifiers = ["Programming Language :: Python :: {}.{}".format(major, minor)
+                              for major, minor in supported_python_versions]
+
+first_supported_version = "{}.{}".format(supported_python_versions[0][0], supported_python_versions[0][1])
+# next minor after the latest supported version
+first_unsupported_version = "{}.{}".format(supported_python_versions[-1][0], supported_python_versions[-1][1] + 1)
+
 setup(name="night_rally",
       version=__versionstr__,
       description="Nightly Benchmark Scripts for Elasticsearch Benchmarks based on Rally",
@@ -50,6 +60,9 @@ def str_from_file(name):
           exclude=("tests*", "external*")
       ),
       include_package_data=True,
+      # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires
+      python_requires=">={},<{}".format(first_supported_version, first_unsupported_version),
+
       package_data={"": ["*.json", "*.yml", "*.cfg", "*.j2"]},
       install_requires=install_requires,
       test_suite="tests",
@@ -72,8 +85,5 @@ def str_from_file(name):
           "Operating System :: POSIX",
           "Programming Language :: Python",
           "Programming Language :: Python :: 3",
-          "Programming Language :: Python :: 3.4",
-          "Programming Language :: Python :: 3.5",
-          "Programming Language :: Python :: 3.6"
-      ],
+      ] + python_version_classifiers,
       zip_safe=False)