Add support for python 3.10

.github/workflows/tests.yml

@@ -17,17 +17,21 @@ jobs:
 
     runs-on: ubuntu-18.04
 
+    strategy:
+      matrix:
+        python-version: [ "3.6", "3.10" ]
+
     env:
       ANSIBLE_ROLES_PATH: "~/.ansible/roles"
       CI_TEST: 1
       TEST: main
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.6
+    - name: Set Up Commcare-Cloud Python Version
       uses: actions/setup-python@v3
       with:
-        python-version: "3.6"
+        python-version: ${{ matrix.python-version }}
 
     - name: Install dependencies
       run: |
@@ -38,10 +42,11 @@ jobs:
         # tries to install from the commcare-cloud-bootstrap.egg_info directory
         pip install -e .[test]
 
-        # Python for deploy-stack in .tests/tests.sh
-        # May be removed when this file is updated to match global_vars/all.yml:python_version
-        curl -sSf --retry 5 -o python-3.9.tar.bz2 'https://storage.googleapis.com/travis-ci-language-archives/python/binaries/ubuntu/18.04/x86_64/python-3.9.tar.bz2'
-        sudo tar xjf python-3.9.tar.bz2 --directory /
+        # setup 3.9 for deploy-stack
+        # may be removed when this file is updated to match global_vars/all.yml:python_version
+        sudo add-apt-repository ppa:deadsnakes/ppa
+        sudo apt update
+        sudo apt install python3.9 python3.9-venv
 
         # Install virtualenv as an isolated global executable for deploy-stack in .tests/tests.sh
         python3 -m pip install --user pipx

Makefile

@@ -9,11 +9,11 @@ PIP_COMPILE = pip-compile --output-file ${REQUIREMENTS} setup.py
 ANSIBLE_ENV=COLUMNS=80
 src/commcare_cloud/help_cache/ansible.txt: export ANSIBLE_CONFIG=src/commcare_cloud/ansible/ansible.cfg
 src/commcare_cloud/help_cache/ansible.txt:
-	$(ANSIBLE_ENV) ansible -h > src/commcare_cloud/help_cache/ansible.txt
+	$(ANSIBLE_ENV) ansible -h | sed 's/^options:/optional arguments:/' > src/commcare_cloud/help_cache/ansible.txt
 
 src/commcare_cloud/help_cache/ansible-playbook.txt: export ANSIBLE_CONFIG=src/commcare_cloud/ansible/ansible.cfg
 src/commcare_cloud/help_cache/ansible-playbook.txt:
-	$(ANSIBLE_ENV) ansible-playbook -h > src/commcare_cloud/help_cache/ansible-playbook.txt
+	$(ANSIBLE_ENV) ansible-playbook -h | sed 's/^options:/optional arguments:/' > src/commcare_cloud/help_cache/ansible-playbook.txt
 
 hosting_docs/source/reference/1-commcare-cloud/commands.md : src/commcare_cloud/* src/commcare_cloud/*/* src/commcare_cloud/*/*/*
 	manage-commcare-cloud make-docs > hosting_docs/source/reference/1-commcare-cloud/commands.md

setup.py

@@ -32,7 +32,7 @@
 ]
 test_deps = [
     'modernize',
-    'nose>=1.3.7',
+    'nose @ git+https://github.com/dimagi/nose.git@06dff28bbe661b9d032ce839ea0ec8e9eaf6f337',
     'parameterized>=0.6.1',
     'requests-mock',
 ]
@@ -57,6 +57,7 @@
             'commcare-cloud = commcare_cloud.commcare_cloud:main',
             'cchq = commcare_cloud.commcare_cloud:main',
             'manage-commcare-cloud = commcare_cloud.manage_commcare_cloud.manage_commcare_cloud:main',
+            'patched-fab = commcare_cloud.patched_fab:main',
         ],
     },
     install_requires=install_deps,

src/commcare_cloud/commands/ansible/ansible_playbook.py

@@ -22,7 +22,7 @@
 from commcare_cloud.commands.command_base import CommandBase, Argument
 from commcare_cloud.environment.main import get_environment
 from commcare_cloud.environment.paths import ANSIBLE_DIR
-from commcare_cloud.parse_help import add_to_help_text, filtered_help_message, ANSIBLE_HELP_OPTIONS_PREFIX
+from commcare_cloud.parse_help import ANSIBLE_HELP_OPTIONS_PREFIX, add_to_help_text, filtered_help_message
 
 
 class AnsiblePlaybook(CommandBase):

src/commcare_cloud/commands/ansible/run_module.py

@@ -15,7 +15,7 @@
     get_user_arg, run_action_with_check_mode)
 from commcare_cloud.commands.command_base import CommandBase, Argument
 from commcare_cloud.environment.main import get_environment
-from commcare_cloud.parse_help import add_to_help_text, filtered_help_message, ANSIBLE_HELP_OPTIONS_PREFIX
+from commcare_cloud.parse_help import ANSIBLE_HELP_OPTIONS_PREFIX, add_to_help_text, filtered_help_message
 from commcare_cloud.environment.paths import ANSIBLE_DIR
 
 NON_POSITIONAL_ARGUMENTS = (

src/commcare_cloud/commands/fab.py

@@ -41,7 +41,7 @@ class _Parser(self.parser.__class__):
             @memoized
             def epilog(self):
                 lines = subprocess.check_output(
-                    ['fab', '-f', FABFILE, '-l'],
+                    ['patched-fab', '-f', FABFILE, '-l'],
                     universal_newlines=True,
                 ).splitlines()
                 return '\n'.join(
@@ -98,7 +98,7 @@ def print_deploy_deprecation():
 
 def exec_fab_command(env_name, *extra_args):
     cmd_parts = (
-        'fab', '-f', FABFILE,
+        'patched-fab', '-f', FABFILE,
         env_name,
     ) + tuple(extra_args)
     cmd = ' '.join(shlex_quote(arg) for arg in cmd_parts)

src/commcare_cloud/parse_help.py

@@ -7,7 +7,7 @@
 
 from six.moves import range
 
-ANSIBLE_HELP_OPTIONS_PREFIX='optional arguments:'
+ANSIBLE_HELP_OPTIONS_PREFIX = 'optional arguments:'
 _HELP_CACHE = os.path.join(os.path.dirname(__file__), 'help_cache')
 _AVAILABLE_HELP_CACHES = {
     'ansible -h': os.path.join(_HELP_CACHE, 'ansible.txt'),
@@ -20,7 +20,7 @@ def _get_help_text(command):
         with open(_AVAILABLE_HELP_CACHES[command], 'r', encoding='utf-8') as f:
             return f.read()
     else:
-        return subprocess.check_output(command, shell=True)
+        return subprocess.check_output(command, shell=True).decode('utf-8')
 
 
 _LARGE_INDENT = '                        '

src/commcare_cloud/patched_fab.py

@@ -0,0 +1,11 @@
+
+def main():
+    patch_fabric_imports()
+    import fabric.main
+    return fabric.main.main()
+
+
+def patch_fabric_imports():
+    import collections
+    from collections.abc import Mapping
+    collections.Mapping = Mapping