From 73016a4fe5170d4eadb46399d406d773bcc3e2c3 Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Tue, 6 Feb 2024 03:39:37 -0500 Subject: [PATCH 1/6] zuul: Update images - Bump Fedora 36 to 39 - Use cloud images for centos stream 8 and 9 --- .zuul.d/jobs.yaml | 14 +++++++------- .zuul.d/nodesets.yaml | 30 +++++++++++++++--------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.zuul.d/jobs.yaml b/.zuul.d/jobs.yaml index 3516bfa..b3636ed 100644 --- a/.zuul.d/jobs.yaml +++ b/.zuul.d/jobs.yaml @@ -36,7 +36,7 @@ - github.com/ansible-community/ara - github.com/ansible-community/ara-collection - name: github.com/ansible/ansible - override-checkout: stable-2.13 + override-checkout: stable-2.16 pre-run: tests/zuul_pre_multinode_networking.yaml - job: @@ -44,7 +44,7 @@ parent: ara-role-database-backends nodeset: ara-database-server-multinode description: | - Deploys the ARA API server on Fedora 36 as well as CentOS Stream 8/9 + Deploys the ARA API server on Fedora 39 as well as CentOS Stream 8/9 and tests it against a central PostgreSQL server installed on CentOS Stream 9. The job exercises the ara_api Ansible role, the ARA Ansible plugins, the ARA API clients as well as the API itself. @@ -56,7 +56,7 @@ parent: ara-role-database-backends nodeset: ara-database-server-multinode description: | - Deploys the ARA API server on Fedora 36 as well as CentOS Stream 8/9 + Deploys the ARA API server on Fedora 39 as well as CentOS Stream 8/9 and tests it against a central MySQL server installed on CentOS Stream 9. The job exercises the ara_api Ansible role, the ARA Ansible plugins, the ARA API clients as well as the API itself. @@ -68,7 +68,7 @@ parent: ara-role-database-backends nodeset: ara-multinode description: | - Deploys the ARA API server on Fedora 36 as well as CentOS Stream 8/9 + Deploys the ARA API server on Fedora 39 as well as CentOS Stream 8/9 and tests it using the distributed sqlite database backend. run: tests/with_distributed_sqlite.yaml @@ -77,7 +77,7 @@ parent: ara-role-integration-base nodeset: ara-multinode description: | - Deploys the ARA API server on Fedora 36 as well as CentOS Stream 8/9 + Deploys the ARA API server on Fedora 39 as well as CentOS Stream 8/9 and tests it running gunicorn with nginx in front managing external authentication. The job exercises the ara_api and ara_frontend_nginx Ansible roles, the @@ -90,7 +90,7 @@ parent: ara-role-integration-base nodeset: ara-multinode description: | - Deploys the ARA API server on Fedora 36 as well as CentOS Stream 8/9 + Deploys the ARA API server on Fedora 39 as well as CentOS Stream 8/9 and tests it running gunicorn with nginx in front managing external authentication with client certificates. The job exercises the ara_api and ara_frontend_nginx Ansible roles, the @@ -103,7 +103,7 @@ - job: name: ara-role-api-fedora-packages parent: ara-role-integration-base - nodeset: ara-fedora-36 + nodeset: ara-fedora-39 description: | Deploys the ARA API server on Fedora 36 using distribution packages for ARA and Ansible. diff --git a/.zuul.d/nodesets.yaml b/.zuul.d/nodesets.yaml index dc39df1..70a83e3 100644 --- a/.zuul.d/nodesets.yaml +++ b/.zuul.d/nodesets.yaml @@ -4,13 +4,13 @@ name: ara-database-server-multinode nodes: - name: database-server - label: ansible-cloud-centos-9-stream-tiny - - name: fedora-36 - label: ansible-cloud-fedora-36-tiny + label: cloud-centos-9-stream + - name: fedora-39 + label: cloud-fedora-39 - name: centos-stream-9 - label: ansible-cloud-centos-9-stream-tiny + label: cloud-centos-9-stream - name: centos-stream-8 - label: ansible-cloud-centos-8-stream-tiny + label: cloud-centos-8-stream # TODO: lacking ubuntu/debian testing coverage because there is no ubuntu image in CI #- name: ubuntu-bionic # label: ubuntu-bionic-1vcpu @@ -20,7 +20,7 @@ - database-server - name: ara-api-server nodes: - - fedora-36 + - fedora-39 - centos-stream-9 - centos-stream-8 # - ubuntu-bionic @@ -30,29 +30,29 @@ - nodeset: name: ara-multinode nodes: - - name: fedora-36 - label: ansible-cloud-fedora-36-tiny + - name: fedora-39 + label: cloud-fedora-39 - name: centos-stream-9 - label: ansible-cloud-centos-9-stream-tiny + label: cloud-centos-9-stream - name: centos-stream-8 - label: ansible-cloud-centos-8-stream-tiny + label: cloud-centos-8-stream # TODO: lacking ubuntu/debian testing coverage because there is no ubuntu image in CI #- name: ubuntu-bionic # label: ubuntu-bionic-1vcpu groups: - name: ara-api-server nodes: - - fedora-36 + - fedora-39 - centos-stream-9 - centos-stream-8 # - ubuntu-bionic - nodeset: - name: ara-fedora-36 + name: ara-fedora-39 nodes: - - name: fedora-36 - label: ansible-cloud-fedora-36-tiny + - name: fedora-39 + label: cloud-fedora-39 groups: - name: ara-api-server nodes: - - fedora-36 + - fedora-39 From 5cc7efa8b5e9883e0e1a71faec0e59c189b4c12d Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Wed, 7 Feb 2024 06:28:11 -0500 Subject: [PATCH 2/6] zuul: remove ara-fedora-39 node It's already defined in ansible-community/ara --- .zuul.d/nodesets.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.zuul.d/nodesets.yaml b/.zuul.d/nodesets.yaml index 70a83e3..263e90a 100644 --- a/.zuul.d/nodesets.yaml +++ b/.zuul.d/nodesets.yaml @@ -47,12 +47,3 @@ - centos-stream-8 # - ubuntu-bionic -- nodeset: - name: ara-fedora-39 - nodes: - - name: fedora-39 - label: cloud-fedora-39 - groups: - - name: ara-api-server - nodes: - - fedora-39 From 5efc63da8a7e42166e445974419dc9676e2a1285 Mon Sep 17 00:00:00 2001 From: Jeffrey van Pelt Date: Tue, 23 Jan 2024 14:10:31 +0100 Subject: [PATCH 3/6] Make the version of Python used by ARA configurable, default to /usr/bin/python3 --- roles/ara_api/tasks/config.yaml | 2 +- .../tasks/database_engine/django.db.backends.mysql.yaml | 2 +- .../tasks/database_engine/django.db.backends.postgresql.yaml | 2 +- roles/ara_api/tasks/install/pypi.yaml | 4 ++-- roles/ara_api/tasks/install/source.yaml | 4 ++-- roles/ara_api/tasks/pre-requirements.yaml | 2 +- roles/ara_api/tasks/wsgi_server/gunicorn.yaml | 2 +- roles/ara_api/vars/CentOS_8.yaml | 4 +++- roles/ara_api/vars/CentOS_9.yaml | 2 ++ roles/ara_api/vars/Fedora.yaml | 2 ++ roles/ara_api/vars/Ubuntu.yaml | 2 ++ 11 files changed, 18 insertions(+), 10 deletions(-) diff --git a/roles/ara_api/tasks/config.yaml b/roles/ara_api/tasks/config.yaml index 56b7cd8..d8f1ca8 100644 --- a/roles/ara_api/tasks/config.yaml +++ b/roles/ara_api/tasks/config.yaml @@ -50,7 +50,7 @@ - name: Generate a random secret key environment: PATH: "{{ path_with_virtualenv }}" - command: python3 -c "from django.utils.crypto import get_random_string; print(get_random_string(length=50))" + command: "{{ ara_api_python_command }} -c 'from django.utils.crypto import get_random_string; print(get_random_string(length=50))'" no_log: "{{ ara_api_secure_logging }}" register: generated_key diff --git a/roles/ara_api/tasks/database_engine/django.db.backends.mysql.yaml b/roles/ara_api/tasks/database_engine/django.db.backends.mysql.yaml index d5f112b..9040bca 100644 --- a/roles/ara_api/tasks/database_engine/django.db.backends.mysql.yaml +++ b/roles/ara_api/tasks/database_engine/django.db.backends.mysql.yaml @@ -43,7 +43,7 @@ name: mysqlclient state: present virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}" - virtualenv_command: /usr/bin/python3 -m venv + virtualenv_command: "{{ ara_api_python_command }} -m venv" - name: Run SQL migrations environment: diff --git a/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml b/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml index e4320e4..b1359d1 100644 --- a/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml +++ b/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml @@ -45,7 +45,7 @@ name: psycopg2<2.9 state: present virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}" - virtualenv_command: /usr/bin/python3 -m venv + virtualenv_command: "{{ ara_api_python_command }} -m venv" - name: Run SQL migrations environment: diff --git a/roles/ara_api/tasks/install/pypi.yaml b/roles/ara_api/tasks/install/pypi.yaml index b5e76a9..fb297c3 100644 --- a/roles/ara_api/tasks/install/pypi.yaml +++ b/roles/ara_api/tasks/install/pypi.yaml @@ -5,7 +5,7 @@ state: "{{ (ara_api_version == 'latest') | ternary('latest', 'present') }}" version: "{{ (ara_api_version != 'latest') | ternary(ara_api_version, omit) }}" virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}" - virtualenv_command: /usr/bin/python3 -m venv + virtualenv_command: "{{ ara_api_python_command }} -m venv" notify: restart ara-api - name: Install python-passlib for managing authentication credentials @@ -13,7 +13,7 @@ name: passlib state: present virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}" - virtualenv_command: /usr/bin/python3 -m venv + virtualenv_command: "{{ ara_api_python_command }} -m venv" when: ara_api_external_auth - name: Prefix the virtualenv bin directory to PATH diff --git a/roles/ara_api/tasks/install/source.yaml b/roles/ara_api/tasks/install/source.yaml index e02244a..a16da6b 100644 --- a/roles/ara_api/tasks/install/source.yaml +++ b/roles/ara_api/tasks/install/source.yaml @@ -27,14 +27,14 @@ name: "{{ ara_api_source_checkout }}[server]" state: present virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}" - virtualenv_command: /usr/bin/python3 -m venv + virtualenv_command: "{{ ara_api_python_command }} -m venv" - name: Install python-passlib for managing authentication credentials pip: name: passlib state: present virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}" - virtualenv_command: /usr/bin/python3 -m venv + virtualenv_command: "{{ ara_api_python_command }} -m venv" when: ara_api_external_auth - name: Prefix the virtualenv bin directory to PATH diff --git a/roles/ara_api/tasks/pre-requirements.yaml b/roles/ara_api/tasks/pre-requirements.yaml index 45bdc3e..6373408 100644 --- a/roles/ara_api/tasks/pre-requirements.yaml +++ b/roles/ara_api/tasks/pre-requirements.yaml @@ -60,7 +60,7 @@ # The ansible_python_version fact might end up retrieving the version of # python2 so we need to explicitely get the version of python 3 available. - name: Validate availability of Python 3.5 - command: /usr/bin/python3 -c 'import sys; print(".".join(map(str, sys.version_info[:2])))' + command: "{{ ara_api_python_command }} -c 'import sys; print(\".\".join(map(str, sys.version_info[:2])))'" changed_when: false failed_when: false register: python_version diff --git a/roles/ara_api/tasks/wsgi_server/gunicorn.yaml b/roles/ara_api/tasks/wsgi_server/gunicorn.yaml index f4c4c23..ffde2d4 100644 --- a/roles/ara_api/tasks/wsgi_server/gunicorn.yaml +++ b/roles/ara_api/tasks/wsgi_server/gunicorn.yaml @@ -21,7 +21,7 @@ name: gunicorn state: present virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}" - virtualenv_command: /usr/bin/python3 -m venv + virtualenv_command: "{{ ara_api_python_command }} -m venv" - name: Get path to gunicorn command: which gunicorn diff --git a/roles/ara_api/vars/CentOS_8.yaml b/roles/ara_api/vars/CentOS_8.yaml index ef466f3..6a5e67b 100644 --- a/roles/ara_api/vars/CentOS_8.yaml +++ b/roles/ara_api/vars/CentOS_8.yaml @@ -24,7 +24,7 @@ ara_distribution_packages: [] # but we want to use the non-system one. Install it if it's missing. ara_api_required_packages: - git - - python36 + - python39 - policycoreutils-python-utils # cronie provides crontab - cronie @@ -41,3 +41,5 @@ ara_api_mysql_packages: - redhat-rpm-config - python36-devel - gcc + +ara_api_python_command: /usr/bin/python3.9 diff --git a/roles/ara_api/vars/CentOS_9.yaml b/roles/ara_api/vars/CentOS_9.yaml index 4a44d07..5d90d7d 100644 --- a/roles/ara_api/vars/CentOS_9.yaml +++ b/roles/ara_api/vars/CentOS_9.yaml @@ -42,3 +42,5 @@ ara_api_mysql_packages: - redhat-rpm-config - python3-devel - gcc + +ara_api_python_command: /usr/bin/python3 diff --git a/roles/ara_api/vars/Fedora.yaml b/roles/ara_api/vars/Fedora.yaml index 3c6777a..1a3107d 100644 --- a/roles/ara_api/vars/Fedora.yaml +++ b/roles/ara_api/vars/Fedora.yaml @@ -40,3 +40,5 @@ ara_api_mysql_packages: - redhat-rpm-config - python3-devel - gcc + +ara_api_python_command: /usr/bin/python3 diff --git a/roles/ara_api/vars/Ubuntu.yaml b/roles/ara_api/vars/Ubuntu.yaml index b5c2da2..57254b0 100644 --- a/roles/ara_api/vars/Ubuntu.yaml +++ b/roles/ara_api/vars/Ubuntu.yaml @@ -37,3 +37,5 @@ ara_api_mysql_packages: - libmariadbclient-dev - python3-dev - gcc + +ara_api_python_command: /usr/bin/python3 From 2d5aad7619212cae9102f8bbe5d7a16a4acaa717 Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Sat, 10 Feb 2024 20:34:38 -0500 Subject: [PATCH 4/6] centos 8: use python39-devel instead of python36-devel We need the development dependencies to install database backend packages with pip. --- roles/ara_api/vars/CentOS_8.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/ara_api/vars/CentOS_8.yaml b/roles/ara_api/vars/CentOS_8.yaml index 6a5e67b..777eaf5 100644 --- a/roles/ara_api/vars/CentOS_8.yaml +++ b/roles/ara_api/vars/CentOS_8.yaml @@ -32,14 +32,14 @@ ara_api_required_packages: ara_api_postgresql_packages: - postgresql - postgresql-devel - - python36-devel + - python39-devel - gcc ara_api_mysql_packages: - mariadb - mariadb-connector-c-devel - redhat-rpm-config - - python36-devel + - python39-devel - gcc ara_api_python_command: /usr/bin/python3.9 From 9c8ca34b6aa031f4cc8fe507b299c80282eac3ad Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Sat, 10 Feb 2024 20:47:30 -0500 Subject: [PATCH 5/6] tests: bump version of postgresql from 10 to 12 django 4.2 requires postgresql 12 and above. --- tests/vars/postgresql_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vars/postgresql_tests.yaml b/tests/vars/postgresql_tests.yaml index eb77755..1804f44 100644 --- a/tests/vars/postgresql_tests.yaml +++ b/tests/vars/postgresql_tests.yaml @@ -14,4 +14,4 @@ ara_api_configure_cron: true # The host is defined dynamically based on the address of the database server # ara_api_database_host: 127.0.0.1 _postgresql_container_name: ara-postgresql-tests -_postgresql_image_name: docker.io/postgres:10 +_postgresql_image_name: docker.io/postgres:12 From 89bd6f4892edad91aab2ad048553c0b0134539e1 Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Sat, 10 Feb 2024 21:06:15 -0500 Subject: [PATCH 6/6] postgresql: unpin the version of psycopg2 ara bumped the version of django to 4.2 allowing us to unpin psycopg2. --- .../database_engine/django.db.backends.postgresql.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml b/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml index b1359d1..937737d 100644 --- a/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml +++ b/roles/ara_api/tasks/database_engine/django.db.backends.postgresql.yaml @@ -36,13 +36,12 @@ Install the following packages before running this role again {{ _postgresql_missing_packages | join(' ') }} -# Django requires psycopg2 when using postgresql -# https://docs.djangoproject.com/en/2.2/ref/databases/#postgresql-notes +# Django requires psycopg2 (or psycopg3) when using postgresql +# https://docs.djangoproject.com/en/5.0/releases/4.2/#psycopg-3-support +# TODO: Consider psycopg3: https://github.com/ansible-community/ara-collection/issues/73 - name: Ensure psycopg2 is installed pip: - # Pin psycopg2 until we upgrade to django 3.2 LTS - # https://github.com/ansible-community/ara/issues/320 - name: psycopg2<2.9 + name: psycopg2 state: present virtualenv: "{{ ara_api_venv | bool | ternary(ara_api_venv_path, omit) }}" virtualenv_command: "{{ ara_api_python_command }} -m venv"