From ab9cad89194befd0ea061a6a449b3755e75e3803 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Tue, 16 Jul 2024 11:30:00 -0700 Subject: [PATCH 01/20] update_job_status() now has option to ignore runtime errors --- qiita_client/qiita_client.py | 10 ++++++++-- qiita_client/tests/test_qiita_client.py | 26 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 881cf5a..e8bc982 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -527,7 +527,7 @@ def get_job_info(self, job_id): logger.debug('Entered QiitaClient.get_job_info()') return self.get("/qiita_db/jobs/%s" % job_id) - def update_job_step(self, job_id, new_step): + def update_job_step(self, job_id, new_step, ignore_error=False): """Updates the current step of the job in the server Parameters @@ -536,10 +536,16 @@ def update_job_step(self, job_id, new_step): The job id new_step : str The new step + ignore_error : bool + Problems communicating w/Qiita will not raise an Error. """ logger.debug('Entered QiitaClient.update_job_step()') json_payload = dumps({'step': new_step}) - self.post("/qiita_db/jobs/%s/step/" % job_id, data=json_payload) + try: + self.post("/qiita_db/jobs/%s/step/" % job_id, data=json_payload) + except RuntimeError as e: + if ignore_error is True: + raise e def complete_job(self, job_id, success, error_msg=None, artifacts_info=None): diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index cfc6ec2..cc91d6f 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -19,6 +19,7 @@ from qiita_client.exceptions import BadRequestError CLIENT_ID = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4' +BAD_CLIENT_ID = 'NOT_A_CLIENT_ID' CLIENT_SECRET = ('J7FfQ7CQdOxuKhQAf1eoGgBAE81Ns8Gu3EKaWFm3IO2JKh' 'AmmCWZuabe0O5Mp28s1') @@ -99,6 +100,9 @@ def setUp(self): self.server_cert = environ.get('QIITA_SERVER_CERT', None) self.tester = QiitaClient("https://localhost:21174", CLIENT_ID, CLIENT_SECRET, server_cert=self.server_cert) + self.bad_tester = QiitaClient("https://localhost:21174", BAD_CLIENT_ID, + CLIENT_SECRET, + server_cert=self.server_cert) self.clean_up_files = [] # making assertRaisesRegex compatible with Python 2.7 and 3.9 @@ -232,6 +236,28 @@ def test_update_job_step(self): obs = self.tester.update_job_step(job_id, new_step) self.assertIsNone(obs) + def test_update_job_step_ignore_failure(self): + job_id = "bcc7ebcd-39c1-43e4-af2d-822e3589f14d" + new_step = "some new step" + + # confirm that update_job_step behaves as before when ignore_error + # parameter absent or set to False. + + with self.assertRaises(RuntimeError): + self.bad_tester.update_job_step(job_id, new_step) + + with self.assertRaises(RuntimeError): + self.bad_tester.update_job_step(job_id, new_step, + ignore_error=False) + + # confirm that when ignore_error is set to True, an Error is NOT + # raised. + try: + self.bad_tester.update_job_step(job_id, new_step, + ignore_error=True) + except RuntimeError: + self.fail("update_job_step raised RuntimeError unexpectedly") + def test_complete_job(self): # Create a new job data = { From ecd4487d337f482e7dc1cc2c2947736e97eeb63c Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Tue, 16 Jul 2024 11:42:26 -0700 Subject: [PATCH 02/20] Updates based on CI failures --- .github/workflows/qiita-ci.yml | 1 + qiita_client/qiita_client.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 50cda0d..42ac65e 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -66,6 +66,7 @@ jobs: conda config --add channels conda-forge conda create -q --yes -n qiita python=3.9 libgfortran numpy nginx cython redis conda activate qiita + python -m pip install --upgrade pip - name: Qiita install shell: bash -l {0} diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index e8bc982..e263987 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -57,7 +57,7 @@ def __init__(self, output_name, artifact_type, files, archive=None): def __eq__(self, other): logger.debug('Entered ArtifactInfo.__eq__()') - if type(self) != type(other): + if type(self) != type(other): # noqa return False if self.output_name != other.output_name or \ self.artifact_type != other.artifact_type or \ From 97dddfc3fd7a00e962707d0d51a619afbd6a4dd2 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Tue, 16 Jul 2024 11:50:47 -0700 Subject: [PATCH 03/20] Updated CI config --- .github/workflows/qiita-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 42ac65e..9fe26db 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -80,6 +80,7 @@ jobs: run: | conda create --yes -n qiita_client python=${{ matrix.python-version }} pip nose flake8 coverage conda activate qiita_client + python -m pip install --upgrade pip pip --quiet install . From 63a62d4ccaca79f2bdf5d10e1944cb5fc95c5687 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Tue, 16 Jul 2024 12:03:27 -0700 Subject: [PATCH 04/20] Migrating CI off Python 2.x Migrating CI off Python 2.x to support latest change. Since latest change is straightforward, current operation using Python 2.7 should not be disrupted. --- .github/workflows/qiita-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 9fe26db..a6129a1 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: - python-version: ["2.7", "3.9"] + python-version: ["3.9", "3.11"] services: postgres: From ef705eb7e6bc42c59e13a656d51a328b1eabfde5 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Tue, 16 Jul 2024 12:18:14 -0700 Subject: [PATCH 05/20] Updated CI configuration based on qp-klp plugin config. --- .github/workflows/qiita-ci.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index a6129a1..c3aaa31 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -82,7 +82,23 @@ jobs: conda activate qiita_client python -m pip install --upgrade pip - pip --quiet install . + export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/server.crt + export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg + pip --quiet install -U pip + + pip --quiet install https://github.com/qiita-spots/qtp-job-output-folder/archive/refs/heads/main.zip + + # pip --quiet install . + pip install . + pip --quiet install coveralls + + export QP_KLP_CONFIG_FP=`pwd`/configuration.json + + configure_qtp_job_output_folder --env-script "source /home/runner/.profile; conda activate klp" --server-cert $QIITA_SERVER_CERT + configure_klp --env-script "source /home/runner/.profile; export QP_KLP_CONFIG_FP=$QP_KLP_CONFIG_FP; conda activate klp" --server-cert $QIITA_SERVER_CERT + + echo "Available Qiita plugins" + ls ~/.qiita_plugins/ - name: Starting Main Services shell: bash -l {0} @@ -122,10 +138,10 @@ jobs: conda activate qiita_client export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/server.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg - - nosetests --with-doctest --with-coverage --cover-package=qiita_client - - - uses: codecov/codecov-action@v1 + export QP_KLP_CONFIG_FP=`pwd`/configuration.json + export PYTHONWARNINGS="ignore:Certificate for localhost has no \`subjectAltName\`" + nosetests --with-doctest --with-coverage -v --cover-package=qiita_client + - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} file: codecov.yml From 75e6ff41d103107d447e47969d587a76a94c1e70 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Tue, 16 Jul 2024 12:24:07 -0700 Subject: [PATCH 06/20] bugfix --- .github/workflows/qiita-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index c3aaa31..d16c461 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -95,7 +95,6 @@ jobs: export QP_KLP_CONFIG_FP=`pwd`/configuration.json configure_qtp_job_output_folder --env-script "source /home/runner/.profile; conda activate klp" --server-cert $QIITA_SERVER_CERT - configure_klp --env-script "source /home/runner/.profile; export QP_KLP_CONFIG_FP=$QP_KLP_CONFIG_FP; conda activate klp" --server-cert $QIITA_SERVER_CERT echo "Available Qiita plugins" ls ~/.qiita_plugins/ From 40c22908c4fc77fcfe446071415fd9c3efa349de Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Tue, 16 Jul 2024 12:33:29 -0700 Subject: [PATCH 07/20] Recently renamed certs --- .github/workflows/qiita-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index d16c461..7fb4d04 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -82,7 +82,7 @@ jobs: conda activate qiita_client python -m pip install --upgrade pip - export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/server.crt + export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_server.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg pip --quiet install -U pip @@ -103,7 +103,7 @@ jobs: shell: bash -l {0} run: | conda activate qiita - export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/server.crt + export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_server.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev/#g" `pwd`/qiita-dev/qiita_core/support_files/config_test.cfg > ${QIITA_CONFIG_FP} @@ -135,7 +135,7 @@ jobs: COVER_PACKAGE: ${{ matrix.cover_package }} run: | conda activate qiita_client - export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/server.crt + export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_server.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export QP_KLP_CONFIG_FP=`pwd`/configuration.json export PYTHONWARNINGS="ignore:Certificate for localhost has no \`subjectAltName\`" From 167c91693effc956118f211214ca210badb7ad4c Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Tue, 16 Jul 2024 12:39:19 -0700 Subject: [PATCH 08/20] Remove 3.11 from test matrix --- .github/workflows/qiita-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 7fb4d04..3adbfb7 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: - python-version: ["3.9", "3.11"] + python-version: ["3.9"] services: postgres: From 2a8138bd109a9b3f491b950a9803477e2d0c6bc4 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Tue, 16 Jul 2024 17:54:54 -0700 Subject: [PATCH 09/20] Added script to add qiita root-ca to environment. Added script to add the root-ca used to sign Qiita's server.crt to qiita_client's environment. The functionality in this small script may be more useful if it becomes part of qiita_client itself. --- .github/workflows/qiita-ci.yml | 9 +++++++-- rootca_insert.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 rootca_insert.py diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 3adbfb7..973122a 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -7,6 +7,7 @@ on: jobs: # derived from https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml main: + # 7/16/24: confirm current ubuntu-latest is still 22.04. runs-on: ubuntu-latest strategy: @@ -84,7 +85,6 @@ jobs: export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_server.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg - pip --quiet install -U pip pip --quiet install https://github.com/qiita-spots/qtp-job-output-folder/archive/refs/heads/main.zip @@ -136,9 +136,14 @@ jobs: run: | conda activate qiita_client export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_server.crt + export QIITA_ROOT_CA=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg - export QP_KLP_CONFIG_FP=`pwd`/configuration.json export PYTHONWARNINGS="ignore:Certificate for localhost has no \`subjectAltName\`" + + # before starting nosetests, add the root ca used to sign qiita's ci_server.crt file + # to the environment's certifi store. This will prevent CERTIFICATE_VERIFY_FAILED + # errors. + python rootca_insert.py $QIITA_ROOT_CA nosetests --with-doctest --with-coverage -v --cover-package=qiita_client - uses: codecov/codecov-action@v3 with: diff --git a/rootca_insert.py b/rootca_insert.py new file mode 100644 index 0000000..59d12e5 --- /dev/null +++ b/rootca_insert.py @@ -0,0 +1,18 @@ +import certifi +from sys import argv + + +def main(ca_fp): + with open(ca_fp, 'rb') as f: + my_ca = f.read() + + ca_file = certifi.where() + + with open(ca_file, 'ab') as f: + f.write(my_ca) + + print("%s updated" % ca_file) + + +if __name__ == '__main__': + main(argv[1]) From 52ee5fa8db860fdd797c3fc991d5945e755d77f3 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Wed, 17 Jul 2024 12:25:45 -0700 Subject: [PATCH 10/20] Update PluginTestCase base class Update PluginTestCase base class to rely on standard environment to verify server credentials. Rely on new script to update standard environment. --- qiita_client/testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_client/testing.py b/qiita_client/testing.py index ef94989..642898e 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -28,7 +28,7 @@ def setUpClass(cls): qiita_port = int(environ.get('QIITA_PORT', 21174)) cls.qclient = QiitaClient( "https://localhost:%d" % qiita_port, cls.client_id, - cls.client_secret, server_cert=cls.server_cert) + cls.client_secret) logger.debug( 'PluginTestCase.setUpClass() token %s' % cls.qclient._token) cls.qclient.post('/apitest/reload_plugins/') From 36d04bc69c3a75d8d4d7e34de3b342bcc998fa82 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Wed, 17 Jul 2024 12:38:12 -0700 Subject: [PATCH 11/20] Updated CI config to avoid installing extra plugins --- .github/workflows/qiita-ci.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 973122a..70fe684 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -82,22 +82,7 @@ jobs: conda create --yes -n qiita_client python=${{ matrix.python-version }} pip nose flake8 coverage conda activate qiita_client python -m pip install --upgrade pip - - export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_server.crt - export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg - - pip --quiet install https://github.com/qiita-spots/qtp-job-output-folder/archive/refs/heads/main.zip - - # pip --quiet install . pip install . - pip --quiet install coveralls - - export QP_KLP_CONFIG_FP=`pwd`/configuration.json - - configure_qtp_job_output_folder --env-script "source /home/runner/.profile; conda activate klp" --server-cert $QIITA_SERVER_CERT - - echo "Available Qiita plugins" - ls ~/.qiita_plugins/ - name: Starting Main Services shell: bash -l {0} From 732acfe4c8c0a38ad25cb6bcf0fc7d089e055fc1 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Wed, 17 Jul 2024 18:33:34 -0700 Subject: [PATCH 12/20] Updates from testing --- qiita_client/plugin.py | 7 ++++- qiita_client/qiita_client.py | 23 +++++++++------ qiita_client/testing.py | 9 +++--- qiita_client/tests/test_plugin.py | 6 ++-- qiita_client/tests/test_qiita_client.py | 37 +++++++++++++------------ 5 files changed, 47 insertions(+), 35 deletions(-) diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index a674eb8..e2c6c0d 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -245,7 +245,12 @@ def __call__(self, server_url, job_id, output_dir): qclient = QiitaClient(server_url, config.get('oauth2', 'CLIENT_ID'), config.get('oauth2', 'CLIENT_SECRET'), - server_cert=config.get('oauth2', 'SERVER_CERT')) + # for this group of tests, confirm optional + # ca_cert parameter works as intended. Setting + # this value will prevent underlying libraries + # from validating the server's cert using + # certifi's pem cache. + ca_cert=environ['QIITA_ROOT_CA']) if job_id == 'register': self._register(qclient) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index e263987..0942ed4 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -166,8 +166,8 @@ class QiitaClient(object): The client id to connect to the Qiita server client_secret : str The client secret id to connect to the Qiita server - server_cert : str, optional - The server certificate, in case that it is not verified + ca_cert : str, optional + CA cert used to sign and verify cert@server_url Methods @@ -175,7 +175,7 @@ class QiitaClient(object): get post """ - def __init__(self, server_url, client_id, client_secret, server_cert=None): + def __init__(self, server_url, client_id, client_secret, ca_cert=None): self._server_url = server_url self._session = requests.Session() @@ -186,16 +186,21 @@ def __init__(self, server_url, client_id, client_secret, server_cert=None): # if certificate verification should be performed or not, or a # string with the path to the certificate file that needs to be used # to verify the identity of the server. - # We are setting this attribute at __init__ time so we can avoid - # executing this if statement for each request issued. - if not server_cert: + # We are setting this attribute at __init__ time to avoid executing + # this if statement for each request issued. + + # As self-signed server certs are no longer allowed in one or more of + # our dependencies, ca_cert (if provided) must now reference a file + # that can be used to verify the certificate used by the server + # referenced by server_url, rather than the server's own certificate. + if not ca_cert: # The server certificate is not provided, use standard certificate # verification methods self._verify = True else: # The server certificate is provided, use it to verify the identity # of the server - self._verify = server_cert + self._verify = ca_cert # Set up oauth2 self._client_id = client_id @@ -543,8 +548,8 @@ def update_job_step(self, job_id, new_step, ignore_error=False): json_payload = dumps({'step': new_step}) try: self.post("/qiita_db/jobs/%s/step/" % job_id, data=json_payload) - except RuntimeError as e: - if ignore_error is True: + except BaseException as e: + if ignore_error is False: raise e def complete_job(self, job_id, success, error_msg=None, diff --git a/qiita_client/testing.py b/qiita_client/testing.py index 642898e..89e7a25 100644 --- a/qiita_client/testing.py +++ b/qiita_client/testing.py @@ -24,11 +24,12 @@ def setUpClass(cls): cls.client_id = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4' cls.client_secret = ('J7FfQ7CQdOxuKhQAf1eoGgBAE81Ns8Gu3EKaWFm3IO2JKh' 'AmmCWZuabe0O5Mp28s1') - cls.server_cert = environ.get('QIITA_SERVER_CERT', None) qiita_port = int(environ.get('QIITA_PORT', 21174)) - cls.qclient = QiitaClient( - "https://localhost:%d" % qiita_port, cls.client_id, - cls.client_secret) + + # do not rely on defining ca_cert for these tests. Instead append + # the appropriate CA cert to certifi's pem file. + cls.qclient = QiitaClient("https://localhost:%d" % qiita_port, + cls.client_id, cls.client_secret) logger.debug( 'PluginTestCase.setUpClass() token %s' % cls.qclient._token) cls.qclient.post('/apitest/reload_plugins/') diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index 05cba7f..9ebc376 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -168,8 +168,7 @@ def html_generator_func(a, b, c, d): validate_func, html_generator_func, atypes) # Generate the config file for the new plugin - tester.generate_config('ls', 'echo', - server_cert=self.server_cert) + tester.generate_config('ls', 'echo') # Ask Qiita to reload the plugins self.qclient.post('/apitest/reload_plugins/') @@ -213,8 +212,7 @@ def func(qclient, job_id, job_params, working_dir): {'out1': 'Demultiplexed'}) tester.register_command(a_cmd) - tester.generate_config('ls', 'echo', - server_cert=self.server_cert) + tester.generate_config('ls', 'echo') self.qclient.post('/apitest/reload_plugins/') tester("https://localhost:21174", 'register', 'ignored') diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index cc91d6f..3f5d66c 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -7,7 +7,7 @@ # ----------------------------------------------------------------------------- from unittest import TestCase, main -from os import environ, remove, close +from os import remove, close from os.path import basename, exists from tempfile import mkstemp from json import dumps @@ -97,12 +97,13 @@ def test_format_payload_error(self): class QiitaClientTests(PluginTestCase): def setUp(self): - self.server_cert = environ.get('QIITA_SERVER_CERT', None) - self.tester = QiitaClient("https://localhost:21174", CLIENT_ID, - CLIENT_SECRET, server_cert=self.server_cert) - self.bad_tester = QiitaClient("https://localhost:21174", BAD_CLIENT_ID, - CLIENT_SECRET, - server_cert=self.server_cert) + # self.server_cert = environ.get('QIITA_SERVER_CERT', None) + self.tester = QiitaClient("https://localhost:21174", + CLIENT_ID, + CLIENT_SECRET) + self.bad_tester = QiitaClient("https://localhost:21174", + BAD_CLIENT_ID, + CLIENT_SECRET) self.clean_up_files = [] # making assertRaisesRegex compatible with Python 2.7 and 3.9 @@ -115,12 +116,11 @@ def tearDown(self): remove(fp) def test_init(self): - obs = QiitaClient("https://localhost:21174", CLIENT_ID, - CLIENT_SECRET, server_cert=self.server_cert) + obs = QiitaClient("https://localhost:21174", CLIENT_ID, CLIENT_SECRET) self.assertEqual(obs._server_url, "https://localhost:21174") self.assertEqual(obs._client_id, CLIENT_ID) self.assertEqual(obs._client_secret, CLIENT_SECRET) - self.assertEqual(obs._verify, self.server_cert) + self.assertEqual(obs._verify, True) def test_get(self): obs = self.tester.get("/qiita_db/artifacts/1/") @@ -243,26 +243,29 @@ def test_update_job_step_ignore_failure(self): # confirm that update_job_step behaves as before when ignore_error # parameter absent or set to False. - with self.assertRaises(RuntimeError): + with self.assertRaises(BaseException): self.bad_tester.update_job_step(job_id, new_step) - with self.assertRaises(RuntimeError): - self.bad_tester.update_job_step(job_id, new_step, + with self.assertRaises(BaseException): + self.bad_tester.update_job_step(job_id, + new_step, ignore_error=False) # confirm that when ignore_error is set to True, an Error is NOT # raised. try: - self.bad_tester.update_job_step(job_id, new_step, + self.bad_tester.update_job_step(job_id, + new_step, ignore_error=True) - except RuntimeError: - self.fail("update_job_step raised RuntimeError unexpectedly") + except BaseException as e: + self.fail("update_job_step() raised an error: %s" % str(e)) def test_complete_job(self): # Create a new job data = { 'user': 'demo@microbio.me', - 'command': dumps(['QIIME', '1.9.1', 'Pick closed-reference OTUs']), + 'command': dumps(['QIIMEq2', '1.9.1', + 'Pick closed-reference OTUs']), 'status': 'running', 'parameters': dumps({"reference": 1, "sortmerna_e_value": 1, From 9aec8fcd1837c4f1c9e5a604f578a009f0906303 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Wed, 17 Jul 2024 18:43:42 -0700 Subject: [PATCH 13/20] Add testing for 3.11 --- .github/workflows/qiita-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 70fe684..1c2680f 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.11"] services: postgres: From b0cb09f5a62453175d0d6dee2e4c80b4e36e7eaf Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Wed, 17 Jul 2024 18:50:17 -0700 Subject: [PATCH 14/20] Removed matrix testing for python 3.11 --- .github/workflows/qiita-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 1c2680f..70fe684 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - python-version: ["3.9", "3.11"] + python-version: ["3.9"] services: postgres: From afd24583e2bfe7de09e63ef4771b66dce933e4c6 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Mon, 22 Jul 2024 13:01:10 -0700 Subject: [PATCH 15/20] test removal of pip updates --- .github/workflows/qiita-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 70fe684..f4ce0f7 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -67,7 +67,6 @@ jobs: conda config --add channels conda-forge conda create -q --yes -n qiita python=3.9 libgfortran numpy nginx cython redis conda activate qiita - python -m pip install --upgrade pip - name: Qiita install shell: bash -l {0} @@ -81,7 +80,6 @@ jobs: run: | conda create --yes -n qiita_client python=${{ matrix.python-version }} pip nose flake8 coverage conda activate qiita_client - python -m pip install --upgrade pip pip install . - name: Starting Main Services From 8b036e2b1fbf79034f9a1f8503941092684d385e Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Mon, 22 Jul 2024 13:07:18 -0700 Subject: [PATCH 16/20] Test removal of some qiita installation commands --- .github/workflows/qiita-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index f4ce0f7..2eda5c8 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -122,12 +122,6 @@ jobs: export QIITA_ROOT_CA=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export PYTHONWARNINGS="ignore:Certificate for localhost has no \`subjectAltName\`" - - # before starting nosetests, add the root ca used to sign qiita's ci_server.crt file - # to the environment's certifi store. This will prevent CERTIFICATE_VERIFY_FAILED - # errors. - python rootca_insert.py $QIITA_ROOT_CA - nosetests --with-doctest --with-coverage -v --cover-package=qiita_client - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} From e55a895457b3258212d048e658a4c2a02c9b7533 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Mon, 22 Jul 2024 13:52:37 -0700 Subject: [PATCH 17/20] Updated based on discussions --- .github/workflows/qiita-ci.yml | 5 ++--- README.md | 4 ++-- qiita_client/plugin.py | 2 +- qiita_client/qiita_client.py | 2 +- qiita_client/tests/test_qiita_client.py | 10 ++++++---- rootca_insert.py | 18 ------------------ 6 files changed, 12 insertions(+), 29 deletions(-) delete mode 100644 rootca_insert.py diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index 2eda5c8..f0bb7e5 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -86,7 +86,7 @@ jobs: shell: bash -l {0} run: | conda activate qiita - export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_server.crt + export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev/#g" `pwd`/qiita-dev/qiita_core/support_files/config_test.cfg > ${QIITA_CONFIG_FP} @@ -118,8 +118,7 @@ jobs: COVER_PACKAGE: ${{ matrix.cover_package }} run: | conda activate qiita_client - export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_server.crt - export QIITA_ROOT_CA=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt + export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg export PYTHONWARNINGS="ignore:Certificate for localhost has no \`subjectAltName\`" - uses: codecov/codecov-action@v3 diff --git a/README.md b/README.md index c4677ba..7dfeedf 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ This package includes the Qiita Client utility library, a library to simplify th How to test this package? ------------------------- In order to test the Qiita Client package, a local installation of Qiita should be running in test mode on the address `https://localhost:21174`, with the default test database created in Qiita's test suite. -Also, if Qiita is running with the default server SSL certificate, you need to export the variable `QIITA_SERVER_CERT` in your environment, so the Qiita Client can perform secure connections against the Qiita server: +Also, if Qiita is running with the default server SSL certificate, you need to export the variable `QIITA_ROOTCA_CERT` in your environment, so the Qiita Client can perform secure connections against the Qiita server: ```bash -export QIITA_SERVER_CERT=/qiita_core/support_files/server.crt +export QIITA_ROOT_CA=/qiita_core/support_files/ci_rootca.crt ``` diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index e2c6c0d..c41ea42 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -250,7 +250,7 @@ def __call__(self, server_url, job_id, output_dir): # this value will prevent underlying libraries # from validating the server's cert using # certifi's pem cache. - ca_cert=environ['QIITA_ROOT_CA']) + ca_cert=environ['QIITA_ROOTCA_CERT']) if job_id == 'register': self._register(qclient) diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index 0942ed4..c697d46 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -57,7 +57,7 @@ def __init__(self, output_name, artifact_type, files, archive=None): def __eq__(self, other): logger.debug('Entered ArtifactInfo.__eq__()') - if type(self) != type(other): # noqa + if type(self) is not type(other): return False if self.output_name != other.output_name or \ self.artifact_type != other.artifact_type or \ diff --git a/qiita_client/tests/test_qiita_client.py b/qiita_client/tests/test_qiita_client.py index 3f5d66c..17184e1 100644 --- a/qiita_client/tests/test_qiita_client.py +++ b/qiita_client/tests/test_qiita_client.py @@ -7,7 +7,7 @@ # ----------------------------------------------------------------------------- from unittest import TestCase, main -from os import remove, close +from os import remove, close, environ from os.path import basename, exists from tempfile import mkstemp from json import dumps @@ -97,7 +97,6 @@ def test_format_payload_error(self): class QiitaClientTests(PluginTestCase): def setUp(self): - # self.server_cert = environ.get('QIITA_SERVER_CERT', None) self.tester = QiitaClient("https://localhost:21174", CLIENT_ID, CLIENT_SECRET) @@ -116,11 +115,14 @@ def tearDown(self): remove(fp) def test_init(self): - obs = QiitaClient("https://localhost:21174", CLIENT_ID, CLIENT_SECRET) + obs = QiitaClient("https://localhost:21174", + CLIENT_ID, + CLIENT_SECRET, + ca_cert=environ['QIITA_ROOT_CA']) self.assertEqual(obs._server_url, "https://localhost:21174") self.assertEqual(obs._client_id, CLIENT_ID) self.assertEqual(obs._client_secret, CLIENT_SECRET) - self.assertEqual(obs._verify, True) + self.assertEqual(obs._verify, environ['QIITA_ROOT_CA']) def test_get(self): obs = self.tester.get("/qiita_db/artifacts/1/") diff --git a/rootca_insert.py b/rootca_insert.py deleted file mode 100644 index 59d12e5..0000000 --- a/rootca_insert.py +++ /dev/null @@ -1,18 +0,0 @@ -import certifi -from sys import argv - - -def main(ca_fp): - with open(ca_fp, 'rb') as f: - my_ca = f.read() - - ca_file = certifi.where() - - with open(ca_file, 'ab') as f: - f.write(my_ca) - - print("%s updated" % ca_file) - - -if __name__ == '__main__': - main(argv[1]) From 802db8a6a8b4535e1a30503e226f1c15fbf3e9df Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Mon, 22 Jul 2024 14:04:52 -0700 Subject: [PATCH 18/20] test change --- qiita_client/plugin.py | 2 +- qiita_client/tests/test_plugin.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index c41ea42..0616a8d 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -379,6 +379,6 @@ def register_command(self, command): PUBLICATIONS = %s [oauth2] -SERVER_CERT = %s +ROOTCA_CERT = %s CLIENT_ID = %s CLIENT_SECRET = %s""" diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index 9ebc376..ce14c5a 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -147,7 +147,7 @@ def html_generator_func(a, b, c, d): 'PUBLICATIONS = \n', '\n', '[oauth2]\n', - 'SERVER_CERT = \n'] + 'ROOTCA_CERT = \n'] # We will test the last 2 lines independently since they're variable # in each test run self.assertEqual(conf[:-2], exp_lines) From d994d89291c7f683fa27bfac80c08cf61129d1c2 Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Mon, 22 Jul 2024 15:24:02 -0700 Subject: [PATCH 19/20] Removed unused config setting --- qiita_client/plugin.py | 1 - qiita_client/tests/test_plugin.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/qiita_client/plugin.py b/qiita_client/plugin.py index 0616a8d..a0fdc53 100644 --- a/qiita_client/plugin.py +++ b/qiita_client/plugin.py @@ -379,6 +379,5 @@ def register_command(self, command): PUBLICATIONS = %s [oauth2] -ROOTCA_CERT = %s CLIENT_ID = %s CLIENT_SECRET = %s""" diff --git a/qiita_client/tests/test_plugin.py b/qiita_client/tests/test_plugin.py index ce14c5a..4981cfa 100644 --- a/qiita_client/tests/test_plugin.py +++ b/qiita_client/tests/test_plugin.py @@ -146,8 +146,7 @@ def html_generator_func(a, b, c, d): 'PLUGIN_TYPE = artifact definition\n', 'PUBLICATIONS = \n', '\n', - '[oauth2]\n', - 'ROOTCA_CERT = \n'] + '[oauth2]\n'] # We will test the last 2 lines independently since they're variable # in each test run self.assertEqual(conf[:-2], exp_lines) From 93a061baf8a8c770de57e3c6ab5bfbb398d806af Mon Sep 17 00:00:00 2001 From: Charles Cowart Date: Tue, 23 Jul 2024 09:57:26 -0700 Subject: [PATCH 20/20] Updates based on feedback --- .github/workflows/qiita-ci.yml | 4 +--- qiita_client/qiita_client.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/qiita-ci.yml b/.github/workflows/qiita-ci.yml index f0bb7e5..ebfd92c 100644 --- a/.github/workflows/qiita-ci.yml +++ b/.github/workflows/qiita-ci.yml @@ -7,7 +7,6 @@ on: jobs: # derived from https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml main: - # 7/16/24: confirm current ubuntu-latest is still 22.04. runs-on: ubuntu-latest strategy: @@ -80,7 +79,7 @@ jobs: run: | conda create --yes -n qiita_client python=${{ matrix.python-version }} pip nose flake8 coverage conda activate qiita_client - pip install . + pip --quiet install . - name: Starting Main Services shell: bash -l {0} @@ -120,7 +119,6 @@ jobs: conda activate qiita_client export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg - export PYTHONWARNINGS="ignore:Certificate for localhost has no \`subjectAltName\`" - uses: codecov/codecov-action@v3 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/qiita_client/qiita_client.py b/qiita_client/qiita_client.py index c697d46..610097f 100644 --- a/qiita_client/qiita_client.py +++ b/qiita_client/qiita_client.py @@ -532,7 +532,7 @@ def get_job_info(self, job_id): logger.debug('Entered QiitaClient.get_job_info()') return self.get("/qiita_db/jobs/%s" % job_id) - def update_job_step(self, job_id, new_step, ignore_error=False): + def update_job_step(self, job_id, new_step, ignore_error=True): """Updates the current step of the job in the server Parameters