From ae5dd8e216bf558e09fe4f457b6c951d7128b04a Mon Sep 17 00:00:00 2001 From: Qiao Qiao <68757394+qiaouchicago@users.noreply.github.com> Date: Fri, 21 Jul 2023 14:00:23 -0500 Subject: [PATCH] DEV 2005 add gitlab ci (#409) * add gitlab ci * update deps * remove requiremtns.txt in tox * get db var from env * format setup.py --------- Co-authored-by: Arthur Collet --- .gitlab-ci.yml | 46 ++++++ .secrets.baseline | 6 +- dev-requirements.in | 10 -- dev-requirements.txt | 238 ++++++-------------------------- setup.py | 13 ++ test/conftest.py | 22 ++- test/helpers.py | 11 +- test/test_admin_script.py | 6 +- test/test_datamodel.py | 9 +- test/test_gdc_postgres_admin.py | 8 +- test/test_node_tagging.py | 13 +- tox.ini | 24 +++- 12 files changed, 164 insertions(+), 242 deletions(-) create mode 100644 .gitlab-ci.yml delete mode 100644 dev-requirements.in diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..e54e051e --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,46 @@ +--- + +include: + - project: nci-gdc/gitlab-templates + ref: 0.2.1 + file: + - templates/global/full.yaml + - templates/python/full.yaml + - templates/common/python.yaml + +tox: + stage: test + parallel: + matrix: + - PYTHON_VERSION: ['3.6', '3.7', '3.8'] + image: ${BASE_CONTAINER_REGISTRY}/python${PYTHON_VERSION}-builder:${BASE_CONTAINER_VERSION} + services: + - name: docker.osdc.io/ncigdc/ci-postgres-13:feat_dev-1924-postgres-init-script-3673ef90 + alias: postgres + tags: + - dind + variables: + # these are for postgres docker + POSTGRES_HOST_AUTH_METHOD: trust + PG_HOST: postgres + PG_INIT_SQL: | + create user test with superuser password 'test'; + create database automated_test with owner test; + create database dev_models with owner test; + before_script: + - !reference [.load_github_key, script] + - pip install "tox<4" + script: + - tox -r -e py + + +# used by release +.python_versions: + parallel: + matrix: + - BUILD_PY_VERSION: [python3.6] + - RELEASE_PY_VERSION: [python3.6] + +release: + before_script: + - git fetch --unshallow || true diff --git a/.secrets.baseline b/.secrets.baseline index e18f4983..2d940361 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2022-10-14T21:28:31Z", + "generated_at": "2023-07-21T18:27:05Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -60,7 +60,7 @@ "hashed_secret": "5d0fa74acf95d1d6bebd0d37f76a94e77d604fd9", "is_secret": false, "is_verified": false, - "line_number": 41, + "line_number": 35, "type": "Basic Auth Credentials" } ], @@ -69,7 +69,7 @@ "hashed_secret": "5d0fa74acf95d1d6bebd0d37f76a94e77d604fd9", "is_secret": false, "is_verified": false, - "line_number": 36, + "line_number": 38, "type": "Basic Auth Credentials" } ] diff --git a/dev-requirements.in b/dev-requirements.in deleted file mode 100644 index f86798c9..00000000 --- a/dev-requirements.in +++ /dev/null @@ -1,10 +0,0 @@ --c requirements.txt - -dataclasses; python_version < '3.7' -notebook -jupyter -jupyter-client -jupyter-console -jupyter-core -pytest -pytest-cov diff --git a/dev-requirements.txt b/dev-requirements.txt index 58ba9fb3..dc5d9aaa 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,246 +2,88 @@ # This file is autogenerated by pip-compile with python 3.6 # To update, run: # -# pip-compile --output-file=dev-requirements.txt dev-requirements.in +# pip-compile --extra=dev --output-file=dev-requirements.txt # -argon2-cffi==21.3.0 - # via notebook -argon2-cffi-bindings==21.2.0 - # via argon2-cffi -async-generator==1.10 - # via nbclient attrs==22.2.0 # via - # -c requirements.txt + # gdcdictionary # jsonschema # pytest -backcall==0.2.0 - # via ipython -bleach==4.1.0 - # via nbconvert cffi==1.15.1 - # via - # -c requirements.txt - # argon2-cffi-bindings + # via cryptography coverage[toml]==6.2 # via pytest-cov -dataclasses==0.8 ; python_version < "3.7" - # via - # -r dev-requirements.in - # argon2-cffi +cryptography==3.4.8 + # via gdc-ng-models decorator==5.1.1 - # via - # -c requirements.txt - # ipython - # traitlets -defusedxml==0.7.1 - # via nbconvert -entrypoints==0.4 - # via - # jupyter-client - # nbconvert + # via gdcdatamodel (setup.py) +gdc-ng-models @ git+https://github.com/NCI-GDC/gdc-ng-models.git@1.6.4 + # via gdcdatamodel (setup.py) +gdcdictionary @ git+https://github.com/NCI-GDC/gdcdictionary.git@2.6.6 + # via gdcdatamodel (setup.py) +graphviz==0.19.1 + # via gdcdatamodel (setup.py) importlib-metadata==4.8.3 # via - # -c requirements.txt # jsonschema # pluggy # pytest iniconfig==1.1.1 # via pytest -ipykernel==5.5.6 - # via - # ipywidgets - # jupyter - # jupyter-console - # notebook - # qtconsole -ipython==7.16.3 - # via - # ipykernel - # ipywidgets - # jupyter-console -ipython-genutils==0.2.0 - # via - # ipykernel - # ipywidgets - # nbformat - # notebook - # qtconsole - # traitlets -ipywidgets==7.7.5 - # via jupyter -jedi==0.17.2 - # via ipython -jinja2==3.0.3 - # via - # nbconvert - # notebook jsonschema==3.2.0 # via - # -c requirements.txt - # nbformat -jupyter==1.0.0 - # via -r dev-requirements.in -jupyter-client==7.1.2 - # via - # -r dev-requirements.in - # ipykernel - # jupyter-console - # nbclient - # notebook - # qtconsole -jupyter-console==6.4.3 - # via - # -r dev-requirements.in - # jupyter -jupyter-core==4.9.2 - # via - # -r dev-requirements.in - # jupyter-client - # nbconvert - # nbformat - # notebook - # qtconsole -jupyterlab-pygments==0.1.2 - # via nbconvert -jupyterlab-widgets==1.1.4 - # via ipywidgets -markupsafe==2.0.1 - # via jinja2 -mistune==0.8.4 - # via nbconvert -nbclient==0.5.9 - # via nbconvert -nbconvert==6.0.7 - # via - # jupyter - # notebook -nbformat==5.1.3 - # via - # nbclient - # nbconvert - # notebook -nest-asyncio==1.5.6 - # via - # jupyter-client - # nbclient - # notebook -notebook==6.4.10 - # via - # -r dev-requirements.in - # jupyter - # widgetsnbextension + # gdcdatamodel (setup.py) + # gdcdictionary packaging==21.3 - # via - # bleach - # pytest - # qtpy -pandocfilters==1.5.0 - # via nbconvert -parso==0.7.1 - # via jedi -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython + # via pytest pluggy==1.0.0 # via pytest -prometheus-client==0.16.0 - # via notebook -prompt-toolkit==3.0.36 +psqlgraph @ git+https://github.com/NCI-GDC/psqlgraph.git@4.0.1 + # via gdcdatamodel (setup.py) +psycopg2==2.9.6 # via - # ipython - # jupyter-console -ptyprocess==0.7.0 - # via - # pexpect - # terminado + # gdc-ng-models + # psqlgraph py==1.11.0 # via pytest pycparser==2.21 - # via - # -c requirements.txt - # cffi -pygments==2.14.0 - # via - # ipython - # jupyter-console - # jupyterlab-pygments - # nbconvert - # qtconsole + # via cffi pyparsing==3.0.9 # via packaging pyrsistent==0.18.0 # via - # -c requirements.txt + # gdcdatamodel (setup.py) # jsonschema pytest==7.0.1 # via - # -r dev-requirements.in + # gdcdatamodel (setup.py) # pytest-cov pytest-cov==4.0.0 - # via -r dev-requirements.in -python-dateutil==2.8.2 - # via jupyter-client -pyzmq==25.0.2 - # via - # jupyter-client - # notebook - # qtconsole -qtconsole==5.2.2 - # via jupyter -qtpy==2.0.1 - # via qtconsole -send2trash==1.8.2 - # via notebook + # via gdcdatamodel (setup.py) +pytz==2020.5 + # via + # gdc-ng-models + # gdcdatamodel (setup.py) +pyyaml==6.0.1 + # via gdcdictionary +rstr==3.0.0 + # via psqlgraph six==1.16.0 + # via jsonschema +sqlalchemy==1.3.24 # via - # -c requirements.txt - # bleach - # jsonschema - # python-dateutil - # traitlets -terminado==0.12.1 - # via notebook -testpath==0.6.0 - # via nbconvert + # gdc-ng-models + # psqlgraph tomli==1.2.3 # via # coverage # pytest -tornado==6.1 - # via - # ipykernel - # jupyter-client - # notebook - # terminado -traitlets==4.3.3 - # via - # ipykernel - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # nbclient - # nbconvert - # nbformat - # notebook - # qtconsole typing-extensions==4.1.1 - # via - # -c requirements.txt - # argon2-cffi - # importlib-metadata -wcwidth==0.2.6 - # via prompt-toolkit -webencodings==0.5.1 - # via bleach -widgetsnbextension==3.6.4 - # via ipywidgets + # via importlib-metadata +xlocal==0.5 + # via psqlgraph zipp==3.6.0 - # via - # -c requirements.txt - # importlib-metadata + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/setup.py b/setup.py index ba965c17..1f630abb 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,19 @@ "gdc-ng-models @ git+https://github.com/NCI-GDC/gdc-ng-models.git@1.6.4#egg=gdc-ng-models", "psqlgraph @ git+https://github.com/NCI-GDC/psqlgraph.git@4.0.1#egg=psqlgraph", ], + extras_require={ + "dev": [ + "pytest", + "pytest-cov", + ], + "jupyter": [ + "notebook", + "jupyter", + "jupyter-client", + "jupyter-console", + "jupyter-core" + ] + }, package_data={ "gdcdatamodel": [ "xml_mappings/*.yaml", diff --git a/test/conftest.py b/test/conftest.py index 4b38beb0..26baefac 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -7,8 +7,7 @@ import random import unittest import uuid -from test.helpers import create_tables, truncate -from test.models import BasicDictionary +from test import helpers, models as test_models import pkg_resources import pytest @@ -18,18 +17,13 @@ from gdcdatamodel import models -models.load_dictionary(BasicDictionary, "basic") +models.load_dictionary(test_models.BasicDictionary, "basic") from gdcdatamodel.models import basic # noqa @pytest.fixture(scope="session") def db_config(): - return { - "host": "localhost", - "user": "test", - "password": "test", - "database": "automated_test", - } + return helpers.DB_CONFIG @pytest.fixture(scope="session") @@ -46,11 +40,11 @@ def tables_created(db_config): ) ) - create_tables(engine) + helpers.create_tables(engine) yield - truncate(engine) + helpers.truncate(engine) @pytest.fixture(scope="session") @@ -125,10 +119,10 @@ def redacted_fixture(g): @pytest.mark.usefixtures("db_class") class BaseTestCase(unittest.TestCase): def setUp(self): - truncate(self.g.engine) + helpers.truncate(self.g.engine) def tearDown(self): - truncate(self.g.engine) + helpers.truncate(self.g.engine) @pytest.fixture(scope="module") @@ -136,7 +130,7 @@ def sample_data(): with pkg_resources.resource_stream(__name__, "schema/data/sample.yaml") as f: graph = yaml.safe_load(f) - f = mocks.GraphFactory(basic, BasicDictionary) + f = mocks.GraphFactory(basic, test_models.BasicDictionary) nodes = f.create_from_nodes_and_edges( nodes=graph["nodes"], edges=graph["edges"], diff --git a/test/helpers.py b/test/helpers.py index 7af0b770..77a9a7fa 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -1,5 +1,6 @@ +import os import psqlgraph -from psqlgraph import Edge, Node, create_all, ext +from psqlgraph import create_all, ext from gdcdatamodel import models @@ -64,3 +65,11 @@ def truncate_ng_tables(conn): for meta in ng_models_metadata: for table in meta.tables: conn.execute(f"DELETE FROM {table}") + + +DB_CONFIG = { + "host": os.getenv("PG_HOST", "localhost"), + "user": os.getenv("PG_HOST", "test"), + "password": os.getenv("PG_PASS", "test"), + "database": os.getenv("PG_NAME","automated_test"), +} diff --git a/test/test_admin_script.py b/test/test_admin_script.py index 78219491..d147626c 100644 --- a/test/test_admin_script.py +++ b/test/test_admin_script.py @@ -6,9 +6,11 @@ from gdcdatamodel import gdc_postgres_admin as pgadmin from gdcdatamodel import models +from test import helpers -def get_base_args(host="localhost", database="automated_test", namespace=None): - return ["-H", host, "-U", "postgres", "-D", database, "-N", namespace or ""] + +def get_base_args(host=helpers.DB_CONFIG.get('host'), database=helpers.DB_CONFIG.get('database'), namespace=None): + return ["-H", host, "-U", helpers.DB_CONFIG.get('user'), "-D", database, "-N", namespace or ""] def get_admin_driver(db_config, namespace=None): diff --git a/test/test_datamodel.py b/test/test_datamodel.py index 47972a6b..7de33954 100644 --- a/test/test_datamodel.py +++ b/test/test_datamodel.py @@ -6,6 +6,7 @@ from psqlgraph.exc import ValidationError from gdcdatamodel import models as md +from test import helpers logging.basicConfig(level=logging.INFO) @@ -13,10 +14,10 @@ class TestDataModel(unittest.TestCase): @classmethod def setUpClass(cls): - host = "localhost" - user = "test" - password = "test" - database = "automated_test" + host = helpers.DB_CONFIG.get('host') + user = helpers.DB_CONFIG.get('user') + password = helpers.DB_CONFIG.get('password') + database = helpers.DB_CONFIG.get('database') cls.g = PsqlGraphDriver(host, user, password, database) cls._clear_tables() diff --git a/test/test_gdc_postgres_admin.py b/test/test_gdc_postgres_admin.py index ca8b8d0d..107b792c 100644 --- a/test/test_gdc_postgres_admin.py +++ b/test/test_gdc_postgres_admin.py @@ -11,6 +11,8 @@ from gdcdatamodel import gdc_postgres_admin as pgadmin from gdcdatamodel import models +from test import helpers + logging.basicConfig() @@ -19,9 +21,9 @@ class TestGDCPostgresAdmin(unittest.TestCase): logger = logging.getLogger("TestGDCPostgresAdmin") logger.setLevel(logging.INFO) - host = "localhost" - user = "postgres" - database = "automated_test" + host = helpers.DB_CONFIG.get('host') + user = helpers.DB_CONFIG.get('user') + database = helpers.DB_CONFIG.get('database') base_args = [ "-H", diff --git a/test/test_node_tagging.py b/test/test_node_tagging.py index 64ce70df..8bc8a623 100644 --- a/test/test_node_tagging.py +++ b/test/test_node_tagging.py @@ -1,4 +1,5 @@ -from test.helpers import create_tables, truncate +import os +from test import helpers import pytest from psqlgraph import PsqlGraphDriver @@ -11,17 +12,17 @@ def bg(): """Fixture for database driver""" cfg = { - "host": "localhost", - "user": "test", - "password": "test", + "host": os.getenv("PG_HOST", "localhost"), + "user": os.getenv("PG_USER", "test"), + "password": os.getenv("PG_PASS", "test"), "database": "dev_models", "package_namespace": "basic", } g = PsqlGraphDriver(**cfg) - create_tables(g.engine, namespace="basic") + helpers.create_tables(g.engine, namespace="basic") yield g - truncate(g.engine, namespace="basic") + helpers.truncate(g.engine, namespace="basic") @pytest.fixture(scope="module") diff --git a/tox.ini b/tox.ini index 3bab1b1c..ad8c7549 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,12 @@ testpaths = test [testenv] +passenv = + PG_* +setenv = + NO_PROXY=localhost,postgres + no_proxy=localhost,postgres deps = - -rrequirements.txt -rdev-requirements.txt commands = pytest -lv --cov gdcdatamodel --cov-report term --cov-report xml --cov-report html --junit-xml test-reports/results.xml {posargs} @@ -18,3 +22,21 @@ deps = requests codacy-coverage commands = python-codacy-coverage -r coverage.xml + +[testenv:publish] +changedir = +passenv = + TWINE_* +skip_install=true +deps = + setuptools_scm + build + twine +install_command = + python -m pip install {opts} {packages} +commands = + python -m setuptools_scm + python -m build + python -m twine check dist/* + python -m twine upload dist/* +commands_post=