Merge pull request #646 from biocommons/630-migrate-to-github-actions…

…-for-testing-and-packaging

630 migrate to GitHub actions for testing and packaging

.github/workflows/python-package.yml

@@ -2,45 +2,40 @@ name: Python package
 
 on:
   push:
-    branches: ["main"]
-    tags: ["*"]
   pull_request:
-    branches: ["main"]
 
 jobs:
-  codechecks:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: pip
-          cache-dependency-path: "**/setup.cfg"
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pytest pytest-cov
-          pip install -e .[test]
-
-      - name: Lint with flake8
-        run: |
-          # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-
-      - name: Format check with isort
-        run: |
-          isort --check src
-
-      - name: Format check with black
-        run: |
-          black --check src
+  # codechecks:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #     - uses: actions/checkout@v3
+
+  #     - name: Set up Python
+  #       uses: actions/setup-python@v4
+  #       with:
+  #         python-version: "3.10"
+  #         cache: "pip"
+  #         cache-dependency-path: "**/setup.cfg"
+
+  #     - name: Install dependencies
+  #       run: |
+  #         pip install -e .[dev]
+
+  #     - name: Lint with flake8
+  #       run: |
+  #         # stop the build if there are Python syntax errors or undefined names
+  #         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+  #         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+  #         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+  #     - name: Format check with isort
+  #       run: |
+  #         isort --check src
+
+  #     - name: Format check with black
+  #       run: |
+  #         black --check src
 
   test:
     runs-on: ubuntu-latest
@@ -57,22 +52,20 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-          cache: pip
+          cache: "pip"
           cache-dependency-path: "**/setup.cfg"
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install pytest pytest-cov
-          pip install -e .[test]
+          pip install -e .[dev]
 
       - name: Test with pytest
         run: |
-          pytest
+          make test
 
   deploy:
     needs:
-      - codechecks
+      # - codechecks # skip for now
       - test
     runs-on: ubuntu-latest
 
@@ -88,7 +81,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          pip install --upgrade pip
           pip install build pytest pytest-cov twine
       - name: Build package
         run: python -m build

.vscode/settings.json

@@ -1,11 +1,12 @@
 {
-    "python.formatting.provider": "yapf",
+    "python.formatting.provider": "black",
     "python.venvPath": "${workspaceFolder}/venv/",
     "python.testing.unittestEnabled": false,
     "python.testing.nosetestsEnabled": false,
     "python.testing.pytestEnabled": true,
     "python.testing.pytestArgs": [
         "--exitfirst",
         "--verbose"
-    ]
+    ],
+    "python.analysis.typeCheckingMode": "off"
 }

Makefile

@@ -2,16 +2,14 @@
 
 .DELETE_ON_ERROR:
 .PHONY: FORCE
-.PRECIOUS :
+.PRECIOUS:
 .SUFFIXES:
 
 SHELL:=/bin/bash -e -o pipefail
 SELF:=$(firstword $(MAKEFILE_LIST))
 
-PKG=hgvs
-PKGD=$(subst .,/,${PKG})
-PYV:=3.9
-VEDIR=venv/${PYV}
+PY_VERSION:=$(shell python3 -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')
+VE_DIR=venv/${PY_VERSION}
 
 TEST_DIRS:=tests
 DOC_TESTS:=docs hgvs ./README.rst
@@ -29,45 +27,42 @@ help:
 ############################################################################
 #= SETUP, INSTALLATION, PACKAGING
 
-#=> venv: make a Python 3 virtual environment
-venv: ${VEDIR}
-venv/%:
-	python$* -mvenv $@; \
-	source $@/bin/activate; \
-	python -m ensurepip --upgrade; \
-	pip install --upgrade pip setuptools
-
-#=> setup: setup/upgrade packages *in current environment*
-.PHONY: setup
-setup: etc/develop.reqs etc/install.reqs
-	if [ -s $(word 1,$^) ]; then pip install --upgrade -r $(word 1,$^); fi
-	if [ -s $(word 2,$^) ]; then pip install --upgrade -r $(word 2,$^); fi
-
 #=> devready: create venv, install prerequisites, install pkg in develop mode
 .PHONY: devready
 devready:
-	make ${VEDIR} && source ${VEDIR}/bin/activate && make setup develop
+	make ${VE_DIR} && source ${VE_DIR}/bin/activate && make develop
 	@echo '#################################################################################'
-	@echo '###  Do not forget to `source ${VEDIR}/bin/activate` to use this environment  ###'
+	@echo '###  Do not forget to `source ${VE_DIR}/bin/activate` to use this environment  ###'
 	@echo '#################################################################################'
 
+#=> venv: make a Python 3 virtual environment
+venv: ${VEDIR}
+${VE_DIR}: venv/%:
+	python$* -mvenv $@; \
+	source $@/bin/activate; \
+	python -m ensurepip --upgrade; \
+	pip install --upgrade pip setuptools wheel
+
 #=> develop: install package in develop mode
-#=> install: install package
-#=> bdist bdist_egg bdist_wheel build sdist: distribution options
-.PHONY: bdist bdist_egg bdist_wheel build build_sphinx sdist install develop
+.PHONY: develop install
 develop:
-	pip install -e .
-bdist bdist_egg bdist_wheel build sdist install: %:
-	python setup.py $@
+	@if [ -z "$${VIRTUAL_ENV}" ]; then echo "Not in a virtual environment; see README.md" 1>&2; exit 1; fi
+	pip install -e .[dev]
 
+#=> install: install package
+install:
+	pip install .
 
+#=> build: make sdist and wheel
+.PHONY: build
+build: %:
+	python -m build
 
 
 ############################################################################
 #= TESTING
 # see test configuration in setup.cfg
 
-
 #=> test: execute tests
 #=> test-code: test code (including embedded doctests)
 #=> test-docs: test example code in docs
@@ -78,15 +73,27 @@ bdist bdist_egg bdist_wheel build sdist install: %:
 # => extra fx issues mapping models normalization parametrize pnd quick regression validation
 .PHONY: test test-code test-docs stest
 test:
-	python setup.py pytest
+	pytest
 test-code:
-	python setup.py pytest --addopts="${TEST_DIRS}"
+	pytest --addopts="${TEST_DIRS}"
 test-docs:
-	python setup.py pytest --addopts="${DOC_TESTS}"
+	pytest --addopts="${DOC_TESTS}"
 stest:
-	python setup.py pytest --addopts="-vvv -s -k ${t}"
+	pytest --addopts="-vvv -s -k ${t}"
 test-%:
-	python setup.py pytest --addopts="-m '$*' ${TEST_DIRS}"
+	pytest --addopts="-m '$*' ${TEST_DIRS}"
+
+#=> test-learn: build test data cache
+#=> test-relearn: destroy and rebuild test data cache
+test-learn:
+	# The appropriate environment should be set up using misc/docker-compose.yml
+	UTA_DB_URL=postgresql://anonymous@localhost:5432/uta/uta_20180821 \
+	HGVS_SEQREPO_URL=http://localhost:5000/seqrepo \
+	HGVS_CACHE_MODE=learn \
+	pytest -s
+test-relearn:
+	rm -f tests/data/cache-py3.hdp
+	make test-learn
 
 #=> tox -- run all tox tests
 tox:
@@ -96,14 +103,13 @@ tox:
 ############################################################################
 #= UTILITY TARGETS
 
-# N.B. Although code is stored in github, I use hg and hg-git on the command line
 #=> reformat: reformat code with yapf and commit
 .PHONY: reformat
 reformat:
-	@if hg sum | grep -qL '^commit:.*modified'; then echo "Repository not clean" 1>&2; exit 1; fi
-	@if hg sum | grep -qL ' applied'; then echo "Repository has applied patches" 1>&2; exit 1; fi
-	yapf -i -r "${PKGD}" tests
-	hg commit -m "reformatted with yapf"
+	@if ! git diff --cached --exit-code >/dev/null; then echo "Repository not clean" 1>&2; exit 1; fi
+	black src tests
+	isort src tests
+	git commit -a -m "reformatted with black and isort"
 
 #=> docs -- make sphinx docs
 .PHONY: docs
@@ -117,28 +123,32 @@ docs: develop
 #=> clean: remove temporary and backup files
 .PHONY: clean
 clean:
-	find . \( -name \*~ -o -name \*.bak \) -print0 | xargs -0r rm
+	rm -frv **/*~ **/*.bak
 	-make -C docs $@
 	-make -C examples $@
 
 #=> cleaner: remove files and directories that are easily rebuilt
 .PHONY: cleaner
 cleaner: clean
-	rm -fr .cache *.egg-info build dist docs/_build htmlcov
-	find . \( -name \*.pyc -o -name \*.orig -o -name \*.rej \) -print0 | xargs -0r rm
-	find . -name __pycache__ -print0 | xargs -0r rm -fr
-	/bin/rm -fr examples/.ipynb_checkpoints
-	/bin/rm -f hgvs.{dot,svg,png,sfood}
-	-make -C docs $@
-	-make -C examples $@
-
-#=> cleanest: remove files and directories that require more time/network fetches to rebuild
+	rm -frv .cache build dist docs/_build
+	rm -frv **/__pycache__
+	rm -frv **/*.egg-info
+	rm -frv **/*.pyc
+	rm -frv **/*.orig
+	rm -frv **/*.rej
+
+#=> cleanest: remove files and directories that are more expensive to rebuild
 .PHONY: cleanest
 cleanest: cleaner
-	rm -fr .eggs .tox venv
+	rm -frv .eggs .tox venv
 	-make -C docs $@
 	-make -C examples $@
 
+#=> distclean: remove untracked files and other detritus
+.PHONY: distclean
+distclean: cleanest
+	git clean -df
+
 
 ############################################################################
 #= HGVS specific

README.rst

@@ -94,14 +94,14 @@ Configuration
 otherwise through environment variables, like so::
 
   # N.B. These are examples. The correct values will depend on your installation
-  $ export UTA_DB_URL=postgresql://anonymous:anonymous@localhost:5432/uta/uta_20180821
+  $ export UTA_DB_URL=postgresql://anonymous:anonymous@localhost:5432/uta/uta_20210129
   $ export HGVS_SEQREPO_DIR=/usr/local/share/seqrepo/latest
 
 Alternatively, if you are unable to pass the postgresql password in the
 UTA_DB_URL environment variable (i.e., generating an auth token), you can set
 UTA_DB_URL to ``postgresql://<user>@<host>/<db>/<schema>`` and set PGPASSWORD. For example::
 
-  $ export UTA_DB_URL=postgresql://anonymous@localhost:5432/uta/uta_20180821 PGPASSWORD=anonymous
+  $ export UTA_DB_URL=postgresql://anonymous@localhost:5432/uta/uta_20210129 PGPASSWORD=anonymous
 
 See the installation instructions for details.
 

docs/modules/config.rst

@@ -51,9 +51,9 @@ The defaults are::
   pool_min = 1
   pool_max = 10
   
-  prd_uta_version = uta_20180821
-  stg_uta_version = uta_20180821
-  dev_uta_version = uta_20180821
+  prd_uta_version = uta_20210129
+  stg_uta_version = uta_20210129
+  dev_uta_version = uta_20210129
   public_host = uta.biocommons.org
   local_host = localhost
   public_prd = postgresql://anonymous:anonymous@${public_host}/uta/${prd_uta_version}

misc/docker-compose.yml

@@ -1,9 +1,9 @@
 # Creates a stack of data services for hgvs and vr-python
 #
 # ```
-# $ docker volume create --name=uta_vol
-# $ docker volume create --name=seqrepo_vol
-# $ docker-compose up
+# docker volume create --name=uta_vol
+# docker volume create --name=seqrepo_vol
+# docker-compose up
 # ```
 
 
@@ -23,6 +23,7 @@ services:
     volumes:
       - seqrepo_vol:/usr/local/share/seqrepo
     network_mode: host
+    command: seqrepo-rest-service /usr/local/share/seqrepo/2021-01-29
 
   uta:
     # Test: