From 9290e24bf045d160e041b38bc6c45d39aeee36b9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 23 Nov 2020 20:22:34 +0100 Subject: [PATCH 1/7] Matrix pytest workflow for py3.7 to py3.9 --- .github/workflows/pytest.yml | 23 +++++++++++++++++++---- setup.cfg | 1 - 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 0a76384..3bd4130 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,6 +1,3 @@ -# TODO add OK_API_KEY secret -# TODO add Slack notification - name: Test suite & diagnostics on: @@ -13,12 +10,26 @@ jobs: pytest: runs-on: ubuntu-latest + strategy: + matrix: + include: + - python-version: "3.7" + run-diagnostics: false + - python-version: "3.8" + run-diagnostics: false + - python-version: "3.9" + run-diagnostics: true + + name: py${{ matrix.python-version }} + steps: - uses: actions/checkout@v2 with: submodules: true - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} - name: Cache Python packages uses: actions/cache@v2 @@ -26,9 +37,10 @@ jobs: path: | ~/.cache/pip ~/.cache/rclone*.zip - key: ubuntu-latest + key: ubuntu-latest-${{ matrix.python-version }} - name: Install rclone for Google Cloud Storage uploads + if: ${{ matrix.run-diagnostics }} env: service_account_json: ${{ secrets.GCS_SERVICE_ACCOUNT_0 }} run: | @@ -56,8 +68,11 @@ jobs: uses: codecov/codecov-action@v1 - name: Create diagnostics and upload to Google Cloud Storage + if: ${{ matrix.run-diagnostics }} run: | item historical diagnostics output/ rclone --config=ci/rclone.conf --progress \ copy output \ gcs:historical-data-ci.transportenergy.org/$GITHUB_RUN_NUMBER/ + + # TODO add Slack notification diff --git a/setup.cfg b/setup.cfg index b4a1537..b1020c9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,6 @@ classifiers = Intended Audience :: Science/Research Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 From 857f7cf4b2f5d873d4f177e6eeb7fb5d747e4a3c Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 23 Nov 2020 20:23:23 +0100 Subject: [PATCH 2/7] =?UTF-8?q?Use=20py3.7=20compatible=20lru=5Fcache()=20?= =?UTF-8?q?in=20historical=E2=80=A6T009?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- item/historical/scripts/T009.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/item/historical/scripts/T009.py b/item/historical/scripts/T009.py index d9b8e8d..67d9d95 100644 --- a/item/historical/scripts/T009.py +++ b/item/historical/scripts/T009.py @@ -24,7 +24,7 @@ ) -@lru_cache +@lru_cache() def service(value): """Determine 'service' dimension based on a vehicle type.""" if value in [ From 58962b2319c76fd225b325269b0cbff1512a7236 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 23 Nov 2020 20:49:56 +0100 Subject: [PATCH 3/7] Update doc/usage.rst with current examples --- doc/usage.rst | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/doc/usage.rst b/doc/usage.rst index 032a522..b06156c 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -4,7 +4,7 @@ Usage The following instructions are not language-specific. Both the Python and R tools will operate on data stored in the following way; see the language-specific sections below for more details. -1. Clone the repository and/or install the code in one or both languages. +1. Install the Python package, or clone the repository and install the code in both languages. 2. Create 1 or more separate directory to contain the input and output files. A simple way to do this is to call ``$ item mkdirs PATH``, where ``PATH`` is a base directory; this will create a tree of directories. @@ -20,6 +20,10 @@ Installation ============ Use `pip `_. +From `PyPI `_:: + + $ pip install transport-energy + From source (for instance, to develop the code locally):: $ git clone --recurse-submodules git@github.com:transportenergy/database.git @@ -28,24 +32,33 @@ From source (for instance, to develop the code locally):: Or, without cloning the repository:: - $ pip install --editable git://github.com/transportenergy/database#egg=item@subdirectory=python + $ pip install --editable git://github.com/transportenergy/database#egg=item@subdirectory=python [doc,hist,tests] Usage ===== -From Python scripts:: +From Python scripts, import the :mod:`item` module or submodules and call specific functions described elsewhere in this documentation. +For instance:: + + from item import historical, model, structure + + # Display information about the cleaning script for + # one historical data set + help(historical.T009) - import item + # Generate a template for model data + structure.make_template() - data = item.load_model_data(1) + # Display metadata (information about a model's data) + model.load_model_regions("itf", version=2) Run tests ========= -Tests in ``item/tests`` can be run with `py.test `_. -The command-line option ``--local-data`` must be defined in order for these tests to work:: +Unit and integration tests in :mod:`item.tests` can be run with `pytest `_. +The command-line option ``--local-data`` can be used to point to local data files, e.g. the model database, not included in the Git repository:: $ py.test --local-data=../../data/model/database item ================ test session starts ================ From d84977682412f376d206dd3b3e976718cb69df1e Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 23 Nov 2020 20:56:28 +0100 Subject: [PATCH 4/7] Adjust test_historical.test_coverage for T000 --- item/tests/test_historical.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/item/tests/test_historical.py b/item/tests/test_historical.py index b1ae5f9..f9c442e 100644 --- a/item/tests/test_historical.py +++ b/item/tests/test_historical.py @@ -92,7 +92,7 @@ def test_run_notebook(dataset_id, tmp_path): assert errors == [] -@pytest.mark.parametrize("dataset_id, N_areas", [(0, 57), (1, 37), (2, 53), (3, 57)]) +@pytest.mark.parametrize("dataset_id, N_areas", [(0, 58), (1, 37), (2, 53), (3, 57)]) def test_coverage(dataset_id, N_areas): """Test the historical.diagnostics.coverage method.""" df = pd.read_csv(fetch_source(dataset_id, use_cache=True)) From 1a71aa1b30c9bde5acb68139e8d8559877ae6f50 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 23 Nov 2020 20:57:24 +0100 Subject: [PATCH 5/7] Add "fail-fast: false" to pytest workflow configuration --- .github/workflows/pytest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 3bd4130..f0b64e7 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -20,6 +20,8 @@ jobs: - python-version: "3.9" run-diagnostics: true + fail-fast: false + name: py${{ matrix.python-version }} steps: From 8c084ec470df2e44bfa2634fad7acc8e03e4b14d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 23 Nov 2020 21:04:39 +0100 Subject: [PATCH 6/7] Mark test_historical.test_fetch() as always slow --- .github/workflows/pytest.yml | 2 +- item/tests/test_historical.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index f0b64e7..79d6bec 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -64,7 +64,7 @@ jobs: - name: Run pytest env: OK_API_KEY: ${{ secrets.OPENKAPSARC_API_KEY }} - run: pytest item --color=yes --cov-report=xml --verbose + run: pytest --color=yes --cov-report=xml --verbose item - name: Upload test coverage to Codecov.io uses: codecov/codecov-action@v1 diff --git a/item/tests/test_historical.py b/item/tests/test_historical.py index f9c442e..db0b14a 100644 --- a/item/tests/test_historical.py +++ b/item/tests/test_historical.py @@ -11,6 +11,7 @@ from item.historical.util import run_notebook +@pytest.mark.slow @pytest.mark.parametrize( "source_id", [ From a115cb9927a93ad3f0e12d2acede989ee47ad8bd Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 23 Nov 2020 21:21:31 +0100 Subject: [PATCH 7/7] Add #43 to whatsnew.rst --- doc/whatsnew.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 83115c4..4e399df 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -11,7 +11,9 @@ What's new? Next release ============ -- Add the :doc:`glossary` page. +- Update :doc:`usage` documentation (:issue:`41`, :pull:`43`). +- Increase minimum Python version to 3.7 and ensure compatibility (:issue:`41`, :pull:`43`). +- Add the :doc:`glossary` page (:pull:`42`). - Correct an error in the input data for :mod:`.T001` (:issue:`32`, :pull:`40`).