Merge pull request #43 from transportenergy/issue/41

Add py3.8 and earlier to CI; improve docs

.github/workflows/pytest.yml

@@ -1,6 +1,3 @@
-# TODO add OK_API_KEY secret
-# TODO add Slack notification
-
 name: Test suite & diagnostics
 
 on:
@@ -13,22 +10,39 @@ jobs:
   pytest:
     runs-on: ubuntu-latest
 
+    strategy:
+      matrix:
+        include:
+        - python-version: "3.7"
+          run-diagnostics: false
+        - python-version: "3.8"
+          run-diagnostics: false
+        - python-version: "3.9"
+          run-diagnostics: true
+
+      fail-fast: false
+
+    name: py${{ matrix.python-version }}
+
     steps:
     - uses: actions/checkout@v2
       with:
         submodules: true
 
     - uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
 
     - name: Cache Python packages
       uses: actions/cache@v2
       with:
         path: |
           ~/.cache/pip
           ~/.cache/rclone*.zip
-        key: ubuntu-latest
+        key: ubuntu-latest-${{ matrix.python-version }}
 
     - name: Install rclone for Google Cloud Storage uploads
+      if: ${{ matrix.run-diagnostics }}
       env:
         service_account_json: ${{ secrets.GCS_SERVICE_ACCOUNT_0 }}
       run: |
@@ -50,14 +64,17 @@ jobs:
     - name: Run pytest
       env:
         OK_API_KEY: ${{ secrets.OPENKAPSARC_API_KEY }}
-      run: pytest item --color=yes --cov-report=xml --verbose
+      run: pytest --color=yes --cov-report=xml --verbose item
 
     - name: Upload test coverage to Codecov.io
       uses: codecov/codecov-action@v1
 
     - name: Create diagnostics and upload to Google Cloud Storage
+      if: ${{ matrix.run-diagnostics }}
       run: |
         item historical diagnostics output/
         rclone --config=ci/rclone.conf --progress \
           copy output \
           gcs:historical-data-ci.transportenergy.org/$GITHUB_RUN_NUMBER/
+
+    # TODO add Slack notification

doc/usage.rst

@@ -4,7 +4,7 @@ Usage
 The following instructions are not language-specific.
 Both the Python and R tools will operate on data stored in the following way; see the language-specific sections below for more details.
 
-1. Clone the repository and/or install the code in one or both languages.
+1. Install the Python package, or clone the repository and install the code in both languages.
 
 2. Create 1 or more separate directory to contain the input and output files.
    A simple way to do this is to call ``$ item mkdirs PATH``, where ``PATH`` is a base directory; this will create a tree of directories.
@@ -20,6 +20,10 @@ Installation
 ============
 
 Use `pip <https://pip.pypa.io/en/stable/>`_.
+From `PyPI <https://pypi.org/project/transport-energy/>`_::
+
+    $ pip install transport-energy
+
 From source (for instance, to develop the code locally)::
 
     $ git clone --recurse-submodules git@github.com:transportenergy/database.git
@@ -28,24 +32,33 @@ From source (for instance, to develop the code locally)::
 
 Or, without cloning the repository::
 
-    $ pip install --editable git://github.com/transportenergy/database#egg=item@subdirectory=python
+    $ pip install --editable git://github.com/transportenergy/database#egg=item@subdirectory=python [doc,hist,tests]
 
 
 Usage
 =====
 
-From Python scripts::
+From Python scripts, import the :mod:`item` module or submodules and call specific functions described elsewhere in this documentation.
+For instance::
+
+    from item import historical, model, structure
+
+    # Display information about the cleaning script for
+    # one historical data set
+    help(historical.T009)
 
-    import item
+    # Generate a template for model data
+    structure.make_template()
 
-    data = item.load_model_data(1)
+    # Display metadata (information about a model's data)
+    model.load_model_regions("itf", version=2)
 
 
 Run tests
 =========
 
-Tests in ``item/tests`` can be run with `py.test <https://pytest.org/>`_.
-The command-line option ``--local-data`` must be defined in order for these tests to work::
+Unit and integration tests in :mod:`item.tests` can be run with `pytest <https://pytest.org/>`_.
+The command-line option ``--local-data`` can be used to point to local data files, e.g. the model database, not included in the Git repository::
 
     $ py.test --local-data=../../data/model/database item
     ================ test session starts ================

doc/whatsnew.rst

@@ -11,7 +11,9 @@ What's new?
 Next release
 ============
 
-- Add the :doc:`glossary` page.
+- Update :doc:`usage` documentation (:issue:`41`, :pull:`43`).
+- Increase minimum Python version to 3.7 and ensure compatibility (:issue:`41`, :pull:`43`).
+- Add the :doc:`glossary` page (:pull:`42`).
 - Correct an error in the input data for :mod:`.T001` (:issue:`32`, :pull:`40`).
 
 

item/historical/scripts/T009.py

@@ -24,7 +24,7 @@
 )
 
 
-@lru_cache
+@lru_cache()
 def service(value):
     """Determine 'service' dimension based on a vehicle type."""
     if value in [

item/tests/test_historical.py

@@ -11,6 +11,7 @@
 from item.historical.util import run_notebook
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize(
     "source_id",
     [
@@ -92,7 +93,7 @@ def test_run_notebook(dataset_id, tmp_path):
     assert errors == []
 
 
-@pytest.mark.parametrize("dataset_id, N_areas", [(0, 57), (1, 37), (2, 53), (3, 57)])
+@pytest.mark.parametrize("dataset_id, N_areas", [(0, 58), (1, 37), (2, 53), (3, 57)])
 def test_coverage(dataset_id, N_areas):
     """Test the historical.diagnostics.coverage method."""
     df = pd.read_csv(fetch_source(dataset_id, use_cache=True))

setup.cfg

@@ -15,7 +15,6 @@ classifiers =
     Intended Audience :: Science/Research
     Programming Language :: Python
     Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9