Merge pull request #64 from transportenergy/hotfix/ci

Update continuous integration configuration
transportenergy · Feb 17, 2021 · cbf4e1b · cbf4e1b
2 parents 01ec2fa + 140cc30
commit cbf4e1b
Show file tree

Hide file tree

Showing 10 changed files with 175 additions and 83 deletions.
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -6,6 +6,10 @@ on:
   pull_request:
     branches: [ master ]
 
+env:
+  gcs_bucket: gcs:data.transportenergy.org/historical/ci/
+  gcs_url: https://storage.googleapis.com/data.transportenergy.org/historical/ci/
+
 jobs:
   pytest:
     runs-on: ubuntu-latest
@@ -20,8 +24,6 @@ jobs:
         - python-version: "3.9"
           run-diagnostics: true
 
-      fail-fast: false
-
     name: py${{ matrix.python-version }}
 
     steps:
@@ -41,25 +43,11 @@ jobs:
           ~/.cache/rclone*.zip
         key: ubuntu-latest-${{ matrix.python-version }}
 
-    - name: Install rclone for Google Cloud Storage uploads
-      if: ${{ matrix.run-diagnostics }}
-      env:
-        service_account_json: ${{ secrets.GCS_SERVICE_ACCOUNT_0 }}
-      run: |
-        mkdir -p $HOME/.cache
-        pushd $HOME/.cache
-        curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip
-        popd
-        unzip $HOME/.cache/rclone-current-linux-amd64.zip
-        ls -d rclone-v* > $GITHUB_PATH
-        echo "$service_account_json" >ci/item-historical-database.json
-
     - name: Upgrade pip, wheel
       run: python -m pip install --upgrade pip wheel
 
     - name: Install the Python package and dependencies
-      # TODO make this work without editable by adjusting path config
-      run: pip install --editable .[doc,hist,tests]
+      run: pip install --editable .[tests]
 
     - name: Run pytest
       env:
@@ -69,12 +57,33 @@ jobs:
     - name: Upload test coverage to Codecov.io
       uses: codecov/codecov-action@v1
 
+    - name: Install rclone
+      if: ${{ matrix.run-diagnostics }}
+      env:
+        service_account_json: ${{ secrets.GCS_SERVICE_ACCOUNT_1 }}
+      run: |
+        mkdir -p $HOME/.cache
+        pushd $HOME/.cache
+        curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip
+        popd
+        unzip $HOME/.cache/rclone-current-linux-amd64.zip
+        ls -d rclone-v* > $GITHUB_PATH
+        echo "$service_account_json" >ci/service-account-key.json
+
     - name: Create diagnostics and upload to Google Cloud Storage
       if: ${{ matrix.run-diagnostics }}
+      env:
+        RCLONE_CONFIG: ci/rclone.conf
       run: |
         item historical diagnostics output/
-        rclone --config=ci/rclone.conf --progress \
-          copy output \
-          gcs:historical-data-ci.transportenergy.org/$GITHUB_RUN_NUMBER/
+        rclone --progress copy output ${{ env.gcs_bucket }}${{ github.run_number }}/
 
-    # TODO add Slack notification
+    - uses: LouisBrunner/checks-action@v1.1.1
+      if: ${{ matrix.run-diagnostics }}
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        name: Upload historical database & diagnostics
+        conclusion: success
+        details_url: ${{ env.gcs_url }}${{ github.run_number }}/index.html
+        output: |
+          {"summary": "${{ env.gcs_url }}${{ github.run_number }}/index.html"}
diff --git a/ci/rclone.conf b/ci/rclone.conf
@@ -1,4 +1,4 @@
 [gcs]
 type = google cloud storage
-service_account_file = ci/item-historical-database.json
+service_account_file = ci/service-account-key.json
 object_acl = publicRead
diff --git a/doc/ci.rst b/doc/ci.rst
@@ -0,0 +1,57 @@
+Continuous integration
+**********************
+
+In software development, the term “continuous integration” (CI) refers to a process to check that pieces of code can be ‘integrated’, e.g. built and run together, and that they perform as expected.
+The process is ‘continuous’ because it is automatically performed for incremental changes in the code, e.g. Git commits.
+CI enables efficient use of resources (in particular, people's time) by reducing or eliminating manual work in performing these checks.
+
+This page details how iTEM uses CI practices to enable **continuous validation**, i.e. to reduce manual work in checking the validity of methods, data, and results related to transport data.
+
+GitHub Actions
+==============
+
+Every pull request to the ``transportenergy/database`` has the following checks run on it.
+The different checks, or *jobs*, are described by *workflow files*:
+
+- :file:`.github/workflows/cq.yaml`
+- :file:`.github/workflows/pytest.yaml`
+
+These files are interpreted by `GitHub Actions <https://docs.github.com/en/actions>`_ to set up and control the jobs (follow the link for extensive documentation).
+
+Test suite & diagnostics
+------------------------
+
+pytest
+   This runs the test suite, which is in the files under :file:`item/tests/`.
+   These are implemented using `pytest <https://docs.pytest.org>`_.
+
+   The *coverage*—details of which statements of the code are executed at least once during the tests, and which are missing—is uploaded to Codecov.
+
+Upload historical database and diagnostics
+   The :doc:`cli` command ``item historical diagnostics`` is run to completely regenerate the historical database and diagnostic calculations based on it.
+   The results are uploaded to Google Cloud Storage, and a link to the :file:`index.html` is published to the pull request on GitHub.
+
+   The file contains the following sections:
+
+   Coverage
+      One file for each input data set that summarizes the data included in the input data.
+
+   Quality
+      The outputs of other :ref:`diagnostic <diagnostics>` checks of intermediate or output data.
+
+   Input data
+      A copy of the input data retrieved from each upstream data source at the time of the build.
+
+Code quality
+------------
+
+black, flake8, mypy
+   Check that the :ref:`code-style` has been properly applied.
+
+Sphinx docs
+   Check that the documentation in :file:`doc/` can be built without errors.
+   The documentation on https://transportenergy.rtfd.io is built separately and hosted by `Read The Docs <https://readthedocs.org>`_.
+   This check is only to flag potential issues in the build.
+
+twine
+   Check that the Python package can be built.
diff --git a/doc/cli.rst b/doc/cli.rst
diff --git a/doc/conf.py b/doc/conf.py
@@ -111,20 +111,16 @@
 
 # -- Options for LaTeX output ------------------------------------------------
 
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
+# latex_elements = {
+#     # The paper size ('letterpaper' or 'a4paper').
+#     "papersize": "letterpaper",
+#     # The font size ('10pt', '11pt' or '12pt').
+#     "pointsize": "10pt",
+#     # Additional stuff for the LaTeX preamble.
+#     "preamble": "",
+#     # Latex figure (float) alignment
+#     "figure_align": "htbp",
+# }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,

diff --git a/doc/developing.rst b/doc/developing.rst
@@ -23,8 +23,10 @@ Repository organization
     - ``item_config_example.yaml`` — an example configuration file.
 
 
-Style guide
-===========
+.. _code-style:
+
+Code style
+==========
 
 - Run the following tools on all new and modified code::
 

diff --git a/doc/historical.rst b/doc/historical.rst
@@ -25,6 +25,8 @@ See the documentation of :func:`~historical.process` for a detailed description
 The notebook name corresponds to the input data set which it handles, e.g. :file:`T001.ipynb`.
 
 
+.. _diagnostics:
+
 Diagnostics
 ===========
 

diff --git a/doc/index.rst b/doc/index.rst
@@ -25,8 +25,8 @@ This documentation, built automatically from the `transportenergy/database GitHu
    model
    historical
    structure
+   ci
    remote
-   cli
    metadata
    glossary
    whatsnew
@@ -56,7 +56,7 @@ The goals for these tools are that:
 License
 =======
 
-Copyright © 2017–2020 iTEM contributors
+Copyright © 2017–2021 iTEM contributors
 
 The software is licensed under the `GNU General Public License, version 3 <http://www.gnu.org/licenses/gpl-3.0.en.html>`_.
 

diff --git a/doc/usage.rst b/doc/usage.rst
@@ -1,9 +1,6 @@
 Usage
 *****
 
-The following instructions are not language-specific.
-Both the Python and R tools will operate on data stored in the following way; see the language-specific sections below for more details.
-
 1. Install the Python package, or clone the repository and install the code in both languages.
 
 2. Create 1 or more separate directory to contain the input and output files.
@@ -13,7 +10,7 @@ Both the Python and R tools will operate on data stored in the following way; se
 3. Copy the file ``item_config_example.yaml`` to ``item_config.yaml`` in any working directory where you intend to use this code.
    Edit the file (see the inline comments) to point to the directories created in #2 above.
 
-4. Use the tools through the :doc:`cli`.
+4. Use the tools through the :ref:`cli`.
 
 
 Installation
@@ -28,19 +25,21 @@ From source (for instance, to develop the code locally)::
 
     $ git clone --recurse-submodules git@github.com:transportenergy/database.git
     $ cd database
-    $ pip install --editable .[doc,hist,tests]
+    $ pip install --editable .[tests]
 
 Or, without cloning the repository::
 
-    $ pip install --editable git://github.com/transportenergy/database#egg=item@subdirectory=python [doc,hist,tests]
+    $ pip install --editable git://github.com/transportenergy/database#egg=item[tests]
 
 
-Usage
-=====
+Code against the API
+====================
 
-From Python scripts, import the :mod:`item` module or submodules and call specific functions described elsewhere in this documentation.
+From **Python** code, import the :mod:`item` module or submodules and call specific functions described elsewhere in this documentation.
 For instance::
 
+.. code-block:: python
+
     from item import historical, model, structure
 
     # Display information about the cleaning script for
@@ -53,6 +52,46 @@ For instance::
     # Display metadata (information about a model's data)
     model.load_model_regions("itf", version=2)
 
+.. _reticulate:
+
+From **R** code, use `reticulate <https://rstudio.github.io/reticulate/>`_ package to access the Python API:
+
+.. code-block:: R
+
+    item <- reticulate::import("item")
+
+    # Display metadata (information about a model's data)
+    item$model$load_model_regions("itf", version=2)
+
+
+.. _cli:
+
+Use the command-line interface
+==============================
+
+The ``item`` executable installed with the package provides a command-line interface (CLI) that can be used to perform some tasks without writing code.
+The CLI provides its own help with the ``--help`` option::
+
+    $ item --help
+    Usage: item [OPTIONS] COMMAND [ARGS]...
+
+      Command-line interface for the iTEM databases.
+
+    Options:
+      --path <KEY> <PATH>  Override data paths (multiple allowed).
+      --help               Show this message and exit.
+
+    Commands:
+      debug       Show debugging information, including paths.
+      help        Show extended help for the command-line tool.
+      historical  Manipulate the historical database.
+      mkdirs      Create a directory tree for the database.
+      model       Manipulate the model database.
+      remote      Access remote data sources.
+      template    Generate the MIP submission template.
+
+Use e.g. ``item historical --help`` to get help specific to one (sub)command.
+
 
 Run tests
 =========
@@ -64,6 +103,8 @@ The command-line option ``--local-data`` can be used to point to local data file
     ================ test session starts ================
     …
 
+See the page :doc:`ci` for further details.
+
 
 .. _usage-cite:
 

diff --git a/setup.cfg b/setup.cfg
@@ -41,10 +41,21 @@ setup_requires =
     setuptools_scm >= 3.5
 
 [options.extras_require]
-doc = sphinx; sphinx-rtd-theme
-eppa = gdx >= 3
-hist = jupyter; nbformat; requests; requests-cache
-tests = pytest; pytest-cov
+doc =
+    sphinx
+    sphinx-rtd-theme
+eppa =
+    gdx >= 3
+hist =
+    jupyter
+    nbformat
+    requests
+    requests-cache
+tests =
+    %(doc)s
+    %(hist)s
+    pytest
+    pytest-cov
 
 [options.entry_points]
 console_scripts =
@@ -71,20 +82,17 @@ omit =
     */tests/*
 
 [isort]
-default_section = THIRDPARTY
-known_first_party = item
-multi_line_output = 3
-include_trailing_comma = True
-force_grid_wrap = 0
-use_parentheses = True
-line_length = 88
+profile = black
 
 [flake8]
 max-line-length = 88
 ignore =
     # line break before binary operator
     W503
 
+[mypy]
+# Empty section required as of mypy 0.800;
+# see https://github.com/python/mypy/issues/9940
 
 [mypy-gdx.*]
 ignore_missing_imports = True