Merge pull request #14 from PTB-PSt1/test_notebooks

Resolve #1 by testing all notebooks against expected output
PTB-M4D · Apr 29, 2021 · aea134e · aea134e
2 parents 6a17a79 + c0053c9
commit aea134e
Show file tree

Hide file tree

Showing 17 changed files with 3,776 additions and 2,404 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,140 @@
+# Python CircleCI 2.1 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-python/ for more details
+#
+version: 2.1
+
+executors:
+  # Define a parameterized executor which accepts two parameters to choose the python
+  # version which will be used for the docker image and the tox actions.
+  tester:
+    working_directory: ~/repo
+    parameters:
+      tag:
+        type: string
+        default: "3.8"
+    docker:
+      - image: circleci/python:<< parameters.tag >>
+
+commands:
+  # Reusable command to prepare the environment for testing.
+  create_folders_and_venv:
+    description: "Prepare everything."
+    parameters:
+      pyenv:
+        type: string
+        default: "py38"
+    steps:
+    # Checkout code.
+    - checkout
+    # Create test-result folder.
+    - run:
+        name: Create test result folder
+        command: |
+          mkdir test-results
+
+    # Create PyDynamic_tutorials virtual environment.
+    - run:
+        name: Create virtual environment
+        command: |
+          python3 -m venv << parameters.pyenv >>
+          source << parameters.pyenv >>/bin/activate
+          pip install --upgrade pip setuptools pip-tools
+
+  # Reusable command to install production dependencies.
+  install_deps:
+    description: "Install dependencies."
+    parameters:
+      pyenv:
+        type: string
+        default: "py38"
+    steps:
+
+    # Install dependencies.
+    - run:
+        name: Install dependencies
+        command: |
+          source << parameters.pyenv >>/bin/activate
+          python -m piptools sync requirements/requirements.txt requirements/requirements-dev.txt
+
+  # Reusable command to conduct actual testing.
+  execute_nbval:
+    description: "Execute pytest with nbval plugin."
+    parameters:
+      pyenv:
+        type: string
+    steps:
+
+    # Call pytest with nbval.
+    - run:
+        name: Perform pytest --nbval testing
+        command: |
+          source << parameters.pyenv >>/bin/activate
+          pytest --nbval --current-env --sanitize-with nbval_sanitization | tee --append test-results/pytest.log
+
+  # Reusable command to store the previously generated test results.
+  store_results:
+    description: "Store test results and artifacts."
+    parameters:
+      pyenv:
+        type: string
+    steps:
+    # Store test results.
+    - store_artifacts:
+        path: test-results
+        destination: test-results
+
+    - store_test_results:
+        path: test-results
+
+workflows:
+  # Create workflow for testing PyDynamic_tutorials.
+  test:
+    jobs:
+      # Create 'test' job to test and install PyDynamic_tutorials for every commit.
+      - test:
+          name: "test_python3.6"
+          tag: "3.6"
+          pyenv: "py36"
+      - test:
+          name: "test_python3.7"
+          tag: "3.7"
+          pyenv: "py37"
+      - test:
+          name: "test_python3.8"
+          tag: "3.8"
+          pyenv: "py38"
+
+jobs:
+
+  # Define one 'test' job with parameters to deal with all desired cases. The
+  # reason for this is the desire to ensure the following for all supported Python
+  # versions referring to the docs: all notebooks are guaranteed to produce the same
+  # output as they contain.
+  test:
+    # Define all parameters, where 'tag' is used for the docker image and 'pyenv' is
+    # the string which is used to identify the current Python version. We reuse
+    # this variable basically everywhere we deal with the virtual environment.
+    parameters:
+      tag:
+        type: string
+        default: "3.8"
+      pyenv:
+        type: string
+        default: "py38"
+
+    # Specify the executor and hand over the docker image tag parameter.
+    executor:
+      name: tester
+      tag: << parameters.tag >>
+
+    # Specify the steps to execute during this test jobs.
+    steps:
+      - create_folders_and_venv:
+          pyenv: << parameters.pyenv >>
+      - install_deps:
+          pyenv: << parameters.pyenv >>
+      - execute_nbval:
+          pyenv: << parameters.pyenv >>
+      - store_results:
+          pyenv: << parameters.pyenv >>
diff --git a/PyDynamic_tutorials/deconvolution/01 Basic measurement data pre-processing.ipynb b/PyDynamic_tutorials/deconvolution/01 Basic measurement data pre-processing.ipynb
diff --git a/PyDynamic_tutorials/deconvolution/02 Preparation of calibration data.ipynb b/PyDynamic_tutorials/deconvolution/02 Preparation of calibration data.ipynb
diff --git a/...amic_tutorials/deconvolution/03 Interpolation and extrapolation of calibration data.ipynb b/...amic_tutorials/deconvolution/03 Interpolation and extrapolation of calibration data.ipynb
diff --git a/PyDynamic_tutorials/deconvolution/04 Calculation of impulse response of hydrophone.ipynb b/PyDynamic_tutorials/deconvolution/04 Calculation of impulse response of hydrophone.ipynb
diff --git a/PyDynamic_tutorials/deconvolution/05 Deconvolution in the frequency domain.ipynb b/PyDynamic_tutorials/deconvolution/05 Deconvolution in the frequency domain.ipynb
diff --git a/PyDynamic_tutorials/deconvolution/06 Regularized deconvolution.ipynb b/PyDynamic_tutorials/deconvolution/06 Regularized deconvolution.ipynb
diff --git a/PyDynamic_tutorials/uncertainty/01 Basic measurement data pre-processing.ipynb b/PyDynamic_tutorials/uncertainty/01 Basic measurement data pre-processing.ipynb
diff --git a/PyDynamic_tutorials/uncertainty/02 Basic interpolation.ipynb b/PyDynamic_tutorials/uncertainty/02 Basic interpolation.ipynb
diff --git a/PyDynamic_tutorials/uncertainty/03 Basic extrapolation.ipynb b/PyDynamic_tutorials/uncertainty/03 Basic extrapolation.ipynb
diff --git a/README.md b/README.md
@@ -14,11 +14,11 @@ included in _PyDynamic_'s codebase subfolders
 To get going with the tutorials you can either start directly in your browser or get
 a local copy and experiment offline on your machine.
 
- ### Quick start in current browser session
+### Quick start in current browser session
 
 To start working in the notebooks directly in the browser, click [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/PTB-PSt1/PyDynamic_tutorials/master).
 
- ### Get a local copy to work offline
+### Get a local copy to work offline
 
 To get started on your local machine, follow these simple steps:
 
@@ -86,7 +86,7 @@ $ jupyter notebook
 
 #### 5. Go to localhost:8888
 
-Usually a browser window will have opened automatically at this point. Otherwise just
+Usually a browser window will have opened automatically at this point. Otherwise, just
 open one yourself and navigate to the printed URL in the console, in our case
 `http://localhost:8888/?token=f368c552e042d48d46ff4c8a094448d0e7681790b0719215`.
 

diff --git a/conftest.py b/conftest.py
@@ -0,0 +1,9 @@
+from nbval.plugin import IPyNbFile
+
+
+def pytest_collectstart(collector):
+    if isinstance(collector, IPyNbFile):
+        collector.skip_compare += 'stderr', \
+                                  'application/javascript', \
+                                  'application/vnd.holoviews_load.v0+json', \
+                                  'text/html',
diff --git a/nbval_sanitization b/nbval_sanitization
@@ -0,0 +1,3 @@
+[regex1]
+regex: \|█+\|\ (\d+\.)?\d+(G|MB|k)\/(\d+\.)?\d+(G|MB|k)\ \[(\d+)?\d+:\d+<(\d+)?\d+:\d+,\ ('\\n ')?(\d+\.)?\d+(G|MB|kB)\/s
+replace: DOWNLOAD_TIME
diff --git a/requirements/requirements-dev.in b/requirements/requirements-dev.in
@@ -7,3 +7,7 @@ sphinx
 nbsphinx
 recommonmark
 sphinx_rtd_theme
+nbval
+# This pinning is needed, since more current versions of Python (> 3.6) are less
+# compatible with more current versions of this package.
+dataclasses<=0.6