Merge pull request #1794 from Saransh-cpp/run-tests-once-using-coverage

Run tests once using coverage on `ubuntu-latest` with Python 3.9

.github/workflows/test_on_push.yml

@@ -72,9 +72,17 @@ jobs:
       if: matrix.os == 'ubuntu-latest'
       run: tox -e pybamm-requires
 
-    - name: Run unit tests for GNU/Linux
+    - name: Run unit tests for GNU/Linux with Python 3.7 and 3.8
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version != 3.9
+      run: python -m tox -e quick
+
+    - name: Run unit tests for GNU/Linux with Python 3.9 and generate coverage report
+      if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.9
+      run: tox -e coverage
+
+    - name: Run integration tests for GNU/Linux
       if: matrix.os == 'ubuntu-latest'
-      run: python -m tox -e tests
+      run: python -m tox -e integration
 
     - name: Run unit tests for Windows and MacOS
       if: matrix.os != 'ubuntu-latest'
@@ -88,10 +96,6 @@ jobs:
       if: matrix.os == 'ubuntu-latest'
       run: tox -e examples
 
-    - name: Install and run coverage
-      if: success() && (matrix.os == 'ubuntu-latest' && matrix.python-version == 3.9)
-      run: tox -e coverage
-
     - name: Upload coverage report
       if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.9
       uses: codecov/codecov-action@v2.1.0

CONTRIBUTING.md

@@ -280,7 +280,7 @@ Major PyBaMM features are showcased in [Jupyter notebooks](https://jupyter.org/)
 
 All example notebooks should be listed in [examples/README.md](https://github.com/pybamm-team/PyBaMM/blob/develop/examples/notebooks/README.md). Please follow the (naming and writing) style of existing notebooks where possible.
 
-Where possible, notebooks are tested daily. A list of slow notebooks (which time-out and fail tests) is maintained in `.slow-books`, these notebooks will be excluded from daily testing.
+All the notebooks are tested daily.
 
 ## Citations
 

run-tests.py

@@ -104,35 +104,20 @@ def run_doc_tests():
         sys.exit(ret)
 
 
-def run_notebook_and_scripts(skip_slow_books=False, executable="python"):
+def run_notebook_and_scripts(executable="python"):
     """
     Runs Jupyter notebook tests. Exits if they fail.
     """
-    # Ignore slow books?
-    ignore_list = []
-    if skip_slow_books and os.path.isfile(".slow-books"):
-        with open(".slow-books", "r") as f:
-            for line in f.readlines():
-                line = line.strip()
-                if not line or line[:1] == "#":
-                    continue
-                if not line.startswith("examples/"):
-                    line = "examples/" + line
-                if not line.endswith(".ipynb"):
-                    line = line + ".ipynb"
-                if not os.path.isfile(line):
-                    raise Exception("Slow notebook note found: " + line)
-                ignore_list.append(line)
 
     # Scan and run
     print("Testing notebooks and scripts with executable `" + str(executable) + "`")
-    if not scan_for_nb_and_scripts("examples", True, executable, ignore_list):
+    if not scan_for_nb_and_scripts("examples", True, executable):
         print("\nErrors encountered in notebooks")
         sys.exit(1)
     print("\nOK")
 
 
-def scan_for_nb_and_scripts(root, recursive=True, executable="python", ignore_list=[]):
+def scan_for_nb_and_scripts(root, recursive=True, executable="python"):
     """
     Scans for, and tests, all notebooks and scripts in a directory.
     """
@@ -142,9 +127,6 @@ def scan_for_nb_and_scripts(root, recursive=True, executable="python", ignore_li
     # Scan path
     for filename in os.listdir(root):
         path = os.path.join(root, filename)
-        if path in ignore_list:
-            print("Skipping slow book: " + path)
-            continue
 
         # Recurse into subdirectories
         if recursive and os.path.isdir(path):
@@ -354,11 +336,6 @@ def export_notebook(ipath, opath):
     parser.add_argument(
         "--examples",
         action="store_true",
-        help="Test only the fast Jupyter notebooks and scripts in `examples`.",
-    )
-    parser.add_argument(
-        "--allexamples",
-        action="store_true",
         help="Test all Jupyter notebooks and scripts in `examples`.",
     )
     parser.add_argument(
@@ -416,12 +393,9 @@ def export_notebook(ipath, opath):
         has_run = True
         run_doc_tests()
     # Notebook tests
-    if args.allexamples:
-        has_run = True
-        run_notebook_and_scripts(executable=interpreter)
     elif args.examples:
         has_run = True
-        run_notebook_and_scripts(True, interpreter)
+        run_notebook_and_scripts(interpreter)
     if args.debook:
         has_run = True
         export_notebook(*args.debook)

tox.ini

@@ -22,6 +22,7 @@ commands =
       tests-!windows-!mac: sh -c "pybamm_install_jax"  # install jax, jaxlib for ubuntu
 	 tests: python run-tests.py --unit --folder all
 	 quick: python run-tests.py --unit
+         integration: python run-tests.py --unit --folder integration
 	 examples: python run-tests.py --examples
 	 dev-!windows-!mac: sh -c "echo export LD_LIBRARY_PATH={env:LD_LIBRARY_PATH} >> {envbindir}/activate"
 	 doctests: python run-tests.py --doctest