Add weekly CI workflow to test all notebooks (#6046)

* Add a test to run all notebooks
* Add a Weekly CI workflow to run all notebooks test
* Temporarily increase cadence to daily runs for validation

.github/workflows/ci-weekly.yml

@@ -0,0 +1,38 @@
+name: Continuous Integration - Weekly
+
+on:
+  schedule:
+    # Checks out master by default.
+    # TODO (#6037) - replace with weekly cadence after verification
+    - cron: '0 0 * * *'
+    # - cron: '0 0 * * 0'
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  notebooks-stable:
+    name: All Notebooks Isolated Test against Cirq stable
+    env:
+      NOTEBOOK_PARTITIONS: 4
+    strategy:
+      matrix:
+        # partitions should be named partition-0 to partition-(NOTEBOOK_PARTITIONS-1)
+        partition: [partition-0, partition-1, partition-2, partition-3]
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: '3.8'
+          architecture: 'x64'
+      - name: Install requirements
+        run: pip install -r dev_tools/requirements/isolated-base.env.txt
+      - name: Notebook tests
+        run: check/pytest -n auto -m weekly dev_tools/notebooks/isolated_notebook_test.py -k ${{matrix.partition}}
+      - uses: actions/upload-artifact@v2
+        if: failure()
+        with:
+          name: notebook-outputs
+          path: out

dev_tools/conftest.py

@@ -28,6 +28,7 @@
 
 def pytest_configure(config):
     config.addinivalue_line("markers", "slow: mark tests as slow")
+    config.addinivalue_line("markers", "weekly: mark tests as run only by weekly automation")
 
 
 def pytest_collection_modifyitems(config, items):
@@ -40,6 +41,11 @@ def pytest_collection_modifyitems(config, items):
         if 'slow' in item.keywords:
             item.add_marker(skip_slow_marker)
 
+    skip_weekly_marker = pytest.mark.skip(reason='only run by weekly automation')
+    for item in items:
+        if 'weekly' in item.keywords:
+            item.add_marker(skip_weekly_marker)
+
 
 @pytest.fixture(scope="session")
 def cloned_env(testrun_uid, worker_id):

dev_tools/notebooks/isolated_notebook_test.py

@@ -14,10 +14,10 @@
 
 # ========================== ISOLATED NOTEBOOK TESTS ============================================
 #
-# In these tests are only changed notebooks are tested. It is assumed that notebooks install cirq
-# conditionally if they can't import cirq. This installation path is the main focus and it is
-# exercised in an isolated virtual environment for each notebook. This is also the path that is
-# tested in the devsite workflows, these tests meant to provide earlier feedback.
+# It is assumed that notebooks install cirq conditionally if they can't import cirq. This
+# installation path is the main focus and it is exercised in an isolated virtual environment for
+# each notebook. This is also the path that is tested in the devsite workflows, these tests meant
+# to provide earlier feedback.
 #
 # In case the dev environment changes or this particular file changes, all notebooks are executed!
 # This can take a long time and even lead to timeout on Github Actions, hence partitioning of the
@@ -155,23 +155,7 @@ def _partitioned_test_cases(notebooks):
     return [(f"partition-{i%n_partitions}", notebook) for i, notebook in enumerate(notebooks)]
 
 
-@pytest.mark.slow
-@pytest.mark.parametrize(
-    "partition, notebook_path",
-    _partitioned_test_cases(filter_notebooks(_list_changed_notebooks(), SKIP_NOTEBOOKS)),
-)
-def test_notebooks_against_released_cirq(partition, notebook_path, cloned_env):
-    """Tests the notebooks in isolated virtual environments.
-
-    In order to speed up the execution of these tests an auxiliary file may be supplied which
-    performs substitutions on the notebook to make it faster.
-
-    Specifically for a notebook file notebook.ipynb, one can supply a file notebook.tst which
-    contains the substitutes.  The substitutions are provide in the form `pattern->replacement`
-    where the pattern is what is matched and replaced. While the pattern is compiled as a
-    regular expression, it is considered best practice to not use complicated regular expressions.
-    Lines in this file that do not have `->` are ignored.
-    """
+def _rewrite_and_run_notebook(notebook_path, cloned_env):
     notebook_file = os.path.basename(notebook_path)
     notebook_rel_dir = os.path.dirname(os.path.relpath(notebook_path, "."))
     out_path = f"out/{notebook_rel_dir}/{notebook_file[:-6]}.out.ipynb"
@@ -213,6 +197,40 @@ def test_notebooks_against_released_cirq(partition, notebook_path, cloned_env):
     os.remove(rewritten_notebook_path)
 
 
+@pytest.mark.slow
+@pytest.mark.parametrize(
+    "partition, notebook_path",
+    _partitioned_test_cases(filter_notebooks(_list_changed_notebooks(), SKIP_NOTEBOOKS)),
+)
+def test_changed_notebooks_against_released_cirq(partition, notebook_path, cloned_env):
+    """Tests changed notebooks in isolated virtual environments.
+
+    In order to speed up the execution of these tests an auxiliary file may be supplied which
+    performs substitutions on the notebook to make it faster.
+
+    Specifically for a notebook file notebook.ipynb, one can supply a file notebook.tst which
+    contains the substitutes.  The substitutions are provide in the form `pattern->replacement`
+    where the pattern is what is matched and replaced. While the pattern is compiled as a
+    regular expression, it is considered best practice to not use complicated regular expressions.
+    Lines in this file that do not have `->` are ignored.
+    """
+    _rewrite_and_run_notebook(notebook_path, cloned_env)
+
+
+@pytest.mark.weekly
+@pytest.mark.parametrize(
+    "partition, notebook_path",
+    _partitioned_test_cases(filter_notebooks(list_all_notebooks(), SKIP_NOTEBOOKS)),
+)
+def test_all_notebooks_against_released_cirq(partition, notebook_path, cloned_env):
+    """Tests all notebooks in isolated virtual environments.
+
+    See `test_changed_notebooks_against_released_cirq` for more details on
+    notebooks execution.
+    """
+    _rewrite_and_run_notebook(notebook_path, cloned_env)
+
+
 @pytest.mark.parametrize("notebook_path", NOTEBOOKS_DEPENDING_ON_UNRELEASED_FEATURES)
 def test_ensure_unreleased_notebooks_install_cirq_pre(notebook_path):
     # utf-8 is important for Windows testing, otherwise characters like ┌──┐ fail on cp1252