diff --git a/.github/workflows/ci-weekly.yml b/.github/workflows/ci-weekly.yml new file mode 100644 index 00000000000..ba15da86403 --- /dev/null +++ b/.github/workflows/ci-weekly.yml @@ -0,0 +1,38 @@ +name: Continuous Integration - Weekly + +on: + schedule: + # Checks out master by default. + # TODO (#6037) - replace with weekly cadence after verification + - cron: '0 0 * * *' + # - cron: '0 0 * * 0' + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + notebooks-stable: + name: All Notebooks Isolated Test against Cirq stable + env: + NOTEBOOK_PARTITIONS: 4 + strategy: + matrix: + # partitions should be named partition-0 to partition-(NOTEBOOK_PARTITIONS-1) + partition: [partition-0, partition-1, partition-2, partition-3] + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + with: + python-version: '3.8' + architecture: 'x64' + - name: Install requirements + run: pip install -r dev_tools/requirements/isolated-base.env.txt + - name: Notebook tests + run: check/pytest -n auto -m weekly dev_tools/notebooks/isolated_notebook_test.py -k ${{matrix.partition}} + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: notebook-outputs + path: out diff --git a/dev_tools/conftest.py b/dev_tools/conftest.py index add63642720..7dbde3b3ef4 100644 --- a/dev_tools/conftest.py +++ b/dev_tools/conftest.py @@ -28,6 +28,7 @@ def pytest_configure(config): config.addinivalue_line("markers", "slow: mark tests as slow") + config.addinivalue_line("markers", "weekly: mark tests as run only by weekly automation") def pytest_collection_modifyitems(config, items): @@ -40,6 +41,11 @@ def pytest_collection_modifyitems(config, items): if 'slow' in item.keywords: item.add_marker(skip_slow_marker) + skip_weekly_marker = pytest.mark.skip(reason='only run by weekly automation') + for item in items: + if 'weekly' in item.keywords: + item.add_marker(skip_weekly_marker) + @pytest.fixture(scope="session") def cloned_env(testrun_uid, worker_id): diff --git a/dev_tools/notebooks/isolated_notebook_test.py b/dev_tools/notebooks/isolated_notebook_test.py index 6ac26b5956b..ad01d67fcd2 100644 --- a/dev_tools/notebooks/isolated_notebook_test.py +++ b/dev_tools/notebooks/isolated_notebook_test.py @@ -14,10 +14,10 @@ # ========================== ISOLATED NOTEBOOK TESTS ============================================ # -# In these tests are only changed notebooks are tested. It is assumed that notebooks install cirq -# conditionally if they can't import cirq. This installation path is the main focus and it is -# exercised in an isolated virtual environment for each notebook. This is also the path that is -# tested in the devsite workflows, these tests meant to provide earlier feedback. +# It is assumed that notebooks install cirq conditionally if they can't import cirq. This +# installation path is the main focus and it is exercised in an isolated virtual environment for +# each notebook. This is also the path that is tested in the devsite workflows, these tests meant +# to provide earlier feedback. # # In case the dev environment changes or this particular file changes, all notebooks are executed! # This can take a long time and even lead to timeout on Github Actions, hence partitioning of the @@ -155,23 +155,7 @@ def _partitioned_test_cases(notebooks): return [(f"partition-{i%n_partitions}", notebook) for i, notebook in enumerate(notebooks)] -@pytest.mark.slow -@pytest.mark.parametrize( - "partition, notebook_path", - _partitioned_test_cases(filter_notebooks(_list_changed_notebooks(), SKIP_NOTEBOOKS)), -) -def test_notebooks_against_released_cirq(partition, notebook_path, cloned_env): - """Tests the notebooks in isolated virtual environments. - - In order to speed up the execution of these tests an auxiliary file may be supplied which - performs substitutions on the notebook to make it faster. - - Specifically for a notebook file notebook.ipynb, one can supply a file notebook.tst which - contains the substitutes. The substitutions are provide in the form `pattern->replacement` - where the pattern is what is matched and replaced. While the pattern is compiled as a - regular expression, it is considered best practice to not use complicated regular expressions. - Lines in this file that do not have `->` are ignored. - """ +def _rewrite_and_run_notebook(notebook_path, cloned_env): notebook_file = os.path.basename(notebook_path) notebook_rel_dir = os.path.dirname(os.path.relpath(notebook_path, ".")) out_path = f"out/{notebook_rel_dir}/{notebook_file[:-6]}.out.ipynb" @@ -213,6 +197,40 @@ def test_notebooks_against_released_cirq(partition, notebook_path, cloned_env): os.remove(rewritten_notebook_path) +@pytest.mark.slow +@pytest.mark.parametrize( + "partition, notebook_path", + _partitioned_test_cases(filter_notebooks(_list_changed_notebooks(), SKIP_NOTEBOOKS)), +) +def test_changed_notebooks_against_released_cirq(partition, notebook_path, cloned_env): + """Tests changed notebooks in isolated virtual environments. + + In order to speed up the execution of these tests an auxiliary file may be supplied which + performs substitutions on the notebook to make it faster. + + Specifically for a notebook file notebook.ipynb, one can supply a file notebook.tst which + contains the substitutes. The substitutions are provide in the form `pattern->replacement` + where the pattern is what is matched and replaced. While the pattern is compiled as a + regular expression, it is considered best practice to not use complicated regular expressions. + Lines in this file that do not have `->` are ignored. + """ + _rewrite_and_run_notebook(notebook_path, cloned_env) + + +@pytest.mark.weekly +@pytest.mark.parametrize( + "partition, notebook_path", + _partitioned_test_cases(filter_notebooks(list_all_notebooks(), SKIP_NOTEBOOKS)), +) +def test_all_notebooks_against_released_cirq(partition, notebook_path, cloned_env): + """Tests all notebooks in isolated virtual environments. + + See `test_changed_notebooks_against_released_cirq` for more details on + notebooks execution. + """ + _rewrite_and_run_notebook(notebook_path, cloned_env) + + @pytest.mark.parametrize("notebook_path", NOTEBOOKS_DEPENDING_ON_UNRELEASED_FEATURES) def test_ensure_unreleased_notebooks_install_cirq_pre(notebook_path): # utf-8 is important for Windows testing, otherwise characters like ┌──┐ fail on cp1252