From 22d7a62bd3a5b1e1ec0358f8256514848a4e1a07 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Fri, 17 Nov 2023 22:26:38 +0100 Subject: [PATCH 01/11] Factor out notebook script calling --- .github/workflows/notebooks_dispatch.yml | 51 +++++++++++++++++++ .../workflows/notebooks_with_irdb_clone.yml | 32 ++---------- .../notebooks_with_irdb_download.yml | 38 +------------- 3 files changed, 57 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/notebooks_dispatch.yml diff --git a/.github/workflows/notebooks_dispatch.yml b/.github/workflows/notebooks_dispatch.yml new file mode 100644 index 00000000..248f1629 --- /dev/null +++ b/.github/workflows/notebooks_dispatch.yml @@ -0,0 +1,51 @@ +name: Notebooks Dispatch + +on: + # Allows wourkflow to be called by other workflows + workflow_call: + inputs: + clone-irdb: + type: boolean + description: If True, use clone of IRDB repo instead of package download. + +jobs: + run_notebooks: + strategy: + matrix: + # Run all operating systems, because this is the first interaction + # that users have with ScopeSim / IRDB. + # However, only use minimum and maximum supported Python version, + # as the IRDB download often fails. + # As of 2023/07/19, this test still often fails. See + # https://github.com/AstarVienna/ScopeSim/issues/254 + # For now, only run on one configuration, as having one green test + # is more useful than 5 green and one red. The others can be enabled + # once we fix the problem properly. + # os: [ubuntu-latest, windows-latest, macos-latest] + # python-version: ['3.8', '3.11'] + os: [ubuntu-latest] + python-version: ["3.11"] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # Install this version of ScopeSim. Otherwise the PyPI version of + # ScopeSim will be installed when the test-requriments + # are installed, because ScopeSim is a dependency of + # ScopeSim_Templates. + pip install . + pip install .[dev,test] + + - name: Run notebooks + env: + CLONE: ${{ inputs.clone-irdb && '--clone-irdb' || '' }} + run: ./runnotebooks.sh $CLONE --delete diff --git a/.github/workflows/notebooks_with_irdb_clone.yml b/.github/workflows/notebooks_with_irdb_clone.yml index 8f68efd8..98ca7367 100644 --- a/.github/workflows/notebooks_with_irdb_clone.yml +++ b/.github/workflows/notebooks_with_irdb_clone.yml @@ -21,31 +21,7 @@ on: jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - # Run only on a minimal subset of the matrix, as this is ran on many - # commits. - os: [ubuntu-latest] - python-version: ['3.11'] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # Install this version of ScopeSim. Otherwise the PyPI version of - # ScopeSim will be installed when the test-requriments - # are installed, because ScopeSim is a dependency of - # ScopeSim_Templates. - pip install . - pip install .[dev,test] - - name: Run notebooks - env: - TQDM_MININTERVAL: 10 - run: ./runnotebooks.sh --checkout-irdb --delete + call-notebooks-with-clone: + uses: ./.github/workflows/notebooks_dispatch.yml + with: + clone-irdb: true diff --git a/.github/workflows/notebooks_with_irdb_download.yml b/.github/workflows/notebooks_with_irdb_download.yml index 3b440c89..f2ad7952 100644 --- a/.github/workflows/notebooks_with_irdb_download.yml +++ b/.github/workflows/notebooks_with_irdb_download.yml @@ -10,39 +10,5 @@ on: jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - # Run all operating systems, because this is the first interaction - # that users have with ScopeSim / IRDB. - # However, only use minimum and maximum supported Python version, - # as the IRDB download often fails. - # As of 2023/07/19, this test still often fails. See - # https://github.com/AstarVienna/ScopeSim/issues/254 - # For now, only run on one configuration, as having one green test - # is more useful than 5 green and one red. The others can be enabled - # once we fix the problem properly. - # os: [ubuntu-latest, windows-latest, macos-latest] - # python-version: ['3.8', '3.11'] - os: [ubuntu-latest] - python-version: ['3.11'] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # Install this version of ScopeSim. Otherwise the PyPI version of - # ScopeSim will be installed when the test-requriments - # are installed, because ScopeSim is a dependency of - # ScopeSim_Templates. - pip install . - pip install .[dev,test] - - name: Run notebooks - # No --checkout-irdb to download the IRDB as a normal end user would. - run: ./runnotebooks.sh --delete + call-notebooks-without-clone: + uses: ./.github/workflows/notebooks_dispatch.yml From f568caf4c876e3c047e8e6767eddda91becff1c4 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Fri, 17 Nov 2023 22:30:12 +0100 Subject: [PATCH 02/11] Remove dev_spectroscopy branch reference --- .github/workflows/notebooks_with_irdb_clone.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/notebooks_with_irdb_clone.yml b/.github/workflows/notebooks_with_irdb_clone.yml index 98ca7367..daec5521 100644 --- a/.github/workflows/notebooks_with_irdb_clone.yml +++ b/.github/workflows/notebooks_with_irdb_clone.yml @@ -5,12 +5,10 @@ on: branches: - master - dev_master - - dev_spectroscopy pull_request: branches: - master - dev_master - - dev_spectroscopy # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 8e207e66d2bf3463d6c4d780d75aa112b608f52c Mon Sep 17 00:00:00 2001 From: teutoburg Date: Sun, 19 Nov 2023 19:05:23 +0100 Subject: [PATCH 03/11] Include tqdm improvements --- .github/workflows/notebooks_dispatch.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/notebooks_dispatch.yml b/.github/workflows/notebooks_dispatch.yml index 248f1629..25407acb 100644 --- a/.github/workflows/notebooks_dispatch.yml +++ b/.github/workflows/notebooks_dispatch.yml @@ -47,5 +47,6 @@ jobs: - name: Run notebooks env: + TQDM_MININTERVAL: 10 CLONE: ${{ inputs.clone-irdb && '--clone-irdb' || '' }} run: ./runnotebooks.sh $CLONE --delete From 380c3c2a0600715c1389c01fa97b4c9bf8007005 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Tue, 19 Dec 2023 18:39:26 +0100 Subject: [PATCH 04/11] Ignore pytest notebook warnings --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d8e6118b..7840ebb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,6 +98,9 @@ filterwarnings = [ "ignore:'cgi'*:DeprecationWarning", "ignore:The py23*:DeprecationWarning", "ignore:datetime.datetime.utcfromtimestamp()*:DeprecationWarning", + # Pytest Notebook stuff + "ignore:The*:pytest.PytestRemovedIn8Warning", + "ignore:Proactor*:RuntimeWarning" ] [tool.coverage.report] From fdd0e1fd51847a1027780950915fabefa1c9906a Mon Sep 17 00:00:00 2001 From: teutoburg Date: Tue, 19 Dec 2023 18:43:03 +0100 Subject: [PATCH 05/11] Use poetry --- .github/workflows/notebooks_dispatch.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/notebooks_dispatch.yml b/.github/workflows/notebooks_dispatch.yml index 25407acb..d9970f57 100644 --- a/.github/workflows/notebooks_dispatch.yml +++ b/.github/workflows/notebooks_dispatch.yml @@ -28,22 +28,21 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Set up Poetry + uses: abatilo/actions-poetry@v2 + with: + poetry-version: "1.7.1" + - name: Install dependencies run: | - python -m pip install --upgrade pip - # Install this version of ScopeSim. Otherwise the PyPI version of - # ScopeSim will be installed when the test-requriments - # are installed, because ScopeSim is a dependency of - # ScopeSim_Templates. - pip install . - pip install .[dev,test] + poetry install --with test --with dev --all-extras - name: Run notebooks env: From b3cafcab48eea8ef8ce26957872fefd12e0699fc Mon Sep 17 00:00:00 2001 From: teutoburg Date: Tue, 19 Dec 2023 18:47:29 +0100 Subject: [PATCH 06/11] Also use poetry for calling jupytext --- runnotebooks.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/runnotebooks.sh b/runnotebooks.sh index 6110d88f..eb43fd99 100755 --- a/runnotebooks.sh +++ b/runnotebooks.sh @@ -38,10 +38,10 @@ do fnpy="${fnnotebook%.ipynb}.py" # Convert .ipynb file to .py. - jupytext --to py "${fnnotebook}" + poetry run jupytext --to py "${fnnotebook}" # Run the python script and quit on first error. - python "${fnpy}" || exit 1 + poetry run python "${fnpy}" || exit 1 echo "- ${fnnotebook}" >> $GITHUB_STEP_SUMMARY # Delete generated files if --delete is specified. From 9dd2b1ec548c9a1a56019903e01e73b8fd643c09 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 20 Dec 2023 13:24:10 +0100 Subject: [PATCH 07/11] Only find notebooks in docs (and not in clones irdb) --- runnotebooks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runnotebooks.sh b/runnotebooks.sh index eb43fd99..3e384e27 100755 --- a/runnotebooks.sh +++ b/runnotebooks.sh @@ -32,7 +32,7 @@ fi echo "## Notebooks tested" >> $GITHUB_STEP_SUMMARY # https://github.com/koalaman/shellcheck/wiki/SC2044 -find . -iname "*.ipynb" -print0 | while IFS= read -r -d '' fnnotebook +find ./docs -iname "*.ipynb" -print0 | while IFS= read -r -d '' fnnotebook do echo "Testing ${fnnotebook} ..." fnpy="${fnnotebook%.ipynb}.py" From 614598eb18c6cfaedbbfcb2f68b707c75de7610f Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 20 Dec 2023 13:43:09 +0100 Subject: [PATCH 08/11] No more irdb symlinks for now --- runnotebooks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runnotebooks.sh b/runnotebooks.sh index 3e384e27..f2c9daa2 100755 --- a/runnotebooks.sh +++ b/runnotebooks.sh @@ -11,7 +11,7 @@ if [[ "x${1}" == "x--clone-irdb" ]] ; then echo "## Symlinks" >> $GITHUB_STEP_SUMMARY # https://github.com/koalaman/shellcheck/wiki/SC2044 - find . -iname "*.ipynb" -printf '%h\0' | sort -z | uniq -z | while IFS= read -r -d '' dirnotebooks; do + find ./docs -iname "*.ipynb" -printf '%h\0' | sort -z | uniq -z | while IFS= read -r -d '' dirnotebooks; do echo "${dirnotebooks}" echo "- ${dirnotebooks}" >> $GITHUB_STEP_SUMMARY dirinstpkgs="${dirnotebooks}/inst_pkgs" From 984f734006fcba284d07766e019e17f1ad2539f6 Mon Sep 17 00:00:00 2001 From: teutoburg Date: Wed, 20 Dec 2023 14:47:38 +0100 Subject: [PATCH 09/11] attempt to fix broken script --- runnotebooks.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runnotebooks.sh b/runnotebooks.sh index f2c9daa2..16f68419 100755 --- a/runnotebooks.sh +++ b/runnotebooks.sh @@ -4,14 +4,14 @@ echo "# Running Notebooks Tests" >> $GITHUB_STEP_SUMMARY if [[ "x${1}" == "x--clone-irdb" ]] ; then # Cloning IRDB - if [[ ! -e irdb ]] ; then + if [[ ! -e inst_pkgs ]] ; then echo "_Cloning IRDB_" >> $GITHUB_STEP_SUMMARY - git clone https://github.com/AstarVienna/irdb.git + git clone https://github.com/AstarVienna/irdb.git inst_pkgs fi echo "## Symlinks" >> $GITHUB_STEP_SUMMARY # https://github.com/koalaman/shellcheck/wiki/SC2044 - find ./docs -iname "*.ipynb" -printf '%h\0' | sort -z | uniq -z | while IFS= read -r -d '' dirnotebooks; do + find . -iname "*.ipynb" -printf '%h\0' | sort -z | uniq -z | while IFS= read -r -d '' dirnotebooks; do echo "${dirnotebooks}" echo "- ${dirnotebooks}" >> $GITHUB_STEP_SUMMARY dirinstpkgs="${dirnotebooks}/inst_pkgs" @@ -23,7 +23,7 @@ if [[ "x${1}" == "x--clone-irdb" ]] ; then fi # Comment out any download_package[s] in the notebooks. - pushd "${dirnotebooks}" || exit 1 + pushd "${dirnotebooks}" sed -i -E 's|"(.*\.download_package)|"#\1|g' -- *.ipynb popd || exit 1 done From 009885d1af967dd9717b9098e3f05e010be952e3 Mon Sep 17 00:00:00 2001 From: Hugo Buddelmeijer Date: Wed, 20 Dec 2023 15:23:14 +0100 Subject: [PATCH 10/11] Fix --clone-irdb mode Also: - Add safe bash header - Make runnable locally --- runnotebooks.sh | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/runnotebooks.sh b/runnotebooks.sh index 16f68419..31a7190e 100755 --- a/runnotebooks.sh +++ b/runnotebooks.sh @@ -1,36 +1,36 @@ #!/usr/bin/env bash -echo "# Running Notebooks Tests" >> $GITHUB_STEP_SUMMARY +# https://betterdev.blog/minimal-safe-bash-script-template/ +set -Eeuo pipefail + +# Set stdout as default so the script can be ran on the commandline. +STEP_SUMMARY="${GITHUB_STEP_SUMMARY:-/dev/stdout}" + +echo "# Running Notebooks Tests" >> $STEP_SUMMARY if [[ "x${1}" == "x--clone-irdb" ]] ; then # Cloning IRDB if [[ ! -e inst_pkgs ]] ; then - echo "_Cloning IRDB_" >> $GITHUB_STEP_SUMMARY + echo "_Cloning IRDB_" >> $STEP_SUMMARY git clone https://github.com/AstarVienna/irdb.git inst_pkgs fi - echo "## Symlinks" >> $GITHUB_STEP_SUMMARY + echo "## Patch notebooks" >> $STEP_SUMMARY # https://github.com/koalaman/shellcheck/wiki/SC2044 - find . -iname "*.ipynb" -printf '%h\0' | sort -z | uniq -z | while IFS= read -r -d '' dirnotebooks; do + find ./docs -iname "*.ipynb" -printf '%h\0' | sort -z | uniq -z | while IFS= read -r -d '' dirnotebooks; do echo "${dirnotebooks}" - echo "- ${dirnotebooks}" >> $GITHUB_STEP_SUMMARY - dirinstpkgs="${dirnotebooks}/inst_pkgs" - if [[ (! -e ./docs/source/examples/inst_pkgs) && (! -L ./docs/source/examples/inst_pkgs) ]] ; then - echo "Creating symlink to irdb: ${dirinstpkgs}" >> $GITHUB_STEP_SUMMARY - ln -s irdb "${dirinstpkgs}" - else - echo "Directory exists, not creating symlink: ${dirinstpkgs}" >> $GITHUB_STEP_SUMMARY - fi - - # Comment out any download_package[s] in the notebooks. - pushd "${dirnotebooks}" + echo "- ${dirnotebooks}" >> $STEP_SUMMARY + pushd "${dirnotebooks}" || exit 1 + # Comment out any download_package[s] in the notebooks. sed -i -E 's|"(.*\.download_package)|"#\1|g' -- *.ipynb + # Comment out explicitly setting the local_packages_path to somewhere. + sed -i -E 's|"(.*__config__\[\\"!SIM.file.local_packages_path)|"#\1|g' -- *.ipynb popd || exit 1 done fi -echo "## Notebooks tested" >> $GITHUB_STEP_SUMMARY +echo "## Notebooks tested" >> $STEP_SUMMARY # https://github.com/koalaman/shellcheck/wiki/SC2044 find ./docs -iname "*.ipynb" -print0 | while IFS= read -r -d '' fnnotebook do @@ -42,12 +42,15 @@ do # Run the python script and quit on first error. poetry run python "${fnpy}" || exit 1 - echo "- ${fnnotebook}" >> $GITHUB_STEP_SUMMARY + echo "- ${fnnotebook}" >> $STEP_SUMMARY # Delete generated files if --delete is specified. # By default do not delete any files. + # The delete functionality is intended to make it easy for developers to test + # all the notebooks on their own machine. if [ "x$1" = "x--delete" ] then + echo "Removing ${fnpy}" rm "${fnpy}" fi From 705e7e68efa1b151673ff28c2a89be535a9e3177 Mon Sep 17 00:00:00 2001 From: Hugo Buddelmeijer Date: Wed, 20 Dec 2023 15:25:46 +0100 Subject: [PATCH 11/11] Remove redundant || exit --- runnotebooks.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runnotebooks.sh b/runnotebooks.sh index 31a7190e..3362a8ae 100755 --- a/runnotebooks.sh +++ b/runnotebooks.sh @@ -20,12 +20,12 @@ if [[ "x${1}" == "x--clone-irdb" ]] ; then find ./docs -iname "*.ipynb" -printf '%h\0' | sort -z | uniq -z | while IFS= read -r -d '' dirnotebooks; do echo "${dirnotebooks}" echo "- ${dirnotebooks}" >> $STEP_SUMMARY - pushd "${dirnotebooks}" || exit 1 + pushd "${dirnotebooks}" # Comment out any download_package[s] in the notebooks. sed -i -E 's|"(.*\.download_package)|"#\1|g' -- *.ipynb # Comment out explicitly setting the local_packages_path to somewhere. sed -i -E 's|"(.*__config__\[\\"!SIM.file.local_packages_path)|"#\1|g' -- *.ipynb - popd || exit 1 + popd done fi @@ -41,7 +41,7 @@ do poetry run jupytext --to py "${fnnotebook}" # Run the python script and quit on first error. - poetry run python "${fnpy}" || exit 1 + poetry run python "${fnpy}" echo "- ${fnnotebook}" >> $STEP_SUMMARY # Delete generated files if --delete is specified.