DM-40142: Infrastructure upgrades

.github/workflows/ci.yaml

@@ -1,106 +1,117 @@
 name: CI
 
 "on":
+  merge_group: {}
+  pull_request: {}
   push:
     branches-ignore:
       # These should always correspond to pull requests, so ignore them for
       # the push trigger and let them be triggered by the pull_request
       # trigger, avoiding running the workflow twice.  This is a minor
       # optimization so there's no need to ensure this is comprehensive.
       - "dependabot/**"
+      - "gh-readonly-queue/**"
       - "renovate/**"
       - "tickets/**"
       - "u/**"
     tags:
       - "*"
-  pull_request: {}
 
 jobs:
-  test:
+  lint:
     runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        python:
-          - "3.11"
+    timeout-minutes: 5
 
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: ${{ matrix.python }}
+          python-version: "3.11"
 
       - name: Run pre-commit
         uses: pre-commit/action@v3.0.0
 
-      - name: Install tox
-        run: pip install tox tox-docker
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
 
-      - name: Cache tox environments
-        id: cache-tox
-        uses: actions/cache@v3
-        with:
-          path: .tox
-          # requirements/*.txt and pyproject.toml have versioning info
-          # that would impact the tox environment.
-          key: tox-${{ matrix.python }}-${{ hashFiles('requirements/*.txt') }}-${{ hashFiles('pyproject.toml') }}
-          restore-keys: |
-            tox-${{ matrix.python }}-${{ hashFiles('requirements/*.txt') }}-
+    strategy:
+      matrix:
+        python:
+          - "3.11"
+
+    steps:
+      - uses: actions/checkout@v3
 
       - name: Run tox
-        run: tox -e docker,coverage-report,typing
+        uses: lsst-sqre/run-tox@v1
+        with:
+          python-version: ${{ matrix.python }}
+          tox-envs: "docker,coverage-report,typing"
+          tox-plugins: "tox-docker"
 
   build:
     runs-on: ubuntu-latest
-    needs: [test]
+    needs: [lint, test]
+    timeout-minutes: 10
 
     # Only do Docker builds of tagged releases and pull requests from ticket
     # branches.  This will still trigger on pull requests from untrusted
     # repositories whose branch names match our tickets/* branch convention,
     # but in this case the build will fail with an error since the secret
     # won't be set.
     if: >
-      startsWith(github.ref, 'refs/tags/')
-      || startsWith(github.head_ref, 'tickets/')
+      github.event_name != 'merge_group'
+      && (startsWith(github.ref, 'refs/tags/')
+          || startsWith(github.head_ref, 'tickets/'))
 
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
-      - name: Define the Docker tag
-        id: vars
-        run: echo ::set-output name=tag::$(scripts/docker-tag.sh)
-
-      - name: Print the tag
-        id: print
-        run: echo ${{ steps.vars.outputs.tag }}
+      - uses: lsst-sqre/build-and-push-to-ghcr@v1
+        id: build
+        with:
+          image: ${{ github.repository }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+  docs:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
 
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v2
+    steps:
+      - uses: actions/checkout@v3
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
+          # Ensure the documentation gets the right version.
+          fetch-depth: 0
 
-      - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Update package lists
+        run: sudo apt-get update
 
-      - name: Build and push
-        uses: docker/build-push-action@v4
+      - name: Install extra packages
+        run: sudo apt install -y graphviz
+
+      - name: Run tox
+        uses: lsst-sqre/run-tox@v1
+        with:
+          python-version: "3.11"
+          tox-envs: "docs,docs-linkcheck"
+
+      # Only attempt documentation uploads for long-lived branches, tagged
+      # releases, and pull requests from ticket branches.  This avoids version
+      # clutter in the docs and failures when a PR doesn't have access to
+      # secrets.
+      - name: Upload to LSST the Docs
+        uses: lsst-sqre/ltd-upload@v1
         with:
-          context: .
-          push: true
-          tags: |
-            lsstsqre/times-square:${{ steps.vars.outputs.tag }}
-            ghcr.io/lsst-sqre/times-square:${{ steps.vars.outputs.tag }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          project: times-square
+          dir: "docs/_build/html"
+          username: ${{ secrets.LTD_USERNAME }}
+          password: ${{ secrets.LTD_PASSWORD }}
+        if: >
+          github.event_name != 'merge_group'
+          && (github.event_name != 'pull_request'
+            || startsWith(github.head_ref, 'tickets/'))

.github/workflows/dependencies.yaml

@@ -0,0 +1,33 @@
+name: Dependency Update
+
+"on":
+  schedule:
+    - cron: "0 12 * * 1"
+  workflow_dispatch: {}
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Run neophile
+        uses: lsst-sqre/run-neophile@v1
+        with:
+          python-version: "3.11"
+          mode: pr
+          types: pre-commit
+          app-id: ${{ secrets.NEOPHILE_APP_ID }}
+          app-secret: ${{ secrets.NEOPHILE_PRIVATE_KEY }}
+
+      - name: Report status
+        if: always()
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "Periodic dependency update for {repo} failed"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}

.github/workflows/periodic-ci.yaml

@@ -0,0 +1,51 @@
+# This is a separate run of the Python test suite that runs from a schedule,
+# doesn't cache the tox environment, and updates pinned dependencies first.
+# The purpose is to test compatibility with the latest versions of
+# dependencies.
+
+name: Periodic CI
+
+"on":
+  schedule:
+    - cron: "0 12 * * 1"
+  workflow_dispatch: {}
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    strategy:
+      matrix:
+        python:
+          - "3.11"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      # Use the oldest supported version of Python to update dependencies,
+      # not the matrixed Python version, since this accurately reflects
+      # how dependencies should later be updated.
+      - name: Run neophile
+        uses: lsst-sqre/run-neophile@v1
+        with:
+          python-version: "3.11"
+          mode: update
+
+      - name: Run tests in tox
+        uses: lsst-sqre/run-tox@v1
+        with:
+          python-version: ${{ matrix.python }}
+          tox-envs: "lint,typing,docker,docs,docs-linkcheck"
+          tox-plugins: "tox-docker"
+          use-cache: false
+
+      - name: Report status
+        if: always()
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: "failure"
+          notification_title: "Periodic test for {repo} failed"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}

.gitignore

@@ -1,3 +1,5 @@
+docs/_static/openapi.json
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.pre-commit-config.yaml

@@ -2,22 +2,17 @@ repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.4.0
     hooks:
+      - id: trailing-whitespace
       - id: check-yaml
       - id: check-toml
 
-  - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.0.278
     hooks:
-      - id: isort
-        additional_dependencies:
-          - toml
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
 
   - repo: https://github.com/psf/black
     rev: 23.3.0
     hooks:
       - id: black
-
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8

CHANGELOG.md

@@ -0,0 +1,92 @@
+# Change log
+
+<!--
+Generate new changelog fragments with: scriv create.
+
+Collect fragments into this file with: scriv collect --version X.Y.Z
+-->
+
+<!-- scriv-insert-here -->
+
+## 0.7.0 (2023-04-19)
+
+### New features
+
+- Adopt `safir.redis.pydantic` for Redis-backed storage.
+- Adopt `safir.github` creating the GitHub App client and modelling of GitHub resources with Pydantic.
+
+### Bug fixes
+
+- Fix handling of disabled pages so that they aren't executed in a GitHub check, and are dropped if they previously existed in the database.
+
+### Other changes
+
+- Update to Python 3.11
+
+### 0.6.0 (2022-08-18)
+
+### New features
+
+- Times Square now exposes information about pages created during GitHub PR check runs:
+
+  - `GET /times-square/api/v1/github-pr/:org/:repo/:sha` provides metadata for a repository's check run in general, such as the contents in the check run and the GitHub pull request or check run.
+  - `GET /times-square/api/v1/github-pr/:org/:repo/:sha/:path` provides metadata about a specific notebook.
+
+  Times Square check runs also link to the pull request preview pages published through Times Square's interface in Squareone.
+
+## 0.5.0 (2022-07-04)
+
+### New features
+
+- Times Square now implements two GitHub check runs for pull requests on notebook repositories:
+
+  - The "YAML config" check validates the structure of YAML configuration files, specifically the `times-square.yaml` repository settings as well as the YAML sidecar files that describe each notebook.
+  - The "Notebook execution" check actually runs notebooks (given their default parameters) with Noteburst, and ensures that they return without error.
+
+  Together, these features will help contributors to Times Square notebook repositories ensure that their contributions work before they merge pull requests.
+
+## 0.4.0 (2022-05-14)
+
+### New features
+
+- Times Square now supports sourcing Jupyter Notebooks from GitHub repositories, in addition to its original mode of API-sourced notebooks. To do this, Times Square acts as a GitHub App integration, and receives webhook events from GitHub. When the Times Square app is installed in a GitHub repository, or a push to the default branch is accomplished, Times Square syncs Jupyter Notebooks from that repository into its page store.
+
+  Times Square-enabled GitHub repositories feature a `times-square.yaml` file that provides settings for the repository as a whole (such as switching the repository on or off) and setting the root path for notebooks. Each notebook (`ipynb` file) also has a corresponding YAML sidecar file that provides metadata for individual notebooks, including the title, description, tags, authors, and the parameter schemas.
+
+- To efficiently process webhook events, Times Square now operates an arq (Redis-backed) distributed queue.
+
+- New API endpoints:
+
+  - `GET /github/webhook` receives webhook events from GitHub (this should not require Gafaelfawr auth since webhooks are internally authenticated)
+  - `GET /v1/github` provides a hierarchical tree of GitHub-backed pages within their GitHub organization, repository, and directory contexts. This endpoint powers Squareone's navigational view.
+  - `GET /v1/github/:path` provides the same data as `/v1/pages/:page`, but uses the GitHub _display path_ as the path argument. An example of a display is `lsst-sqre/times-square-demo/matplotlib/gaussian2d`.
+
+  The `GET /v1/github` and `GET /v1/pages/:page` endpoints both include a new `github` field with metadata specific to GitHub-backed pages.
+
+## 0.3.0 (2022-03-31)
+
+### New features
+
+- This release adds a new `/v1/pages/:page/htmlstatus` endpoint that provides information about the availability of HTML for a specific page instance (set of page parameters). The Times Square UI monitors this endpoint to determine if an HTML rendering is available and when to refresh the iframe element displaying the HTML rendering.
+
+- The `htmlstatus` endpoint includes a SHA256 hash of the HTML, which is now stored alongside the HTML in the Redis cache. This hash can be used to invalidate expired HTML renderings for pages with a finite time-to-live setting.
+
+- This release adds a new `times-square reset-html` command to the command line interface for clearing the Redis cache of HTML renderings (primarily useful during development).
+
+## 0.2.0 (2022-03-15)
+
+### New features
+
+- Set up the `/v1/` HTTP API, along with core services, domain models and storage for managing pages and interfacing with Noteburst for notebook execution.
+
+## 0.2.0 (2022-03-15)
+
+### New features
+
+- Set up the `/v1/` HTTP API, along with core services, domain models and storage for managing pages and interfacing with Noteburst for notebook execution.
+
+## 0.1.0 (2021-11-17)
+
+### New features
+
+- Initial application set up.