WIP: Add registry-watcher image and separate docker-compose services

.dockerignore

@@ -1,4 +1,19 @@
+/target/
 /.rustwide
 /.rustwide-docker
+/Justfile
+/justfiles/
+/LICENSE
+/README.md
+/docker-compose.yml
+/dockerfiles/
+/docs/
 /ignored
-**/target
+/mcps
+/triagebot.toml
+/clippy.toml
+/.env
+/.env.*
+/.envrc
+/.docker.env
+archive_cache

.editorconfig

@@ -8,5 +8,9 @@ insert_final_newline = true
 indent_style = space
 indent_size = 4
 
+[Justfile]
+indent_size = 2
+
 [*.js]
 max_line_length = 100
+

.github/workflows/ci.yml

@@ -32,28 +32,26 @@ jobs:
         uses: raven-actions/actionlint@v2
         with:
           files: .github/workflow/*
-          flags: "-ignore SC2086" # ignore some shellcheck errors
+          flags: "-ignore SC2086"  # ignore some shellcheck errors
 
       - name: install `just`
         run: sudo snap install --edge --classic just
 
+      - uses: cargo-bins/cargo-binstall@main
+
       - name: restore build & cargo cache
         uses: Swatinem/rust-cache@v2
         with:
           prefix-key: ${{ env.RUST_CACHE_KEY }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Launch postgres
-        run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+        run: just ensure_db_and_s3_are_running
 
       - name: install SQLX CLI
-        run: cargo install sqlx-cli --no-default-features --features postgres
+        run: cargo binstall sqlx-cli
 
       - name: run database migrations
         run: cargo sqlx migrate run --database-url $DOCSRS_DATABASE_URL
@@ -68,46 +66,32 @@ jobs:
             --database-url $DOCSRS_DATABASE_URL \
             --target-version 0
 
-      - name: Clean up the database
-        run: docker compose down --volumes
+      - name: shut down test environment
+        if: ${{ always() }}
+        run: just compose-down-and-wipe
 
   test:
-    env:
-      SQLX_OFFLINE: 1
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
 
+      - name: install `just`
+        run: sudo snap install --edge --classic just
+
       - name: restore build & cargo cache
         uses: Swatinem/rust-cache@v2
         with:
           prefix-key: ${{ env.RUST_CACHE_KEY }}
 
-      - name: Build
-        run: cargo build --workspace --locked
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-      - name: Launch postgres and min.io
-        run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
-
-      - name: run workspace tests
-        run: |
-          cargo test --workspace --locked --no-fail-fast
+      - name: run tests
+        run: just run-tests run-builder-tests
 
-      - name: run slow tests
-        env:
-          DOCSRS_INCLUDE_DEFAULT_TARGETS: true
-        run: |
-          cargo test --locked -- --ignored --test-threads=1
-
-      - name: Clean up the database
-        run: docker compose down --volumes
+      - name: shut down test environment
+        if: ${{ always() }}
+        run: just compose-down-and-wipe
 
   GUI_test:
     runs-on: ubuntu-latest
@@ -119,43 +103,26 @@ jobs:
         with:
           prefix-key: ${{ env.RUST_CACHE_KEY }}
 
-      - name: Launch postgres and min.io
-        run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
-
-      - name: Run GUI tests
-        run: ./dockerfiles/run-gui-tests.sh
-
-      - name: Clean up the database
-        run: docker compose down --volumes
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
+      - name: install `just`
+        run: sudo snap install --edge --classic just
 
-    steps:
-      - uses: actions/checkout@v5
-      - name: update rust toolchain
-        run: rustup component add rustfmt
+      - name: Run GUI tests
+        run: just run-gui-tests
 
-      - run: cargo fmt -- --check
+      - name: shut down test environment
+        if: ${{ always() }}
+        run: just compose-down-and-wipe
 
-  clippy:
-    name: Clippy
+  rslint:
+    name: rust linters
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v5
 
-      - name: update rust toolchain
-        run: rustup component add clippy
-
       - name: install `just`
         run: sudo snap install --edge --classic just
 
@@ -164,6 +131,9 @@ jobs:
         with:
           prefix-key: ${{ env.RUST_CACHE_KEY }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - run: just lint
 
   eslint:

.github/workflows/docker.yml

@@ -1,13 +1,45 @@
 name: Docker
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  schedule:
+    - cron: "0 0 * * *"
 
 jobs:
   docker:
+    strategy:
+      matrix:
+        target: [
+          "web-server",
+          "build-server",
+          "registry-watcher",
+          "cli"
+        ]
     name: Test docker image builds
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
 
-      - name: Build the Docker image
-        run: docker build -t docs-rs -f dockerfiles/Dockerfile .
+      - name: setup docker buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: build docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: "./dockerfiles/Dockerfile"
+          platforms: linux/amd64
+          target: ${{ matrix.target }}
+          build-args: |
+            GIT_SHA=${{ github.sha }}
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: false
+
+          # TODO: later we would set `push: true` and also provide nice tags
+          # for the images.
+          # Unclear is how the deploy would work then.

.gitignore

@@ -1,5 +1,6 @@
 /ignored
 /.env
+/.docker.env
 /src/web/badge/Cargo.lock
 target
 *.css
@@ -10,5 +11,5 @@ target
 .vagrant
 .rustwide
 .rustwide-docker
-.archive_cache
+archive_cache
 .workspace

Justfile

@@ -1,18 +1,10 @@
+set shell := ["bash", "-Eeuo", "pipefail", "-c"]
+
 # List available commands
 _default:
     just --list
 
-sqlx-prepare ADDITIONAL_ARGS="":
-  cargo sqlx prepare \
-    --database-url $DOCSRS_DATABASE_URL \
-    --workspace {{ ADDITIONAL_ARGS }} \
-    -- --all-targets --all-features
-
-sqlx-check:
-  just sqlx-prepare "--check"
-
-lint: 
-  cargo clippy --all-features --all-targets --workspace --locked -- -D warnings
-
-lint-js *args:
-  deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }}
+import 'justfiles/cli.just'
+import 'justfiles/utils.just'
+import 'justfiles/services.just'
+import 'justfiles/testing.just'

NOTES.md

@@ -0,0 +1,87 @@
+# PR desc
+
+- the `Justfile` commands are built so they would even work on a 100% fresh
+  setup, and will set up and initialize everything they need.
+- I removed `index: Index` from the context. There is no need to clone the
+  crates.io index on a web- or build-server. The handful of places where we then
+  still need the index just create the obj.
+- there was an error I had when calling `Index::peek_changes` from
+  `crates-index-diff`. In there, we're using the github fastpath to check if
+  there are new commits in the repo, without having to actually `git pull` from
+  the remote. When I called `.peek_changes` in an async context, this lead to
+  tokio errors because inside `crates-index-diff` we're using
+  `reqwest::blocking`. Odd thing is: I couldn't find out why this doesn't fail
+  on production. It might have started failing just after the config/context
+  rewrite, which is not deployed yet.
+  [#2937](https://github.com/rust-lang/docs.rs/pull/2937)
+- unsure if the prefix mount should also be a docker volume, for performance.
+  Not sure how often we actually have to look at the contents?
+
+  ## docker image
+
+for now, these are production images, changing files will not auto-reload /
+build the image. We could decide to do that layer. Also I don't want to copy the
+".git" folder into the image, just for the version number. I made the SHA a
+build-arg / env and used these in our codebase.
+
+The fallback to fetching the has from the repo still exists, we might be able to
+drop this functionality at some point.
+
+## profiles
+
+- default: just db & s3
+
+runs & configures by default:
+
+- `db` -> postgres db
+- `s3` -> minio
+
+optional profile: `web`:
+
+- `web` -> webserver
+
+optional profile: `builder`:
+
+- `builder-a` -> build-server 1
+- `builder-b` -> build-server 2 ( two parallel build-servers, sharing nothing
+  apart from the build queue they access)
+
+optional profile: `watcher`:
+
+- `registry-watcher` ->
+
+* crates.io registry watcher
+* repo-stats updater
+* cdn invalidator
+* release-rebuild-enqueuer
+
+optional profile: `metrics`:
+
+- `prometheus` -> configured prometheus instance
+
+optional profile: `full`: all of the above.
+
+Services purely for manual usage with `docker compose run` are:
+
+- `cli`: to run simple CLI commands that only need the database & S3
+- `builder-cli`: to run CLI commands that need the build environment.
+- `registry-watcher-cli`: to run CLI commands that need the crates.io index.
+
+CAVEATS:
+
+- the build-servers have to run on the `linux/amd64` platform, while it doesn't
+  matter for the rest of the services. This means for example on a Mac, the
+  layers will be cached separately, once for `linux/amd64` and once for
+  `linux/arm64`. Only alternative would be to build everything for `amd64`, but
+  that would imply a performance impact on the services that don't need it.
+- volumes: typically docker-native volumes are faster than mounts, but sometimes
+  annoying to inspect for debugging.
+
+For now we choose:
+
+- docker-native for DB, S3, rustwide workspace, crates.io index
+- mounts for prefix, code mounts
+- prometheus scrape config is set to collect from the web server, the registry
+  watcher, and the build servers. Scraping is not dynamic, so the local
+  prometheus server will try to fetch from all service instances (web, watcher,
+  builder), and just error in case the specific server isn't accessible.