From 57949f7e83e0caf2e9373196ed37c81c56ceab40 Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Wed, 15 Nov 2023 15:16:17 +0100 Subject: [PATCH 01/47] Update readme for docker-compose v2 --- README.md | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b678d9a38..42627c854 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,7 @@ mkdir -p ignored/cratesfyi-prefix/crates.io-index # Builds the docs.rs binary SQLX_OFFLINE=1 cargo build # Start the external services. -# It may be `docker compose` in newer versions -docker-compose up -d db s3 +docker compose up -d db s3 # anything that doesn't run via docker-compose needs the settings defined in # .env. Either via `. ./.env` as below, or via any dotenv shell integration. . ./.env @@ -129,7 +128,7 @@ npm install browser-ui-test ### Pure docker-compose -If you have trouble with the above commands, consider using `docker-compose up --build`, +If you have trouble with the above commands, consider using `docker compose up --build`, which uses docker-compose for the web server as well. This will not cache dependencies - in particular, you'll have to rebuild all 400 whenever the lockfile changes - but makes sure that you're in a known environment so you should have fewer problems getting started. @@ -137,11 +136,18 @@ but makes sure that you're in a known environment so you should have fewer probl You can also use the `web` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows): ```sh # run a build for a single crate -docker-compose run web build crate regex 1.3.1 +docker compose run web build crate regex 1.3.1 # or build essential files -docker-compose run web build add-essential-files +docker compose run web build add-essential-files # rebuild the web container when you changed code. -docker-compose build web +docker compose up -d web --build +``` + +You can also run other commands like the setup above from within the container: + +```sh +docker compose run --rm cli database migrate +docker compose run --rm cli build update-toolchain ``` Note that running tests is not supported when using pure docker-compose. @@ -163,14 +169,14 @@ Three services are defined: #### Rebuilding Containers -To rebuild the site, run `docker-compose build`. +To rebuild the site, run `docker compose build`. Note that docker-compose caches the build even if you change the source code, so this will be necessary anytime you make changes. If you want to completely clean up the database, don't forget to remove the volumes too: ```sh -$ docker-compose down --volumes +$ docker compose down --volumes ``` #### FAQ @@ -184,7 +190,7 @@ This is probably because you have `git.autocrlf` set to true, ##### I see the error `/opt/rustwide/cargo-home/bin/cargo: cannot execute binary file: Exec format error` when running builds. -You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker-compose run web build [...]` workaround described above. +You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run web build [...]` workaround described above. See [rustwide#41](https://github.com/rust-lang/rustwide/issues/41) for more details about supporting more platforms directly. @@ -212,11 +218,11 @@ cargo run -- start-web-server ```sh # Builds and adds it into database # This is the main command to build and add a documentation into docs.rs. -# For example, `docker-compose run web build crate regex 1.1.6` +# For example, `docker compose run web build crate regex 1.1.6` cargo run -- build crate # alternatively, via the web container -docker-compose run web build crate +docker compose run web build crate # Builds every crate on crates.io and adds them into database # (beware: this may take months to finish) From 6eac43e203032402292706cddff73abb391e2866 Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Fri, 13 Oct 2023 23:13:12 +0200 Subject: [PATCH 02/47] Update CI to use docker-compose v2 --- dockerfiles/run-gui-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/run-gui-tests.sh b/dockerfiles/run-gui-tests.sh index 8e556b849..1f4420739 100755 --- a/dockerfiles/run-gui-tests.sh +++ b/dockerfiles/run-gui-tests.sh @@ -5,7 +5,7 @@ set -e # Just in case it's running, we stop the web server. docker compose stop web -docker compose up -d db s3 +docker compose up --wait --wait-timeout 30 db s3 # If we have a .env file, we need to temporarily move it so # it doesn't make sqlx fail compilation. From ce187a3119d5bc832905d4a57390e67e7c89c6a1 Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Sat, 21 Oct 2023 13:29:54 +0200 Subject: [PATCH 03/47] Don't bind internal docker-compose services to external IPs --- README.md | 6 +++--- docker-compose.yml | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 42627c854..894e7d56a 100644 --- a/README.md +++ b/README.md @@ -161,9 +161,9 @@ Three services are defined: | name | access | credentials | description | |------|-------------------------------------------------|----------------------------|----------------------------------------| -| web | http://localhost:3000 | N/A | A container running the docs.rs binary | -| db | postgresql://cratesfyi:password@localhost:15432 | - | Postgres database used by web | -| s3 | http://localhost:9000 | `cratesfyi` - `secret_key` | MinIO (simulates AWS S3) used by web | +| web | http://0.0.0.0:3000 | N/A | A container running the docs.rs binary | +| db | postgresql://cratesfyi:password@127.0.0.1:15432 | - | Postgres database used by web | +| s3 | http://127.0.0.1:9000 | `cratesfyi` - `secret_key` | MinIO (simulates AWS S3) used by web | [docker-compose.yml]: ./docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml index 78ad3cbad..7ba45c05a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,7 +51,7 @@ services: POSTGRES_PASSWORD: password ports: # Use a non-standard port on the host to avoid conflicting with existing postgres servers - - "15432:5432" + - "127.0.0.1:15432:5432" healthcheck: test: ["CMD", "pg_isready", "--username", "cratesfyi"] interval: 10s @@ -66,8 +66,8 @@ services: minio server /data --console-address ":9001"; " ports: - - "9000:9000" - - "9001:9001" + - "127.0.0.1:9000:9000" + - "127.0.0.1:9001:9001" volumes: - minio-data:/data environment: @@ -91,7 +91,7 @@ services: context: ./dockerfiles dockerfile: ./Dockerfile-prometheus ports: - - "9090:9090" + - "127.0.0.1:9090:9090" healthcheck: test: ["CMD", "curl", "--silent", "--fail", "localhost:9090/-/ready"] From d2ed9e487aa9ddd9540afe6ebe48bf9bae2a036e Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Wed, 15 Nov 2023 15:53:34 +0100 Subject: [PATCH 04/47] Run separate web server, registry watcher and build servers in docker-compose --- .github/workflows/ci.yml | 35 ++------ .github/workflows/docker.yml | 6 +- .gitignore | 1 + README.md | 39 +++++--- docker-compose.yml | 166 +++++++++++++++++++++++++---------- dockerfiles/Dockerfile | 88 ++++++++++++++----- dockerfiles/entrypoint.sh | 38 -------- 7 files changed, 218 insertions(+), 155 deletions(-) delete mode 100755 dockerfiles/entrypoint.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4603d797f..70d3a8e7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,13 +44,8 @@ jobs: - name: Launch postgres run: | - cp .env.sample .env - mkdir -p ${DOCSRS_PREFIX}/public-html - docker compose up -d db - # Give the database enough time to start up - sleep 5 - # Make sure the database is actually working - psql "${DOCSRS_DATABASE_URL}" + touch .docker.env + docker compose up --wait --wait-timeout 30 db - name: install SQLX CLI run: cargo install sqlx-cli --no-default-features --features postgres @@ -68,9 +63,6 @@ jobs: --database-url $DOCSRS_DATABASE_URL \ --target-version 0 - - name: Clean up the database - run: docker compose down --volumes - test: env: SQLX_OFFLINE: 1 @@ -88,13 +80,8 @@ jobs: - name: Launch postgres and min.io run: | - cp .env.sample .env - mkdir -p ${DOCSRS_PREFIX}/public-html - docker compose up -d db s3 - # Give the database enough time to start up - sleep 5 - # Make sure the database is actually working - psql "${DOCSRS_DATABASE_URL}" + touch .docker.env + docker compose up --wait --wait-timeout 30 db s3 - name: run workspace tests run: | @@ -106,9 +93,6 @@ jobs: run: | cargo test --locked -- --ignored --test-threads=1 - - name: Clean up the database - run: docker compose down --volumes - GUI_test: runs-on: ubuntu-latest steps: @@ -121,19 +105,12 @@ jobs: - name: Launch postgres and min.io run: | - cp .env.sample .env - mkdir -p ${DOCSRS_PREFIX}/public-html - docker compose up -d db s3 - # Give the database enough time to start up - sleep 5 - # Make sure the database is actually working - psql "${DOCSRS_DATABASE_URL}" + touch .docker.env + docker compose up --wait --wait-timeout 30 db s3 - name: Run GUI tests run: ./dockerfiles/run-gui-tests.sh - - name: Clean up the database - run: docker compose down --volumes fmt: name: Rustfmt diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 10c3c03b9..e81ed1f0b 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,5 +9,7 @@ jobs: steps: - uses: actions/checkout@v5 - - name: Build the Docker image - run: docker build -t docs-rs -f dockerfiles/Dockerfile . + - run: docker build --target web-server -f dockerfiles/Dockerfile . + - run: docker build --target build-server -f dockerfiles/Dockerfile . + - run: docker build --target registry-watcher -f dockerfiles/Dockerfile . + - run: docker build --target cli -f dockerfiles/Dockerfile . diff --git a/.gitignore b/.gitignore index 99643b163..9d9ef06ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /ignored /.env +/.docker.env /src/web/badge/Cargo.lock target *.css diff --git a/README.md b/README.md index 894e7d56a..b78180783 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ mkdir -p ignored/cratesfyi-prefix/crates.io-index # Builds the docs.rs binary SQLX_OFFLINE=1 cargo build # Start the external services. -docker compose up -d db s3 +docker compose up --wait db s3 # anything that doesn't run via docker-compose needs the settings defined in # .env. Either via `. ./.env` as below, or via any dotenv shell integration. . ./.env @@ -133,21 +133,32 @@ which uses docker-compose for the web server as well. This will not cache dependencies - in particular, you'll have to rebuild all 400 whenever the lockfile changes - but makes sure that you're in a known environment so you should have fewer problems getting started. -You can also use the `web` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows): +You'll need to `touch .docker.env` first, this file can have any environment +variable overrides you want to use in docker containers. + +You can also use the `builder-a` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows): + ```sh +# update the toolchain +docker compose run --rm builder-a build update-toolchain # run a build for a single crate -docker compose run web build crate regex 1.3.1 -# or build essential files -docker compose run web build add-essential-files -# rebuild the web container when you changed code. -docker compose up -d web --build +docker compose run --rm builder-a build crate regex 1.3.1 +# rebuild containers when you changed code. +docker compose up --wait --build ``` -You can also run other commands like the setup above from within the container: +You can also run other non-build commands like the setup steps above, or queueing crates for the background builders from within the `cli` container: ```sh docker compose run --rm cli database migrate -docker compose run --rm cli build update-toolchain +docker compose run --rm cli queue add regex 1.3.1 +``` + +If the command needs the crates.io-index clone then it must be run from within +a `registry-watcher` container: + +```sh +docker compose run --rm registry-watcher queue set-last-seen-reference --head ``` Note that running tests is not supported when using pure docker-compose. @@ -169,7 +180,7 @@ Three services are defined: #### Rebuilding Containers -To rebuild the site, run `docker compose build`. +To rebuild the site, run `docker compose --profile all build`. Note that docker-compose caches the build even if you change the source code, so this will be necessary anytime you make changes. @@ -190,7 +201,7 @@ This is probably because you have `git.autocrlf` set to true, ##### I see the error `/opt/rustwide/cargo-home/bin/cargo: cannot execute binary file: Exec format error` when running builds. -You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run web build [...]` workaround described above. +You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run --rm builder-a build [...]` workaround described above. See [rustwide#41](https://github.com/rust-lang/rustwide/issues/41) for more details about supporting more platforms directly. @@ -218,11 +229,11 @@ cargo run -- start-web-server ```sh # Builds and adds it into database # This is the main command to build and add a documentation into docs.rs. -# For example, `docker compose run web build crate regex 1.1.6` +# For example, `docker compose run --rm builder-a build crate regex 1.1.6` cargo run -- build crate -# alternatively, via the web container -docker compose run web build crate +# alternatively, within docker-compose containers +docker compose run --rm builder-a build crate # Builds every crate on crates.io and adds them into database # (beware: this may take months to finish) diff --git a/docker-compose.yml b/docker-compose.yml index 7ba45c05a..935f58048 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,44 +1,126 @@ -version: "3" +x-healthcheck: &healthcheck-interval + interval: 10s + timeout: 1s + start_interval: 1s + start_period: 10s + +x-environment: &environment + RUST_BACKTRACE: true + + DOCSRS_PREFIX: /opt/docsrs/prefix + DOCSRS_COMPILER_METRICS_PATH: /opt/docsrs/prefix/metrics + + DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db + DOCSRS_MIN_POOL_IDLE: 1 + DOCSRS_MIN_POOL_SIZE: 2 + DOCSRS_MAX_POOL_SIZE: 10 + DOCSRS_MAX_LEGACY_POOL_SIZE: 10 + + DOCSRS_STORAGE_BACKEND: s3 + + S3_ENDPOINT: http://s3:9000 + AWS_ACCESS_KEY_ID: cratesfyi + AWS_SECRET_ACCESS_KEY: secret_key + + DOCSRS_RENDER_THREADS: 2 + + DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide + DOCSRS_DOCKER: true + DOCSRS_DOCKER_IMAGE: ghcr.io/rust-lang/crates-build-env/linux-micro + DOCSRS_BUILD_CPU_LIMIT: 2 + DOCSRS_INCLUDE_DEFAULT_TARGETS: false + +x-build: &build + context: . + dockerfile: ./dockerfiles/Dockerfile + args: + PROFILE: dev + PROFILE_DIR: debug + +x-builder: &builder + build: + <<: *build + target: build-server + platform: "linux/amd64" + depends_on: + - db + - s3 + environment: *environment + env_file: + - .docker.env + healthcheck: + <<: *healthcheck-interval + test: curl --silent --fail localhost:3000/about/metrics + services: web: build: - context: . - dockerfile: ./dockerfiles/Dockerfile - args: - PROFILE: dev - PROFILE_DIR: debug + <<: *build + target: web-server platform: "linux/amd64" depends_on: - db - s3 ports: - - "3000:3000" - # for metrics - expose: ["3000"] + - "3000:80" + environment: *environment + env_file: + - .docker.env + healthcheck: + <<: *healthcheck-interval + test: curl --silent --fail localhost:80/about/metrics + + # Include the registry watcher with + # `docker compose --profile watch up --build --wait` + registry-watcher: + build: + <<: *build + target: registry-watcher + platform: "linux/amd64" + depends_on: + - db volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - - ".rustwide-docker:/opt/docsrs/rustwide" - "cratesio-index:/opt/docsrs/prefix/crates.io-index" - - "./ignored/cratesfyi-prefix/metrics:/opt/docsrs/prefix/metrics" - "./static:/opt/docsrs/static:ro" - environment: - DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide - DOCSRS_COMPILER_METRICS_PATH: /opt/docsrs/prefix/metrics - DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db - DOCSRS_STORAGE_BACKEND: s3 - S3_ENDPOINT: http://s3:9000 - AWS_ACCESS_KEY_ID: cratesfyi - AWS_SECRET_ACCESS_KEY: secret_key - DOCSRS_MAX_LEGACY_POOL_SIZE: 10 - DOCSRS_MAX_POOL_SIZE: 10 - DOCSRS_MIN_POOL_IDLE: 1 + environment: *environment env_file: - - .env + - .docker.env + profiles: + - watch + - all healthcheck: - test: ["CMD", "curl", "--silent", "--fail", "localhost:3000"] - interval: 10s - timeout: 5s - retries: 10 + <<: *healthcheck-interval + test: curl --silent --fail localhost:3000/about/metrics + + builder-a: + <<: *builder + volumes: + - ".rustwide-docker/builder-a:/opt/docsrs/rustwide" + - "./ignored/cratesfyi-prefix/metrics:/opt/docsrs/prefix/metrics" + - "/var/run/docker.sock:/var/run/docker.sock" + + builder-b: + <<: *builder + volumes: + - ".rustwide-docker/builder-b:/opt/docsrs/rustwide" + - "./ignored/cratesfyi-prefix/metrics:/opt/docsrs/prefix/metrics" + - "/var/run/docker.sock:/var/run/docker.sock" + + cli: + build: + <<: *build + target: cli + platform: "linux/amd64" + depends_on: + - db + - s3 + volumes: + - "cratesio-index:/opt/docsrs/prefix/crates.io-index" + environment: *environment + env_file: + - .docker.env + profiles: + - all db: build: @@ -53,10 +135,8 @@ services: # Use a non-standard port on the host to avoid conflicting with existing postgres servers - "127.0.0.1:15432:5432" healthcheck: - test: ["CMD", "pg_isready", "--username", "cratesfyi"] - interval: 10s - timeout: 5s - retries: 10 + <<: *healthcheck-interval + test: pg_isready --username cratesfyi s3: image: minio/minio @@ -74,17 +154,8 @@ services: MINIO_ROOT_USER: cratesfyi MINIO_ROOT_PASSWORD: secret_key healthcheck: - test: - [ - "CMD", - "curl", - "--silent", - "--fail", - "localhost:9000/minio/health/ready", - ] - interval: 10s - timeout: 5s - retries: 10 + <<: *healthcheck-interval + test: mc ready local prometheus: build: @@ -93,11 +164,8 @@ services: ports: - "127.0.0.1:9090:9090" healthcheck: - test: - ["CMD", "curl", "--silent", "--fail", "localhost:9090/-/ready"] - interval: 10s - timeout: 5s - retries: 10 + <<: *healthcheck-interval + test: promtool check healthy gui_tests: build: @@ -108,6 +176,8 @@ services: - "host.docker.internal:host-gateway" volumes: - "${PWD}:/build/out" + profiles: + - all volumes: postgres-data: {} diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index cce7ccf16..d49df8c80 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -73,44 +73,84 @@ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ && DEBIAN_FRONTEND=noninteractive apt-get install -y \ ca-certificates \ + curl \ tini \ && rm -rf /var/lib/apt/lists/* +WORKDIR /srv/docsrs + +# Tini is a small init binary to properly handle signals +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] +CMD ["start-web-server", "0.0.0.0:80"] + ARG PROFILE_DIR=release COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin COPY static /srv/docsrs/static COPY templates /srv/docsrs/templates COPY vendor /srv/docsrs/vendor -WORKDIR /srv/docsrs +######################## +# Build server stage # +######################## + +FROM ubuntu:22.04 AS build-server + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + ca-certificates \ + tini \ + curl \ + docker.io \ + build-essential \ + gcc \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + # Tini is a small init binary to properly handle signals -CMD ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "start-web-server", "0.0.0.0:80"] +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] +CMD ["start-build-server"] -################## -# Output stage # -################## +ARG PROFILE_DIR=release +COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin + +############################ +# Registry watcher stage # +############################ -FROM ubuntu:22.04 AS output +FROM ubuntu:22.04 AS registry-watcher -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - git \ - libmagic1 \ - docker.io \ - ca-certificates \ - build-essential \ - gcc \ - pkg-config \ - libssl-dev +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + ca-certificates \ + tini \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Tini is a small init binary to properly handle signals +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] +CMD ["start-registry-watcher", "--repository-stats-updater=enabled", "--cdn-invalidator=enabled"] + +ARG PROFILE_DIR=release +COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin + +############### +# CLI stage # +############### + +FROM ubuntu:22.04 AS cli + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + ca-certificates \ + tini \ + && rm -rf /var/lib/apt/lists/* -RUN mkdir -p /opt/docsrs/prefix +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] ARG PROFILE_DIR=release COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin -COPY static /opt/docsrs/static -COPY templates /opt/docsrs/templates -COPY dockerfiles/entrypoint.sh /opt/docsrs/ -COPY vendor /opt/docsrs/vendor - -WORKDIR /opt/docsrs -ENTRYPOINT ["/opt/docsrs/entrypoint.sh"] -CMD ["daemon", "--registry-watcher=disabled"] diff --git a/dockerfiles/entrypoint.sh b/dockerfiles/entrypoint.sh deleted file mode 100755 index b8fdc61aa..000000000 --- a/dockerfiles/entrypoint.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -set -euv - -export DOCSRS_PREFIX=/opt/docsrs/prefix -export DOCSRS_DOCKER=true -export DOCSRS_LOG=${DOCSRS_LOG-"docs-rs,rustwide=info"} -export PATH="$PATH:/build/target/release" - -# Try migrating the database multiple times if it fails -# This avoids the docker container crashing the first time it's started with -# docker-compose, as PostgreSQL needs some time to initialize. -set +e -failed=0 -while true; do - if ! cratesfyi database migrate; then - ((failed=failed + 1)) - if [ "${failed}" -eq 5 ]; then - exit 1 - fi - echo "failed to migrate the database" - echo "waiting 1 second..." - sleep 1 - else - break - fi -done -set -e - -if ! [ -d "${DOCSRS_PREFIX}/crates.io-index/.git" ]; then - git clone ${REGISTRY_URL:-https://github.com/rust-lang/crates.io-index} "${DOCSRS_PREFIX}/crates.io-index" - # Prevent new crates built before the container creation to be built - git --git-dir="$DOCSRS_PREFIX/crates.io-index/.git" branch crates-index-diff_last-seen -fi - -cratesfyi build update-toolchain --only-first-time - -cratesfyi "$@" From b392a6d619bff34b93871c8315b1ce9a6f9778a1 Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Tue, 19 Mar 2024 14:59:29 +0100 Subject: [PATCH 05/47] Ignore more files that are irrelevant to building a docker image --- .dockerignore | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index b96afe2a7..29b8ca6ad 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,12 @@ +/target/ /.rustwide /.rustwide-docker +/Justfile +/LICENSE +/README.md +/docker-compose.yml +/docs/ /ignored -**/target +/mcps +/triagebot.toml +/clippy.toml From 1c2fafc9dc715f22fcb02fbdf144d60d9e35988a Mon Sep 17 00:00:00 2001 From: Wim Looman Date: Sat, 17 May 2025 12:46:21 +0200 Subject: [PATCH 06/47] Better document initializing the docker-compose environment --- Justfile | 27 +++++++++++++++++++++++++++ README.md | 24 ++++++++++++++++++++---- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/Justfile b/Justfile index 14016ee71..eff68df25 100644 --- a/Justfile +++ b/Justfile @@ -16,3 +16,30 @@ lint: lint-js *args: deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} + +# Initialize the docker compose database +[group('compose')] +compose-migrate: + docker compose run --build --rm cli database migrate + +# Update last seen reference to the current index head, to only build newly published crates +[group('compose')] +compose-queue-head: + docker compose run --build --rm cli queue set-last-seen-reference --head + +# Launch base docker services, ensuring the database is migrated +[group('compose')] +compose-up: + just compose-migrate + docker compose up --build -d + +# Launch base docker services and registry watcher, ensuring the database is migrated +[group('compose')] +compose-up-watch: + just compose-migrate + docker compose --profile watch up --build -d + +# Shutdown docker services and cleanup all temporary volumes +[group('compose')] +compose-down: + docker compose --profile all down --volumes --remove-orphans diff --git a/README.md b/README.md index b78180783..881d5eabe 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,13 @@ This will not cache dependencies - in particular, you'll have to rebuild all 400 but makes sure that you're in a known environment so you should have fewer problems getting started. You'll need to `touch .docker.env` first, this file can have any environment -variable overrides you want to use in docker containers. +variable overrides you want to use in docker containers. Then run the migrations +before launching the main services: + +```sh +docker compose run --build --rm cli database migrate +docker compose up --build -d +``` You can also use the `builder-a` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows): @@ -154,15 +160,25 @@ docker compose run --rm cli database migrate docker compose run --rm cli queue add regex 1.3.1 ``` -If the command needs the crates.io-index clone then it must be run from within -a `registry-watcher` container: +If you want to run the registry watcher, you'll need to first set the "last seen +reference" from the registry index, e.g. to set it to the current head so only +newly published crates are built: + +```sh +docker compose run --rm cli queue set-last-seen-reference --head +``` + +Then enable the docker-compose profile that includes the watcher: ```sh -docker compose run --rm registry-watcher queue set-last-seen-reference --head +docker compose --profile watch up --build -d ``` Note that running tests is not supported when using pure docker-compose. +Some of the above commands are included in the `Justfile` for ease of use, +check the `[compose]` group in `just --list`. + Please file bugs for any trouble you have running docs.rs! ### Docker-Compose From a2d77db5996d1a3dd017671350675838ebd3aa1a Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sat, 1 Nov 2025 23:17:22 +0100 Subject: [PATCH 07/47] update dockerfile, use external GIT_SHA, update CI flow --- .github/workflows/docker.yml | 28 ++++++++++-- build.rs | 25 ++++++----- dockerfiles/Dockerfile | 83 +++++++++++++++++++++++++----------- src/lib.rs | 15 ++++++- src/registry_api.rs | 8 +--- src/repositories/github.rs | 8 ++-- src/repositories/gitlab.rs | 8 ++-- src/repositories/mod.rs | 6 --- 8 files changed, 122 insertions(+), 59 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e81ed1f0b..417ea6fd8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,12 +4,32 @@ on: [push, pull_request] jobs: docker: + strategy: + matrix: + target: [ + "web-server", + "build-server", + "registry-watcher", + "cli" + ] name: Test docker image builds runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - - run: docker build --target web-server -f dockerfiles/Dockerfile . - - run: docker build --target build-server -f dockerfiles/Dockerfile . - - run: docker build --target registry-watcher -f dockerfiles/Dockerfile . - - run: docker build --target cli -f dockerfiles/Dockerfile . + - name: setup docker buildx + uses: docker/setup-buildx-action@v3 + + - name: build docker image + uses: docker/build-push-action@v6 + with: + context: . + file: "./dockerfiles/Dockerfile" + platforms: linux/amd64,linux/arm64 + target: ${{ matrix.target }} + build_args: | + GIT_SHA=${{ github.sha }} + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + push: false diff --git a/build.rs b/build.rs index ae2f06821..7c7548e9c 100644 --- a/build.rs +++ b/build.rs @@ -71,7 +71,7 @@ mod tracked { fn main() -> Result<()> { let out_dir = env::var("OUT_DIR").context("missing OUT_DIR")?; let out_dir = Path::new(&out_dir); - write_git_version(out_dir)?; + read_git_version()?; compile_sass(out_dir)?; write_known_targets(out_dir)?; compile_syntax(out_dir).context("could not compile syntax files")?; @@ -81,16 +81,21 @@ fn main() -> Result<()> { Ok(()) } -fn write_git_version(out_dir: &Path) -> Result<()> { - let maybe_hash = get_git_hash()?; - let git_hash = maybe_hash.as_deref().unwrap_or("???????"); - - let build_date = time::OffsetDateTime::now_utc().date(); +fn read_git_version() -> Result<()> { + if let Ok(v) = env::var("GIT_SHA") { + // first try to read an externally provided git SAH, e.g., from CI + println!("cargo:rustc-env=GIT_SHA={v}"); + } else { + // then try to read the git repo. + let maybe_hash = get_git_hash()?; + let git_hash = maybe_hash.as_deref().unwrap_or("???????"); + println!("cargo:rustc-env=GIT_SHA={git_hash}"); + } - std::fs::write( - out_dir.join("git_version"), - format!("({git_hash} {build_date})"), - )?; + println!( + "cargo:rustc-env=BUILD_DATE={}", + time::OffsetDateTime::now_utc().date(), + ); Ok(()) } diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index d49df8c80..76c426fd7 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -9,19 +9,34 @@ # Build stage # ################# -FROM ubuntu:22.04 AS build +FROM rust:1.91-slim-trixie AS build + +ENV DEBIAN_FRONTEND=noninteractive # Install packaged dependencies -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential git curl cmake gcc g++ pkg-config libmagic-dev \ - libssl-dev zlib1g-dev ca-certificates mold clang - -# Install the stable toolchain with rustup -RUN curl https://sh.rustup.rs >/tmp/rustup-init && \ - chmod +x /tmp/rustup-init && \ - /tmp/rustup-init -y --no-modify-path --default-toolchain stable --profile minimal +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + git \ + curl \ + cmake \ + gcc \ + g++ \ + pkg-config \ + libmagic-dev \ + libssl-dev \ + zlib1g-dev \ + ca-certificates \ + mold \ + clang + ENV PATH=/root/.cargo/bin:$PATH +# get the git SHA from the build args, for our generated version numbers +ARG GIT_SHA=dev +ENV GIT_SHA=$GIT_SHA + # Configure linking to use mold instead for speed (need to use clang because gcc # is too old on this image) ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang @@ -34,8 +49,7 @@ ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS=-Clink-arg=-fuse-ld=mold WORKDIR /build COPY benches benches COPY Cargo.lock Cargo.toml ./ -COPY crates/metadata crates/metadata/ -COPY crates/font-awesome-as-a-crate crates/font-awesome-as-a-crate +COPY crates crates RUN mkdir -p src/bin && \ echo "fn main() {}" > src/bin/cratesfyi.rs && \ echo "fn main() {}" > build.rs @@ -50,7 +64,6 @@ RUN cargo build --profile=$PROFILE # source code didn't change thanks to mtime weirdness. RUN rm -rf src build.rs -COPY .git .git COPY build.rs build.rs RUN touch build.rs COPY src src/ @@ -67,14 +80,19 @@ RUN cargo build --profile=$PROFILE # Web server stage # ###################### -FROM ubuntu:22.04 AS web-server +FROM debian:trixie-slim AS web-server +ENV DEBIAN_FRONTEND=noninteractive + +# hadolint ignore=DL3008 RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + && apt-get update \ + && apt-get install -y \ + --no-install-recommends \ ca-certificates \ curl \ tini \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* WORKDIR /srv/docsrs @@ -86,18 +104,20 @@ CMD ["start-web-server", "0.0.0.0:80"] ARG PROFILE_DIR=release COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin COPY static /srv/docsrs/static -COPY templates /srv/docsrs/templates COPY vendor /srv/docsrs/vendor ######################## # Build server stage # ######################## -FROM ubuntu:22.04 AS build-server +FROM debian:trixie-slim AS build-server + +ENV DEBIAN_FRONTEND=noninteractive +# hadolint ignore=DL3008 RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + && apt-get install -y \ + --no-install-recommends \ ca-certificates \ tini \ curl \ @@ -106,6 +126,7 @@ RUN apt-get update \ gcc \ pkg-config \ libssl-dev \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* # Tini is a small init binary to properly handle signals @@ -119,15 +140,19 @@ COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin # Registry watcher stage # ############################ -FROM ubuntu:22.04 AS registry-watcher +FROM debian:trixie-slim AS registry-watcher +ENV DEBIAN_FRONTEND=noninteractive + +# hadolint ignore=DL3008 RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + && apt-get install -y \ + --no-install-recommends \ ca-certificates \ tini \ curl \ git \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* # Tini is a small init binary to properly handle signals @@ -141,15 +166,23 @@ COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin # CLI stage # ############### -FROM ubuntu:22.04 AS cli +FROM debian:trixie-slim AS cli + +ENV DEBIAN_FRONTEND=noninteractive +# hadolint ignore=DL3008 RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + && apt-get install -y \ + --no-install-recommends \ ca-certificates \ tini \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* +WORKDIR /srv/docsrs +# copy migrations so we can run them via CLI script +COPY migrations migrations/ + ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] ARG PROFILE_DIR=release diff --git a/src/lib.rs b/src/lib.rs index 32ab281e0..fc9d1423f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -56,8 +56,21 @@ pub(crate) static GLOBAL_ALERT: Option = Some(GlobalAlert { /// commit hash and build date pub const BUILD_VERSION: &str = concat!( env!("CARGO_PKG_VERSION"), + " (", + env!("GIT_SHA"), " ", - include_str!(concat!(env!("OUT_DIR"), "/git_version")) + env!("BUILD_DATE"), + " )" +); + +pub const APP_USER_AGENT: &str = concat!( + env!("CARGO_PKG_NAME"), + " ", + " (", + env!("GIT_SHA"), + " ", + env!("BUILD_DATE"), + " )" ); /// Where rustdoc's static files are stored in S3. diff --git a/src/registry_api.rs b/src/registry_api.rs index 74fe7f6dc..f856980b9 100644 --- a/src/registry_api.rs +++ b/src/registry_api.rs @@ -1,4 +1,4 @@ -use crate::{error::Result, utils::retry_async}; +use crate::{APP_USER_AGENT, error::Result, utils::retry_async}; use anyhow::{Context, anyhow, bail}; use chrono::{DateTime, Utc}; use reqwest::header::{ACCEPT, HeaderValue, USER_AGENT}; @@ -8,12 +8,6 @@ use std::fmt; use tracing::instrument; use url::Url; -const APP_USER_AGENT: &str = concat!( - env!("CARGO_PKG_NAME"), - " ", - include_str!(concat!(env!("OUT_DIR"), "/git_version")) -); - #[derive(Debug)] pub struct RegistryApi { api_base: Url, diff --git a/src/repositories/github.rs b/src/repositories/github.rs index 986ac2c84..e37a73a20 100644 --- a/src/repositories/github.rs +++ b/src/repositories/github.rs @@ -9,9 +9,11 @@ use reqwest::{ use serde::Deserialize; use tracing::{trace, warn}; -use crate::repositories::{ - APP_USER_AGENT, FetchRepositoriesResult, RateLimitReached, Repository, RepositoryForge, - RepositoryName, +use crate::{ + APP_USER_AGENT, + repositories::{ + FetchRepositoriesResult, RateLimitReached, Repository, RepositoryForge, RepositoryName, + }, }; const GRAPHQL_UPDATE: &str = "query($ids: [ID!]!) { diff --git a/src/repositories/gitlab.rs b/src/repositories/gitlab.rs index c1fb70b95..c09c7c280 100644 --- a/src/repositories/gitlab.rs +++ b/src/repositories/gitlab.rs @@ -10,9 +10,11 @@ use std::collections::HashSet; use std::str::FromStr; use tracing::warn; -use crate::repositories::{ - APP_USER_AGENT, FetchRepositoriesResult, RateLimitReached, Repository, RepositoryForge, - RepositoryName, +use crate::{ + APP_USER_AGENT, + repositories::{ + FetchRepositoriesResult, RateLimitReached, Repository, RepositoryForge, RepositoryName, + }, }; const GRAPHQL_UPDATE: &str = "query($ids: [ID!]!) { diff --git a/src/repositories/mod.rs b/src/repositories/mod.rs index 9a6932390..2376e33cf 100644 --- a/src/repositories/mod.rs +++ b/src/repositories/mod.rs @@ -5,12 +5,6 @@ pub use self::updater::{ FetchRepositoriesResult, Repository, RepositoryForge, RepositoryStatsUpdater, }; -pub const APP_USER_AGENT: &str = concat!( - env!("CARGO_PKG_NAME"), - " ", - include_str!(concat!(env!("OUT_DIR"), "/git_version")) -); - #[derive(Debug, thiserror::Error)] #[error("rate limit reached")] struct RateLimitReached; From 44ceaba2073b0a1413ea946a2054ece7af6c441e Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sat, 1 Nov 2025 23:20:29 +0100 Subject: [PATCH 08/47] fix build-args docker CI --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 417ea6fd8..f23d9c126 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,7 +27,7 @@ jobs: file: "./dockerfiles/Dockerfile" platforms: linux/amd64,linux/arm64 target: ${{ matrix.target }} - build_args: | + build-args: | GIT_SHA=${{ github.sha }} load: true cache-from: type=gha From fadd3eec0efc5afdb2714195c74272564ce15385 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sat, 1 Nov 2025 23:22:58 +0100 Subject: [PATCH 09/47] only one platform for now --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f23d9c126..eb2f58beb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: with: context: . file: "./dockerfiles/Dockerfile" - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64 target: ${{ matrix.target }} build-args: | GIT_SHA=${{ github.sha }} From 4f0568180b962ce47d58d8478376b7084e1c2e9b Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 01:33:08 +0100 Subject: [PATCH 10/47] WIP --- Justfile | 93 +++++++++++++++++--- docker-compose.yml | 169 ++++++++++++++++++++++++++----------- dockerfiles/Dockerfile | 7 +- dockerfiles/prometheus.yml | 3 +- 4 files changed, 208 insertions(+), 64 deletions(-) diff --git a/Justfile b/Justfile index eff68df25..898ffa553 100644 --- a/Justfile +++ b/Justfile @@ -17,29 +17,96 @@ lint: lint-js *args: deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} +_touch-docker-env: + touch .docker.env + +# clean up docker images, volumes & other local artifacts from this docker-compose +# config +[group('compose')] +cleanup: + docker compose down --volumes --remove-orphans --rmi local + rm -rf .rustwide-docker/ && mkdir -p .rustwide-docker + rm -rf ignored/ && mkdir -p ignored + + +# run any CLI command in its own one-off docker container. Args are passed to the container. +[group('compose')] +compose-cli *args: _touch-docker-env + # ensure dependencies are running. + # docker compose ignores dependencies from the yaml file + # when we use `compose run`. + # + # no-op if they are already running. + docker compose up -d db s3 --wait + # run the CLI with the provided args. + + # not suited for commands that need: + # * the crates.io index, or + # * need to run builds. + docker compose run --build --rm cli {{ args }} + +# run builder CLI command in its own one-off docker container. +[group('compose')] +compose-builder-cli *args: _touch-docker-env + docker compose up -d db s3 --wait + docker compose run --build --rm builder-cli {{ args }} + +# run registry-watcher CLI command in its own one-off docker container. +[group('compose')] +compose-registry-watcher-cli *args: _touch-docker-env + docker compose up -d db s3 --wait + docker compose run --build --rm registry-watcher-cli {{ args }} + # Initialize the docker compose database [group('compose')] -compose-migrate: - docker compose run --build --rm cli database migrate +compose-cli-migrate: + just compose-cli database migrate # Update last seen reference to the current index head, to only build newly published crates [group('compose')] -compose-queue-head: - docker compose run --build --rm cli queue set-last-seen-reference --head +compose-cli-queue-head: + just compose-cli queue set-last-seen-reference --head + +# run migrations, then launch one or more docker compose profiles in the background +[group('compose')] +compose-up *profiles: _touch-docker-env compose-cli-migrate + if [ {{profiles.len()}} -eq 0 ]; then + echo "❌ Error: You must specify at least one profile, e.g.:"; + echo " just compose-up web"; + echo " just compose-up web builder"; + exit 1; + fi + + docker compose \ + {{ profiles | map(p => "--profile " + p) | join(" ") }} \ + up --build -d + +# Launch web server in the background +[group('compose')] +compose-up-web: + just compose-up web + +# Launch two build servers in the background +[group('compose')] +compose-up-builder: + just compose-up builder + +# Launch registry watcher in the background +[group('compose')] +compose-up-watcher: + just compose-up watcher -# Launch base docker services, ensuring the database is migrated +# Launch prometheus server in the background [group('compose')] -compose-up: - just compose-migrate - docker compose up --build -d +compose-up-metrics: + just compose-up metrics -# Launch base docker services and registry watcher, ensuring the database is migrated +# Launch everything in the background [group('compose')] -compose-up-watch: - just compose-migrate - docker compose --profile watch up --build -d +compose-up-full: + just compose-up full # Shutdown docker services and cleanup all temporary volumes [group('compose')] compose-down: - docker compose --profile all down --volumes --remove-orphans + docker compose --profile full down --remove-orphans diff --git a/docker-compose.yml b/docker-compose.yml index 935f58048..53dc29c57 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,44 @@ +# main docker-compose file for local development. +# +# runs & configures by default: +# * `db` -> postgres db +# * `s3` -> minio +# +# optional profile: `web`: +# * `web` -> webserver +# +# optional profile: `builder`: +# * `builder-a` -> build-server 1 +# * `builder-b` -> build-server 2 +# ( two parallel build-servers, sharing nothing apart +# from the build queue they access) +# +# optional profile: `watcher`: +# * `registry-watcher` -> crates.io registry watcher + repo-stats updater + +# cdn invalidator +# +# optional profile: `metrics`: +# * `prometheus` -> configured prometheus instance +# +# scrape config is set to collect from the web- & buildservers. +# +# optional profile: `full`: all of the above. +# +# Services purely for manual usage with `docker compose run` are: +# * `cli`: to run simple CLI commands that only need the database & S3 +# * `builder-cli`: to run CLI commands that need the build environment. +# * `registry-watcher-cli`: to run CLI commands that need the crates.io index. +# +# CAVEATS: +# * the build-servers have to run on the `linux/amd64` platform, +# while it doesn't matter for the rest of the services. +# This means for example on a Mac, the layers will be +# cached separately, once for `linux/amd64` and once for +# `linux/arm64`. +# Only alternative would be to build everything for `amd64`, but +# that would imply a performance impact on the services that don't +# need it. + x-healthcheck: &healthcheck-interval interval: 10s timeout: 1s @@ -6,9 +47,9 @@ x-healthcheck: &healthcheck-interval x-environment: &environment RUST_BACKTRACE: true + RUST_LOG: debug DOCSRS_PREFIX: /opt/docsrs/prefix - DOCSRS_COMPILER_METRICS_PATH: /opt/docsrs/prefix/metrics DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db DOCSRS_MIN_POOL_IDLE: 1 @@ -22,14 +63,6 @@ x-environment: &environment AWS_ACCESS_KEY_ID: cratesfyi AWS_SECRET_ACCESS_KEY: secret_key - DOCSRS_RENDER_THREADS: 2 - - DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide - DOCSRS_DOCKER: true - DOCSRS_DOCKER_IMAGE: ghcr.io/rust-lang/crates-build-env/linux-micro - DOCSRS_BUILD_CPU_LIMIT: 2 - DOCSRS_INCLUDE_DEFAULT_TARGETS: false - x-build: &build context: . dockerfile: ./dockerfiles/Dockerfile @@ -41,86 +74,125 @@ x-builder: &builder build: <<: *build target: build-server + # builders only work with linux/amd64 platform: "linux/amd64" depends_on: - db - s3 - environment: *environment + environment: + <<: *environment + DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide + DOCSRS_COMPILER_METRICS_PATH: /opt/docsrs/prefix/metrics + DOCSRS_DOCKER: true + DOCSRS_DOCKER_IMAGE: ghcr.io/rust-lang/crates-build-env/linux-micro + DOCSRS_BUILD_CPU_LIMIT: 2 + DOCSRS_INCLUDE_DEFAULT_TARGETS: false env_file: - .docker.env healthcheck: <<: *healthcheck-interval test: curl --silent --fail localhost:3000/about/metrics +x-registry-watcher: ®istry-watcher + build: + <<: *build + target: registry-watcher + depends_on: + - db + - s3 + volumes: + - "./static:/opt/docsrs/static:ro" + environment: *environment + env_file: + - .docker.env + healthcheck: + <<: *healthcheck-interval + test: "true" # no metrics for now in the watcher + services: web: build: <<: *build target: web-server - platform: "linux/amd64" depends_on: - db - s3 ports: - "3000:80" - environment: *environment + environment: + <<: *environment + DOCSRS_RENDER_THREADS: 2 env_file: - .docker.env healthcheck: <<: *healthcheck-interval test: curl --silent --fail localhost:80/about/metrics + profiles: + - web + - full - # Include the registry watcher with - # `docker compose --profile watch up --build --wait` registry-watcher: - build: - <<: *build - target: registry-watcher - platform: "linux/amd64" - depends_on: - - db + <<: *registry-watcher volumes: - - "cratesio-index:/opt/docsrs/prefix/crates.io-index" - - "./static:/opt/docsrs/static:ro" - environment: *environment - env_file: - - .docker.env + - "./ignored/docker-registry-watcher/prefix:/opt/docsrs/prefix" profiles: - - watch - - all - healthcheck: - <<: *healthcheck-interval - test: curl --silent --fail localhost:3000/about/metrics + - watcher + - full + + registry-watcher-cli: + <<: *registry-watcher + volumes: + - "./ignored/docker-registry-watcher-cli/prefix:/opt/docsrs/prefix" + profiles: + - manual builder-a: <<: *builder volumes: - ".rustwide-docker/builder-a:/opt/docsrs/rustwide" - - "./ignored/cratesfyi-prefix/metrics:/opt/docsrs/prefix/metrics" + - "./ignored/docker-builder-a/prefix:/opt/docsrs/prefix" - "/var/run/docker.sock:/var/run/docker.sock" + profiles: + - builder + - full builder-b: <<: *builder volumes: - ".rustwide-docker/builder-b:/opt/docsrs/rustwide" - - "./ignored/cratesfyi-prefix/metrics:/opt/docsrs/prefix/metrics" + - "./ignored/docker-builder-b/prefix:/opt/docsrs/prefix" - "/var/run/docker.sock:/var/run/docker.sock" + profiles: + - builder + - full + + builder-cli: + <<: *builder + volumes: + - ".rustwide-docker/builder-cli:/opt/docsrs/rustwide" + - "./ignored/docker-builder-cli/prefix:/opt/docsrs/prefix" + - "/var/run/docker.sock:/var/run/docker.sock" + profiles: + - manual cli: build: <<: *build target: cli - platform: "linux/amd64" depends_on: + # only for clarifycation. + # when using "docker compose run", these dependencies are ignored, + # we handle this in our `just` commands. - db - s3 - volumes: - - "cratesio-index:/opt/docsrs/prefix/crates.io-index" environment: *environment + volumes: + - "./ignored/docker-cli/prefix:/opt/docsrs/prefix" env_file: - .docker.env profiles: - - all + # CLI should not be run as background daemon, just manually + - manual db: build: @@ -132,7 +204,8 @@ services: POSTGRES_USER: cratesfyi POSTGRES_PASSWORD: password ports: - # Use a non-standard port on the host to avoid conflicting with existing postgres servers + # Use a non-standard port on the host to avoid conflicting + # with existing postgres servers - "127.0.0.1:15432:5432" healthcheck: <<: *healthcheck-interval @@ -163,23 +236,21 @@ services: dockerfile: ./Dockerfile-prometheus ports: - "127.0.0.1:9090:9090" + # we intentionally don't define depends_on here. + # While the scrapers are configured to fetch from eventually running + # web or build-servers, adding these as dependency would mean we can't + # test metrics just with a webserver. + # Prometheus will just scrape from the working endpoints, and skip/error + # on the broken ones. healthcheck: <<: *healthcheck-interval test: promtool check healthy - - gui_tests: - build: - context: . - dockerfile: ./dockerfiles/Dockerfile-gui-tests - network_mode: "host" - extra_hosts: - - "host.docker.internal:host-gateway" - volumes: - - "${PWD}:/build/out" profiles: - - all + # we rarely need to test with actual prometheus, so always running + # it is a waste. + - metrics + - full volumes: postgres-data: {} minio-data: {} - cratesio-index: {} diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 76c426fd7..0d12ab30c 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -165,6 +165,11 @@ COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin ############### # CLI stage # ############### +# This stage is used to run one-off commands like database migrations. +# not suited for commands that need: +# * the crates.io index, or +# * need to run builds. +# for these, use the build-server stage, or registry-watcher stage instead. FROM debian:trixie-slim AS cli @@ -181,7 +186,7 @@ RUN apt-get update \ WORKDIR /srv/docsrs # copy migrations so we can run them via CLI script -COPY migrations migrations/ +# COPY migrations migrations/ ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] diff --git a/dockerfiles/prometheus.yml b/dockerfiles/prometheus.yml index 08a069372..55cfd9a4b 100644 --- a/dockerfiles/prometheus.yml +++ b/dockerfiles/prometheus.yml @@ -11,4 +11,5 @@ scrape_configs: - job_name: "docs.rs" metrics_path: "/about/metrics" static_configs: - - targets: ["web:3000"] + # registry watcher doesn't have metrics yet. + - targets: ["web:80", "builder-a:3000", "builder-b:3000"] From c2d21fc291bd7fc11a95e5330d388563c8d4ae42 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 01:59:32 +0100 Subject: [PATCH 11/47] no index --- Justfile | 37 +++++++++--------------------------- src/bin/cratesfyi.rs | 9 +++++---- src/context.rs | 13 +------------ src/index.rs | 10 ++++++++++ src/utils/consistency/mod.rs | 9 ++++++--- src/utils/daemon.rs | 3 ++- 6 files changed, 33 insertions(+), 48 deletions(-) diff --git a/Justfile b/Justfile index 898ffa553..7c35a7fef 100644 --- a/Justfile +++ b/Justfile @@ -28,34 +28,24 @@ cleanup: rm -rf .rustwide-docker/ && mkdir -p .rustwide-docker rm -rf ignored/ && mkdir -p ignored +_compose-cli service_name *args: _touch-docker-env + docker compose up -d db s3 --wait + docker compose run --build --rm {{ service_name }} {{ args }} # run any CLI command in its own one-off docker container. Args are passed to the container. [group('compose')] -compose-cli *args: _touch-docker-env - # ensure dependencies are running. - # docker compose ignores dependencies from the yaml file - # when we use `compose run`. - # - # no-op if they are already running. - docker compose up -d db s3 --wait - # run the CLI with the provided args. - - # not suited for commands that need: - # * the crates.io index, or - # * need to run builds. - docker compose run --build --rm cli {{ args }} +compose-cli *args: _touch-docker-env + just _compose-cli cli {{ args }} # run builder CLI command in its own one-off docker container. [group('compose')] compose-builder-cli *args: _touch-docker-env - docker compose up -d db s3 --wait - docker compose run --build --rm builder-cli {{ args }} + just _compose-cli builder-cli {{ args }} # run registry-watcher CLI command in its own one-off docker container. [group('compose')] compose-registry-watcher-cli *args: _touch-docker-env - docker compose up -d db s3 --wait - docker compose run --build --rm registry-watcher-cli {{ args }} + just _compose-cli registry-watcher-cli {{ args }} # Initialize the docker compose database [group('compose')] @@ -65,21 +55,12 @@ compose-cli-migrate: # Update last seen reference to the current index head, to only build newly published crates [group('compose')] compose-cli-queue-head: - just compose-cli queue set-last-seen-reference --head + just compose-registry-watcher-cli queue set-last-seen-reference --head # run migrations, then launch one or more docker compose profiles in the background [group('compose')] compose-up *profiles: _touch-docker-env compose-cli-migrate - if [ {{profiles.len()}} -eq 0 ]; then - echo "❌ Error: You must specify at least one profile, e.g.:"; - echo " just compose-up web"; - echo " just compose-up web builder"; - exit 1; - fi - - docker compose \ - {{ profiles | map(p => "--profile " + p) | join(" ") }} \ - up --build -d + docker compose {{ prepend("--profile ", profiles) }} up --build -d # Launch web server in the background [group('compose')] diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 61c4dc1dd..412e3b3a2 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -1,7 +1,7 @@ use anyhow::{Context as _, Result, anyhow}; use clap::{Parser, Subcommand, ValueEnum}; use docs_rs::{ - Config, Context, PackageKind, RustwideBuilder, + Config, Context, Index, PackageKind, RustwideBuilder, db::{self, CrateId, Overrides, add_path_into_database}, start_background_metrics_webserver, start_web_server, utils::{ @@ -202,8 +202,8 @@ impl CommandLine { start_background_metrics_webserver(Some(metric_server_socket_addr), &ctx)?; ctx.runtime.block_on(async move { - docs_rs::utils::watch_registry(&ctx.async_build_queue, &ctx.config, ctx.index) - .await + let index = Arc::new(Index::from_config(&ctx.config)?); + docs_rs::utils::watch_registry(&ctx.async_build_queue, &ctx.config, index).await })?; } Self::StartBuildServer { @@ -297,8 +297,9 @@ impl QueueSubcommand { let reference = match (reference, head) { (Some(reference), false) => reference, (None, true) => { + let index = Index::from_config(&ctx.config)?; println!("Fetching changes to set reference to HEAD"); - let (_, oid) = ctx.index.diff()?.peek_changes()?; + let (_, oid) = index.diff()?.peek_changes()?; oid } (_, _) => unreachable!(), diff --git a/src/context.rs b/src/context.rs index 008d1a3b7..631d3e583 100644 --- a/src/context.rs +++ b/src/context.rs @@ -2,7 +2,7 @@ use crate::cdn::CdnBackend; use crate::db::Pool; use crate::repositories::RepositoryStatsUpdater; use crate::{ - AsyncBuildQueue, AsyncStorage, BuildQueue, Config, Index, InstanceMetrics, RegistryApi, + AsyncBuildQueue, AsyncStorage, BuildQueue, Config, InstanceMetrics, RegistryApi, ServiceMetrics, Storage, }; use anyhow::Result; @@ -19,7 +19,6 @@ pub struct Context { pub pool: Pool, pub service_metrics: Arc, pub instance_metrics: Arc, - pub index: Arc, pub registry_api: Arc, pub repository_stats_updater: Arc, pub runtime: runtime::Handle, @@ -67,15 +66,6 @@ impl Context { let cdn = Arc::new(CdnBackend::new(&config).await); - let index = Arc::new({ - let path = config.registry_index_path.clone(); - if let Some(registry_url) = config.registry_url.clone() { - Index::from_url(path, registry_url) - } else { - Index::new(path) - }? - }); - let runtime = runtime::Handle::current(); // sync wrappers around build-queue & storage async resources let build_queue = Arc::new(BuildQueue::new(runtime.clone(), async_build_queue.clone())); @@ -90,7 +80,6 @@ impl Context { pool: pool.clone(), service_metrics: Arc::new(ServiceMetrics::new()?), instance_metrics, - index, registry_api: Arc::new(RegistryApi::new( config.registry_api_host.clone(), config.crates_io_api_call_retries, diff --git a/src/index.rs b/src/index.rs index a3ca021f1..ff7ad7962 100644 --- a/src/index.rs +++ b/src/index.rs @@ -1,3 +1,4 @@ +use crate::Config; use crate::error::Result; use crate::utils::report_error; use anyhow::Context; @@ -12,6 +13,15 @@ pub struct Index { } impl Index { + pub fn from_config(config: &Config) -> Result { + let path = config.registry_index_path.clone(); + Ok(if let Some(registry_url) = config.registry_url.clone() { + Index::from_url(path, registry_url) + } else { + Index::new(path) + }?) + } + pub fn from_url(path: PathBuf, url: String) -> Result { crates_index_diff::Index::from_path_or_cloned_with_options( &path, diff --git a/src/utils/consistency/mod.rs b/src/utils/consistency/mod.rs index acb90588d..355f46aee 100644 --- a/src/utils/consistency/mod.rs +++ b/src/utils/consistency/mod.rs @@ -1,4 +1,4 @@ -use crate::{Context, db::delete, utils::spawn_blocking}; +use crate::{Context, Index, db::delete, utils::spawn_blocking}; use anyhow::{Context as _, Result}; use itertools::Itertools; use tracing::{info, warn}; @@ -33,8 +33,11 @@ pub async fn run_check(ctx: &Context, dry_run: bool) -> Result<()> { tracing::info!("Loading data from index..."); let index_data = spawn_blocking({ - let index = ctx.index.clone(); - move || index::load(&index) + let config = ctx.config.clone(); + move || { + let index = Index::from_config(&config)?; + index::load(&index) + } }) .await .context("Loading crate data from index for consistency check")?; diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index 7699774e6..f5f6b97de 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -55,7 +55,8 @@ pub async fn watch_registry( fn start_registry_watcher(context: &Context) -> Result<(), Error> { let build_queue = context.async_build_queue.clone(); let config = context.config.clone(); - let index = context.index.clone(); + + let index = Arc::new(Index::from_config(&config)?); context.runtime.spawn(async move { // space this out to prevent it from clashing against the queue-builder thread on launch From 3334a090afbf52f502da21ae5a9aa0a7cde9e560 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 02:06:33 +0100 Subject: [PATCH 12/47] web works --- docker-compose.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 53dc29c57..8584f1901 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -102,7 +102,16 @@ x-registry-watcher: ®istry-watcher - s3 volumes: - "./static:/opt/docsrs/static:ro" - environment: *environment + - "crates-io-index:/opt/docsrs/crates.io-index" + environment: + <<: *environment + # provide registry url so we clone the index on the first try + REGISTRY_URL: https://github.com/rust-lang/crates.io-index.git + # the crates.io index can be shared between the registry watcher & its + # CLI service. + # So we configure it to be separate from the prefix. + # Also on a native docker volume for performance + REGISTRY_INDEX_PATH: /opt/docsrs/crates.io-index env_file: - .docker.env healthcheck: @@ -254,3 +263,4 @@ services: volumes: postgres-data: {} minio-data: {} + crates-io-index: {} From 5a016876a60b58377efdad51c204f2840a15f2df Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 02:17:47 +0100 Subject: [PATCH 13/47] udpates --- Justfile | 2 +- docker-compose.yml | 18 +++++++++++++----- dockerfiles/Dockerfile | 4 ++-- dockerfiles/prometheus.yml | 2 +- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Justfile b/Justfile index 7c35a7fef..2bdf6fe5e 100644 --- a/Justfile +++ b/Justfile @@ -60,7 +60,7 @@ compose-cli-queue-head: # run migrations, then launch one or more docker compose profiles in the background [group('compose')] compose-up *profiles: _touch-docker-env compose-cli-migrate - docker compose {{ prepend("--profile ", profiles) }} up --build -d + docker compose {{ prepend("--profile ", profiles) }} up --build -d --wait # Launch web server in the background [group('compose')] diff --git a/docker-compose.yml b/docker-compose.yml index 8584f1901..14260cbd9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,6 +38,11 @@ # Only alternative would be to build everything for `amd64`, but # that would imply a performance impact on the services that don't # need it. +# * volumes: typically docker-native volumes are faster than mounts, but +# sometimes annoying to inspect for debugging. +# For now we choose: +# * docker-native for DB, S3, rustwide workspace, crates.io index +# * mounts for prefix, code mounts x-healthcheck: &healthcheck-interval interval: 10s @@ -127,7 +132,7 @@ services: - db - s3 ports: - - "3000:80" + - "3000:3000" environment: <<: *environment DOCSRS_RENDER_THREADS: 2 @@ -135,7 +140,7 @@ services: - .docker.env healthcheck: <<: *healthcheck-interval - test: curl --silent --fail localhost:80/about/metrics + test: curl --silent --fail localhost:3000/about/metrics profiles: - web - full @@ -158,7 +163,7 @@ services: builder-a: <<: *builder volumes: - - ".rustwide-docker/builder-a:/opt/docsrs/rustwide" + - "rustwide-builder-a:/opt/docsrs/rustwide" - "./ignored/docker-builder-a/prefix:/opt/docsrs/prefix" - "/var/run/docker.sock:/var/run/docker.sock" profiles: @@ -168,7 +173,7 @@ services: builder-b: <<: *builder volumes: - - ".rustwide-docker/builder-b:/opt/docsrs/rustwide" + - "rustwide-builder-b:/opt/docsrs/rustwide" - "./ignored/docker-builder-b/prefix:/opt/docsrs/prefix" - "/var/run/docker.sock:/var/run/docker.sock" profiles: @@ -178,7 +183,7 @@ services: builder-cli: <<: *builder volumes: - - ".rustwide-docker/builder-cli:/opt/docsrs/rustwide" + - "rustwide-builder-cli:/opt/docsrs/rustwide" - "./ignored/docker-builder-cli/prefix:/opt/docsrs/prefix" - "/var/run/docker.sock:/var/run/docker.sock" profiles: @@ -264,3 +269,6 @@ volumes: postgres-data: {} minio-data: {} crates-io-index: {} + rustwide-builder-a: {} + rustwide-builder-b: {} + rustwide-builder-cli: {} diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 0d12ab30c..4f6b5abcb 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -99,7 +99,7 @@ WORKDIR /srv/docsrs # Tini is a small init binary to properly handle signals ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] -CMD ["start-web-server", "0.0.0.0:80"] +CMD ["start-web-server", "0.0.0.0:3000"] ARG PROFILE_DIR=release COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin @@ -131,7 +131,7 @@ RUN apt-get update \ # Tini is a small init binary to properly handle signals ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] -CMD ["start-build-server"] +CMD ["start-build-server", "0.0.0.0:3000"] ARG PROFILE_DIR=release COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin diff --git a/dockerfiles/prometheus.yml b/dockerfiles/prometheus.yml index 55cfd9a4b..ce51b1bb3 100644 --- a/dockerfiles/prometheus.yml +++ b/dockerfiles/prometheus.yml @@ -12,4 +12,4 @@ scrape_configs: metrics_path: "/about/metrics" static_configs: # registry watcher doesn't have metrics yet. - - targets: ["web:80", "builder-a:3000", "builder-b:3000"] + - targets: ["web:3000", "builder-a:3000", "builder-b:3000"] From 64ce22d35079ab9bf1b28d2cbfaa673e1893e061 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 02:41:25 +0100 Subject: [PATCH 14/47] WIP --- docker-compose.yml | 11 +++-------- dockerfiles/Dockerfile | 2 +- dockerfiles/prometheus.yml | 2 +- src/bin/cratesfyi.rs | 3 +-- src/build_queue.rs | 17 ++++++++++++----- src/index.rs | 12 ++++++++++++ 6 files changed, 30 insertions(+), 17 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 14260cbd9..d86bc0a1b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -107,11 +107,10 @@ x-registry-watcher: ®istry-watcher - s3 volumes: - "./static:/opt/docsrs/static:ro" - - "crates-io-index:/opt/docsrs/crates.io-index" + - "./ignored/docker-registry-watcher/prefix:/opt/docsrs/prefix" + - crates-io-index:/opt/docsrs/crates.io-index environment: <<: *environment - # provide registry url so we clone the index on the first try - REGISTRY_URL: https://github.com/rust-lang/crates.io-index.git # the crates.io index can be shared between the registry watcher & its # CLI service. # So we configure it to be separate from the prefix. @@ -121,7 +120,7 @@ x-registry-watcher: ®istry-watcher - .docker.env healthcheck: <<: *healthcheck-interval - test: "true" # no metrics for now in the watcher + test: curl --silent --fail localhost:3000/about/metrics services: web: @@ -147,16 +146,12 @@ services: registry-watcher: <<: *registry-watcher - volumes: - - "./ignored/docker-registry-watcher/prefix:/opt/docsrs/prefix" profiles: - watcher - full registry-watcher-cli: <<: *registry-watcher - volumes: - - "./ignored/docker-registry-watcher-cli/prefix:/opt/docsrs/prefix" profiles: - manual diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 4f6b5abcb..a9e2cd554 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -157,7 +157,7 @@ RUN apt-get update \ # Tini is a small init binary to properly handle signals ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] -CMD ["start-registry-watcher", "--repository-stats-updater=enabled", "--cdn-invalidator=enabled"] +CMD ["start-registry-watcher", "0.0.0.0:3000", "--repository-stats-updater=enabled", "--cdn-invalidator=enabled", "--queue-rebuilds=enabled"] ARG PROFILE_DIR=release COPY --from=build /build/target/$PROFILE_DIR/cratesfyi /usr/local/bin diff --git a/dockerfiles/prometheus.yml b/dockerfiles/prometheus.yml index ce51b1bb3..cabac00fb 100644 --- a/dockerfiles/prometheus.yml +++ b/dockerfiles/prometheus.yml @@ -12,4 +12,4 @@ scrape_configs: metrics_path: "/about/metrics" static_configs: # registry watcher doesn't have metrics yet. - - targets: ["web:3000", "builder-a:3000", "builder-b:3000"] + - targets: ["web:3000", "registry-watcher:3000", "builder-a:3000", "builder-b:3000"] diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 412e3b3a2..da509d0d4 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -299,8 +299,7 @@ impl QueueSubcommand { (None, true) => { let index = Index::from_config(&ctx.config)?; println!("Fetching changes to set reference to HEAD"); - let (_, oid) = index.diff()?.peek_changes()?; - oid + index.last_reference()? } (_, _) => unreachable!(), }; diff --git a/src/build_queue.rs b/src/build_queue.rs index ec7855846..b3ba79936 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -13,7 +13,7 @@ use sqlx::Connection as _; use std::collections::HashMap; use std::sync::Arc; use tokio::runtime; -use tracing::{debug, error, info, instrument}; +use tracing::{debug, error, info, instrument, warn}; /// The static priority for background rebuilds. /// Used when queueing rebuilds, and when rendering them @@ -228,10 +228,17 @@ impl AsyncBuildQueue { pub async fn get_new_crates(&self, index: &Index) -> Result { let diff = index.diff()?; - let last_seen_reference = self - .last_seen_reference() - .await? - .context("no last_seen_reference set in database")?; + let last_seen_reference = if let Some(oid) = self.last_seen_reference().await? { + oid + } else { + warn!( + "no last-seen reference found in our database. We assume a fresh install and + set the latest reference (HEAD) as last. This means we will then start to queue + builds for new crates only from now on, and not for all existing crates." + ); + index.last_reference()? + }; + diff.set_last_seen_reference(last_seen_reference)?; let (changes, new_reference) = diff.peek_changes_ordered()?; diff --git a/src/index.rs b/src/index.rs index ff7ad7962..80258c3ab 100644 --- a/src/index.rs +++ b/src/index.rs @@ -6,6 +6,7 @@ use crates_index_diff::gix; use std::path::PathBuf; use std::process::Command; use std::sync::atomic::AtomicBool; +use tracing::info; pub struct Index { path: PathBuf, @@ -16,8 +17,10 @@ impl Index { pub fn from_config(config: &Config) -> Result { let path = config.registry_index_path.clone(); Ok(if let Some(registry_url) = config.registry_url.clone() { + info!(path=%path.display(), registry_url, "using custom registry index"); Index::from_url(path, registry_url) } else { + info!(path=%path.display(), "using default registry index"); Index::new(path) }?) } @@ -94,4 +97,13 @@ impl Index { pub fn repository_url(&self) -> Option<&str> { self.repository_url.as_deref() } + + /// get the last refernce oid in the crates.io index repository. + /// + /// Specifically not the "last seen" ref, but the actual last in + /// the index. + pub fn last_reference(&self) -> Result { + let (_, oid) = self.diff()?.peek_changes()?; + Ok(oid) + } } From a07071959f947733b3c67e4123d80b522288d3de Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 03:31:15 +0100 Subject: [PATCH 15/47] watcher works, also from scratch --- docker-compose.yml | 5 ++++- src/bin/cratesfyi.rs | 7 +++++-- src/build_queue.rs | 47 ++++++++++++++++++++++++++++++-------------- src/index.rs | 19 ++++++++++++------ src/utils/daemon.rs | 2 +- 5 files changed, 55 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d86bc0a1b..0f1125f33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,7 +52,7 @@ x-healthcheck: &healthcheck-interval x-environment: &environment RUST_BACKTRACE: true - RUST_LOG: debug + DOCSRS_LOG: docs_rs=trace,tower_http=debug,rustwide=info,aws_smithy_http=trace,axum=debug DOCSRS_PREFIX: /opt/docsrs/prefix @@ -116,6 +116,9 @@ x-registry-watcher: ®istry-watcher # So we configure it to be separate from the prefix. # Also on a native docker volume for performance REGISTRY_INDEX_PATH: /opt/docsrs/crates.io-index + # configure the rebuild-queuer + DOCSRS_MAX_QUEUED_REBUILDS: 10 + env_file: - .docker.env healthcheck: diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index da509d0d4..56c891e72 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -3,6 +3,7 @@ use clap::{Parser, Subcommand, ValueEnum}; use docs_rs::{ Config, Context, Index, PackageKind, RustwideBuilder, db::{self, CrateId, Overrides, add_path_into_database}, + index::IndexExt as _, start_background_metrics_webserver, start_web_server, utils::{ ConfigName, get_config, get_crate_pattern_and_priority, list_crate_priorities, @@ -200,9 +201,11 @@ impl CommandLine { } start_background_metrics_webserver(Some(metric_server_socket_addr), &ctx)?; + // Index::from_config might use `tokio::block_on, so `from_config` might fail + // when called inside an async context. + let index = Arc::new(Index::from_config(&ctx.config)?); ctx.runtime.block_on(async move { - let index = Arc::new(Index::from_config(&ctx.config)?); docs_rs::utils::watch_registry(&ctx.async_build_queue, &ctx.config, index).await })?; } @@ -299,7 +302,7 @@ impl QueueSubcommand { (None, true) => { let index = Index::from_config(&ctx.config)?; println!("Fetching changes to set reference to HEAD"); - index.last_reference()? + index.diff()?.latest_commit_reference()? } (_, _) => unreachable!(), }; diff --git a/src/build_queue.rs b/src/build_queue.rs index b3ba79936..e8d34729e 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -2,8 +2,11 @@ use crate::Context; use crate::db::{CrateId, Pool, delete_crate, delete_version, update_latest_version_id}; use crate::docbuilder::PackageKind; use crate::error::Result; +use crate::index::IndexExt as _; use crate::storage::AsyncStorage; -use crate::utils::{ConfigName, get_config, get_crate_priority, report_error, retry, set_config}; +use crate::utils::{ + ConfigName, get_config, get_crate_priority, report_error, retry, set_config, spawn_blocking, +}; use crate::{BuildPackageSummary, cdn}; use crate::{Config, Index, InstanceMetrics, RustwideBuilder}; use anyhow::Context as _; @@ -225,23 +228,37 @@ impl AsyncBuildQueue { /// Updates registry index repository and adds new crates into build queue. /// /// Returns the number of crates added - pub async fn get_new_crates(&self, index: &Index) -> Result { - let diff = index.diff()?; + pub async fn get_new_crates(&self, index: Arc) -> Result { + let last_seen_reference = self.last_seen_reference().await?; + + let (changes, last_seen_reference, new_reference) = spawn_blocking({ + let index = index.clone(); + move || { + // this needs to be in spawn_blocking because crates-index-diff might use + // a blocking `reqwest` client when doing `.peek_changes_ordered()`. + let diff = index.diff()?; + + let last_seen_reference = if let Some(oid) = last_seen_reference { + oid + } else { + warn!( + "no last-seen reference found in our database. We assume a fresh install and + set the latest reference (HEAD) as last. This means we will then start to queue + builds for new releases only from now on, and not for all existing releases." + ); + // this also calls `.peek_changes()`, so might execute blocking http calls, + // which lead to panics when used in an async context. + diff.latest_commit_reference()? + }; - let last_seen_reference = if let Some(oid) = self.last_seen_reference().await? { - oid - } else { - warn!( - "no last-seen reference found in our database. We assume a fresh install and - set the latest reference (HEAD) as last. This means we will then start to queue - builds for new crates only from now on, and not for all existing crates." - ); - index.last_reference()? - }; + diff.set_last_seen_reference(last_seen_reference)?; - diff.set_last_seen_reference(last_seen_reference)?; + let (changes, new_reference) = diff.peek_changes_ordered()?; - let (changes, new_reference) = diff.peek_changes_ordered()?; + Ok((changes, last_seen_reference, new_reference)) + } + }) + .await?; let mut conn = self.db.get_async().await?; let mut crates_added = 0; diff --git a/src/index.rs b/src/index.rs index 80258c3ab..a25db07f1 100644 --- a/src/index.rs +++ b/src/index.rs @@ -16,13 +16,13 @@ pub struct Index { impl Index { pub fn from_config(config: &Config) -> Result { let path = config.registry_index_path.clone(); - Ok(if let Some(registry_url) = config.registry_url.clone() { + if let Some(registry_url) = config.registry_url.clone() { info!(path=%path.display(), registry_url, "using custom registry index"); Index::from_url(path, registry_url) } else { info!(path=%path.display(), "using default registry index"); Index::new(path) - }?) + } } pub fn from_url(path: PathBuf, url: String) -> Result { @@ -97,13 +97,20 @@ impl Index { pub fn repository_url(&self) -> Option<&str> { self.repository_url.as_deref() } +} - /// get the last refernce oid in the crates.io index repository. +pub trait IndexExt { + /// get the reference oid of the latest commit in the crates.io + /// index repository (aka, the `HEAD`). /// - /// Specifically not the "last seen" ref, but the actual last in + /// Specifically not the "last seen" ref, but the actual latest in /// the index. - pub fn last_reference(&self) -> Result { - let (_, oid) = self.diff()?.peek_changes()?; + fn latest_commit_reference(&self) -> Result; +} + +impl IndexExt for crates_index_diff::Index { + fn latest_commit_reference(&self) -> Result { + let (_, oid) = self.peek_changes()?; Ok(oid) } } diff --git a/src/utils/daemon.rs b/src/utils/daemon.rs index f5f6b97de..3e2b0652a 100644 --- a/src/utils/daemon.rs +++ b/src/utils/daemon.rs @@ -31,7 +31,7 @@ pub async fn watch_registry( } else { debug!("Checking new crates"); match build_queue - .get_new_crates(&index) + .get_new_crates(index.clone()) .await .context("Failed to get new crates") { From 5ca9eccb209afc439ebc508783de511f328a4b45 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 03:42:17 +0100 Subject: [PATCH 16/47] comments --- Justfile | 2 ++ docker-compose.yml | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Justfile b/Justfile index 2bdf6fe5e..86938835d 100644 --- a/Justfile +++ b/Justfile @@ -24,6 +24,8 @@ _touch-docker-env: # config [group('compose')] cleanup: + # FIXME: the images sometimes can't be deleted? because in-use? + # But all containers are stopped? docker compose down --volumes --remove-orphans --rmi local rm -rf .rustwide-docker/ && mkdir -p .rustwide-docker rm -rf ignored/ && mkdir -p ignored diff --git a/docker-compose.yml b/docker-compose.yml index 0f1125f33..0b5e3053c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,13 +14,18 @@ # from the build queue they access) # # optional profile: `watcher`: -# * `registry-watcher` -> crates.io registry watcher + repo-stats updater + -# cdn invalidator +# * `registry-watcher` -> +# - crates.io registry watcher +# - repo-stats updater +# - cdn invalidator +# - release-rebuild-enqueuer # # optional profile: `metrics`: # * `prometheus` -> configured prometheus instance # -# scrape config is set to collect from the web- & buildservers. +# prometheus scrape config is set to collect from the web server, the registry watcher, and +# the build servers. We assume there are no metrics from the CLIs, as we do in +# prod. # # optional profile: `full`: all of the above. # From 9209bda87106fdff8df000c97f50d57c09dfe8de Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 03:43:28 +0100 Subject: [PATCH 17/47] update docker ci flow --- .github/workflows/docker.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index eb2f58beb..ff5314d3e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,6 +1,12 @@ name: Docker -on: [push, pull_request] +on: + push: + branches: + - master + pull_request: + schedule: + - cron: "0 0 * * *" jobs: docker: From f47b37c36d7235e9dad569ee015b12bf1177e83c Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 03:51:03 +0100 Subject: [PATCH 18/47] re-add gui-test service --- Justfile | 11 ++++++----- docker-compose.yml | 12 ++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Justfile b/Justfile index 86938835d..227cb200b 100644 --- a/Justfile +++ b/Justfile @@ -39,6 +39,11 @@ _compose-cli service_name *args: _touch-docker-env compose-cli *args: _touch-docker-env just _compose-cli cli {{ args }} +# Initialize the docker compose database +[group('compose')] +compose-cli-migrate: + just compose-cli database migrate + # run builder CLI command in its own one-off docker container. [group('compose')] compose-builder-cli *args: _touch-docker-env @@ -49,16 +54,12 @@ compose-builder-cli *args: _touch-docker-env compose-registry-watcher-cli *args: _touch-docker-env just _compose-cli registry-watcher-cli {{ args }} -# Initialize the docker compose database -[group('compose')] -compose-cli-migrate: - just compose-cli database migrate - # Update last seen reference to the current index head, to only build newly published crates [group('compose')] compose-cli-queue-head: just compose-registry-watcher-cli queue set-last-seen-reference --head + # run migrations, then launch one or more docker compose profiles in the background [group('compose')] compose-up *profiles: _touch-docker-env compose-cli-migrate diff --git a/docker-compose.yml b/docker-compose.yml index 0b5e3053c..327d07e6d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -268,6 +268,18 @@ services: - metrics - full + gui_tests: + build: + context: . + dockerfile: ./dockerfiles/Dockerfile-gui-tests + network_mode: "host" + extra_hosts: + - "host.docker.internal:host-gateway" + volumes: + - "${PWD}:/build/out" + profiles: + - manual + volumes: postgres-data: {} minio-data: {} From c086127ccf54444989431b6a07168901dca7b620 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 04:10:21 +0100 Subject: [PATCH 19/47] fixes --- Justfile | 33 ++++++++++++++++++++++++++++++-- dockerfiles/Dockerfile | 7 ++++++- dockerfiles/Dockerfile-gui-tests | 20 +++++++++++++++---- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/Justfile b/Justfile index 227cb200b..5c5b21615 100644 --- a/Justfile +++ b/Justfile @@ -44,11 +44,41 @@ compose-cli *args: _touch-docker-env compose-cli-migrate: just compose-cli database migrate +# add a release to the build queue +[group('compose')] +compose-cli-set-toolchain *args: + just compose-cli queue add {{ args }} + # run builder CLI command in its own one-off docker container. [group('compose')] compose-builder-cli *args: _touch-docker-env just _compose-cli builder-cli {{ args }} +# set the nightly rust version to be used for builds. Format: `nightly-YYYY-MM-DD` +[group('compose')] +compose-cli-set-toolchain NAME: + just compose-builder-cli build set-toolchain {{ NAME }} + +# update the toolchain in the builders +[group('compose')] +compose-cli-update-toolchain: + just compose-builder-cli build update-toolchain + +# build & the toolchain shared essential files +[group('compose')] +compose-cli-add-essential-files: + just compose-builder-cli build add-essential-files + +# build & the toolchain shared essential files +[group('compose')] +compose-cli-build-crate *args: + just compose-builder-cli build crate {{ args }} + +# Update last seen reference to the current index head, to only build newly published crates +[group('compose')] +compose-cli-queue-head: + just compose-registry-watcher-cli queue set-last-seen-reference --head + # run registry-watcher CLI command in its own one-off docker container. [group('compose')] compose-registry-watcher-cli *args: _touch-docker-env @@ -59,11 +89,10 @@ compose-registry-watcher-cli *args: _touch-docker-env compose-cli-queue-head: just compose-registry-watcher-cli queue set-last-seen-reference --head - # run migrations, then launch one or more docker compose profiles in the background [group('compose')] compose-up *profiles: _touch-docker-env compose-cli-migrate - docker compose {{ prepend("--profile ", profiles) }} up --build -d --wait + docker compose {{ prepend("--profile ", profiles) }} up --build -d --wait --remove-orphans # Launch web server in the background [group('compose')] diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index a9e2cd554..432773b62 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -109,6 +109,11 @@ COPY vendor /srv/docsrs/vendor ######################## # Build server stage # ######################## +# * includes docker-cli, but expects the docker engine to be mapped into +# the container from the outside, e.g. via +# +# volumes: +# - "/var/run/docker.sock:/var/run/docker.sock" FROM debian:trixie-slim AS build-server @@ -121,7 +126,7 @@ RUN apt-get update \ ca-certificates \ tini \ curl \ - docker.io \ + docker-cli \ build-essential \ gcc \ pkg-config \ diff --git a/dockerfiles/Dockerfile-gui-tests b/dockerfiles/Dockerfile-gui-tests index 162614564..e584603bb 100644 --- a/dockerfiles/Dockerfile-gui-tests +++ b/dockerfiles/Dockerfile-gui-tests @@ -1,11 +1,23 @@ FROM ubuntu:22.04 AS build +ENV DEBIAN_FRONTEND=noninteractive + # Install packaged dependencies -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - build-essential git curl cmake gcc g++ pkg-config libmagic-dev \ - libssl-dev zlib1g-dev ca-certificates +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + git \ + curl \ + cmake \ + gcc \ + g++ \ + pkg-config \ + libmagic-dev \ + libssl-dev \ + zlib1g-dev \ + ca-certificates -RUN apt-get update && apt-get install -y \ +RUN apt-get install -y \ ca-certificates \ curl \ docker.io \ From 766ccb4af06016736ce919667840ae907394c3f9 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 04:22:59 +0100 Subject: [PATCH 20/47] fixes --- Justfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Justfile b/Justfile index 5c5b21615..969f10e15 100644 --- a/Justfile +++ b/Justfile @@ -46,7 +46,7 @@ compose-cli-migrate: # add a release to the build queue [group('compose')] -compose-cli-set-toolchain *args: +compose-cli-queue-add *args: just compose-cli queue add {{ args }} # run builder CLI command in its own one-off docker container. @@ -74,11 +74,6 @@ compose-cli-add-essential-files: compose-cli-build-crate *args: just compose-builder-cli build crate {{ args }} -# Update last seen reference to the current index head, to only build newly published crates -[group('compose')] -compose-cli-queue-head: - just compose-registry-watcher-cli queue set-last-seen-reference --head - # run registry-watcher CLI command in its own one-off docker container. [group('compose')] compose-registry-watcher-cli *args: _touch-docker-env From a04ab0ae23ef65587ec2a031eda70722537b54f4 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 05:44:47 +0100 Subject: [PATCH 21/47] fix edege-cas for add_essential_files in update-toollchai. --- Justfile | 15 +++-- docker-compose.yml | 2 +- src/bin/cratesfyi.rs | 14 +---- src/build_queue.rs | 33 +---------- src/docbuilder/rustwide_builder.rs | 89 +++++++++++++++++++++++++++++- src/utils/queue_builder.rs | 5 +- 6 files changed, 107 insertions(+), 51 deletions(-) diff --git a/Justfile b/Justfile index 969f10e15..22508487b 100644 --- a/Justfile +++ b/Justfile @@ -24,11 +24,16 @@ _touch-docker-env: # config [group('compose')] cleanup: - # FIXME: the images sometimes can't be deleted? because in-use? - # But all containers are stopped? - docker compose down --volumes --remove-orphans --rmi local - rm -rf .rustwide-docker/ && mkdir -p .rustwide-docker - rm -rf ignored/ && mkdir -p ignored + # FIXME onlt delete containers in our project + docker container prune --force + # FIXME onlt delete volumesin our project + docker volume prune -f -a + + # FIXME: the images sometimes can't be deleted? because in-use? + # But all containers are stopped? + docker compose down --volumes --remove-orphans --rmi local + rm -rf .rustwide-docker/ && mkdir -p .rustwide-docker + rm -rf ignored/ && mkdir -p ignored _compose-cli service_name *args: _touch-docker-env docker compose up -d db s3 --wait diff --git a/docker-compose.yml b/docker-compose.yml index 327d07e6d..17f772dd6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -180,7 +180,7 @@ services: - "./ignored/docker-builder-b/prefix:/opt/docsrs/prefix" - "/var/run/docker.sock:/var/run/docker.sock" profiles: - - builder + - builderaaa - full builder-cli: diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 56c891e72..c2bf0589d 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -439,6 +439,8 @@ impl BuildSubcommand { } => { let mut builder = rustwide_builder()?; + builder.update_toolchain_and_add_essential_files()?; + if let Some(path) = local { builder .build_local_package(&path) @@ -477,17 +479,7 @@ impl BuildSubcommand { return Ok(()); } - rustwide_builder()? - .update_toolchain() - .context("failed to update toolchain")?; - - rustwide_builder()? - .purge_caches() - .context("failed to purge caches")?; - - rustwide_builder()? - .add_essential_files() - .context("failed to add essential files")?; + rustwide_builder()?.update_toolchain_and_add_essential_files()?; } Self::AddEssentialFiles => { diff --git a/src/build_queue.rs b/src/build_queue.rs index e8d34729e..ebc3a9505 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -601,35 +601,6 @@ impl BuildQueue { Ok(()) } - fn update_toolchain(&self, builder: &mut RustwideBuilder) -> Result<()> { - let updated = retry( - || { - builder - .update_toolchain() - .context("downloading new toolchain failed") - }, - 3, - )?; - - if updated { - // toolchain has changed, purge caches - retry( - || { - builder - .purge_caches() - .context("purging rustwide caches failed") - }, - 3, - )?; - - builder - .add_essential_files() - .context("adding essential files failed")?; - } - - Ok(()) - } - /// Builds the top package from the queue. Returns whether there was a package in the queue. /// /// Note that this will return `Ok(true)` even if the package failed to build. @@ -662,8 +633,8 @@ impl BuildQueue { return Err(err); } - if let Err(err) = self - .update_toolchain(&mut *builder) + if let Err(err) = builder + .update_toolchain_and_add_essential_files() .context("Updating toolchain failed, locking queue") { report_error(&err); diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 5f0242e58..af68c4a5c 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -17,7 +17,7 @@ use crate::storage::{ rustdoc_json_path, source_archive_path, }; use crate::utils::{ - CargoMetadata, ConfigName, copy_dir_all, get_config, parse_rustc_version, report_error, + CargoMetadata, ConfigName, copy_dir_all, get_config, parse_rustc_version, report_error, retry, set_config, }; use crate::{AsyncStorage, Config, Context, InstanceMetrics, RegistryApi, Storage}; @@ -170,11 +170,46 @@ impl RustwideBuilder { Ok(()) } + #[instrument(skip_all)] + pub fn update_toolchain_and_add_essential_files(&mut self) -> Result<()> { + info!("try updating the toolchain"); + let updated = retry( + || { + self.update_toolchain() + .context("downloading new toolchain failed") + }, + 3, + )?; + + debug!(updated, "toolchain update check complete"); + + if updated { + // toolchain has changed, purge caches + retry( + || { + self.purge_caches() + .context("purging rustwide caches failed") + }, + 3, + )?; + + self.add_essential_files() + .context("adding essential files failed")?; + } + + Ok(()) + } + + #[instrument(skip_all)] pub fn update_toolchain(&mut self) -> Result { self.toolchain = self.runtime.block_on(async { let mut conn = self.db.get_async().await?; get_configured_toolchain(&mut conn).await })?; + debug!( + configured_toolchain = self.toolchain.to_string(), + "configured toolchain" + ); // For CI builds, a lot of the normal update_toolchain things don't apply. // CI builds are only for one platform (https://forge.rust-lang.org/infra/docs/rustc-ci.html#try-builds) @@ -243,7 +278,57 @@ impl RustwideBuilder { } } - let has_changed = old_version != Some(self.rustc_version()?); + let new_version = self.rustc_version()?; + debug!(new_version, "detected new rustc version"); + let mut has_changed = old_version.as_ref() != Some(&new_version); + + if !has_changed { + // This fixes an edge-case on a fresh build server. + // + // It seems like on the fresh server, there _is_ a recent nightly toolchain + // installed, which we then updated with necessary components in this + // method. + // + // But: *for this toolchain, we never ran `add_essential_files`*, because it + // was not installed by us. + // + // Now the culprit: even through we "fix" the previously installed nightly toolchain + // with the needed components & targets, we return "updated = false", since the actual + // version number didn't change. + // + // As a result, `BuildQueue::update_toolchain` will not call `add_essential_files`, + // which then means we don't have the toolchain-shared static files on our S3 bucket. + // + // The workaround specifically for `add_essential_files` is the following: + // + // After `add_essential_files` is finished, it sets `ConfigName::RustcVersion` in the + // database to the version it uploaded the essential files for. + // + // This means, if `ConfigName::RustcVersion` is empty, or different from the current new + // version, we can set `updated = true` too. + // + // I feel like there are more edge-cases, but for now this is OK. + // + // Alternative would have been to run `build update-toolchain --only-first-time` + // in a newly created `ENTRYPOINT` script for the build-server. I leaned to towards + // a more self-contained solution which doesn't need docker at all, and also would work + // if you run the build-server directly on your machine. + + let rustc_version = self.runtime.block_on({ + let pool = self.db.clone(); + async move { + let mut conn = pool + .get_async() + .await + .context("failed to get a database connection")?; + + get_config::(&mut conn, ConfigName::RustcVersion).await + } + })?; + + has_changed = rustc_version.is_none() || rustc_version != Some(new_version); + } + Ok(has_changed) } diff --git a/src/utils/queue_builder.rs b/src/utils/queue_builder.rs index 8a1d90502..292cedfc0 100644 --- a/src/utils/queue_builder.rs +++ b/src/utils/queue_builder.rs @@ -6,10 +6,13 @@ use std::time::Duration; use std::{fs, io, path::Path, thread}; use tracing::{debug, error, warn}; +/// the main build-server loop pub fn queue_builder(context: &Context, mut builder: RustwideBuilder) -> Result<(), Error> { loop { let temp_dir = &context.config.temp_dir; - if let Err(e) = remove_tempdirs(temp_dir) { + if temp_dir.exists() + && let Err(e) = remove_tempdirs(temp_dir) + { report_error(&anyhow::anyhow!(e).context(format!( "failed to clean temporary directory {:?}", temp_dir From c9a5b657235f8d6f67366e3ad05bd15f0895ede8 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 06:16:27 +0100 Subject: [PATCH 22/47] fixes --- Justfile | 4 ++++ dockerfiles/run-gui-tests.sh | 5 +++++ src/docbuilder/rustwide_builder.rs | 9 ++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 22508487b..eb03426cd 100644 --- a/Justfile +++ b/Justfile @@ -123,3 +123,7 @@ compose-up-full: [group('compose')] compose-down: docker compose --profile full down --remove-orphans + +[group('compose')] +compose-logs *services: + docker compose --profile full logs -f {{ services }} diff --git a/dockerfiles/run-gui-tests.sh b/dockerfiles/run-gui-tests.sh index 1f4420739..acf1ea592 100755 --- a/dockerfiles/run-gui-tests.sh +++ b/dockerfiles/run-gui-tests.sh @@ -2,6 +2,11 @@ set -e +# FIXME: @syphar, it might be easy to update the script with the new +# docker compose setup to make it run on non-linux machines. +# Perhasp we could even replace the script just with some "Justfile" +# tasks. + # Just in case it's running, we stop the web server. docker compose stop web diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index af68c4a5c..9ef0e105a 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -201,7 +201,7 @@ impl RustwideBuilder { } #[instrument(skip_all)] - pub fn update_toolchain(&mut self) -> Result { + fn update_toolchain(&mut self) -> Result { self.toolchain = self.runtime.block_on(async { let mut conn = self.db.get_async().await?; get_configured_toolchain(&mut conn).await @@ -313,6 +313,13 @@ impl RustwideBuilder { // in a newly created `ENTRYPOINT` script for the build-server. I leaned to towards // a more self-contained solution which doesn't need docker at all, and also would work // if you run the build-server directly on your machine. + // Fixing it here also means the startup of the actual build-server including its + // metrics collection endpoints don't be delayed. Generally shoudl doesn't be + // a differene how much time is neede on a fresh build-server, between picking the + // up from the queue, and actually starting to build the release. In the old + // solution, the entrypoint would do the toolchain-update & add-essential files + // before even starting the build-server, now we're roughly doing the same thing + // inside the main builder loop. let rustc_version = self.runtime.block_on({ let pool = self.db.clone(); From b69feb4c8b73b37fed3ae4f0c4389a2de778e5e6 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 09:06:07 +0100 Subject: [PATCH 23/47] fix cleanup command --- .editorconfig | 4 ++++ Justfile | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/.editorconfig b/.editorconfig index da7d8221e..d9432f1f2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,5 +8,9 @@ insert_final_newline = true indent_style = space indent_size = 4 +[Justfile] +indent_size = 2 + [*.js] max_line_length = 100 + diff --git a/Justfile b/Justfile index eb03426cd..25f3843de 100644 --- a/Justfile +++ b/Justfile @@ -1,7 +1,11 @@ +set shell := ["bash", "-euxo", "pipefail", "-c"] + + # List available commands _default: just --list + sqlx-prepare ADDITIONAL_ARGS="": cargo sqlx prepare \ --database-url $DOCSRS_DATABASE_URL \ @@ -24,16 +28,30 @@ _touch-docker-env: # config [group('compose')] cleanup: - # FIXME onlt delete containers in our project - docker container prune --force - # FIXME onlt delete volumesin our project - docker volume prune -f -a - - # FIXME: the images sometimes can't be deleted? because in-use? - # But all containers are stopped? - docker compose down --volumes --remove-orphans --rmi local - rm -rf .rustwide-docker/ && mkdir -p .rustwide-docker - rm -rf ignored/ && mkdir -p ignored + #!/usr/bin/env bash + + PROJECT_NAME=$(docker compose config | yq -r '.name') + + # stop & remove all docker containters, if they belong to this compose project. + docker ps -a \ + --filter "label=com.docker.compose.project=$PROJECT_NAME" \ + --format json | jq -r '[.ID, .Names] | @tsv' | \ + while IFS=$'\t' read -r cid cname; do + + echo "stopping and removing container ${cname} ($cid)" + docker container stop "$cid" >/dev/null + docker container rm "$cid" >/dev/null + done + + # more cleanup with docker compose down + # 1. volumes: remove named and anonymous volumes declared in the `volumes` section of the compose file. + # 2. orphans: remove containers for services not defined in the compose file. + # 3. rmi local: remove images that don't have a tag (local build) + docker compose down --volumes --remove-orphans --rmi local + + # remove some of our own data directories, when they are mapped to the filesystem. + + rm -rf ignored/ && mkdir -p ignored _compose-cli service_name *args: _touch-docker-env docker compose up -d db s3 --wait From 0b19812a217ad1bfa65435e55f9edaa552fe92fc Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 09:43:50 +0100 Subject: [PATCH 24/47] last changes --- Justfile | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/Justfile b/Justfile index 25f3843de..3e26be52d 100644 --- a/Justfile +++ b/Justfile @@ -1,4 +1,4 @@ -set shell := ["bash", "-euxo", "pipefail", "-c"] +set shell := ["bash", "-euo", "pipefail", "-c"] # List available commands @@ -28,31 +28,16 @@ _touch-docker-env: # config [group('compose')] cleanup: - #!/usr/bin/env bash - - PROJECT_NAME=$(docker compose config | yq -r '.name') - - # stop & remove all docker containters, if they belong to this compose project. - docker ps -a \ - --filter "label=com.docker.compose.project=$PROJECT_NAME" \ - --format json | jq -r '[.ID, .Names] | @tsv' | \ - while IFS=$'\t' read -r cid cname; do - - echo "stopping and removing container ${cname} ($cid)" - docker container stop "$cid" >/dev/null - docker container rm "$cid" >/dev/null - done - - # more cleanup with docker compose down + # Docker cleanup with docker compose down # 1. volumes: remove named and anonymous volumes declared in the `volumes` section of the compose file. - # 2. orphans: remove containers for services not defined in the compose file. + # 2. orphans: remove containers for services not defined in the compose file any more. # 3. rmi local: remove images that don't have a tag (local build) - docker compose down --volumes --remove-orphans --rmi local + docker compose --profile full down --volumes --remove-orphans --rmi local # remove some of our own data directories, when they are mapped to the filesystem. - rm -rf ignored/ && mkdir -p ignored + _compose-cli service_name *args: _touch-docker-env docker compose up -d db s3 --wait docker compose run --build --rm {{ service_name }} {{ args }} @@ -74,7 +59,7 @@ compose-cli-queue-add *args: # run builder CLI command in its own one-off docker container. [group('compose')] -compose-builder-cli *args: _touch-docker-env +compose-builder-cli *args: _touch-docker-env compose-cli-migrate just _compose-cli builder-cli {{ args }} # set the nightly rust version to be used for builds. Format: `nightly-YYYY-MM-DD` @@ -99,7 +84,7 @@ compose-cli-build-crate *args: # run registry-watcher CLI command in its own one-off docker container. [group('compose')] -compose-registry-watcher-cli *args: _touch-docker-env +compose-registry-watcher-cli *args: _touch-docker-env compose-cli-migrate just _compose-cli registry-watcher-cli {{ args }} # Update last seen reference to the current index head, to only build newly published crates From c5c1a790ee0d93b8e089b513c6f8e303cf8a2d7e Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 09:45:29 +0100 Subject: [PATCH 25/47] migratiou --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 3e26be52d..01d0fd0dc 100644 --- a/Justfile +++ b/Justfile @@ -54,7 +54,7 @@ compose-cli-migrate: # add a release to the build queue [group('compose')] -compose-cli-queue-add *args: +compose-cli-queue-add *args compose-cli-migrate: just compose-cli queue add {{ args }} # run builder CLI command in its own one-off docker container. From 927fd562a2b95a5fea6ba0839128605caf664fda Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 09:53:17 +0100 Subject: [PATCH 26/47] wip --- Justfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 01d0fd0dc..d6fb167be 100644 --- a/Justfile +++ b/Justfile @@ -54,7 +54,7 @@ compose-cli-migrate: # add a release to the build queue [group('compose')] -compose-cli-queue-add *args compose-cli-migrate: +compose-cli-queue-add *args: compose-cli-migrate just compose-cli queue add {{ args }} # run builder CLI command in its own one-off docker container. @@ -127,6 +127,7 @@ compose-up-full: compose-down: docker compose --profile full down --remove-orphans +# stream logs from all services running in docker-compose. Optionally specify services to tail logs from. [group('compose')] compose-logs *services: docker compose --profile full logs -f {{ services }} From 8249190b0897ebb7713097229a89af4f3728c50c Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 10:06:43 +0100 Subject: [PATCH 27/47] cleanup --- Justfile | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/Justfile b/Justfile index d6fb167be..3bb855bfd 100644 --- a/Justfile +++ b/Justfile @@ -24,63 +24,53 @@ lint-js *args: _touch-docker-env: touch .docker.env -# clean up docker images, volumes & other local artifacts from this docker-compose -# config -[group('compose')] -cleanup: - # Docker cleanup with docker compose down - # 1. volumes: remove named and anonymous volumes declared in the `volumes` section of the compose file. - # 2. orphans: remove containers for services not defined in the compose file any more. - # 3. rmi local: remove images that don't have a tag (local build) - docker compose --profile full down --volumes --remove-orphans --rmi local - - # remove some of our own data directories, when they are mapped to the filesystem. - rm -rf ignored/ && mkdir -p ignored - - _compose-cli service_name *args: _touch-docker-env + # dependencies in the docker-compose file are ignored + # here. Instead we explicitly start any dependent services first. docker compose up -d db s3 --wait + # run the CLI command docker compose run --build --rm {{ service_name }} {{ args }} -# run any CLI command in its own one-off docker container. Args are passed to the container. +# run any CLI command in its own one-off `cli` docker container. Args are passed to the container. [group('compose')] -compose-cli *args: _touch-docker-env +compose-cli *args: _touch-docker-env compose-cli-migrate just _compose-cli cli {{ args }} # Initialize the docker compose database [group('compose')] compose-cli-migrate: - just compose-cli database migrate + # intentially not using `compose-cli`, otherweise we have a cycle. + just _compose-cli cli database migrate # add a release to the build queue [group('compose')] -compose-cli-queue-add *args: compose-cli-migrate +compose-cli-queue-add *args: just compose-cli queue add {{ args }} # run builder CLI command in its own one-off docker container. [group('compose')] -compose-builder-cli *args: _touch-docker-env compose-cli-migrate - just _compose-cli builder-cli {{ args }} +compose-cli-builder *args: _touch-docker-env compose-cli-migrate + just _compose-cli cli-builder {{ args }} # set the nightly rust version to be used for builds. Format: `nightly-YYYY-MM-DD` [group('compose')] compose-cli-set-toolchain NAME: - just compose-builder-cli build set-toolchain {{ NAME }} + just compose-cli-builder build set-toolchain {{ NAME }} # update the toolchain in the builders [group('compose')] compose-cli-update-toolchain: - just compose-builder-cli build update-toolchain + just compose-cli-builder build update-toolchain # build & the toolchain shared essential files [group('compose')] compose-cli-add-essential-files: - just compose-builder-cli build add-essential-files + just compose-cli-builder build add-essential-files # build & the toolchain shared essential files [group('compose')] compose-cli-build-crate *args: - just compose-builder-cli build crate {{ args }} + just compose-cli-builder build crate {{ args }} # run registry-watcher CLI command in its own one-off docker container. [group('compose')] @@ -117,16 +107,26 @@ compose-up-watcher: compose-up-metrics: just compose-up metrics -# Launch everything in the background +# Launch everything, all at once, in the background [group('compose')] compose-up-full: just compose-up full -# Shutdown docker services and cleanup all temporary volumes +# Shutdown docker services, keep containers & volumes alive. [group('compose')] compose-down: docker compose --profile full down --remove-orphans + +# Shutdown docker services, then +# clean up docker images, volumes & other local artifacts from +# this docker-compose project +[group('compose')] +compose-down-and-wipe: + docker compose --profile full down --volumes --remove-orphans --rmi local + rm -rf ignored/ && mkdir -p ignored + + # stream logs from all services running in docker-compose. Optionally specify services to tail logs from. [group('compose')] compose-logs *services: From 2455dbbb9217f5b6c6bb345caeb54bb04225fdbe Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 11:52:49 +0100 Subject: [PATCH 28/47] notes --- .github/workflows/ci.yml | 10 ++- .github/workflows/docker.yml | 4 ++ Justfile | 132 ++--------------------------------- NOTES.md | 68 ++++++++++++++++++ docker-compose.yml | 11 +-- justfiles/cli.just | 77 ++++++++++++++++++++ justfiles/development.just | 16 +++++ justfiles/services.just | 48 +++++++++++++ 8 files changed, 232 insertions(+), 134 deletions(-) create mode 100644 NOTES.md create mode 100644 justfiles/cli.just create mode 100644 justfiles/development.just create mode 100644 justfiles/services.just diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70d3a8e7c..0d8b6a208 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,11 +32,13 @@ jobs: uses: raven-actions/actionlint@v2 with: files: .github/workflow/* - flags: "-ignore SC2086" # ignore some shellcheck errors + flags: "-ignore SC2086" # ignore some shellcheck errors - name: install `just` run: sudo snap install --edge --classic just + - uses: cargo-bins/cargo-binstall@main + - name: restore build & cargo cache uses: Swatinem/rust-cache@v2 with: @@ -48,7 +50,7 @@ jobs: docker compose up --wait --wait-timeout 30 db - name: install SQLX CLI - run: cargo install sqlx-cli --no-default-features --features postgres + run: cargo binstall sqlx-cli --no-default-features --features postgres - name: run database migrations run: cargo sqlx migrate run --database-url $DOCSRS_DATABASE_URL @@ -70,6 +72,9 @@ jobs: steps: - uses: actions/checkout@v5 + - name: install `just` + run: sudo snap install --edge --classic just + - name: restore build & cargo cache uses: Swatinem/rust-cache@v2 with: @@ -118,6 +123,7 @@ jobs: steps: - uses: actions/checkout@v5 + - name: update rust toolchain run: rustup component add rustfmt diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ff5314d3e..65d298045 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,3 +39,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max push: false + + # TODO: later we would set `push: true` and also provide nice tags + # for the images. + # Unclear is how the deploy would work then. diff --git a/Justfile b/Justfile index 3bb855bfd..fe36bca66 100644 --- a/Justfile +++ b/Justfile @@ -1,133 +1,9 @@ -set shell := ["bash", "-euo", "pipefail", "-c"] - +set shell := ["bash", "-Eeuo", "pipefail", "-c"] # List available commands _default: just --list - -sqlx-prepare ADDITIONAL_ARGS="": - cargo sqlx prepare \ - --database-url $DOCSRS_DATABASE_URL \ - --workspace {{ ADDITIONAL_ARGS }} \ - -- --all-targets --all-features - -sqlx-check: - just sqlx-prepare "--check" - -lint: - cargo clippy --all-features --all-targets --workspace --locked -- -D warnings - -lint-js *args: - deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} - -_touch-docker-env: - touch .docker.env - -_compose-cli service_name *args: _touch-docker-env - # dependencies in the docker-compose file are ignored - # here. Instead we explicitly start any dependent services first. - docker compose up -d db s3 --wait - # run the CLI command - docker compose run --build --rm {{ service_name }} {{ args }} - -# run any CLI command in its own one-off `cli` docker container. Args are passed to the container. -[group('compose')] -compose-cli *args: _touch-docker-env compose-cli-migrate - just _compose-cli cli {{ args }} - -# Initialize the docker compose database -[group('compose')] -compose-cli-migrate: - # intentially not using `compose-cli`, otherweise we have a cycle. - just _compose-cli cli database migrate - -# add a release to the build queue -[group('compose')] -compose-cli-queue-add *args: - just compose-cli queue add {{ args }} - -# run builder CLI command in its own one-off docker container. -[group('compose')] -compose-cli-builder *args: _touch-docker-env compose-cli-migrate - just _compose-cli cli-builder {{ args }} - -# set the nightly rust version to be used for builds. Format: `nightly-YYYY-MM-DD` -[group('compose')] -compose-cli-set-toolchain NAME: - just compose-cli-builder build set-toolchain {{ NAME }} - -# update the toolchain in the builders -[group('compose')] -compose-cli-update-toolchain: - just compose-cli-builder build update-toolchain - -# build & the toolchain shared essential files -[group('compose')] -compose-cli-add-essential-files: - just compose-cli-builder build add-essential-files - -# build & the toolchain shared essential files -[group('compose')] -compose-cli-build-crate *args: - just compose-cli-builder build crate {{ args }} - -# run registry-watcher CLI command in its own one-off docker container. -[group('compose')] -compose-registry-watcher-cli *args: _touch-docker-env compose-cli-migrate - just _compose-cli registry-watcher-cli {{ args }} - -# Update last seen reference to the current index head, to only build newly published crates -[group('compose')] -compose-cli-queue-head: - just compose-registry-watcher-cli queue set-last-seen-reference --head - -# run migrations, then launch one or more docker compose profiles in the background -[group('compose')] -compose-up *profiles: _touch-docker-env compose-cli-migrate - docker compose {{ prepend("--profile ", profiles) }} up --build -d --wait --remove-orphans - -# Launch web server in the background -[group('compose')] -compose-up-web: - just compose-up web - -# Launch two build servers in the background -[group('compose')] -compose-up-builder: - just compose-up builder - -# Launch registry watcher in the background -[group('compose')] -compose-up-watcher: - just compose-up watcher - -# Launch prometheus server in the background -[group('compose')] -compose-up-metrics: - just compose-up metrics - -# Launch everything, all at once, in the background -[group('compose')] -compose-up-full: - just compose-up full - -# Shutdown docker services, keep containers & volumes alive. -[group('compose')] -compose-down: - docker compose --profile full down --remove-orphans - - -# Shutdown docker services, then -# clean up docker images, volumes & other local artifacts from -# this docker-compose project -[group('compose')] -compose-down-and-wipe: - docker compose --profile full down --volumes --remove-orphans --rmi local - rm -rf ignored/ && mkdir -p ignored - - -# stream logs from all services running in docker-compose. Optionally specify services to tail logs from. -[group('compose')] -compose-logs *services: - docker compose --profile full logs -f {{ services }} +import 'justfiles/cli.just' +import 'justfiles/services.just' +import 'justfiles/development.just' diff --git a/NOTES.md b/NOTES.md new file mode 100644 index 000000000..b18acc823 --- /dev/null +++ b/NOTES.md @@ -0,0 +1,68 @@ +# PR desc + +- the `Justfile` commands are built so they would even work on a 100% fresh + setup, and will set up and initialize everything they need. + + ## docker image + +for now, these are production images, changing files will not auto-reload / +build the image. We could decide to do that layer. + +## profiles + +- default: just db & s3 + +runs & configures by default: + +- `db` -> postgres db +- `s3` -> minio + +optional profile: `web`: + +- `web` -> webserver + +optional profile: `builder`: + +- `builder-a` -> build-server 1 +- `builder-b` -> build-server 2 ( two parallel build-servers, sharing nothing + apart from the build queue they access) + +optional profile: `watcher`: + +- `registry-watcher` -> + +* crates.io registry watcher +* repo-stats updater +* cdn invalidator +* release-rebuild-enqueuer + +optional profile: `metrics`: + +- `prometheus` -> configured prometheus instance + +optional profile: `full`: all of the above. + +Services purely for manual usage with `docker compose run` are: + +- `cli`: to run simple CLI commands that only need the database & S3 +- `builder-cli`: to run CLI commands that need the build environment. +- `registry-watcher-cli`: to run CLI commands that need the crates.io index. + +CAVEATS: + +- the build-servers have to run on the `linux/amd64` platform, while it doesn't + matter for the rest of the services. This means for example on a Mac, the + layers will be cached separately, once for `linux/amd64` and once for + `linux/arm64`. Only alternative would be to build everything for `amd64`, but + that would imply a performance impact on the services that don't need it. +- volumes: typically docker-native volumes are faster than mounts, but sometimes + annoying to inspect for debugging. + +For now we choose: + +- docker-native for DB, S3, rustwide workspace, crates.io index +- mounts for prefix, code mounts +- prometheus scrape config is set to collect from the web server, the registry + watcher, and the build servers. Scraping is not dynamic, so the local + prometheus server will try to fetch from all service instances (web, watcher, + builder), and just error in case the specific server isn't accessible. diff --git a/docker-compose.yml b/docker-compose.yml index 17f772dd6..8069e29c9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,10 +23,6 @@ # optional profile: `metrics`: # * `prometheus` -> configured prometheus instance # -# prometheus scrape config is set to collect from the web server, the registry watcher, and -# the build servers. We assume there are no metrics from the CLIs, as we do in -# prod. -# # optional profile: `full`: all of the above. # # Services purely for manual usage with `docker compose run` are: @@ -45,9 +41,16 @@ # need it. # * volumes: typically docker-native volumes are faster than mounts, but # sometimes annoying to inspect for debugging. +# # For now we choose: +# # * docker-native for DB, S3, rustwide workspace, crates.io index # * mounts for prefix, code mounts +# * prometheus scrape config is set to collect from the web server, the +# registry watcher, and the build servers. Scraping is not dynamic, so +# the local prometheus server will try to fetch from all service +# instances (web, watcher, builder), and just error in case the specific +# server isn't accessible. x-healthcheck: &healthcheck-interval interval: 10s diff --git a/justfiles/cli.just b/justfiles/cli.just new file mode 100644 index 000000000..f856b1599 --- /dev/null +++ b/justfiles/cli.just @@ -0,0 +1,77 @@ +# a collection of just commands to wrap various docs.rs CLI commands, +# and run then in a once-off docker container. +# _Which_ container depends on the command itself and its dependencies. + +# low-level helper to run any CLI command in its own one-off docker container, +# ensuring that `db` and `s3` are running. +_cli service_name +args: _touch-docker-env + # dependencies in the docker-cli file are ignored + # here. Instead we explicitly start any dependent services first. + docker compose up -d db s3 --wait + # run the CLI command + docker compose run --build --rm {{ service_name }} {{ args }} + +# run any CLI command in its own one-off `cli` docker container. Args are passed to the container. +# Only for commands that just need `db` and `s3` and minimal dependencies. +[group('cli')] +cli +args: _touch-docker-env cli-db-migrate + just _cli cli {{ args }} + +# Initialize the `docs.rs` database +[group('cli')] +cli-db-migrate: + # intentially not using `cli`, because it has a dependency on `cli-db-migrate`. + # Otherwise we would have a stack overflow / infinite recursion. + # + # TODO: potential optimization: only run the container when we have to + # run migrations? + just _cli cli database migrate + +# add a release to the build queue +[group('cli')] +cli-queue-add +args: + # only does things with the database, so can use the lightweight `cli` container. + just cli queue add {{ args }} + +# run builder CLI command in its own one-off `build-server` docker container. +# Uses a separate builder-cli container & workspace. +[group('cli')] +cli-build +args: _touch-docker-env cli-db-migrate + just _cli cli-builder {{ args }} + +# set the nightly rust version to be used for builds. Format: `nightly-YYYY-MM-DD` +[group('cli')] +cli-build-set-toolchain name only_first_time="false": + FLAG="" + if [ "{{only_first_time}}" = "true" ]; then FLAG="--only-first-time"; fi + just cli-builder build set-toolchain {{ name }} $FLAG + +# update the toolchain in the builders +[group('cli')] +cli-build-update-toolchain: + just cli-builder build update-toolchain + +# build & upload toolchain shared static resources +[group('cli')] +cli-build-add-essential-files: + just cli-builder build add-essential-files + +# build a release +[group('cli')] +cli-build-crate name version build_priority="5": + just cli-builder build crate {{ name }} {{ version }} {{ build_priority }} + +# run registry-watcher CLI command in its own one-off `registry-watcher` docker container. +[group('cli')] +cli-watcher +args: _touch-docker-env cli-db-migrate + just _cli registry-watcher-cli {{ args }} + +# Update last seen reference to the given hash, or the current `HEAD`. +[group('cli')] +cli-queue-reset-last-seen-ref REF="--head": + just cli-watcher queue set-last-seen-reference {{ REF }} + +# synchronize the crates.io index with our own database. +[group('cli')] +cli-db-synchronize +args: + just cli-watcher database synchronize {{ args }} diff --git a/justfiles/development.just b/justfiles/development.just new file mode 100644 index 000000000..2138f756f --- /dev/null +++ b/justfiles/development.just @@ -0,0 +1,16 @@ +# just commands for CI & local development + +sqlx-prepare *args: + cargo sqlx prepare \ + --database-url $DOCSRS_DATABASE_URL \ + --workspace {{ args }} \ + -- --all-targets --all-features + +sqlx-check: + just sqlx-prepare -check + +lint: + cargo clippy --all-features --all-targets --workspace --locked -- -D warnings + +lint-js *args: + deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} diff --git a/justfiles/services.just b/justfiles/services.just new file mode 100644 index 000000000..28f99e8b3 --- /dev/null +++ b/justfiles/services.just @@ -0,0 +1,48 @@ +_touch-docker-env: + touch .docker.env + +# run migrations, then launch one or more docker compose profiles in the background +[group('compose')] +compose-up *profiles: _touch-docker-env cli-db-migrate + docker compose {{ prepend("--profile ", profiles) }} up --build -d --wait --remove-orphans + +# Launch web server in the background +[group('compose')] +compose-up-web: + just compose-up web + +# Launch two build servers in the background +[group('compose')] +compose-up-builder: + just compose-up builder + +# Launch registry watcher in the background +[group('compose')] +compose-up-watcher: + just compose-up watcher + +# Launch prometheus server in the background +[group('compose')] +compose-up-metrics: + just compose-up metrics + +# Launch everything, all at once, in the background +[group('compose')] +compose-up-full: + just compose-up full + +# Shutdown docker services, keep containers & volumes alive. +[group('compose')] +compose-down: + docker compose --profile full down --remove-orphans + +# Shutdown docker services, then clean up docker images, volumes & other local artifacts from this docker-compose project +[group('compose')] +compose-down-and-wipe: + docker compose --profile full --profile manual down --volumes --remove-orphans --rmi local + rm -rf ignored/ && mkdir -p ignored + +# stream logs from all services running in docker-compose. Optionally specify services to tail logs from. +[group('compose')] +compose-logs *services: + docker compose --profile full logs -f {{ services }} From d7e37f15132f0ef62480baa76efecda330e0eb05 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 11:54:20 +0100 Subject: [PATCH 29/47] start migration --- Justfile | 2 +- dockerfiles/run-gui-tests.sh | 49 -------------------------- justfiles/development.just | 16 --------- justfiles/testing.just | 67 ++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 66 deletions(-) delete mode 100755 dockerfiles/run-gui-tests.sh delete mode 100644 justfiles/development.just create mode 100644 justfiles/testing.just diff --git a/Justfile b/Justfile index fe36bca66..0a315f067 100644 --- a/Justfile +++ b/Justfile @@ -6,4 +6,4 @@ _default: import 'justfiles/cli.just' import 'justfiles/services.just' -import 'justfiles/development.just' +import 'justfiles/testing.just' diff --git a/dockerfiles/run-gui-tests.sh b/dockerfiles/run-gui-tests.sh deleted file mode 100755 index acf1ea592..000000000 --- a/dockerfiles/run-gui-tests.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# FIXME: @syphar, it might be easy to update the script with the new -# docker compose setup to make it run on non-linux machines. -# Perhasp we could even replace the script just with some "Justfile" -# tasks. - -# Just in case it's running, we stop the web server. -docker compose stop web - -docker compose up --wait --wait-timeout 30 db s3 - -# If we have a .env file, we need to temporarily move it so -# it doesn't make sqlx fail compilation. -if [ -f .env ]; then - mv .env .tmp.env -fi - -# We add the information we need. -cargo run -- database migrate -cargo run -- build update-toolchain -cargo run -- build crate sysinfo 0.23.4 -cargo run -- build crate sysinfo 0.23.5 -cargo run -- build crate libtest 0.0.1 -cargo run -- build add-essential-files - -if [ -f .tmp.env ]; then - mv .tmp.env .env -fi - -# In case we don't have a `.env`, we create one. -if [ ! -f .env ]; then - cp .env.sample .env -fi - -. .env - -set +e # We disable the "exit right away if command failed" setting. -cargo run -- start-web-server & -SERVER_PID=$! - -# status="docker run . -v `pwd`:/build/out:ro gui_tests" -docker compose build gui_tests -docker compose run --rm --remove-orphans gui_tests -status=$? -kill $SERVER_PID -exit $status diff --git a/justfiles/development.just b/justfiles/development.just deleted file mode 100644 index 2138f756f..000000000 --- a/justfiles/development.just +++ /dev/null @@ -1,16 +0,0 @@ -# just commands for CI & local development - -sqlx-prepare *args: - cargo sqlx prepare \ - --database-url $DOCSRS_DATABASE_URL \ - --workspace {{ args }} \ - -- --all-targets --all-features - -sqlx-check: - just sqlx-prepare -check - -lint: - cargo clippy --all-features --all-targets --workspace --locked -- -D warnings - -lint-js *args: - deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} diff --git a/justfiles/testing.just b/justfiles/testing.just new file mode 100644 index 000000000..e3701737f --- /dev/null +++ b/justfiles/testing.just @@ -0,0 +1,67 @@ +# just commands for CI & local development + +sqlx-prepare *args: + cargo sqlx prepare \ + --database-url $DOCSRS_DATABASE_URL \ + --workspace {{ args }} \ + -- --all-targets --all-features + +sqlx-check: + just sqlx-prepare -check + +lint: + cargo clippy --all-features --all-targets --workspace --locked -- -D warnings + +lint-js *args: + deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} + +gui-tests: + #!/usr/bin/env bash + + set -e + + # FIXME: @syphar, it might be easy to update the script with the new + # docker compose setup to make it run on non-linux machines. + # Perhasp we could even replace the script just with some "Justfile" + # tasks. + + # Just in case it's running, we stop the web server. + docker compose stop web + + docker compose up --wait --wait-timeout 30 db s3 + + # If we have a .env file, we need to temporarily move it so + # it doesn't make sqlx fail compilation. + if [ -f .env ]; then + mv .env .tmp.env + fi + + # We add the information we need. + cargo run -- database migrate + cargo run -- build update-toolchain + cargo run -- build crate sysinfo 0.23.4 + cargo run -- build crate sysinfo 0.23.5 + cargo run -- build crate libtest 0.0.1 + cargo run -- build add-essential-files + + if [ -f .tmp.env ]; then + mv .tmp.env .env + fi + + # In case we don't have a `.env`, we create one. + if [ ! -f .env ]; then + cp .env.sample .env + fi + + . .env + + set +e # We disable the "exit right away if command failed" setting. + cargo run -- start-web-server & + SERVER_PID=$! + + # status="docker run . -v `pwd`:/build/out:ro gui_tests" + docker compose build gui_tests + docker compose run --rm --remove-orphans gui_tests + status=$? + kill $SERVER_PID + exit $status From adf06b9f5dd8bc2f8476893a562f0d615a8e03c4 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 12:51:19 +0100 Subject: [PATCH 30/47] gui tests pass --- docker-compose.yml | 15 +++++++++ dockerfiles/Dockerfile-gui-tests | 17 +++++----- justfiles/cli.just | 18 +++++------ justfiles/services.just | 5 +++ justfiles/testing.just | 54 +++++--------------------------- 5 files changed, 44 insertions(+), 65 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8069e29c9..d5c47928e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,16 @@ # instances (web, watcher, builder), and just error in case the specific # server isn't accessible. +x-docker-cache: &docker-cache + # shared configuration to cache docker layers across CI runs. + # can just always be in in the `docker-compose.yml`, and will + # just do nothing if we're not on GHA. + # So feel free to add more techniques if needed. + cache_from: + - type=gha + cache_to: + - type=gha,mode=max + x-healthcheck: &healthcheck-interval interval: 10s timeout: 1s @@ -78,6 +88,7 @@ x-environment: &environment x-build: &build context: . + <<: *docker-cache dockerfile: ./dockerfiles/Dockerfile args: PROFILE: dev @@ -218,6 +229,7 @@ services: build: context: ./dockerfiles dockerfile: ./Dockerfile-postgres + <<: *docker-cache volumes: - postgres-data:/var/lib/postgresql/data environment: @@ -254,6 +266,7 @@ services: build: context: ./dockerfiles dockerfile: ./Dockerfile-prometheus + <<: *docker-cache ports: - "127.0.0.1:9090:9090" # we intentionally don't define depends_on here. @@ -272,9 +285,11 @@ services: - full gui_tests: + platform: "linux/amd64" build: context: . dockerfile: ./dockerfiles/Dockerfile-gui-tests + <<: *docker-cache network_mode: "host" extra_hosts: - "host.docker.internal:host-gateway" diff --git a/dockerfiles/Dockerfile-gui-tests b/dockerfiles/Dockerfile-gui-tests index e584603bb..95826ef77 100644 --- a/dockerfiles/Dockerfile-gui-tests +++ b/dockerfiles/Dockerfile-gui-tests @@ -1,8 +1,9 @@ -FROM ubuntu:22.04 AS build +FROM node:22-trixie-slim ENV DEBIAN_FRONTEND=noninteractive # Install packaged dependencies +# hadolint ignore=DL3008 RUN apt-get update && \ apt-get install -y --no-install-recommends \ build-essential \ @@ -17,6 +18,7 @@ RUN apt-get update && \ zlib1g-dev \ ca-certificates +# hadolint ignore=DL3008 RUN apt-get install -y \ ca-certificates \ curl \ @@ -28,8 +30,9 @@ RUN apt-get install -y \ xz-utils # Install dependencies for chromium browser +# hadolint ignore=DL3008 RUN apt-get install -y \ - gconf-service \ + # gconf-service \ libasound2 \ libatk1.0-0 \ libatk-bridge2.0-0 \ @@ -41,8 +44,8 @@ RUN apt-get install -y \ libfontconfig1 \ libgbm-dev \ libgcc1 \ - libgconf-2-4 \ - libgdk-pixbuf2.0-0 \ + # libgconf-2-4 \ + # libgdk-pixbuf2.0-0 \ libglib2.0-0 \ libgtk-3-0 \ libnspr4 \ @@ -63,16 +66,12 @@ RUN apt-get install -y \ libxss1 \ libxtst6 \ fonts-liberation \ - libappindicator1 \ + # libappindicator1 \ libnss3 \ lsb-release \ xdg-utils \ wget -RUN curl -sL https://nodejs.org/dist/v22.13.1/node-v22.13.1-linux-x64.tar.xz | tar -xJ -ENV PATH="/node-v22.13.1-linux-x64/bin:${PATH}" -ENV NODE_PATH="/node-v22.13.1-linux-x64/lib/node_modules/" - WORKDIR /build RUN mkdir out diff --git a/justfiles/cli.just b/justfiles/cli.just index f856b1599..4220574e7 100644 --- a/justfiles/cli.just +++ b/justfiles/cli.just @@ -4,7 +4,7 @@ # low-level helper to run any CLI command in its own one-off docker container, # ensuring that `db` and `s3` are running. -_cli service_name +args: _touch-docker-env +_cli service_name *args: _touch-docker-env # dependencies in the docker-cli file are ignored # here. Instead we explicitly start any dependent services first. docker compose up -d db s3 --wait @@ -37,29 +37,29 @@ cli-queue-add +args: # Uses a separate builder-cli container & workspace. [group('cli')] cli-build +args: _touch-docker-env cli-db-migrate - just _cli cli-builder {{ args }} + just _cli builder-cli {{ args }} # set the nightly rust version to be used for builds. Format: `nightly-YYYY-MM-DD` [group('cli')] cli-build-set-toolchain name only_first_time="false": FLAG="" if [ "{{only_first_time}}" = "true" ]; then FLAG="--only-first-time"; fi - just cli-builder build set-toolchain {{ name }} $FLAG + just cli-build build set-toolchain {{ name }} $FLAG # update the toolchain in the builders [group('cli')] cli-build-update-toolchain: - just cli-builder build update-toolchain + just cli-build build update-toolchain # build & upload toolchain shared static resources [group('cli')] cli-build-add-essential-files: - just cli-builder build add-essential-files + just cli-build build add-essential-files # build a release [group('cli')] -cli-build-crate name version build_priority="5": - just cli-builder build crate {{ name }} {{ version }} {{ build_priority }} +cli-build-crate name version: + just cli-build build crate {{ name }} {{ version }} # run registry-watcher CLI command in its own one-off `registry-watcher` docker container. [group('cli')] @@ -68,8 +68,8 @@ cli-watcher +args: _touch-docker-env cli-db-migrate # Update last seen reference to the given hash, or the current `HEAD`. [group('cli')] -cli-queue-reset-last-seen-ref REF="--head": - just cli-watcher queue set-last-seen-reference {{ REF }} +cli-queue-reset-last-seen-ref ref="--head": + just cli-watcher queue set-last-seen-reference {{ ref }} # synchronize the crates.io index with our own database. [group('cli')] diff --git a/justfiles/services.just b/justfiles/services.just index 28f99e8b3..0f3535f36 100644 --- a/justfiles/services.just +++ b/justfiles/services.just @@ -1,6 +1,11 @@ _touch-docker-env: touch .docker.env +_ensure_db_and_s3_are_running: + # dependencies in the docker-cli file are ignored + # here. Instead we explicitly start any dependent services first. + docker compose up -d db s3 --wait + # run migrations, then launch one or more docker compose profiles in the background [group('compose')] compose-up *profiles: _touch-docker-env cli-db-migrate diff --git a/justfiles/testing.just b/justfiles/testing.just index e3701737f..4298e07a5 100644 --- a/justfiles/testing.just +++ b/justfiles/testing.just @@ -15,53 +15,13 @@ lint: lint-js *args: deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} -gui-tests: - #!/usr/bin/env bash - - set -e - - # FIXME: @syphar, it might be easy to update the script with the new - # docker compose setup to make it run on non-linux machines. - # Perhasp we could even replace the script just with some "Justfile" - # tasks. - - # Just in case it's running, we stop the web server. - docker compose stop web - - docker compose up --wait --wait-timeout 30 db s3 - - # If we have a .env file, we need to temporarily move it so - # it doesn't make sqlx fail compilation. - if [ -f .env ]; then - mv .env .tmp.env - fi - +run-gui-tests: _ensure_db_and_s3_are_running cli-db-migrate compose-up-web # We add the information we need. - cargo run -- database migrate - cargo run -- build update-toolchain - cargo run -- build crate sysinfo 0.23.4 - cargo run -- build crate sysinfo 0.23.5 - cargo run -- build crate libtest 0.0.1 - cargo run -- build add-essential-files - - if [ -f .tmp.env ]; then - mv .tmp.env .env - fi - - # In case we don't have a `.env`, we create one. - if [ ! -f .env ]; then - cp .env.sample .env - fi - - . .env + # just cli-build-update-toolchain + # just cli-build-crate sysinfo 0.23.4 + # just cli-build-crate sysinfo 0.23.5 + # just cli-build-crate libtest 0.0.1 + # just cli-build-add-essential-files - set +e # We disable the "exit right away if command failed" setting. - cargo run -- start-web-server & - SERVER_PID=$! + just _cli gui_tests - # status="docker run . -v `pwd`:/build/out:ro gui_tests" - docker compose build gui_tests - docker compose run --rm --remove-orphans gui_tests - status=$? - kill $SERVER_PID - exit $status From 83111bd602e9cef20c690731761bf1e0143d6f76 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 13:04:54 +0100 Subject: [PATCH 31/47] WIP --- .github/workflows/ci.yml | 28 +++++++++++++++++----------- Justfile | 1 + justfiles/cli.just | 32 ++++++++++++++++++++++++-------- justfiles/services.just | 8 -------- justfiles/testing.just | 14 ++++++-------- justfiles/utils.just | 9 +++++++++ 6 files changed, 57 insertions(+), 35 deletions(-) create mode 100644 justfiles/utils.just diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d8b6a208..8f4fcbc8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,9 +45,7 @@ jobs: prefix-key: ${{ env.RUST_CACHE_KEY }} - name: Launch postgres - run: | - touch .docker.env - docker compose up --wait --wait-timeout 30 db + run: just ensure_db_and_s3_are_running - name: install SQLX CLI run: cargo binstall sqlx-cli --no-default-features --features postgres @@ -65,6 +63,10 @@ jobs: --database-url $DOCSRS_DATABASE_URL \ --target-version 0 + - name: shut down test environment + if: ${{ always() }} + run: just compose-down-and-wipe + test: env: SQLX_OFFLINE: 1 @@ -84,9 +86,7 @@ jobs: run: cargo build --workspace --locked - name: Launch postgres and min.io - run: | - touch .docker.env - docker compose up --wait --wait-timeout 30 db s3 + run: just ensure_db_and_s3_are_running - name: run workspace tests run: | @@ -98,6 +98,10 @@ jobs: run: | cargo test --locked -- --ignored --test-threads=1 + - name: shut down test environment + if: ${{ always() }} + run: just compose-down-and-wipe + GUI_test: runs-on: ubuntu-latest steps: @@ -108,13 +112,15 @@ jobs: with: prefix-key: ${{ env.RUST_CACHE_KEY }} - - name: Launch postgres and min.io - run: | - touch .docker.env - docker compose up --wait --wait-timeout 30 db s3 + - name: install `just` + run: sudo snap install --edge --classic just - name: Run GUI tests - run: ./dockerfiles/run-gui-tests.sh + run: just run-gui-tests + + - name: shut down test environment + if: ${{ always() }} + run: just compose-down-and-wipe fmt: diff --git a/Justfile b/Justfile index 0a315f067..d6c9a3e14 100644 --- a/Justfile +++ b/Justfile @@ -5,5 +5,6 @@ _default: just --list import 'justfiles/cli.just' +import 'justfiles/utils.just' import 'justfiles/services.just' import 'justfiles/testing.just' diff --git a/justfiles/cli.just b/justfiles/cli.just index 4220574e7..9cd3ecdd1 100644 --- a/justfiles/cli.just +++ b/justfiles/cli.just @@ -1,24 +1,30 @@ # a collection of just commands to wrap various docs.rs CLI commands, -# and run then in a once-off docker container. +# and run them in a one-off docker container. # _Which_ container depends on the command itself and its dependencies. +# Most service containers have their corresponding CLI container: +# * web -> cli +# * builder-x -> builder-cli +# * registry-watcher -> registry-watcher-cli # low-level helper to run any CLI command in its own one-off docker container, # ensuring that `db` and `s3` are running. -_cli service_name *args: _touch-docker-env - # dependencies in the docker-cli file are ignored - # here. Instead we explicitly start any dependent services first. - docker compose up -d db s3 --wait - # run the CLI command +_cli service_name *args: _touch-docker-env ensure_db_and_s3_are_running + # dependencies in the docker-compose file are ignored + # when running a one-off service with `docker compose run`. + # Instead we explicitly start any dependent services first via + # `ensure_db_and_s3_are_running`. + docker compose run --build --rm {{ service_name }} {{ args }} # run any CLI command in its own one-off `cli` docker container. Args are passed to the container. -# Only for commands that just need `db` and `s3` and minimal dependencies. +# Only for commands that just need `db` and `s3` and minimal system dependencies. [group('cli')] cli +args: _touch-docker-env cli-db-migrate just _cli cli {{ args }} # Initialize the `docs.rs` database [group('cli')] +[group('database')] cli-db-migrate: # intentially not using `cli`, because it has a dependency on `cli-db-migrate`. # Otherwise we would have a stack overflow / infinite recursion. @@ -29,18 +35,22 @@ cli-db-migrate: # add a release to the build queue [group('cli')] +[group('queue')] cli-queue-add +args: # only does things with the database, so can use the lightweight `cli` container. just cli queue add {{ args }} # run builder CLI command in its own one-off `build-server` docker container. -# Uses a separate builder-cli container & workspace. +# Uses a separate builder-cli container & workspace that doesn't conflict +# with the continiously running build-servers. [group('cli')] +[group('build')] cli-build +args: _touch-docker-env cli-db-migrate just _cli builder-cli {{ args }} # set the nightly rust version to be used for builds. Format: `nightly-YYYY-MM-DD` [group('cli')] +[group('build')] cli-build-set-toolchain name only_first_time="false": FLAG="" if [ "{{only_first_time}}" = "true" ]; then FLAG="--only-first-time"; fi @@ -48,30 +58,36 @@ cli-build-set-toolchain name only_first_time="false": # update the toolchain in the builders [group('cli')] +[group('build')] cli-build-update-toolchain: just cli-build build update-toolchain # build & upload toolchain shared static resources [group('cli')] +[group('build')] cli-build-add-essential-files: just cli-build build add-essential-files # build a release [group('cli')] +[group('build')] cli-build-crate name version: just cli-build build crate {{ name }} {{ version }} # run registry-watcher CLI command in its own one-off `registry-watcher` docker container. [group('cli')] +[group('registry-watcher')] cli-watcher +args: _touch-docker-env cli-db-migrate just _cli registry-watcher-cli {{ args }} # Update last seen reference to the given hash, or the current `HEAD`. [group('cli')] +[group('queue')] cli-queue-reset-last-seen-ref ref="--head": just cli-watcher queue set-last-seen-reference {{ ref }} # synchronize the crates.io index with our own database. [group('cli')] +[group('database')] cli-db-synchronize +args: just cli-watcher database synchronize {{ args }} diff --git a/justfiles/services.just b/justfiles/services.just index 0f3535f36..dcd6593b7 100644 --- a/justfiles/services.just +++ b/justfiles/services.just @@ -1,11 +1,3 @@ -_touch-docker-env: - touch .docker.env - -_ensure_db_and_s3_are_running: - # dependencies in the docker-cli file are ignored - # here. Instead we explicitly start any dependent services first. - docker compose up -d db s3 --wait - # run migrations, then launch one or more docker compose profiles in the background [group('compose')] compose-up *profiles: _touch-docker-env cli-db-migrate diff --git a/justfiles/testing.just b/justfiles/testing.just index 4298e07a5..f01a4c42b 100644 --- a/justfiles/testing.just +++ b/justfiles/testing.just @@ -15,13 +15,11 @@ lint: lint-js *args: deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} -run-gui-tests: _ensure_db_and_s3_are_running cli-db-migrate compose-up-web - # We add the information we need. - # just cli-build-update-toolchain - # just cli-build-crate sysinfo 0.23.4 - # just cli-build-crate sysinfo 0.23.5 - # just cli-build-crate libtest 0.0.1 - # just cli-build-add-essential-files +run-gui-tests: ensure_db_and_s3_are_running cli-db-migrate compose-up-web + just cli-build-update-toolchain + just cli-build-crate sysinfo 0.23.4 + just cli-build-crate sysinfo 0.23.5 + just cli-build-crate libtest 0.0.1 + just cli-build-add-essential-files just _cli gui_tests - diff --git a/justfiles/utils.just b/justfiles/utils.just new file mode 100644 index 000000000..3b5fed443 --- /dev/null +++ b/justfiles/utils.just @@ -0,0 +1,9 @@ + +ensure_db_and_s3_are_running: _touch-docker-env + # dependencies in the docker-cli file are ignored + # here. Instead we explicitly start any dependent services first. + docker compose up -d db s3 --wait + +_touch-docker-env: + touch .docker.env + From 5bb7d655f9dea8f97aedb34c1e28cb7ffee24ad5 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 13:11:07 +0100 Subject: [PATCH 32/47] comments --- justfiles/cli.just | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/justfiles/cli.just b/justfiles/cli.just index 9cd3ecdd1..9ffe8c125 100644 --- a/justfiles/cli.just +++ b/justfiles/cli.just @@ -26,7 +26,7 @@ cli +args: _touch-docker-env cli-db-migrate [group('cli')] [group('database')] cli-db-migrate: - # intentially not using `cli`, because it has a dependency on `cli-db-migrate`. + # intentially not using `cli` recipe, because it has a dependency on `cli-db-migrate`. # Otherwise we would have a stack overflow / infinite recursion. # # TODO: potential optimization: only run the container when we have to @@ -48,7 +48,8 @@ cli-queue-add +args: cli-build +args: _touch-docker-env cli-db-migrate just _cli builder-cli {{ args }} -# set the nightly rust version to be used for builds. Format: `nightly-YYYY-MM-DD` +# set the nightly rust version to be used for builds. Format: `nightly-YYYY-MM-DD` +# or just `nightly` for always using the latest nightly. [group('cli')] [group('build')] cli-build-set-toolchain name only_first_time="false": @@ -86,7 +87,7 @@ cli-watcher +args: _touch-docker-env cli-db-migrate cli-queue-reset-last-seen-ref ref="--head": just cli-watcher queue set-last-seen-reference {{ ref }} -# synchronize the crates.io index with our own database. +# find differences between crates.io and our own database, and fix them on our side. [group('cli')] [group('database')] cli-db-synchronize +args: From 527883e521a00f06d2178fe1c61390e3be16425f Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 13:17:17 +0100 Subject: [PATCH 33/47] try fix cargo binstall --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f4fcbc8b..ee0b1d06a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,7 @@ jobs: run: just ensure_db_and_s3_are_running - name: install SQLX CLI - run: cargo binstall sqlx-cli --no-default-features --features postgres + run: cargo binstall sqlx-cli - name: run database migrations run: cargo sqlx migrate run --database-url $DOCSRS_DATABASE_URL From 4b1a2ed8c6be57358c556cffb30c82433d919696 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 13:17:27 +0100 Subject: [PATCH 34/47] fix debian package install --- dockerfiles/Dockerfile-gui-tests | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/dockerfiles/Dockerfile-gui-tests b/dockerfiles/Dockerfile-gui-tests index 95826ef77..7ce888b4a 100644 --- a/dockerfiles/Dockerfile-gui-tests +++ b/dockerfiles/Dockerfile-gui-tests @@ -16,23 +16,13 @@ RUN apt-get update && \ libmagic-dev \ libssl-dev \ zlib1g-dev \ - ca-certificates - -# hadolint ignore=DL3008 -RUN apt-get install -y \ - ca-certificates \ - curl \ - docker.io \ - gcc \ - git \ - libssl-dev \ - pkg-config \ - xz-utils + ca-certificates \ + docker.io \ + xz-utils # Install dependencies for chromium browser # hadolint ignore=DL3008 -RUN apt-get install -y \ - # gconf-service \ +RUN apt-get install -y --no-install-recommends \ libasound2 \ libatk1.0-0 \ libatk-bridge2.0-0 \ @@ -44,8 +34,6 @@ RUN apt-get install -y \ libfontconfig1 \ libgbm-dev \ libgcc1 \ - # libgconf-2-4 \ - # libgdk-pixbuf2.0-0 \ libglib2.0-0 \ libgtk-3-0 \ libnspr4 \ @@ -66,12 +54,14 @@ RUN apt-get install -y \ libxss1 \ libxtst6 \ fonts-liberation \ - # libappindicator1 \ libnss3 \ lsb-release \ xdg-utils \ wget +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /build RUN mkdir out From c1b28932dcd34e9ef561ac596b001f262f152b71 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 13:58:09 +0100 Subject: [PATCH 35/47] more linting, tesst --- .github/workflows/ci.yml | 16 ---------------- dockerfiles/prometheus.yml | 1 - justfiles/services.just | 16 +++++++++++++++- justfiles/testing.just | 31 +++++++++++++++++++++++++++++-- 4 files changed, 44 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee0b1d06a..6f69bd404 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,19 +122,6 @@ jobs: if: ${{ always() }} run: just compose-down-and-wipe - - fmt: - name: Rustfmt - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v5 - - - name: update rust toolchain - run: rustup component add rustfmt - - - run: cargo fmt -- --check - clippy: name: Clippy runs-on: ubuntu-latest @@ -142,9 +129,6 @@ jobs: steps: - uses: actions/checkout@v5 - - name: update rust toolchain - run: rustup component add clippy - - name: install `just` run: sudo snap install --edge --classic just diff --git a/dockerfiles/prometheus.yml b/dockerfiles/prometheus.yml index cabac00fb..082320e44 100644 --- a/dockerfiles/prometheus.yml +++ b/dockerfiles/prometheus.yml @@ -11,5 +11,4 @@ scrape_configs: - job_name: "docs.rs" metrics_path: "/about/metrics" static_configs: - # registry watcher doesn't have metrics yet. - targets: ["web:3000", "registry-watcher:3000", "builder-a:3000", "builder-b:3000"] diff --git a/justfiles/services.just b/justfiles/services.just index dcd6593b7..576e86b80 100644 --- a/justfiles/services.just +++ b/justfiles/services.just @@ -37,7 +37,21 @@ compose-down: [group('compose')] compose-down-and-wipe: docker compose --profile full --profile manual down --volumes --remove-orphans --rmi local - rm -rf ignored/ && mkdir -p ignored + + # When testing this in CI, I had permission issues when trying to remove this folder. + # Likely it's related to the docker container runnning as a different (root?) user, so + # these files in the `ignored/` folder belong to `root`. + + # so we just try if we can use passwordless `sudo`: + if sudo -n true 2>/dev/null; then + # if yes, we can use `sudo` to delete the folder. + sudo -n rm -rf ignored/ + else + # otherwise we try deleting as the current user + rm -rf ignored/ || { echo 'Failed to remove ignored/ - skipping' >&2; } + fi + + mkdir -p ignored # stream logs from all services running in docker-compose. Optionally specify services to tail logs from. [group('compose')] diff --git a/justfiles/testing.just b/justfiles/testing.just index f01a4c42b..ef14dd454 100644 --- a/justfiles/testing.just +++ b/justfiles/testing.just @@ -9,8 +9,35 @@ sqlx-prepare *args: sqlx-check: just sqlx-prepare -check -lint: - cargo clippy --all-features --all-targets --workspace --locked -- -D warnings +# Format the code using `cargo fmt`. +format: + rustup component add rustfmt + # like this we get both the non-zero exit code, and the local code is + # formatted. + cargo fmt --all -- --check || { cargo fmt --all && exit 1; } + +clippy *args: + rustup component add clippy + cargo clippy \ + --all-features \ + --all-targets \ + --workspace \ + --locked \ + {{ args}} \ + -- -D warnings + +clippy-fix: + just clippy --fix --allow-dirty --allow-staged + +# run all linters, for local development & CI +lint: format + #!/usr/bin/env bash + if [ "$GITHUB_ACTIONS" = "true" ]; then + just clippy + else + just clippy_fix + fi + lint-js *args: deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} From 46b4ad9c7153f15f2c4db9ce2c7edf9713810b8c Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 14:03:45 +0100 Subject: [PATCH 36/47] fix --- justfiles/services.just | 2 ++ 1 file changed, 2 insertions(+) diff --git a/justfiles/services.just b/justfiles/services.just index 576e86b80..493cea812 100644 --- a/justfiles/services.just +++ b/justfiles/services.just @@ -36,6 +36,8 @@ compose-down: # Shutdown docker services, then clean up docker images, volumes & other local artifacts from this docker-compose project [group('compose')] compose-down-and-wipe: + #!/usr/bin/env bash + docker compose --profile full --profile manual down --volumes --remove-orphans --rmi local # When testing this in CI, I had permission issues when trying to remove this folder. From ebf8193b6cf6ae0359e0e610e0aab8dbc3391958 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 14:20:32 +0100 Subject: [PATCH 37/47] small fixes, comments --- dockerfiles/Dockerfile | 2 +- justfiles/cli.just | 2 +- justfiles/services.just | 2 +- src/docbuilder/rustwide_builder.rs | 30 +++++++++++++++++++----------- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 432773b62..ee428c401 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -191,7 +191,7 @@ RUN apt-get update \ WORKDIR /srv/docsrs # copy migrations so we can run them via CLI script -# COPY migrations migrations/ +COPY migrations migrations/ ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"] diff --git a/justfiles/cli.just b/justfiles/cli.just index 9ffe8c125..a954b1275 100644 --- a/justfiles/cli.just +++ b/justfiles/cli.just @@ -90,5 +90,5 @@ cli-queue-reset-last-seen-ref ref="--head": # find differences between crates.io and our own database, and fix them on our side. [group('cli')] [group('database')] -cli-db-synchronize +args: +cli-db-synchronize *args: just cli-watcher database synchronize {{ args }} diff --git a/justfiles/services.just b/justfiles/services.just index 493cea812..8ed0518b3 100644 --- a/justfiles/services.just +++ b/justfiles/services.just @@ -31,7 +31,7 @@ compose-up-full: # Shutdown docker services, keep containers & volumes alive. [group('compose')] compose-down: - docker compose --profile full down --remove-orphans + docker compose --profile full --profile manual down --remove-orphans # Shutdown docker services, then clean up docker images, volumes & other local artifacts from this docker-compose project [group('compose')] diff --git a/src/docbuilder/rustwide_builder.rs b/src/docbuilder/rustwide_builder.rs index 9ef0e105a..7d402d24e 100644 --- a/src/docbuilder/rustwide_builder.rs +++ b/src/docbuilder/rustwide_builder.rs @@ -286,14 +286,14 @@ impl RustwideBuilder { // This fixes an edge-case on a fresh build server. // // It seems like on the fresh server, there _is_ a recent nightly toolchain - // installed, which we then updated with necessary components in this - // method. + // installed. In this case, this method will just install necessary components and + // doc-targets/platforms. // - // But: *for this toolchain, we never ran `add_essential_files`*, because it + // But: *for this local old toolchain, we never ran `add_essential_files`*, because it // was not installed by us. // // Now the culprit: even through we "fix" the previously installed nightly toolchain - // with the needed components & targets, we return "updated = false", since the actual + // with the needed components & targets, we return "updated = false", since the // version number didn't change. // // As a result, `BuildQueue::update_toolchain` will not call `add_essential_files`, @@ -302,7 +302,7 @@ impl RustwideBuilder { // The workaround specifically for `add_essential_files` is the following: // // After `add_essential_files` is finished, it sets `ConfigName::RustcVersion` in the - // database to the version it uploaded the essential files for. + // config database to the rustc version it uploaded the essential files for. // // This means, if `ConfigName::RustcVersion` is empty, or different from the current new // version, we can set `updated = true` too. @@ -310,13 +310,21 @@ impl RustwideBuilder { // I feel like there are more edge-cases, but for now this is OK. // // Alternative would have been to run `build update-toolchain --only-first-time` - // in a newly created `ENTRYPOINT` script for the build-server. I leaned to towards - // a more self-contained solution which doesn't need docker at all, and also would work - // if you run the build-server directly on your machine. + // in a newly created `ENTRYPOINT` script for the build-server. This is how it was + // done in the previous (one-dockerfile-and-process-for-everything) approach. + // The `entrypoint.sh` script did call `add-essential-files --only-first-time`. + // + // Problem with that approach: this approach postpones the boot process of the + // build-server, where docker and later the infra will try to check with a HTTP + // endpoint to see if the build server is ready. + // + // So I leaned to towards a more self-contained solution which doesn't need docker + // at all, and also would work if you run the build-server directly on your machine. + // // Fixing it here also means the startup of the actual build-server including its - // metrics collection endpoints don't be delayed. Generally shoudl doesn't be - // a differene how much time is neede on a fresh build-server, between picking the - // up from the queue, and actually starting to build the release. In the old + // metrics collection endpoints don't be delayed. Generally should doesn't be + // a differene how much time is needed on a fresh build-server, between picking the + // release up from the queue, and actually starting to build the release. In the old // solution, the entrypoint would do the toolchain-update & add-essential files // before even starting the build-server, now we're roughly doing the same thing // inside the main builder loop. From ba503c060a3b5f0dfc2b89f6680d9fabb40b6cee Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 14:31:52 +0100 Subject: [PATCH 38/47] log msg --- justfiles/services.just | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/justfiles/services.just b/justfiles/services.just index 8ed0518b3..b92af498d 100644 --- a/justfiles/services.just +++ b/justfiles/services.just @@ -46,10 +46,10 @@ compose-down-and-wipe: # so we just try if we can use passwordless `sudo`: if sudo -n true 2>/dev/null; then - # if yes, we can use `sudo` to delete the folder. + echo "deleting ignored/ folder with sudo" sudo -n rm -rf ignored/ else - # otherwise we try deleting as the current user + echo "trying to delete ignored/ folder with current user." rm -rf ignored/ || { echo 'Failed to remove ignored/ - skipping' >&2; } fi From 52e52aa4f8b02f12ec1486eb9835fcf5278f9dbd Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 14:42:30 +0100 Subject: [PATCH 39/47] notes --- NOTES.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/NOTES.md b/NOTES.md index b18acc823..2f7888387 100644 --- a/NOTES.md +++ b/NOTES.md @@ -2,11 +2,28 @@ - the `Justfile` commands are built so they would even work on a 100% fresh setup, and will set up and initialize everything they need. +- I removed `index: Index` from the context. There is no need to clone the + crates.io index on a web- or build-server. The handful of places where we then + still need the index just create the obj. +- there was an error I had when calling `Index::peek_changes` from + `crates-index-diff`. In there, we're using the github fastpath to check if + there are new commits in the repo, without having to actually `git pull` from + the remote. When I called `.peek_changes` in an async context, this lead to + tokio errors because inside `crates-index-diff` we're using + `reqwest::blocking`. Odd thing is: I couldn't find out why this doesn't fail + on production. It might have started failing just after the config/context + rewrite, which is not deployed yet. + [#2937](https://github.com/rust-lang/docs.rs/pull/2937) ## docker image for now, these are production images, changing files will not auto-reload / -build the image. We could decide to do that layer. +build the image. We could decide to do that layer. Also I don't want to copy the +".git" folder into the image, just for the version number. I made the SHA a +build-arg / env and used these in our codebase. + +The fallback to fetching the has from the repo still exists, we might be able to +drop this functionality at some point. ## profiles From 8fb53322e42c04a5e9b1bdbb303b687006db5225 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 15:27:38 +0100 Subject: [PATCH 40/47] try fixes --- .dockerignore | 7 +++++++ .gitignore | 2 +- docker-compose.yml | 22 +++++++++++++++++----- justfiles/testing.just | 10 +++++----- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/.dockerignore b/.dockerignore index 29b8ca6ad..a3cf24819 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,11 +2,18 @@ /.rustwide /.rustwide-docker /Justfile +/justfiles/ /LICENSE /README.md /docker-compose.yml +/dockerfiles/ /docs/ /ignored /mcps /triagebot.toml /clippy.toml +/.env +/.env.* +/.envrc +/.docker.env +archive_cache diff --git a/.gitignore b/.gitignore index 9d9ef06ba..3ab5f4610 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,5 @@ target .vagrant .rustwide .rustwide-docker -.archive_cache +archive_cache .workspace diff --git a/docker-compose.yml b/docker-compose.yml index d5c47928e..447e0c41b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,7 +98,10 @@ x-builder: &builder build: <<: *build target: build-server - # builders only work with linux/amd64 + # build servers only work with linux/amd64 for now. + # This makes them much slower when working on a mac, but + # it needs some work and digging into the builder to fix + # this. platform: "linux/amd64" depends_on: - db @@ -125,7 +128,6 @@ x-registry-watcher: ®istry-watcher - db - s3 volumes: - - "./static:/opt/docsrs/static:ro" - "./ignored/docker-registry-watcher/prefix:/opt/docsrs/prefix" - crates-io-index:/opt/docsrs/crates.io-index environment: @@ -175,6 +177,7 @@ services: registry-watcher-cli: <<: *registry-watcher profiles: + # watcher-CLI should not be run as background daemon, just manually - manual builder-a: @@ -182,6 +185,8 @@ services: volumes: - "rustwide-builder-a:/opt/docsrs/rustwide" - "./ignored/docker-builder-a/prefix:/opt/docsrs/prefix" + # this exposes the docker engine from the host machine + # to the build-server inside the container. - "/var/run/docker.sock:/var/run/docker.sock" profiles: - builder @@ -192,9 +197,11 @@ services: volumes: - "rustwide-builder-b:/opt/docsrs/rustwide" - "./ignored/docker-builder-b/prefix:/opt/docsrs/prefix" + # this exposes the docker engine from the host machine + # to the build-server inside the container. - "/var/run/docker.sock:/var/run/docker.sock" profiles: - - builderaaa + - builder - full builder-cli: @@ -202,8 +209,11 @@ services: volumes: - "rustwide-builder-cli:/opt/docsrs/rustwide" - "./ignored/docker-builder-cli/prefix:/opt/docsrs/prefix" + # this exposes the docker engine from the host machine + # to the build-server inside the container. - "/var/run/docker.sock:/var/run/docker.sock" profiles: + # builder-CLI should not be run as background daemon, just manually - manual cli: @@ -211,8 +221,8 @@ services: <<: *build target: cli depends_on: - # only for clarifycation. - # when using "docker compose run", these dependencies are ignored, + # only for clarification. + # When using "docker compose run", these dependencies are ignored, # we handle this in our `just` commands. - db - s3 @@ -296,6 +306,8 @@ services: volumes: - "${PWD}:/build/out" profiles: + # gui_tests should not be run as background daemon. + # Just run via `just run-gui-tests`. - manual volumes: diff --git a/justfiles/testing.just b/justfiles/testing.just index ef14dd454..b0e5b2efe 100644 --- a/justfiles/testing.just +++ b/justfiles/testing.just @@ -43,10 +43,10 @@ lint-js *args: deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} run-gui-tests: ensure_db_and_s3_are_running cli-db-migrate compose-up-web - just cli-build-update-toolchain - just cli-build-crate sysinfo 0.23.4 - just cli-build-crate sysinfo 0.23.5 - just cli-build-crate libtest 0.0.1 - just cli-build-add-essential-files + # just cli-build-update-toolchain + # just cli-build-crate sysinfo 0.23.4 + # just cli-build-crate sysinfo 0.23.5 + # just cli-build-crate libtest 0.0.1 + # just cli-build-add-essential-files just _cli gui_tests From 12e8c9de8bb123766adec8b16e9a6c74f80ac6e5 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 15:28:14 +0100 Subject: [PATCH 41/47] fix --- justfiles/testing.just | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/justfiles/testing.just b/justfiles/testing.just index b0e5b2efe..ef14dd454 100644 --- a/justfiles/testing.just +++ b/justfiles/testing.just @@ -43,10 +43,10 @@ lint-js *args: deno run -A npm:eslint@9 static templates gui-tests eslint.config.js {{ args }} run-gui-tests: ensure_db_and_s3_are_running cli-db-migrate compose-up-web - # just cli-build-update-toolchain - # just cli-build-crate sysinfo 0.23.4 - # just cli-build-crate sysinfo 0.23.5 - # just cli-build-crate libtest 0.0.1 - # just cli-build-add-essential-files + just cli-build-update-toolchain + just cli-build-crate sysinfo 0.23.4 + just cli-build-crate sysinfo 0.23.5 + just cli-build-crate libtest 0.0.1 + just cli-build-add-essential-files just _cli gui_tests From bb8baa910bcc0446e36c077bcad9e5696a8ab444 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 16:58:59 +0100 Subject: [PATCH 42/47] test --- .github/workflows/ci.yml | 23 ++++------------------- NOTES.md | 2 ++ justfiles/testing.just | 16 ++++++++++++++++ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f69bd404..2a8661317 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,8 +68,6 @@ jobs: run: just compose-down-and-wipe test: - env: - SQLX_OFFLINE: 1 runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -82,21 +80,8 @@ jobs: with: prefix-key: ${{ env.RUST_CACHE_KEY }} - - name: Build - run: cargo build --workspace --locked - - - name: Launch postgres and min.io - run: just ensure_db_and_s3_are_running - - - name: run workspace tests - run: | - cargo test --workspace --locked --no-fail-fast - - - name: run slow tests - env: - DOCSRS_INCLUDE_DEFAULT_TARGETS: true - run: | - cargo test --locked -- --ignored --test-threads=1 + - name: run tests + run: just run-tests run-builder-tests - name: shut down test environment if: ${{ always() }} @@ -122,8 +107,8 @@ jobs: if: ${{ always() }} run: just compose-down-and-wipe - clippy: - name: Clippy + rslint: + name: rust linters runs-on: ubuntu-latest steps: diff --git a/NOTES.md b/NOTES.md index 2f7888387..5c23acda1 100644 --- a/NOTES.md +++ b/NOTES.md @@ -14,6 +14,8 @@ on production. It might have started failing just after the config/context rewrite, which is not deployed yet. [#2937](https://github.com/rust-lang/docs.rs/pull/2937) +- unsure if the prefix mount should also be a docker volume, for performance. + Not sure how often we actually have to look at the contents? ## docker image diff --git a/justfiles/testing.just b/justfiles/testing.just index ef14dd454..fb483d6e9 100644 --- a/justfiles/testing.just +++ b/justfiles/testing.just @@ -50,3 +50,19 @@ run-gui-tests: ensure_db_and_s3_are_running cli-db-migrate compose-up-web just cli-build-add-essential-files just _cli gui_tests + +# build binaries for all tests +build-tests: + cargo test --no-run --workspace --locked + + +run-tests: ensure_db_and_s3_are_running build-tests + cargo test --workspace --locked --no-fail-fast + +run-builder-tests: ensure_db_and_s3_are_running build-tests + #!/bin/bash + set -euo pipefail + + export DOCSRS_INCLUDE_DEFAULT_TARGETS=true + + cargo test --locked -- --ignored --test-threads=1 From 03feb7672ad04e7471bdba51a4f66354f61e7964 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Sun, 2 Nov 2025 18:41:07 +0100 Subject: [PATCH 43/47] more buildx --- .github/workflows/ci.yml | 12 ++++++++++++ justfiles/testing.just | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a8661317..34ed6d7f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,9 @@ jobs: with: prefix-key: ${{ env.RUST_CACHE_KEY }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Launch postgres run: just ensure_db_and_s3_are_running @@ -80,6 +83,9 @@ jobs: with: prefix-key: ${{ env.RUST_CACHE_KEY }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: run tests run: just run-tests run-builder-tests @@ -97,6 +103,9 @@ jobs: with: prefix-key: ${{ env.RUST_CACHE_KEY }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: install `just` run: sudo snap install --edge --classic just @@ -122,6 +131,9 @@ jobs: with: prefix-key: ${{ env.RUST_CACHE_KEY }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - run: just lint eslint: diff --git a/justfiles/testing.just b/justfiles/testing.just index fb483d6e9..b075d0dfb 100644 --- a/justfiles/testing.just +++ b/justfiles/testing.just @@ -23,7 +23,7 @@ clippy *args: --all-targets \ --workspace \ --locked \ - {{ args}} \ + {{ args }} \ -- -D warnings clippy-fix: From 52f824a24925fd23651a8cca4a21ce8e02e96bfe Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 7 Nov 2025 15:44:17 +0100 Subject: [PATCH 44/47] wip --- .github/workflows/docker.yml | 2 + Justfile | 1 + dockerfiles/Example | 79 ++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 dockerfiles/Example diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 65d298045..b08cd0324 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,6 +26,8 @@ jobs: - name: setup docker buildx uses: docker/setup-buildx-action@v3 + # FIXME: we have to set the release profile here somehoe? + - name: build docker image uses: docker/build-push-action@v6 with: diff --git a/Justfile b/Justfile index d6c9a3e14..fc8320dbb 100644 --- a/Justfile +++ b/Justfile @@ -1,4 +1,5 @@ set shell := ["bash", "-Eeuo", "pipefail", "-c"] +set ignore-comments # List available commands _default: diff --git a/dockerfiles/Example b/dockerfiles/Example new file mode 100644 index 000000000..d3b46970c --- /dev/null +++ b/dockerfiles/Example @@ -0,0 +1,79 @@ +# syntax=docker/dockerfile:1.7 + +ARG RUST_VERSION=1.82 +ARG APP=/app + +######################## +# deps base (warm cache) +######################## +FROM rust:${RUST_VERSION} AS deps-base +WORKDIR ${APP} + +COPY Cargo.toml Cargo.lock ./ +RUN mkdir -p src && echo "fn main(){}" > src/main.rs +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ + --mount=type=cache,target=${APP}/target,id=cargo-target,sharing=locked \ + cargo build --release +RUN rm -rf src + +######################## +# build: web +######################## +FROM rust:${RUST_VERSION} AS build-web +WORKDIR ${APP} +COPY --from=deps-base ${APP}/ ./ +COPY src ./src +# or copy only needed subdirs if workspace +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ + --mount=type=cache,target=${APP}/target,id=cargo-target,sharing=locked \ + cargo build --release --bin web + +######################## +# build: worker +######################## +FROM rust:${RUST_VERSION} AS build-worker +WORKDIR ${APP} +COPY --from=deps-base ${APP}/ ./ +COPY src ./src +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ + --mount=type=cache,target=${APP}/target,id=cargo-target,sharing=locked \ + cargo build --release --bin worker + +######################## +# build: cron +######################## +FROM rust:${RUST_VERSION} AS build-cron +WORKDIR ${APP} +COPY --from=deps-base ${APP}/ ./ +COPY src ./src +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ + --mount=type=cache,target=${APP}/target,id=cargo-target,sharing=locked \ + cargo build --release --bin cron + +######################## +# build: migrate +######################## +FROM rust:${RUST_VERSION} AS build-migrate +WORKDIR ${APP} +COPY --from=deps-base ${APP}/ ./ +COPY src ./src +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ + --mount=type=cache,target=${APP}/target,id=cargo-target,sharing=locked \ + cargo build --release --bin migrate + +######################## +# runtime +######################## +FROM gcr.io/distroless/cc-debian12 AS runtime +WORKDIR /app +COPY --from=build-web /app/target/release/web /usr/local/bin/web +COPY --from=build-worker /app/target/release/worker /usr/local/bin/worker +COPY --from=build-cron /app/target/release/cron /usr/local/bin/cron +COPY --from=build-migrate /app/target/release/migrate /usr/local/bin/migrate +ENTRYPOINT ["/usr/local/bin/web"] + + +#docker buildx build \ +# --cache-from=type=local,src=.buildx-cache \ +# --cache-to=type=local,dest=.buildx-cache,mode=max \ +# -t your/image:tag . From 0f4a9c3f145754e4fb9d596ad85b9d81216aa079 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 7 Nov 2025 17:26:48 +0100 Subject: [PATCH 45/47] build release profile --- .github/workflows/docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b08cd0324..0c830c25f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -26,8 +26,6 @@ jobs: - name: setup docker buildx uses: docker/setup-buildx-action@v3 - # FIXME: we have to set the release profile here somehoe? - - name: build docker image uses: docker/build-push-action@v6 with: @@ -37,6 +35,8 @@ jobs: target: ${{ matrix.target }} build-args: | GIT_SHA=${{ github.sha }} + PROFILE=release + PROFILE_DIR=release load: true cache-from: type=gha cache-to: type=gha,mode=max From c9b2f41f0941bee6879167d24a03c47508aadb9b Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 7 Nov 2025 17:26:59 +0100 Subject: [PATCH 46/47] try cache mounts for cache --- dockerfiles/Dockerfile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index ee428c401..3160d17fd 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1 + # To produce a smaller image this Dockerfile contains two separate stages: in # the first one all the build dependencies are installed and docs.rs is built, # while in the second one just the runtime dependencies are installed, with the @@ -5,6 +7,7 @@ # # As of 2019-10-29 this reduces the image from 2.8GB to 500 MB. + ################# # Build stage # ################# @@ -54,10 +57,10 @@ RUN mkdir -p src/bin && \ echo "fn main() {}" > src/bin/cratesfyi.rs && \ echo "fn main() {}" > build.rs -RUN cargo fetch - ARG PROFILE=release -RUN cargo build --profile=$PROFILE +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ + --mount=type=cache,target=/build/target,id=cargo-target,sharing=locked \ + cargo build --profile=$PROFILE # Dependencies are now cached, copy the actual source code and do another full # build. The touch on all the .rs files is needed, otherwise cargo assumes the @@ -74,7 +77,9 @@ COPY assets assets/ COPY .sqlx .sqlx/ COPY migrations migrations/ -RUN cargo build --profile=$PROFILE +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ + --mount=type=cache,target=/build/target,id=cargo-target,sharing=locked \ + cargo build --profile=$PROFILE ###################### # Web server stage # From 7eda979d596b3c902d168194ca05b48a26f4c021 Mon Sep 17 00:00:00 2001 From: Denis Cornehl Date: Fri, 7 Nov 2025 17:27:02 +0100 Subject: [PATCH 47/47] cleanup --- dockerfiles/Example | 79 --------------------------------------------- 1 file changed, 79 deletions(-) delete mode 100644 dockerfiles/Example diff --git a/dockerfiles/Example b/dockerfiles/Example deleted file mode 100644 index d3b46970c..000000000 --- a/dockerfiles/Example +++ /dev/null @@ -1,79 +0,0 @@ -# syntax=docker/dockerfile:1.7 - -ARG RUST_VERSION=1.82 -ARG APP=/app - -######################## -# deps base (warm cache) -######################## -FROM rust:${RUST_VERSION} AS deps-base -WORKDIR ${APP} - -COPY Cargo.toml Cargo.lock ./ -RUN mkdir -p src && echo "fn main(){}" > src/main.rs -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ - --mount=type=cache,target=${APP}/target,id=cargo-target,sharing=locked \ - cargo build --release -RUN rm -rf src - -######################## -# build: web -######################## -FROM rust:${RUST_VERSION} AS build-web -WORKDIR ${APP} -COPY --from=deps-base ${APP}/ ./ -COPY src ./src -# or copy only needed subdirs if workspace -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ - --mount=type=cache,target=${APP}/target,id=cargo-target,sharing=locked \ - cargo build --release --bin web - -######################## -# build: worker -######################## -FROM rust:${RUST_VERSION} AS build-worker -WORKDIR ${APP} -COPY --from=deps-base ${APP}/ ./ -COPY src ./src -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ - --mount=type=cache,target=${APP}/target,id=cargo-target,sharing=locked \ - cargo build --release --bin worker - -######################## -# build: cron -######################## -FROM rust:${RUST_VERSION} AS build-cron -WORKDIR ${APP} -COPY --from=deps-base ${APP}/ ./ -COPY src ./src -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ - --mount=type=cache,target=${APP}/target,id=cargo-target,sharing=locked \ - cargo build --release --bin cron - -######################## -# build: migrate -######################## -FROM rust:${RUST_VERSION} AS build-migrate -WORKDIR ${APP} -COPY --from=deps-base ${APP}/ ./ -COPY src ./src -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry,sharing=locked \ - --mount=type=cache,target=${APP}/target,id=cargo-target,sharing=locked \ - cargo build --release --bin migrate - -######################## -# runtime -######################## -FROM gcr.io/distroless/cc-debian12 AS runtime -WORKDIR /app -COPY --from=build-web /app/target/release/web /usr/local/bin/web -COPY --from=build-worker /app/target/release/worker /usr/local/bin/worker -COPY --from=build-cron /app/target/release/cron /usr/local/bin/cron -COPY --from=build-migrate /app/target/release/migrate /usr/local/bin/migrate -ENTRYPOINT ["/usr/local/bin/web"] - - -#docker buildx build \ -# --cache-from=type=local,src=.buildx-cache \ -# --cache-to=type=local,dest=.buildx-cache,mode=max \ -# -t your/image:tag .