diff --git a/ci/deploy/devsite.sh b/ci/deploy/devsite.sh index 9fd858a78f89b..e112a1cb42fe5 100755 --- a/ci/deploy/devsite.sh +++ b/ci/deploy/devsite.sh @@ -13,6 +13,12 @@ set -euo pipefail +# rustdoc emits ~13k tiny HTML files per docset, so the S3 syncs below are bound +# by per-object request latency rather than bandwidth. The aws CLI defaults to +# only 10 concurrent requests; bump it so the many small files upload in +# parallel. +aws configure set default.s3.max_concurrent_requests 100 + cargo about generate ci/deploy/licenses.hbs > misc/www/licenses.html aws s3 cp --recursive misc/www/ s3://materialize-dev-website/ @@ -20,10 +26,27 @@ aws s3 cp --recursive misc/www/ s3://materialize-dev-website/ # We exclude all of these pages from search engines for SEO purposes. We don't # want to spend our crawl budget on these pages, nor have these pages appear # ahead of our marketing content. -RUSTDOCFLAGS="--html-in-header $PWD/ci/deploy/noindex.html" bin/doc -RUSTDOCFLAGS="--html-in-header $PWD/ci/deploy/noindex.html" bin/doc --document-private-items -aws s3 sync --size-only target-xcompile/doc/ s3://materialize-dev-website/api/rust -aws s3 sync --size-only target-xcompile/doc/ s3://materialize-dev-website/api/rust-private +# +# `api/rust` gets the public docs; `api/rust-private` gets the docs with private +# items. We build each docset into its own target dir and build+upload them in +# parallel. --delete makes each prefix a clean mirror (and purges private pages +# previously uploaded to api/rust). +export RUSTDOCFLAGS="--html-in-header $PWD/ci/deploy/noindex.html" + +( + CARGO_TARGET_DIR=target-xcompile bin/doc + aws s3 sync --size-only --delete target-xcompile/doc/ s3://materialize-dev-website/api/rust +) & +rust_pid=$! + +( + CARGO_TARGET_DIR=target-xcompile-private bin/doc --document-private-items + aws s3 sync --size-only --delete target-xcompile-private/doc/ s3://materialize-dev-website/api/rust-private +) & +rust_private_pid=$! + +wait "$rust_pid" +wait "$rust_private_pid" bin/pydoc aws s3 sync --size-only --delete target/pydoc/ s3://materialize-dev-website/api/python