From 2547b68a9373b4645dcd7c2056d4373521fd149d Mon Sep 17 00:00:00 2001
From: Matheus Izvekov <mizvekov@gmail.com>
Date: Wed, 1 Oct 2025 15:10:51 -0300
Subject: [PATCH] ci: improve the 'Publish website' step

This makes that step use rsync to upload the website and demos.

Besides supporting compression, rsync also only sends the changes
over the wire.

This improves the performance of that step by a wide margin, reducing
normal upload times from almost 9 minutes down to about 20s for the most
common changes.

This uses rsync's ability to also remove stale files, to make sure
we don't leave old files behind. (This ignores the directories which are
handled separately, like llvm+clang and demos).

There were some pretty stale things left behind that this now removes,
such as a sitemap.xml which was generated last year.
---
 .github/workflows/ci.yml | 18 +++++++++---------
 docs/website/robots.txt  |  2 ++
 2 files changed, 11 insertions(+), 9 deletions(-)
 create mode 100755 docs/website/robots.txt

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a31ddefb9..3c24266be 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -581,7 +581,7 @@ jobs:
         uses: alandefreitas/cpp-actions/package-install@v1.8.12
         id: package-install
         with:
-          apt-get: build-essential asciidoctor cmake bzip2 git
+          apt-get: build-essential asciidoctor cmake bzip2 git rsync
 
       - name: Clone MrDocs
         uses: actions/checkout@v4
@@ -711,6 +711,7 @@ jobs:
           node render.js
           mkdir -p ../../build/website
           cp index.html ../../build/website/index.html
+          cp robots.txt ../../build/website/robots.txt
           cp styles.css ../../build/website/styles.css
           cp -r assets ../../build/website/assets
 
@@ -963,6 +964,7 @@ jobs:
         env:
           SSH_AUTH_SOCK: /tmp/ssh_agent.sock
         run: |
+          set -euvx
           # Add SSH key
           mkdir -p /home/runner/.ssh
           ssh-keyscan dev-websites.cpp.al >> /home/runner/.ssh/known_hosts
@@ -972,17 +974,15 @@ jobs:
           ssh-agent -a $SSH_AUTH_SOCK > /dev/null
           ssh-add /home/runner/.ssh/github_actions
 
+          rsyncopts=(--recursive --delete --links --times --chmod=D0755,F0755 --compress --compress-choice=zstd --rsh="ssh -o StrictHostKeyChecking=no" --human-readable)
+          website_dir="ubuntu@dev-websites.cpp.al:/var/www/mrdox.com"
+          demo_dir="$website_dir/demos/${{ github.ref_name }}"
+
           # Copy files: This step will copy the landing page and the documentation to www.mrdocs.com
-          chmod 755 -R $(pwd)/build/website
-          scp -o StrictHostKeyChecking=no -r $(pwd)/build/website/* ubuntu@dev-websites.cpp.al:/var/www/mrdox.com/
-          
-          # Remove previous demos associated with this tag
-          demo_dir="/var/www/mrdox.com/demos/${{ github.ref_name }}"
-          ssh -o StrictHostKeyChecking=no ubuntu@dev-websites.cpp.al "rm -rf $demo_dir/boost-url; mkdir -p $demo_dir/boost-url"
+          time rsync "${rsyncopts[@]}" --exclude=llvm+clang/ --exclude=demos/ $(pwd)/build/website/ "$website_dir"/
 
           # Copy demos: This step will copy the demos to www.mrdocs.com/demos
-          chmod 755 -R $(pwd)/demos
-          scp -o StrictHostKeyChecking=no -r $(pwd)/demos/* ubuntu@dev-websites.cpp.al:$demo_dir/
+          time rsync "${rsyncopts[@]}" $(pwd)/demos/ "$demo_dir"/
 
       - name: Create changelog
         uses: alandefreitas/cpp-actions/create-changelog@v1.8.12
diff --git a/docs/website/robots.txt b/docs/website/robots.txt
new file mode 100755
index 000000000..c2a49f4fb
--- /dev/null
+++ b/docs/website/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Allow: /