From 9fce278cc18fb0c98c5f63c43ad20c8d10cfe5a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Mon, 9 Oct 2023 14:23:30 +0200 Subject: [PATCH 01/12] feat(udpates.jenkins.io): also sync content into Azure File Share and R2 buckets --- site/publish.sh | 70 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 13 deletions(-) diff --git a/site/publish.sh b/site/publish.sh index e05c560d0..79178b7fe 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -3,24 +3,68 @@ # Used later for rsyncing updates UPDATES_SITE="updates.jenkins.io" RSYNC_USER="www-data" +UPDATES_R2_BUCKETS="westeurope-updates-jenkins-io" +UPDATES_R2_ENDPOINT="https://8d1838a43923148c5cee18ccc356a594.r2.cloudflarestorage.com" +if [[ -z "$ROOT_FOLDER" ]]; then + ROOT_FOLDER="/home/jenkins/lemeurherve/pr-745" # TODO: remove after debug +fi + +echo "ROOT_FOLDER: ${ROOT_FOLDER}" wget --no-verbose -O jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 || { echo "Failed to download jq" >&2 ; exit 1; } chmod +x jq || { echo "Failed to make jq executable" >&2 ; exit 1; } export PATH=.:$PATH -"$( dirname "$0" )/generate.sh" ./www2 ./download +# "$( dirname "$0" )/generate.sh" "${ROOT_FOLDER}"/www2 ./download # push plugins to mirrors.jenkins-ci.org -chmod -R a+r download -rsync -avz --size-only download/plugins/ ${RSYNC_USER}@${UPDATES_SITE}:/srv/releases/jenkins/plugins - -# Invoke a minimal mirrorsync to mirrorbits which will use the 'recent-releases.json' file as input -ssh ${RSYNC_USER}@${UPDATES_SITE} "cat > /tmp/update-center2-rerecent-releases.json" < www2/experimental/recent-releases.json -ssh ${RSYNC_USER}@${UPDATES_SITE} "/srv/releases/sync-recent-releases.sh /tmp/update-center2-rerecent-releases.json" - -# push generated index to the production servers -# 'updates' come from tool installer generator, so leave that alone, but otherwise -# delete old sites -chmod -R a+r www2 -rsync -acvz www2/ --exclude=/updates --delete ${RSYNC_USER}@${UPDATES_SITE}:/var/www/${UPDATES_SITE} +# chmod -R a+r download +# rsync -avz --size-only download/plugins/ ${RSYNC_USER}@${UPDATES_SITE}:/srv/releases/jenkins/plugins + +# # Invoke a minimal mirrorsync to mirrorbits which will use the 'recent-releases.json' file as input +# ssh ${RSYNC_USER}@${UPDATES_SITE} "cat > /tmp/update-center2-rerecent-releases.json" < www2/experimental/recent-releases.json +# ssh ${RSYNC_USER}@${UPDATES_SITE} "/srv/releases/sync-recent-releases.sh /tmp/update-center2-rerecent-releases.json" + +# # push generated index to the production servers +# # 'updates' come from tool installer generator, so leave that alone, but otherwise +# # delete old sites +chmod -R a+r "${ROOT_FOLDER}"/www2 +# # rsync -acvz www2/ --exclude=/updates --delete ${RSYNC_USER}@${UPDATES_SITE}:/var/www/${UPDATES_SITE} + +# ### TODO: cleanup original commands above when https://github.com/jenkins-infra/helpdesk/issues/2649 is ready for production + +# Original-like rsync to pkg VM for testing and timing purposes +time rsync -acvz "${ROOT_FOLDER}"/www2/ --exclude=/updates --delete --stats ${RSYNC_USER}@${UPDATES_SITE}:/tmp/lemeurherve/pr-745/www/${UPDATES_SITE} + +### Above ^: not to be modified + +### Below: parallelise + +# copy & transform simlinks into referent file/dir +# time rsync -acvz --copy-links --safe-links --stats "${ROOT_FOLDER}"/www2/ --exclude=/updates --delete "${ROOT_FOLDER}"/www3/ +time rsync -acvz --no-links --stats "${ROOT_FOLDER}"/www2/ --exclude=/updates --delete "${ROOT_FOLDER}"/www3/ + +ls -l "${ROOT_FOLDER}"/www3/ +ls -l "${ROOT_FOLDER}"/www3/current + +# Sync Azure File Share content +time azcopy sync "${ROOT_FOLDER}"/www3/ "${UPDATES_FILE_SHARE_URL}" --recursive=true --delete-destination=true --exclude-path="updates" + +# Debug +echo "= azcopy sync done." + +# Sync CloudFlare R2 buckets content using the updates-jenkins-io profile, excluding 'updates' folder which comes from tool installer generator +time aws s3 sync "${ROOT_FOLDER}"/www3/ s3://"${UPDATES_R2_BUCKETS}"/ --profile updates-jenkins-io --no-progress --size-only --no-follow-symlinks --exclude="updates/*" --endpoint-url "${UPDATES_R2_ENDPOINT}" +# aws s3 cp "${ROOT_FOLDER}"/www2/ s3://"${UPDATES_R2_BUCKETS}"/ --profile updates-jenkins-io --no-progress --no-follow-symlinks --exclude="updates/*" --endpoint-url "${UPDATES_R2_ENDPOINT}" + + +## TODO: test if needed rclone both rsync VM and R2 bucket(s) replacing these 2 calls + +# Debug +echo "= aws sync done." + +# # /TIME sync, used by mirrorbits to know the last update date to take in account +# date +%s > ./www2/TIME +# aws s3 cp ./www2/TIME s3://"${UPDATES_R2_BUCKETS}"/ --profile updates-jenkins-io --endpoint-url "${UPDATES_R2_ENDPOINT}" +# azcopy cp ./www2/TIME "${UPDATES_FILE_SHARE_URL}" --overwrite=true From 3aca271082ba9dfde4dc03f03abcd546702eacde Mon Sep 17 00:00:00 2001 From: smerle33 Date: Wed, 18 Oct 2023 14:50:13 +0200 Subject: [PATCH 02/12] parrallilization --- site/publish.sh | 67 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/site/publish.sh b/site/publish.sh index 79178b7fe..61c44c276 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -9,6 +9,9 @@ if [[ -z "$ROOT_FOLDER" ]]; then ROOT_FOLDER="/home/jenkins/lemeurherve/pr-745" # TODO: remove after debug fi +# parallel added within the permanent trusted agent here : https://github.com/jenkins-infra/jenkins-infra/blob/production/dist/profile/manifests/buildagent.pp +command -v parallel >/dev/null 2>&1 || { echo "ERROR: parralel command not found. Exiting."; exit 1; } + echo "ROOT_FOLDER: ${ROOT_FOLDER}" wget --no-verbose -O jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 || { echo "Failed to download jq" >&2 ; exit 1; } @@ -34,35 +37,65 @@ chmod -R a+r "${ROOT_FOLDER}"/www2 # ### TODO: cleanup original commands above when https://github.com/jenkins-infra/helpdesk/issues/2649 is ready for production -# Original-like rsync to pkg VM for testing and timing purposes -time rsync -acvz "${ROOT_FOLDER}"/www2/ --exclude=/updates --delete --stats ${RSYNC_USER}@${UPDATES_SITE}:/tmp/lemeurherve/pr-745/www/${UPDATES_SITE} +#### no need to remove the symlinks as the `azcopy sync`for symlinks is not yet supported and we use `--no-follow-symlinks` for `aws s3 sync`` +# Perform a copy with dereference symlink (object storage do not support symlinks) +# copy & transform simlinks into referent file/dir +time rsync -acvz \ + --copy-links `# derefence symlinks` \ + --safe-links `# ignore symlinks outside of copied tree` \ + --stats `# add verbose statistics` \ + "${ROOT_FOLDER}"/www2/ --exclude=updates/ --delete "${ROOT_FOLDER}"/www3/ +## "${ROOT_FOLDER}"/www3/ doesn't have symlinks already +## "${ROOT_FOLDER}"/www2/ still have symlinks +### Below: parallelise +echo '--------------------------- Launch Parallelization -----------------------' -### Above ^: not to be modified -### Below: parallelise +## define function +function parallelfunction() { + echo "=== parallelfunction: $1" -# copy & transform simlinks into referent file/dir -# time rsync -acvz --copy-links --safe-links --stats "${ROOT_FOLDER}"/www2/ --exclude=/updates --delete "${ROOT_FOLDER}"/www3/ -time rsync -acvz --no-links --stats "${ROOT_FOLDER}"/www2/ --exclude=/updates --delete "${ROOT_FOLDER}"/www3/ + case $1 in + rsync*) + # keep exclude as from www2 with symlinks + time rsync -acz "${ROOT_FOLDER}"/www2/ --exclude=/updates --delete --stats ${RSYNC_USER}@${UPDATES_SITE}:/tmp/lemeurherve/pr-745/www/${UPDATES_SITE} + ;; -ls -l "${ROOT_FOLDER}"/www3/ -ls -l "${ROOT_FOLDER}"/www3/current + azsync*) + # Sync Azure File Share content (using www3 to avoid symlinks) + time azcopy sync "${ROOT_FOLDER}"/www3/ "${UPDATES_FILE_SHARE_URL}" --recursive=true --delete-destination=true + ;; -# Sync Azure File Share content -time azcopy sync "${ROOT_FOLDER}"/www3/ "${UPDATES_FILE_SHARE_URL}" --recursive=true --delete-destination=true --exclude-path="updates" + s3sync*) + # Sync CloudFlare R2 buckets content using the updates-jenkins-io profile, excluding 'updates' folder which comes from tool installer generator (using www3 to avoid symlinks) + time aws s3 sync "${ROOT_FOLDER}"/www3/ s3://"${UPDATES_R2_BUCKETS}"/ --profile updates-jenkins-io --no-progress --no-follow-symlinks --size-only --exclude '.htaccess' --endpoint-url "${UPDATES_R2_ENDPOINT}" + ;; -# Debug -echo "= azcopy sync done." + *) + echo -n "unknown" + ;; + esac + +} + +## need to export variables used within the functions above +export UPDATES_SITE +export RSYNC_USER +export UPDATES_R2_BUCKETS +export UPDATES_R2_ENDPOINT +export ROOT_FOLDER + +## export function to use with parallel +export -f parallelfunction +parallel --halt-on-error now,fail=1 parallelfunction ::: rsync azsync s3sync -# Sync CloudFlare R2 buckets content using the updates-jenkins-io profile, excluding 'updates' folder which comes from tool installer generator -time aws s3 sync "${ROOT_FOLDER}"/www3/ s3://"${UPDATES_R2_BUCKETS}"/ --profile updates-jenkins-io --no-progress --size-only --no-follow-symlinks --exclude="updates/*" --endpoint-url "${UPDATES_R2_ENDPOINT}" -# aws s3 cp "${ROOT_FOLDER}"/www2/ s3://"${UPDATES_R2_BUCKETS}"/ --profile updates-jenkins-io --no-progress --no-follow-symlinks --exclude="updates/*" --endpoint-url "${UPDATES_R2_ENDPOINT}" +# wait for all deferred task +echo '=============================== all done ============================' ## TODO: test if needed rclone both rsync VM and R2 bucket(s) replacing these 2 calls # Debug -echo "= aws sync done." # # /TIME sync, used by mirrorbits to know the last update date to take in account # date +%s > ./www2/TIME From df5d525298f409b2a67e89cd4b991cc10679e5a0 Mon Sep 17 00:00:00 2001 From: Damien Duportal Date: Fri, 3 Nov 2023 07:47:20 +0100 Subject: [PATCH 03/12] add a mirrorscan step at the end of the script --- site/publish.sh | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/site/publish.sh b/site/publish.sh index 61c44c276..bce34bbd9 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -8,6 +8,7 @@ UPDATES_R2_ENDPOINT="https://8d1838a43923148c5cee18ccc356a594.r2.cloudflarestora if [[ -z "$ROOT_FOLDER" ]]; then ROOT_FOLDER="/home/jenkins/lemeurherve/pr-745" # TODO: remove after debug fi +export AWS_DEFAULT_REGION=auto # parallel added within the permanent trusted agent here : https://github.com/jenkins-infra/jenkins-infra/blob/production/dist/profile/manifests/buildagent.pp command -v parallel >/dev/null 2>&1 || { echo "ERROR: parralel command not found. Exiting."; exit 1; } @@ -33,18 +34,22 @@ export PATH=.:$PATH # # 'updates' come from tool installer generator, so leave that alone, but otherwise # # delete old sites chmod -R a+r "${ROOT_FOLDER}"/www2 -# # rsync -acvz www2/ --exclude=/updates --delete ${RSYNC_USER}@${UPDATES_SITE}:/var/www/${UPDATES_SITE} +# TIME sync, used by mirrorbits to know the last update date to take in account +date +%s > "${ROOT_FOLDER}"/www2/TIME -# ### TODO: cleanup original commands above when https://github.com/jenkins-infra/helpdesk/issues/2649 is ready for production +## Commented out: original rsync command to PKG VM (should be in the parallelized step below) +# rsync -acvz www2/ --exclude=/updates --delete ${RSYNC_USER}@${UPDATES_SITE}:/var/www/${UPDATES_SITE} -#### no need to remove the symlinks as the `azcopy sync`for symlinks is not yet supported and we use `--no-follow-symlinks` for `aws s3 sync`` +#### No need to remove the symlinks as the `azcopy sync` for symlinks is not yet supported and we use `--no-follow-symlinks` for `aws s3 sync`` # Perform a copy with dereference symlink (object storage do not support symlinks) # copy & transform simlinks into referent file/dir time rsync -acvz \ --copy-links `# derefence symlinks` \ --safe-links `# ignore symlinks outside of copied tree` \ --stats `# add verbose statistics` \ - "${ROOT_FOLDER}"/www2/ --exclude=updates/ --delete "${ROOT_FOLDER}"/www3/ + --exclude='updates' \ + --delete \ + "${ROOT_FOLDER}"/www2/ "${ROOT_FOLDER}"/www3/ ## "${ROOT_FOLDER}"/www3/ doesn't have symlinks already ## "${ROOT_FOLDER}"/www2/ still have symlinks ### Below: parallelise @@ -58,17 +63,23 @@ function parallelfunction() { case $1 in rsync*) # keep exclude as from www2 with symlinks - time rsync -acz "${ROOT_FOLDER}"/www2/ --exclude=/updates --delete --stats ${RSYNC_USER}@${UPDATES_SITE}:/tmp/lemeurherve/pr-745/www/${UPDATES_SITE} + time rsync -acz "${ROOT_FOLDER}"/www2/ --exclude='updates' --delete --stats ${RSYNC_USER}@${UPDATES_SITE}:/tmp/lemeurherve/pr-745/www/${UPDATES_SITE} ;; azsync*) # Sync Azure File Share content (using www3 to avoid symlinks) - time azcopy sync "${ROOT_FOLDER}"/www3/ "${UPDATES_FILE_SHARE_URL}" --recursive=true --delete-destination=true + time azcopy sync "${ROOT_FOLDER}"/www3/ "https://updatesjenkinsio.file.core.windows.net/updates-jenkins-io/?${UPDATES_FILE_SHARE_QUERY_STRING}" --recursive=true --delete-destination=true ;; s3sync*) + ## Note: AWS CLI is configured through environment variables (from Jenkins credentials) - https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html # Sync CloudFlare R2 buckets content using the updates-jenkins-io profile, excluding 'updates' folder which comes from tool installer generator (using www3 to avoid symlinks) - time aws s3 sync "${ROOT_FOLDER}"/www3/ s3://"${UPDATES_R2_BUCKETS}"/ --profile updates-jenkins-io --no-progress --no-follow-symlinks --size-only --exclude '.htaccess' --endpoint-url "${UPDATES_R2_ENDPOINT}" + time aws s3 sync "${ROOT_FOLDER}"/www3/ s3://"${UPDATES_R2_BUCKETS}"/ \ + --no-progress \ + --no-follow-symlinks \ + --size-only \ + --exclude '.htaccess' \ + --endpoint-url "${UPDATES_R2_ENDPOINT}" ;; *) @@ -84,20 +95,16 @@ export RSYNC_USER export UPDATES_R2_BUCKETS export UPDATES_R2_ENDPOINT export ROOT_FOLDER +export UPDATES_FILE_SHARE_QUERY_STRING ## export function to use with parallel export -f parallelfunction parallel --halt-on-error now,fail=1 parallelfunction ::: rsync azsync s3sync - # wait for all deferred task echo '=============================== all done ============================' -## TODO: test if needed rclone both rsync VM and R2 bucket(s) replacing these 2 calls - -# Debug - -# # /TIME sync, used by mirrorbits to know the last update date to take in account -# date +%s > ./www2/TIME -# aws s3 cp ./www2/TIME s3://"${UPDATES_R2_BUCKETS}"/ --profile updates-jenkins-io --endpoint-url "${UPDATES_R2_ENDPOINT}" -# azcopy cp ./www2/TIME "${UPDATES_FILE_SHARE_URL}" --overwrite=true +## Trigger a mirror scan on mirrorbits +# Requires a valid kubernetes credential file at $KUBECONFIG or $HOME/.kube/config by default +pod_name="$(kubectl --namespace=updates-jenkins-io --no-headers=true get pod --output=name | grep mirrorbits-lite | head -n1)" +kubectl --namespace=updates-jenkins-io exec "${pod_name}" --container=mirrorbits-lite -- mirrorbits scan -all -enable -timeout=120 From 7a8a2555976015306ec457358c576f04600c03e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 17 Nov 2023 13:50:31 +0100 Subject: [PATCH 04/12] cleanup --- site/publish.sh | 127 ++++++++++++++++++++++++++++-------------------- 1 file changed, 74 insertions(+), 53 deletions(-) diff --git a/site/publish.sh b/site/publish.sh index bce34bbd9..0e53b421d 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -1,80 +1,85 @@ #!/bin/bash -ex +## Environment variables that could be configured at the job level: +# - OPT_IN_SYNC_FS_R2: (optional) Set it to "optin" to also update azure.updates.jenkins.io Files Share and R2 buckets + # Used later for rsyncing updates UPDATES_SITE="updates.jenkins.io" RSYNC_USER="www-data" -UPDATES_R2_BUCKETS="westeurope-updates-jenkins-io" + +# For syncing R2 buckets with aws-cli configured through environment variables (from Jenkins credentials) +# https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html +export AWS_DEFAULT_REGION="auto" +UPDATES_R2_BUCKET="westeurope-updates-jenkins-io" UPDATES_R2_ENDPOINT="https://8d1838a43923148c5cee18ccc356a594.r2.cloudflarestorage.com" -if [[ -z "$ROOT_FOLDER" ]]; then - ROOT_FOLDER="/home/jenkins/lemeurherve/pr-745" # TODO: remove after debug -fi -export AWS_DEFAULT_REGION=auto -# parallel added within the permanent trusted agent here : https://github.com/jenkins-infra/jenkins-infra/blob/production/dist/profile/manifests/buildagent.pp -command -v parallel >/dev/null 2>&1 || { echo "ERROR: parralel command not found. Exiting."; exit 1; } +# For syncing Azure File Share +UPDATES_FILE_SHARE_URL_AND_PATH="https://updatesjenkinsio.file.core.windows.net/updates-jenkins-io/" -echo "ROOT_FOLDER: ${ROOT_FOLDER}" +# For triggering a mirror scan on mirrorbits +MIRRORBITS_POD_NAME_PREFIX="mirrorbits-lite" +MIRRORBITS_CONTAINER_NAME="mirrorbits-lite" +MIRRORBITS_NAMESPACE="updates-jenkins-io" +## Install jq, required by generate.sh script wget --no-verbose -O jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 || { echo "Failed to download jq" >&2 ; exit 1; } chmod +x jq || { echo "Failed to make jq executable" >&2 ; exit 1; } export PATH=.:$PATH -# "$( dirname "$0" )/generate.sh" "${ROOT_FOLDER}"/www2 ./download +## Generate the content of 'www2' and 'download' folders +"$( dirname "$0" )/generate.sh" ./www2 ./download +## 'download' folder processing # push plugins to mirrors.jenkins-ci.org -# chmod -R a+r download -# rsync -avz --size-only download/plugins/ ${RSYNC_USER}@${UPDATES_SITE}:/srv/releases/jenkins/plugins +chmod -R a+r ./download +rsync -avz --size-only download/plugins/ ${RSYNC_USER}@${UPDATES_SITE}:/srv/releases/jenkins/plugins -# # Invoke a minimal mirrorsync to mirrorbits which will use the 'recent-releases.json' file as input -# ssh ${RSYNC_USER}@${UPDATES_SITE} "cat > /tmp/update-center2-rerecent-releases.json" < www2/experimental/recent-releases.json -# ssh ${RSYNC_USER}@${UPDATES_SITE} "/srv/releases/sync-recent-releases.sh /tmp/update-center2-rerecent-releases.json" +# Invoke a minimal mirrorsync to mirrorbits which will use the 'recent-releases.json' file as input +ssh ${RSYNC_USER}@${UPDATES_SITE} "cat > /tmp/update-center2-rerecent-releases.json" < www2/experimental/recent-releases.json +ssh ${RSYNC_USER}@${UPDATES_SITE} "/srv/releases/sync-recent-releases.sh /tmp/update-center2-rerecent-releases.json" -# # push generated index to the production servers -# # 'updates' come from tool installer generator, so leave that alone, but otherwise -# # delete old sites -chmod -R a+r "${ROOT_FOLDER}"/www2 -# TIME sync, used by mirrorbits to know the last update date to take in account -date +%s > "${ROOT_FOLDER}"/www2/TIME +## 'www2' folder processing +chmod -R a+r./ www2 -## Commented out: original rsync command to PKG VM (should be in the parallelized step below) -# rsync -acvz www2/ --exclude=/updates --delete ${RSYNC_USER}@${UPDATES_SITE}:/var/www/${UPDATES_SITE} +# TIME sync, used by mirrorbits to know the last update date to take in account +date +%s > ./www2/TIME -#### No need to remove the symlinks as the `azcopy sync` for symlinks is not yet supported and we use `--no-follow-symlinks` for `aws s3 sync`` +## No need to remove the symlinks as the `azcopy sync` for symlinks is not yet supported and we use `--no-follow-symlinks` for `aws s3 sync` # Perform a copy with dereference symlink (object storage do not support symlinks) # copy & transform simlinks into referent file/dir -time rsync -acvz \ +rsync --archive --checksum --verbose --compress \ --copy-links `# derefence symlinks` \ --safe-links `# ignore symlinks outside of copied tree` \ --stats `# add verbose statistics` \ - --exclude='updates' \ - --delete \ - "${ROOT_FOLDER}"/www2/ "${ROOT_FOLDER}"/www3/ -## "${ROOT_FOLDER}"/www3/ doesn't have symlinks already -## "${ROOT_FOLDER}"/www2/ still have symlinks -### Below: parallelise -echo '--------------------------- Launch Parallelization -----------------------' - + --exclude='updates' `# populated by https://github.com/jenkins-infra/crawler` \ + --delete `# delete old sites` \ + www2/ www3/ -## define function function parallelfunction() { echo "=== parallelfunction: $1" case $1 in rsync*) - # keep exclude as from www2 with symlinks - time rsync -acz "${ROOT_FOLDER}"/www2/ --exclude='updates' --delete --stats ${RSYNC_USER}@${UPDATES_SITE}:/tmp/lemeurherve/pr-745/www/${UPDATES_SITE} + # Push generated index to the production server + time rsync --archive --checksum --verbose --compress \ + --exclude=/updates `# populated by https://github.com/jenkins-infra/crawler` \ + --delete `# delete old sites` \ + --stats `# add verbose statistics` \ + www2/ ${RSYNC_USER}@${UPDATES_SITE}:/var/www/${UPDATES_SITE} ;; azsync*) - # Sync Azure File Share content (using www3 to avoid symlinks) - time azcopy sync "${ROOT_FOLDER}"/www3/ "https://updatesjenkinsio.file.core.windows.net/updates-jenkins-io/?${UPDATES_FILE_SHARE_QUERY_STRING}" --recursive=true --delete-destination=true + # Sync Azure File Share content using www3 to avoid symlinks + time azcopy sync ./www3/ "${UPDATES_FILE_SHARE_URL_AND_PATH}?${UPDATES_FILE_SHARE_QUERY_STRING}" \ + --recursive=true \ + --delete-destination=true ;; s3sync*) - ## Note: AWS CLI is configured through environment variables (from Jenkins credentials) - https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html - # Sync CloudFlare R2 buckets content using the updates-jenkins-io profile, excluding 'updates' folder which comes from tool installer generator (using www3 to avoid symlinks) - time aws s3 sync "${ROOT_FOLDER}"/www3/ s3://"${UPDATES_R2_BUCKETS}"/ \ + # Sync CloudFlare R2 buckets content excluding 'updates' folder from www3 sync (without symlinks) + # as this folder is populated by https://github.com/jenkins-infra/crawler/blob/master/Jenkinsfile + time aws s3 sync ./www3/ s3://"${UPDATES_R2_BUCKET}"/ \ --no-progress \ --no-follow-symlinks \ --size-only \ @@ -83,28 +88,44 @@ function parallelfunction() { ;; *) - echo -n "unknown" + echo -n "Warning: unknown parameter" ;; - esac + esac } -## need to export variables used within the functions above +# Export local variables used in parallelfunction export UPDATES_SITE export RSYNC_USER -export UPDATES_R2_BUCKETS +export UPDATES_R2_BUCKET export UPDATES_R2_ENDPOINT -export ROOT_FOLDER -export UPDATES_FILE_SHARE_QUERY_STRING +export UPDATES_FILE_SHARE_URL_AND_PATH -## export function to use with parallel +# Export function to use it with parallel export -f parallelfunction -parallel --halt-on-error now,fail=1 parallelfunction ::: rsync azsync s3sync -# wait for all deferred task -echo '=============================== all done ============================' +echo '----------------------- Launch synchronisation(s) -----------------------' +if [[ $OPT_IN_SYNC_FS_R2 == 'optin' ]] +then + # Sync updates.jenkins.io and azure.updates.jenkins.io + parallel --halt-on-error now,fail=1 parallelfunction ::: rsync azsync s3sync +else + # Sync only updates.jenkins.io + parallel --halt-on-error now,fail=1 parallelfunction ::: rsync + + # ## If we prefer to avoid parallel when not opt-in, we can replace the previous instruction by the following one: + # # push generated index to the production server + # rsync --archive --checksum --verbose --compress \ + # --exclude=/updates `# populated by https://github.com/jenkins-infra/crawler` \ + # --delete `# delete old sites` \ + # --stats `# add verbose statistics` \ + # ./www2/ ${RSYNC_USER}@${UPDATES_SITE}:/var/www/${UPDATES_SITE} +fi + +# Wait for all deferred tasks +echo '============================ all done ============================' -## Trigger a mirror scan on mirrorbits +echo '== Triggering a mirror scan on mirrorbits...' # Requires a valid kubernetes credential file at $KUBECONFIG or $HOME/.kube/config by default -pod_name="$(kubectl --namespace=updates-jenkins-io --no-headers=true get pod --output=name | grep mirrorbits-lite | head -n1)" -kubectl --namespace=updates-jenkins-io exec "${pod_name}" --container=mirrorbits-lite -- mirrorbits scan -all -enable -timeout=120 +pod_name="$(kubectl --namespace=${MIRRORBITS_NAMESPACE} --no-headers=true get pod --output=name | grep "${MIRRORBITS_POD_NAME_PREFIX}" | head -n1)" +kubectl --namespace=${MIRRORBITS_NAMESPACE} --container="${MIRRORBITS_CONTAINER_NAME}" exec "${pod_name}" -- mirrorbits scan -all -enable -timeout=120 From 87b9814bf006002323ffb18beb416918cd73098b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 17 Nov 2023 14:10:02 +0100 Subject: [PATCH 05/12] fix typo --- site/publish.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/publish.sh b/site/publish.sh index 0e53b421d..bc3bb617c 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -40,7 +40,7 @@ ssh ${RSYNC_USER}@${UPDATES_SITE} "cat > /tmp/update-center2-rerecent-releases.j ssh ${RSYNC_USER}@${UPDATES_SITE} "/srv/releases/sync-recent-releases.sh /tmp/update-center2-rerecent-releases.json" ## 'www2' folder processing -chmod -R a+r./ www2 +chmod -R a+r ./www2 # TIME sync, used by mirrorbits to know the last update date to take in account date +%s > ./www2/TIME From 70c1d91f994983bfb012a464fd0f030ced066877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 17 Nov 2023 15:38:27 +0100 Subject: [PATCH 06/12] array of R2 buckets --- site/publish.sh | 65 +++++++++++++++++++++++-------------------------- 1 file changed, 31 insertions(+), 34 deletions(-) diff --git a/site/publish.sh b/site/publish.sh index bc3bb617c..6472a9112 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -7,19 +7,9 @@ UPDATES_SITE="updates.jenkins.io" RSYNC_USER="www-data" -# For syncing R2 buckets with aws-cli configured through environment variables (from Jenkins credentials) +# For syncing R2 buckets aws-cli is configured through environment variables (from Jenkins credentials) # https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html export AWS_DEFAULT_REGION="auto" -UPDATES_R2_BUCKET="westeurope-updates-jenkins-io" -UPDATES_R2_ENDPOINT="https://8d1838a43923148c5cee18ccc356a594.r2.cloudflarestorage.com" - -# For syncing Azure File Share -UPDATES_FILE_SHARE_URL_AND_PATH="https://updatesjenkinsio.file.core.windows.net/updates-jenkins-io/" - -# For triggering a mirror scan on mirrorbits -MIRRORBITS_POD_NAME_PREFIX="mirrorbits-lite" -MIRRORBITS_CONTAINER_NAME="mirrorbits-lite" -MIRRORBITS_NAMESPACE="updates-jenkins-io" ## Install jq, required by generate.sh script wget --no-verbose -O jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 || { echo "Failed to download jq" >&2 ; exit 1; } @@ -71,20 +61,25 @@ function parallelfunction() { azsync*) # Sync Azure File Share content using www3 to avoid symlinks - time azcopy sync ./www3/ "${UPDATES_FILE_SHARE_URL_AND_PATH}?${UPDATES_FILE_SHARE_QUERY_STRING}" \ + time azcopy sync ./www3/ "https://updatesjenkinsio.file.core.windows.net/updates-jenkins-io/?${UPDATES_FILE_SHARE_QUERY_STRING}" \ --recursive=true \ --delete-destination=true ;; s3sync*) + # Retrieve the R2 bucket and the R2 endpoint from the task name passed as argument, minus "s3sync" prefix + updates_r2_bucket_and_endpoint="${1#s3sync}" + r2_bucket=${updates_r2_bucket_and_endpoint%|*} + r2_endpoint=${updates_r2_bucket_and_endpoint#*|} + # Sync CloudFlare R2 buckets content excluding 'updates' folder from www3 sync (without symlinks) # as this folder is populated by https://github.com/jenkins-infra/crawler/blob/master/Jenkinsfile - time aws s3 sync ./www3/ s3://"${UPDATES_R2_BUCKET}"/ \ + time aws s3 sync ./www3/ s3://"${r2_bucket}"/ \ --no-progress \ --no-follow-symlinks \ --size-only \ --exclude '.htaccess' \ - --endpoint-url "${UPDATES_R2_ENDPOINT}" + --endpoint-url "${r2_endpoint}" ;; *) @@ -97,35 +92,37 @@ function parallelfunction() { # Export local variables used in parallelfunction export UPDATES_SITE export RSYNC_USER -export UPDATES_R2_BUCKET -export UPDATES_R2_ENDPOINT -export UPDATES_FILE_SHARE_URL_AND_PATH # Export function to use it with parallel export -f parallelfunction -echo '----------------------- Launch synchronisation(s) -----------------------' -if [[ $OPT_IN_SYNC_FS_R2 == 'optin' ]] +# Sync only updates.jenkins.io by default +tasks=("rsync") + +# Sync updates.jenkins.io and azure.updates.jenkins.io File Share and R2 bucket(s) if the flag is set +if [[ $OPT_IN_SYNC_FS_R2 == "optin" ]] then - # Sync updates.jenkins.io and azure.updates.jenkins.io - parallel --halt-on-error now,fail=1 parallelfunction ::: rsync azsync s3sync -else - # Sync only updates.jenkins.io - parallel --halt-on-error now,fail=1 parallelfunction ::: rsync - - # ## If we prefer to avoid parallel when not opt-in, we can replace the previous instruction by the following one: - # # push generated index to the production server - # rsync --archive --checksum --verbose --compress \ - # --exclude=/updates `# populated by https://github.com/jenkins-infra/crawler` \ - # --delete `# delete old sites` \ - # --stats `# add verbose statistics` \ - # ./www2/ ${RSYNC_USER}@${UPDATES_SITE}:/var/www/${UPDATES_SITE} + # Add File Share sync to the tasks + tasks+=("azsync") + + # Add each R2 bucket sync to the tasks + updates_r2_bucket_and_endpoint_pairs=("westeurope-updates-jenkins-io|https://8d1838a43923148c5cee18ccc356a594.r2.cloudflarestorage.com") + for r2_bucket_and_endpoint_pair in "${updates_r2_bucket_and_endpoint_pairs[@]}" + do + tasks+=("s3sync${r2_bucket_and_endpoint_pair}") + done fi +echo '----------------------- Launch synchronisation(s) -----------------------' +parallel --halt-on-error now,fail=1 parallelfunction ::: "${tasks[@]}" + # Wait for all deferred tasks echo '============================ all done ============================' echo '== Triggering a mirror scan on mirrorbits...' +# Kubernetes namespace of mirrorbits +mirrorbits_namespace="updates-jenkins-io" + # Requires a valid kubernetes credential file at $KUBECONFIG or $HOME/.kube/config by default -pod_name="$(kubectl --namespace=${MIRRORBITS_NAMESPACE} --no-headers=true get pod --output=name | grep "${MIRRORBITS_POD_NAME_PREFIX}" | head -n1)" -kubectl --namespace=${MIRRORBITS_NAMESPACE} --container="${MIRRORBITS_CONTAINER_NAME}" exec "${pod_name}" -- mirrorbits scan -all -enable -timeout=120 +pod_name="$(kubectl --namespace=${mirrorbits_namespace} --no-headers=true get pod --output=name | grep mirrorbits-lite | head -n1)" +kubectl --namespace=${mirrorbits_namespace} --container=mirrorbits-lite exec "${pod_name}" -- mirrorbits scan -all -enable -timeout=120 From 0f75f158f32698b7818f02d156e3699eb7490b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 17 Nov 2023 15:56:33 +0100 Subject: [PATCH 07/12] move new operations to the opt-in block --- site/publish.sh | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/site/publish.sh b/site/publish.sh index 6472a9112..bf3d81aad 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -32,20 +32,6 @@ ssh ${RSYNC_USER}@${UPDATES_SITE} "/srv/releases/sync-recent-releases.sh /tmp/up ## 'www2' folder processing chmod -R a+r ./www2 -# TIME sync, used by mirrorbits to know the last update date to take in account -date +%s > ./www2/TIME - -## No need to remove the symlinks as the `azcopy sync` for symlinks is not yet supported and we use `--no-follow-symlinks` for `aws s3 sync` -# Perform a copy with dereference symlink (object storage do not support symlinks) -# copy & transform simlinks into referent file/dir -rsync --archive --checksum --verbose --compress \ - --copy-links `# derefence symlinks` \ - --safe-links `# ignore symlinks outside of copied tree` \ - --stats `# add verbose statistics` \ - --exclude='updates' `# populated by https://github.com/jenkins-infra/crawler` \ - --delete `# delete old sites` \ - www2/ www3/ - function parallelfunction() { echo "=== parallelfunction: $1" @@ -102,6 +88,20 @@ tasks=("rsync") # Sync updates.jenkins.io and azure.updates.jenkins.io File Share and R2 bucket(s) if the flag is set if [[ $OPT_IN_SYNC_FS_R2 == "optin" ]] then + # TIME sync, used by mirrorbits to know the last update date to take in account + date +%s > ./www2/TIME + + ## No need to remove the symlinks as the `azcopy sync` for symlinks is not yet supported and we use `--no-follow-symlinks` for `aws s3 sync` + # Perform a copy with dereference symlink (object storage do not support symlinks) + # copy & transform simlinks into referent file/dir + rsync --archive --checksum --verbose --compress \ + --copy-links `# derefence symlinks` \ + --safe-links `# ignore symlinks outside of copied tree` \ + --stats `# add verbose statistics` \ + --exclude='updates' `# populated by https://github.com/jenkins-infra/crawler` \ + --delete `# delete old sites` \ + www2/ www3/ + # Add File Share sync to the tasks tasks+=("azsync") From c15b967ffd9ef8e6c8829d8b3bb2ed1f83034835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 17 Nov 2023 15:58:46 +0100 Subject: [PATCH 08/12] put mirrorbits triggering behind the opt-in flag --- site/publish.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/site/publish.sh b/site/publish.sh index bf3d81aad..26a51ddea 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -119,10 +119,14 @@ parallel --halt-on-error now,fail=1 parallelfunction ::: "${tasks[@]}" # Wait for all deferred tasks echo '============================ all done ============================' -echo '== Triggering a mirror scan on mirrorbits...' -# Kubernetes namespace of mirrorbits -mirrorbits_namespace="updates-jenkins-io" +# Trigger a mirror scan on mirrorbits if the flag is set +if [[ $OPT_IN_SYNC_FS_R2 == "optin" ]] +then + echo '== Triggering a mirror scan on mirrorbits...' + # Kubernetes namespace of mirrorbits + mirrorbits_namespace="updates-jenkins-io" -# Requires a valid kubernetes credential file at $KUBECONFIG or $HOME/.kube/config by default -pod_name="$(kubectl --namespace=${mirrorbits_namespace} --no-headers=true get pod --output=name | grep mirrorbits-lite | head -n1)" -kubectl --namespace=${mirrorbits_namespace} --container=mirrorbits-lite exec "${pod_name}" -- mirrorbits scan -all -enable -timeout=120 + # Requires a valid kubernetes credential file at $KUBECONFIG or $HOME/.kube/config by default + pod_name="$(kubectl --namespace=${mirrorbits_namespace} --no-headers=true get pod --output=name | grep mirrorbits-lite | head -n1)" + kubectl --namespace=${mirrorbits_namespace} --container=mirrorbits-lite exec "${pod_name}" -- mirrorbits scan -all -enable -timeout=120 +fi From f3c68198834f81404a269deda0d5a36edda5f37f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= <91831478+lemeurherve@users.noreply.github.com> Date: Fri, 17 Nov 2023 18:29:30 +0100 Subject: [PATCH 09/12] Apply suggestions from code review Co-authored-by: Damien Duportal --- site/publish.sh | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/site/publish.sh b/site/publish.sh index 26a51ddea..f07882967 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -9,7 +9,7 @@ RSYNC_USER="www-data" # For syncing R2 buckets aws-cli is configured through environment variables (from Jenkins credentials) # https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html -export AWS_DEFAULT_REGION="auto" +export AWS_DEFAULT_REGION='auto' ## Install jq, required by generate.sh script wget --no-verbose -O jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 || { echo "Failed to download jq" >&2 ; exit 1; } @@ -22,7 +22,7 @@ export PATH=.:$PATH ## 'download' folder processing # push plugins to mirrors.jenkins-ci.org -chmod -R a+r ./download +chmod -R a+r download rsync -avz --size-only download/plugins/ ${RSYNC_USER}@${UPDATES_SITE}:/srv/releases/jenkins/plugins # Invoke a minimal mirrorsync to mirrorbits which will use the 'recent-releases.json' file as input @@ -30,7 +30,7 @@ ssh ${RSYNC_USER}@${UPDATES_SITE} "cat > /tmp/update-center2-rerecent-releases.j ssh ${RSYNC_USER}@${UPDATES_SITE} "/srv/releases/sync-recent-releases.sh /tmp/update-center2-rerecent-releases.json" ## 'www2' folder processing -chmod -R a+r ./www2 +chmod -R a+r www2 function parallelfunction() { echo "=== parallelfunction: $1" @@ -42,7 +42,7 @@ function parallelfunction() { --exclude=/updates `# populated by https://github.com/jenkins-infra/crawler` \ --delete `# delete old sites` \ --stats `# add verbose statistics` \ - www2/ ${RSYNC_USER}@${UPDATES_SITE}:/var/www/${UPDATES_SITE} + ./www2/ "${RSYNC_USER}@${UPDATES_SITE}:/var/www/${UPDATES_SITE}" ;; azsync*) @@ -60,7 +60,7 @@ function parallelfunction() { # Sync CloudFlare R2 buckets content excluding 'updates' folder from www3 sync (without symlinks) # as this folder is populated by https://github.com/jenkins-infra/crawler/blob/master/Jenkinsfile - time aws s3 sync ./www3/ s3://"${r2_bucket}"/ \ + time aws s3 sync ./www3/ "s3://${r2_bucket}/" \ --no-progress \ --no-follow-symlinks \ --size-only \ @@ -69,7 +69,7 @@ function parallelfunction() { ;; *) - echo -n "Warning: unknown parameter" + echo -n 'Warning: unknown parameter' ;; esac @@ -82,28 +82,31 @@ export RSYNC_USER # Export function to use it with parallel export -f parallelfunction +# parallel added within the permanent trusted agent here: +# https://github.com/jenkins-infra/jenkins-infra/blob/production/dist/profile/manifests/buildagent.pp +command -v parallel >/dev/null 2>&1 || { echo 'ERROR: parralel command not found. Exiting.'; exit 1; } + # Sync only updates.jenkins.io by default -tasks=("rsync") +tasks=('rsync') # Sync updates.jenkins.io and azure.updates.jenkins.io File Share and R2 bucket(s) if the flag is set -if [[ $OPT_IN_SYNC_FS_R2 == "optin" ]] +if [[ ${OPT_IN_SYNC_FS_R2} == 'optin' ]] then # TIME sync, used by mirrorbits to know the last update date to take in account date +%s > ./www2/TIME ## No need to remove the symlinks as the `azcopy sync` for symlinks is not yet supported and we use `--no-follow-symlinks` for `aws s3 sync` # Perform a copy with dereference symlink (object storage do not support symlinks) - # copy & transform simlinks into referent file/dir - rsync --archive --checksum --verbose --compress \ - --copy-links `# derefence symlinks` \ - --safe-links `# ignore symlinks outside of copied tree` \ - --stats `# add verbose statistics` \ - --exclude='updates' `# populated by https://github.com/jenkins-infra/crawler` \ - --delete `# delete old sites` \ - www2/ www3/ + rm -rf ./www3/ # Cleanup + + rsync --archive --verbose \ + --copy-links `# derefence symlinks` \ + --safe-links `# ignore symlinks outside of copied tree` \ + --exclude='updates' `# Exclude ALL 'updates' directories, not only the root /updates (because symlink dereferencing create additional directories` \ + ./www2/ ./www3/ # Add File Share sync to the tasks - tasks+=("azsync") + tasks+=('azsync') # Add each R2 bucket sync to the tasks updates_r2_bucket_and_endpoint_pairs=("westeurope-updates-jenkins-io|https://8d1838a43923148c5cee18ccc356a594.r2.cloudflarestorage.com") @@ -120,13 +123,13 @@ parallel --halt-on-error now,fail=1 parallelfunction ::: "${tasks[@]}" echo '============================ all done ============================' # Trigger a mirror scan on mirrorbits if the flag is set -if [[ $OPT_IN_SYNC_FS_R2 == "optin" ]] +if [[ ${OPT_IN_SYNC_FS_R2} == 'optin' ]] then echo '== Triggering a mirror scan on mirrorbits...' # Kubernetes namespace of mirrorbits - mirrorbits_namespace="updates-jenkins-io" + mirrorbits_namespace='updates-jenkins-io' # Requires a valid kubernetes credential file at $KUBECONFIG or $HOME/.kube/config by default - pod_name="$(kubectl --namespace=${mirrorbits_namespace} --no-headers=true get pod --output=name | grep mirrorbits-lite | head -n1)" - kubectl --namespace=${mirrorbits_namespace} --container=mirrorbits-lite exec "${pod_name}" -- mirrorbits scan -all -enable -timeout=120 + pod_name=$(kubectl --namespace="${mirrorbits_namespace}" --no-headers=true get pod --output=name | grep mirrorbits-lite | head -n1) + kubectl --namespace="${mirrorbits_namespace}" --container=mirrorbits-lite exec "${pod_name}" -- mirrorbits scan -all -enable -timeout=120 fi From 189e6d0793bab918c1bf4fc0fc1219300c7a80c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 1 Dec 2023 15:55:07 +0100 Subject: [PATCH 10/12] fixup add missing azcopy exclude-path arg --- site/publish.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/site/publish.sh b/site/publish.sh index f07882967..70398c909 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -49,6 +49,7 @@ function parallelfunction() { # Sync Azure File Share content using www3 to avoid symlinks time azcopy sync ./www3/ "https://updatesjenkinsio.file.core.windows.net/updates-jenkins-io/?${UPDATES_FILE_SHARE_QUERY_STRING}" \ --recursive=true \ + --exclude-path="updates" `# populated by https://github.com/jenkins-infra/crawler` \ --delete-destination=true ;; From 626182337168800b8b3bab72919d45e7a75d1423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Thu, 7 Mar 2024 20:44:46 +0100 Subject: [PATCH 11/12] mirrorbits pod name doesn't contain "-lite" anymore --- site/publish.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/site/publish.sh b/site/publish.sh index 70398c909..8abf2a75a 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -131,6 +131,6 @@ then mirrorbits_namespace='updates-jenkins-io' # Requires a valid kubernetes credential file at $KUBECONFIG or $HOME/.kube/config by default - pod_name=$(kubectl --namespace="${mirrorbits_namespace}" --no-headers=true get pod --output=name | grep mirrorbits-lite | head -n1) - kubectl --namespace="${mirrorbits_namespace}" --container=mirrorbits-lite exec "${pod_name}" -- mirrorbits scan -all -enable -timeout=120 + pod_name=$(kubectl --namespace="${mirrorbits_namespace}" --no-headers=true get pod --output=name | grep mirrorbits | head -n1) + kubectl --namespace="${mirrorbits_namespace}" --container=mirrorbits exec "${pod_name}" -- mirrorbits scan -all -enable -timeout=120 fi From ebfac65aaca04e42e8c17ec9bd42c233f0a72c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Le=20Meur?= Date: Fri, 8 Mar 2024 12:14:58 +0100 Subject: [PATCH 12/12] short-lived SAS token by using get-fileshare-signed-url.sh --- site/publish.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/site/publish.sh b/site/publish.sh index 8abf2a75a..0085f1f6f 100755 --- a/site/publish.sh +++ b/site/publish.sh @@ -46,8 +46,11 @@ function parallelfunction() { ;; azsync*) + # Script stored in /usr/local/bin used to generate a signed file share URL with a short-lived SAS token + # Source: https://github.com/jenkins-infra/pipeline-library/blob/master/resources/get-fileshare-signed-url.sh + fileShareUrl=$(get-fileshare-signed-url.sh) # Sync Azure File Share content using www3 to avoid symlinks - time azcopy sync ./www3/ "https://updatesjenkinsio.file.core.windows.net/updates-jenkins-io/?${UPDATES_FILE_SHARE_QUERY_STRING}" \ + time azcopy sync ./www3/ "${fileShareUrl}" \ --recursive=true \ --exclude-path="updates" `# populated by https://github.com/jenkins-infra/crawler` \ --delete-destination=true @@ -80,6 +83,12 @@ function parallelfunction() { export UPDATES_SITE export RSYNC_USER +# Export variables used in parallelfunction/azsync/get-fileshare-signed-url.sh +export STORAGE_FILESHARE=updates-jenkins-io +export STORAGE_NAME=updatesjenkinsio +export STORAGE_DURATION_IN_MINUTE=5 # duration of the short-lived SAS token +export STORAGE_PERMISSIONS=dlrw + # Export function to use it with parallel export -f parallelfunction