ci: publish and use s3 wheels by commit sha

.gitlab/benchmarks/steps/build-baseline.sh

@@ -2,20 +2,30 @@
 set -e -o pipefail
 
 # If we have a tag (e.g. v2.21.1), then use the PyPI published wheel
-# Otherwise, build the wheel from commit hash
+# Otherwise, try to download from S3 by commit SHA, or build the wheel from scratch
 if [[ -n "${BASELINE_TAG}" ]];
 then
   python3.9 -m pip download --no-deps "ddtrace==${BASELINE_TAG:1}"
 else
-  ulimit -c unlimited
-  curl -sSf https://sh.rustup.rs | sh -s -- -y;
-  export PATH="$HOME/.cargo/bin:$PATH"
-  echo "Building wheel for ${BASELINE_BRANCH}:${BASELINE_COMMIT_SHA}"
-  git checkout "${BASELINE_COMMIT_SHA}"
-  mkdir ./tmp
-  PYO3_PYTHON=python3.9 CIBW_BUILD=1 python3.9 -m pip wheel --no-deps -w ./tmp/ ./
-  for wheel in ./tmp/*.whl;
-  do
-    auditwheel repair "$wheel" --plat "manylinux2014_x86_64" -w ./
-  done
+  # Try to download the wheel from S3 using the baseline commit SHA
+  S3_BUCKET="dd-trace-py-builds"
+  S3_INDEX_URL="https://${S3_BUCKET}.s3.amazonaws.com/${BASELINE_COMMIT_SHA}/index.html"
+
+  echo "Attempting to download wheel from S3 index: ${S3_INDEX_URL}"
+  if python3.9 -m pip download --no-deps --index-url "${S3_INDEX_URL}" ddtrace 2>/dev/null; then
+    echo "Successfully downloaded wheel from S3"
+  else
+    echo "Failed to download from S3, building wheel from scratch..."
+    ulimit -c unlimited
+    curl -sSf https://sh.rustup.rs | sh -s -- -y;
+    export PATH="$HOME/.cargo/bin:$PATH"
+    echo "Building wheel for ${BASELINE_BRANCH}:${BASELINE_COMMIT_SHA}"
+    git checkout "${BASELINE_COMMIT_SHA}"
+    mkdir ./tmp
+    PYO3_PYTHON=python3.9 CIBW_BUILD=1 python3.9 -m pip wheel --no-deps -w ./tmp/ ./
+    for wheel in ./tmp/*.whl;
+    do
+      auditwheel repair "$wheel" --plat "manylinux2014_x86_64" -w ./
+    done
+  fi
 fi

.gitlab/package.yml

@@ -74,21 +74,12 @@ publish-wheels-to-s3:
     - echo "Found ${#WHEELS[@]} wheel(s):"
     - printf ' - %s\n' "${WHEELS[@]}"
 
-    - |
-      VERSION=$(grep '^version = ' pyproject.toml | head -1 | sed -E 's/version = "(.+)"/\1/')
-
-      if [ -z "${VERSION:-}" ]; then
-        echo "ERROR: VERSION is not defined in pyproject.toml!"
-        exit 1
-      fi
-
-    - printf 'Detected version %s\n' ${VERSION}
-
-    # Upload all wheels to pipeline-id prefix
+    # Upload all wheels to commit prefix and pipeline-id prefix
+    - aws s3 cp --recursive --exclude "*" --include "*.whl" pywheels "s3://${BUCKET}/${CI_COMMIT_SHA}/"
     - aws s3 cp --recursive --exclude "*" --include "*.whl" pywheels "s3://${BUCKET}/${CI_PIPELINE_ID}/"
 
     - |
-      VERSION_ENC="${VERSION//+/%2B}"
+      S3_BASE_COMMIT="https://${BUCKET}.s3.amazonaws.com/${CI_COMMIT_SHA}"
       S3_BASE_PIPE="https://${BUCKET}.s3.amazonaws.com/${CI_PIPELINE_ID}"
 
       generate_index_html() {
@@ -105,13 +96,17 @@ publish-wheels-to-s3:
       }
 
       # Generate both minimal indexes
+      generate_index_html "index.commit.html"
       generate_index_html "index.pipeline.html"
 
       # Upload to each S3 prefix
+      aws s3 cp "index.commit.html" "s3://${BUCKET}/${CI_COMMIT_SHA}/index.html" --content-type text/html
       aws s3 cp "index.pipeline.html" "s3://${BUCKET}/${CI_PIPELINE_ID}/index.html" --content-type text/html
 
       # Print the clickable URLs
+      COMMIT_INDEX_URL="${S3_BASE_COMMIT}/index.html"
       PIPE_INDEX_URL="${S3_BASE_PIPE}/index.html"
+      echo "S3 index (commit):   ${COMMIT_INDEX_URL}"
       echo "S3 index (pipeline): ${PIPE_INDEX_URL}"