Use the new SCCACHE_PID_FILE envvar so the sccache daemon doesn't randomly change pids

.github/actions/build-and-test-feature/action.yml

@@ -5,6 +5,7 @@ description: Test feature
 inputs:
   args: {type: string, required: true}
   gh_token: {type: string, defaut: '', required: false}
+  aws_idp_url: {type: string, defaut: '', required: false}
   aws_role_arn: {type: string, defaut: '', required: false}
   rw_sccache_bucket: {type: string, defaut: '', required: false}
   rw_sccache_region: {type: string, defaut: '', required: false}
@@ -31,6 +32,7 @@ runs:
         VAULT_S3_TTL: "900" # 15 minutes
         ARGS: ${{ inputs.args }}
         gh_token: "${{ inputs.gh_token }}"
+        aws_idp_url: "${{ inputs.aws_idp_url }}"
         aws_role_arn: "${{ inputs.aws_role_arn }}"
         rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}"
         rw_sccache_region: "${{ inputs.rw_sccache_region }}"

.github/workflows/build-and-test-feature.yml

@@ -37,7 +37,8 @@ jobs:
         with:
           args: "${{ inputs.args }}"
           gh_token: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN }}"
-          aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs' || '' }}"
+          aws_idp_url: ${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'https://token.rapids.nvidia.com' || '' }}
+          aws_role_arn: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'arn:aws:iam::279114543810:role/rapids-token-sccache-devs' || '' }}"
           rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}"
           rw_sccache_region: "${{ vars.AWS_REGION }}"
           sccache_dist_scheduler_url: "sccache.rapids.nvidia.com"

features/src/utils/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "devcontainer-utils",
   "id": "utils",
-  "version": "26.8.0",
+  "version": "26.8.1",
   "description": "A feature to install RAPIDS devcontainer utility scripts",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

features/src/utils/opt/devcontainer/bin/creds/s3/gh/generate.sh

@@ -10,20 +10,9 @@ _creds_github_generate() {
     if ! test -n "${AWS_ROLE_ARN:+x}" \
     || ! test -n "${SCCACHE_BUCKET:+x}" \
     || ! gh nv-gha-aws --help >/dev/null 2>&1; then
-        exit 1;
+        return 1;
     fi
 
-    # Remove existing credentials in case nv-gha-aws declines to issue new ones.
-    if test -w ~/.aws; then
-        rm -rf ~/.aws/{stamp,config,credentials};
-    fi
-
-    SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}";
-
-    devcontainer-utils-creds-s3-persist - <<< \
-        --bucket="${SCCACHE_BUCKET:-}"        \
-        --region="${SCCACHE_REGION:-}"        ;
-
     # Initialize the GitHub CLI with the appropriate user scopes
     # shellcheck disable=SC1091
     . devcontainer-utils-init-github-cli;
@@ -41,7 +30,7 @@ _creds_github_generate() {
     ))";
 
     if test "${#user_orgs[@]}" -eq 0; then
-        exit 1;
+        return 1;
     fi
 
     local org;
@@ -55,6 +44,8 @@ _creds_github_generate() {
         --idp-url "${AWS_IDP_URL:-https://token.gha-runners.nvidia.com}"
     );
 
+    mkdir -p ~/.aws;
+
     for org in "${user_orgs[@]}"; do
         generated_at="$(date '+%s')";
         if gh nv-gha-aws org "${org}" "${nv_gha_aws_args[@]}" >"${HOME}/.aws/credentials" 2>>/var/log/devcontainer-utils/creds-s3.log; then
@@ -67,7 +58,25 @@ _creds_github_generate() {
         fi
     done
 
-    exit 1;
+    return 1;
 }
 
-_creds_github_generate "$@" <&0;
+if ! _creds_github_generate "$@" <&0; then
+
+    # Remove existing credentials in case nv-gha-aws declines to issue new ones.
+    if test -w ~/.aws; then
+        rm -rf ~/.aws/{stamp,config,credentials};
+    fi
+
+    devcontainer-utils-creds-s3-persist - <<<                 \
+        --bucket="${SCCACHE_BUCKET:-}"                        \
+        --region="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}" ;
+
+    # shellcheck disable=SC1090
+    . /etc/profile.d/*-devcontainer-utils.sh;
+
+    exit 1;
+fi
+
+# shellcheck disable=SC1090
+. /etc/profile.d/*-devcontainer-utils.sh;

features/src/utils/opt/devcontainer/bin/creds/s3/persist.sh

@@ -91,19 +91,19 @@ ________EOF
 
     if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then
         if test -w ~/.aws/credentials; then
-            reset_envvar "AWS_SESSION_TOKEN";
+            reset_envvar "AWS_SECRET_ACCESS_KEY";
             cat <<< "aws_secret_access_key=${aws_secret_access_key}" >> ~/.aws/credentials
         else
-            export_envvar "AWS_SESSION_TOKEN" "${aws_secret_access_key}";
+            export_envvar "AWS_SECRET_ACCESS_KEY" "${aws_secret_access_key}";
         fi
     fi
 
     if ! grep -qE "^$" <<< "${aws_session_token:-}"; then
         if test -w ~/.aws/credentials; then
-            reset_envvar "AWS_SECRET_ACCESS_KEY";
+            reset_envvar "AWS_SESSION_TOKEN";
             cat <<< "aws_session_token=${aws_session_token}" >> ~/.aws/credentials
         else
-            export_envvar "AWS_SECRET_ACCESS_KEY" "${aws_session_token}";
+            export_envvar "AWS_SESSION_TOKEN" "${aws_session_token}";
         fi
     fi
 

features/src/utils/opt/devcontainer/bin/creds/s3/test.sh

@@ -32,7 +32,10 @@ _creds_s3_test() {
     local aws_session_token="${AWS_SESSION_TOKEN:-"$(sed -n 's/aws_session_token=//p' ~/.aws/credentials 2>/dev/null)"}";
     local aws_secret_access_key="${AWS_SECRET_ACCESS_KEY:-"$(sed -n 's/aws_secret_access_key=//p' ~/.aws/credentials 2>/dev/null)"}";
 
-    devcontainer-utils-stop-sccache --kill -p 4220 || true;
+    if ! timeout --preserve-status --kill-after=20s 15s \
+         devcontainer-utils-stop-sccache -p 4220; then
+         devcontainer-utils-stop-sccache -p 4220 --kill || :
+    fi
 
     local result=0;
 
@@ -52,7 +55,12 @@ _creds_s3_test() {
        result=1;
     fi
 
-    devcontainer-utils-stop-sccache --kill -p 4220 || true;
+    devcontainer-utils-stop-sccache --kill -p 4220 || :
+
+    if ! timeout --preserve-status --kill-after=20s 15s \
+         devcontainer-utils-stop-sccache -p 4220; then
+         devcontainer-utils-stop-sccache -p 4220 --kill || :
+    fi
 
     if test "$result" -eq 0; then
         local logfile="${SCCACHE_ERROR_LOG:-/tmp/sccache.log}";

features/src/utils/opt/devcontainer/bin/creds/s3/vault/generate.sh

@@ -11,27 +11,17 @@ _creds_vault_generate() {
 
     if ! test -n "${VAULT_HOST:+x}" \
     || ! test -n "${SCCACHE_BUCKET:+x}"; then
-        exit 1;
+        return 1;
     fi
 
     SCCACHE_REGION="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}";
 
-    # Remove existing credentials in case vault declines to issue new ones.
-    if test -w ~/.aws; then
-        rm -rf ~/.aws/{stamp,config,credentials};
-    fi
-
-    devcontainer-utils-creds-s3-persist - <<< " \
-        --bucket '${SCCACHE_BUCKET:-}'          \
-        --region '${SCCACHE_REGION:-}'          \
-    ";
-
     # Initialize the GitHub CLI with the appropriate user scopes
     # shellcheck disable=SC1091
     . devcontainer-utils-init-github-cli;
 
     if ! test -n "${GITHUB_USER:+x}"; then
-        exit 1;
+        return 1;
     fi
 
     # Check whether the user is in one of the allowed GitHub orgs
@@ -47,7 +37,7 @@ _creds_vault_generate() {
     )";
 
     if test "${#user_orgs}" -eq 0; then
-        exit 1;
+        return 1;
     fi
 
     cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
@@ -65,7 +55,7 @@ ____EOF
         cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
 Your GitHub user was not recognized by vault. Skipping.
 ________EOF
-        exit 1;
+        return 1;
     fi
 
     cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
@@ -97,14 +87,14 @@ ____EOF
         cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
 Failed to retrieve AWS S3 credentials. Skipping.
 ________EOF
-        exit 1;
+        return 1;
     fi
 
     if grep -qE "^null$" <<< "${aws_secret_access_key:-null}"; then
         cat <<________EOF | tee -a /var/log/devcontainer-utils/creds-s3.log >&2
 Failed to retrieve AWS S3 credentials. Skipping.
 ________EOF
-        exit 1;
+        return 1;
     fi
 
     cat <<____EOF | tee -a /var/log/devcontainer-utils/creds-s3.log
@@ -127,7 +117,22 @@ ____EOF
     fi
 }
 
-_creds_vault_generate "$@";
+if ! _creds_vault_generate "$@" <&0; then
+
+    # Remove existing credentials in case vault declines to issue new ones.
+    if test -w ~/.aws; then
+        rm -rf ~/.aws/{stamp,config,credentials};
+    fi
+
+    devcontainer-utils-creds-s3-persist - <<<                 \
+        --bucket="${SCCACHE_BUCKET:-}"                        \
+        --region="${SCCACHE_REGION:-${AWS_DEFAULT_REGION:-}}" ;
+
+    # shellcheck disable=SC1090
+    . /etc/profile.d/*-devcontainer-utils.sh;
+
+    exit 1;
+fi
 
 # shellcheck disable=SC1090
 . /etc/profile.d/*-devcontainer-utils.sh;

features/src/utils/opt/devcontainer/bin/post-attach-command.sh

@@ -17,10 +17,6 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:+x}"; then
 
     # Update sccache client configuration to enable/disable sccache-dist
     if test -n "${DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST:+x}"; then
-        devcontainer-utils-install-sccache                   \
-            --repo "${SCCACHE_REPOSITORY:-rapidsai/sccache}" \
-            --version "${SCCACHE_VERSION:-rapids}"           \
-        ;
         if test -n "${SCCACHE_DIST_AUTH_TOKEN:+x}"; then
             devcontainer-utils-init-sccache-dist        \
                 --enable-sccache-dist - <<< "           \
@@ -39,6 +35,6 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_ATTACH_COMMAND:+x}"; then
         fi
     elif command -V sccache >/dev/null 2>&1; then
         # Start the sccache client
-        devcontainer-utils-start-sccache --kill-all;
+        devcontainer-utils-start-sccache;
     fi
 fi

features/src/utils/opt/devcontainer/bin/post-start-command.sh

@@ -7,8 +7,17 @@ if ! test -n "${SKIP_DEVCONTAINER_UTILS_POST_START_COMMAND:+x}"; then
 
     # Fast parallel `chown -R`
     find ~/ /var/log/devcontainer-utils/ -not -user coder -print0 2>/dev/null \
-  | sudo xargs -0 -r -n1 -P"$(nproc --all)" chown "$(id -u):$(id -g)" 2>/dev/null \
- || true;
+  | sudo xargs -0 -r -n1 -P"$(nproc --all)" chown "$(id -u):$(id -g)" 2>/dev/null &
+
+    if test -n "${DEVCONTAINER_UTILS_ENABLE_SCCACHE_DIST:+x}"; then
+        # Install latest sccache client
+        devcontainer-utils-install-sccache                   \
+            --repo "${SCCACHE_REPOSITORY:-rapidsai/sccache}" \
+            --version "${SCCACHE_VERSION:-rapids}"           \
+        ;
+    fi
+
+    wait || :
 
     # shellcheck disable=SC1091
     . devcontainer-utils-init-git;

features/src/utils/opt/devcontainer/bin/sccache/dist/init.sh

@@ -75,7 +75,7 @@ _init_sccache_dist() {
     done
 
     # Restart the sccache client with the new configuration
-    devcontainer-utils-start-sccache --kill-all;
+    devcontainer-utils-start-sccache;
 
     # Verify sccache-dist status and configuration
     if sccache --dist-status 2>/dev/null | jq -er '.SchedulerStatus? != null' >/dev/null 2>&1; then

features/src/utils/opt/devcontainer/bin/sccache/start.sh

@@ -58,15 +58,21 @@ _start_sccache() {
     else
         # Start the sccache server in the background
         RUST_LOG_STYLE="never"                  \
+        SCCACHE_PID_FILE="${pidfile}"           \
         SCCACHE_ERROR_LOG="${logfile}"          \
         SCCACHE_SERVER_LOG="${log_lvl}"         \
         SCCACHE_SERVER_PORT="${sccache_port}"   \
         sccache --start-server 1>&2 2>/dev/null \
       | tee "$logfile";
-        # Write the pid to the pidfile
-        pgrep sccache | sort -n | head -n1 | tee "${pidfile}" >/dev/null;
+
+        # If the pidfile doesn't exist, write it
+        if ! test -s "${pidfile}"; then
+            pgrep sccache | sort -n | head -n1 | tee "${pidfile}" >/dev/null;
+        fi
+
         # Increase the open file limit so users can do `make -j(ulimit -n)`
         prlimit --nofile=$(ulimit -Hn):$(ulimit -Hn) --pid "$(cat "${pidfile}")";
+
         echo "=== sccache logfile: $logfile ===" >&2;
         echo "=== sccache pidfile: $pidfile ===" >&2;
     fi

features/src/utils/opt/devcontainer/bin/sccache/stop.sh

@@ -33,31 +33,32 @@ _stop_sccache() {
 
     if test -n "${a:-${kill_all:+x}}"; then
         # Shutdown all sccache processes forcefully
-        sudo pkill -9 sccache >/dev/null 2>&1 || true;
+        sudo pkill -9 sccache >/dev/null 2>&1 || :
+        rm -f /tmp/sccache.*.pid 2>/dev/null || :
     elif test -n "${k:-${kill:+x}}" && test -f "${pidfile}"; then
         # Shutdown the sccache process on `$sccache_port` forcefully
-        sudo pkill -9 --pidfile "${pidfile}" >/dev/null 2>&1 || true;
+        sudo pkill -9 --pidfile "${pidfile}" >/dev/null 2>&1 || :
+        rm -f "${pidfile}" 2>/dev/null || :
     else
         # Shutdown gracefully
         SCCACHE_SERVER_PORT="${sccache_port}" \
-        sccache --stop-server >/dev/null 2>&1 || true;
+        sccache --stop-server >/dev/null 2>&1 || :
         if test -f "${pidfile}"; then
             # Wait for the server to shutdown
             if command -V pidwait >/dev/null 2>&1; then
-                pidwait --pidfile "${pidfile}" >/dev/null 2>&1 || true;
+                pidwait --pidfile "${pidfile}" >/dev/null 2>&1 || :
             else
                 while IFS= read -r pid; do
                     if test -n "${pid:+x}"; then
                         while test -e "/proc/${pid}"; do
-                            sleep 0.1;
+                            sleep 1;
                         done
                     fi
                 done < "${pidfile}"
             fi
+            rm -f "${pidfile}" 2>/dev/null || :
         fi
     fi
-
-    rm -f "${pidfile}" 2>/dev/null || true;
 }
 
 _stop_sccache "$@" <&0;

features/test/utils/scenarios.json

@@ -5,6 +5,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_idp_url": "${localEnv:aws_idp_url}",
       "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
@@ -51,6 +52,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_idp_url": "${localEnv:aws_idp_url}",
       "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
@@ -97,6 +99,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_idp_url": "${localEnv:aws_idp_url}",
       "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
@@ -143,6 +146,7 @@
     "containerEnv": {
       "gh_token": "${localEnv:gh_token}",
       "vault_host": "${localEnv:vault_host}",
+      "aws_idp_url": "${localEnv:aws_idp_url}",
       "aws_role_arn": "${localEnv:aws_role_arn}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",