From 0210346c2f15a6ac6d88af02b3dfcd67d449ac3c Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 08:13:17 +0900 Subject: [PATCH 01/11] hidemetadata --- .github/scripts/upload_benchmark_results.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index ec49d20..a0fb504 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -162,9 +162,9 @@ def get_benchmark_metadata( "repo": repo_name, "head_branch": head_branch, "head_sha": head_sha, - "workflow_id": os.getenv("WORKFLOW_ID", timestamp), - "run_attempt": os.getenv("RUN_ATTEMPT", 1), - "job_id": os.getenv("JOB_ID", timestamp), + "workflow_id": os.getenv("GITHUB_RUN_ID", timestamp), + "run_attempt": os.getenv("GITHUB_RUN_ATTEMPT", 1), + "job_id": os.getenv("GITHUB_JOB", timestamp), } From 9d9ee5abc382ccde8ebaffa98d7cb5ca91667d27 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 08:47:28 +0900 Subject: [PATCH 02/11] hidemetadata --- .github/scripts/upload_benchmark_results.py | 6 +++--- .github/workflows/sglang-benchmark.yml | 17 ++++++++++++++--- .github/workflows/vllm-benchmark.yml | 11 ++++++++++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index a0fb504..7d04b7c 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -162,9 +162,9 @@ def get_benchmark_metadata( "repo": repo_name, "head_branch": head_branch, "head_sha": head_sha, - "workflow_id": os.getenv("GITHUB_RUN_ID", timestamp), - "run_attempt": os.getenv("GITHUB_RUN_ATTEMPT", 1), - "job_id": os.getenv("GITHUB_JOB", timestamp), + "workflow_id": os.getenv("WORKFLOW_RUN_ID", timestamp), + "run_attempt": os.getenv("RUN_ATTEMPT", 1), + "job_id": os.getenv("JOB_ID", timestamp), } diff --git a/.github/workflows/sglang-benchmark.yml b/.github/workflows/sglang-benchmark.yml index 06cbab9..22c6dce 100644 --- a/.github/workflows/sglang-benchmark.yml +++ b/.github/workflows/sglang-benchmark.yml @@ -197,7 +197,7 @@ jobs: for tag in $(git for-each-ref --sort=-creatordate --format '%(refname:short)' refs/tags); do candidate_image="lmsysorg/sglang:${tag}${IMAGE_SUFFIX}" echo "Checking: $candidate_image" - + if docker manifest inspect "$candidate_image" >/dev/null 2>&1; then SELECTED_TAG="$tag" DOCKER_IMAGE="$candidate_image" @@ -206,7 +206,7 @@ jobs: break fi done - + # Fallback to latest if no tagged version found if [[ -z "$SELECTED_TAG" ]]; then echo "No tagged images found, using latest" @@ -275,17 +275,28 @@ jobs: role-duration-seconds: 18000 aws-region: us-east-1 + - name: Get workflow job id + id: get-job-id + uses: pytorch/test-infra/.github/actions/get-workflow-job-id@main + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Upload the benchmark results if: always() env: BENCHMARK_RESULTS: sglang-benchmarks/benchmarks/results MODELS: ${{ matrix.models }} + WORKFLOW_RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + JOB_ID: ${{ steps.get-job-id.outputs.job-id || '0' }} run: | set -eux sudo chown -R ${UID} "${BENCHMARK_RESULTS}" || true ls -lah "${BENCHMARK_RESULTS}" || echo "Results directory not found" + echo "## workflow info: ${WORKFLOW_RUN_ID} ${RUN_ATTEMPT} ${JOB_ID}" + SANITIZED_DEVICE_TYPE=$(echo "${DEVICE_TYPE// /_}" | sed "s/[^[:alnum:].-]/_/g") SANITIZED_MODELS="${MODELS//\//_}" @@ -302,7 +313,7 @@ jobs: echo "⚠️ No benchmark results found in ${BENCHMARK_RESULTS}" >> $GITHUB_STEP_SUMMARY fi - python3 .github/scripts/upload_benchmark_results.py \ + python3 .github/scripts/_results.py \ --repo-name sgl-project/sglang \ --benchmark-name "SGLang benchmark" \ --benchmark-results "${BENCHMARK_RESULTS}" \ diff --git a/.github/workflows/vllm-benchmark.yml b/.github/workflows/vllm-benchmark.yml index 24358a4..31b04af 100644 --- a/.github/workflows/vllm-benchmark.yml +++ b/.github/workflows/vllm-benchmark.yml @@ -115,7 +115,7 @@ jobs: elif command -v hl-smi; then DEVICE_NAME=hpu hl-smi - else + else arch=$(uname -m) case "$arch" in @@ -344,10 +344,19 @@ jobs: role-duration-seconds: 18000 aws-region: us-east-1 + - name: Get workflow job id + id: get-job-id + uses: pytorch/test-infra/.github/actions/get-workflow-job-id@main + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Upload the benchmark results env: BENCHMARK_RESULTS: vllm-benchmarks/vllm/benchmarks/results MODELS: ${{ matrix.models }} + WORKFLOW_RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} + JOB_ID: ${{ steps.get-job-id.outputs.job-id || '0' }} run: | set -eux From b98a951357b23fb9a13471a856937359723ab2c6 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 08:48:41 +0900 Subject: [PATCH 03/11] hidemetadata --- .github/workflows/vllm-benchmark.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/vllm-benchmark.yml b/.github/workflows/vllm-benchmark.yml index 31b04af..bd11358 100644 --- a/.github/workflows/vllm-benchmark.yml +++ b/.github/workflows/vllm-benchmark.yml @@ -363,6 +363,8 @@ jobs: sudo chown -R ${UID} "${BENCHMARK_RESULTS}" ls -lah "${BENCHMARK_RESULTS}" + echo "## workflow info: ${WORKFLOW_RUN_ID} ${RUN_ATTEMPT} ${JOB_ID}" + SANITIZED_DEVICE_TYPE=$(echo "${DEVICE_TYPE// /_}" | sed "s/[^[:alnum:].-]/_/g") SANITIZED_MODELS="${MODELS//\//_}" From 99ab37979ddc3cefa5403cfeaac7dce185b76700 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 08:51:23 +0900 Subject: [PATCH 04/11] hidemetadata --- .github/workflows/sglang-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sglang-benchmark.yml b/.github/workflows/sglang-benchmark.yml index 22c6dce..901bd33 100644 --- a/.github/workflows/sglang-benchmark.yml +++ b/.github/workflows/sglang-benchmark.yml @@ -313,7 +313,7 @@ jobs: echo "⚠️ No benchmark results found in ${BENCHMARK_RESULTS}" >> $GITHUB_STEP_SUMMARY fi - python3 .github/scripts/_results.py \ + python3 .github/scripts/upload_benchmark_results.py \ --repo-name sgl-project/sglang \ --benchmark-name "SGLang benchmark" \ --benchmark-results "${BENCHMARK_RESULTS}" \ From cc0a5126d4442a4d4b888901d6bd4d933e1d8952 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 08:53:58 +0900 Subject: [PATCH 05/11] hidemetadata --- .github/workflows/sglang-benchmark.yml | 2 +- .github/workflows/vllm-benchmark.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sglang-benchmark.yml b/.github/workflows/sglang-benchmark.yml index 901bd33..b776c24 100644 --- a/.github/workflows/sglang-benchmark.yml +++ b/.github/workflows/sglang-benchmark.yml @@ -288,7 +288,7 @@ jobs: MODELS: ${{ matrix.models }} WORKFLOW_RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} - JOB_ID: ${{ steps.get-job-id.outputs.job-id || '0' }} + JOB_ID: ${{ steps.get-job-id.outputs.job-id }} run: | set -eux diff --git a/.github/workflows/vllm-benchmark.yml b/.github/workflows/vllm-benchmark.yml index bd11358..f09f72f 100644 --- a/.github/workflows/vllm-benchmark.yml +++ b/.github/workflows/vllm-benchmark.yml @@ -356,7 +356,7 @@ jobs: MODELS: ${{ matrix.models }} WORKFLOW_RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} - JOB_ID: ${{ steps.get-job-id.outputs.job-id || '0' }} + JOB_ID: ${{ steps.get-job-id.outputs.job-id }} run: | set -eux From 211c84138a1ec42caa593581934e05bf40dd6ff9 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 08:56:50 +0900 Subject: [PATCH 06/11] hidemetadata --- .github/workflows/vllm-benchmark.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/vllm-benchmark.yml b/.github/workflows/vllm-benchmark.yml index f09f72f..4924d9d 100644 --- a/.github/workflows/vllm-benchmark.yml +++ b/.github/workflows/vllm-benchmark.yml @@ -101,6 +101,12 @@ jobs: python-version: '3.12' cache: 'pip' + - name: Get workflow job id + id: get-job-id + uses: pytorch/test-infra/.github/actions/get-workflow-job-id@main + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Check if the device is supported shell: bash run: | @@ -344,12 +350,6 @@ jobs: role-duration-seconds: 18000 aws-region: us-east-1 - - name: Get workflow job id - id: get-job-id - uses: pytorch/test-infra/.github/actions/get-workflow-job-id@main - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Upload the benchmark results env: BENCHMARK_RESULTS: vllm-benchmarks/vllm/benchmarks/results From 6fb1af5fdd93a46c36f7f0a183529cc752d6baab Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 09:09:55 +0900 Subject: [PATCH 07/11] hidemetadata --- .github/workflows/sglang-benchmark.yml | 11 +++++------ .github/workflows/vllm-benchmark.yml | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sglang-benchmark.yml b/.github/workflows/sglang-benchmark.yml index b776c24..b680b96 100644 --- a/.github/workflows/sglang-benchmark.yml +++ b/.github/workflows/sglang-benchmark.yml @@ -99,6 +99,11 @@ jobs: python-version: '3.12' cache: 'pip' + - name: Get workflow job id + id: get-job-id + uses: pytorch/test-infra/.github/actions/get-workflow-job-id@36562d6c43fa914f7bdef67ce23e5c31f1387b2e + with: + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Check if the device is supported shell: bash @@ -275,12 +280,6 @@ jobs: role-duration-seconds: 18000 aws-region: us-east-1 - - name: Get workflow job id - id: get-job-id - uses: pytorch/test-infra/.github/actions/get-workflow-job-id@main - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Upload the benchmark results if: always() env: diff --git a/.github/workflows/vllm-benchmark.yml b/.github/workflows/vllm-benchmark.yml index 4924d9d..8c1a048 100644 --- a/.github/workflows/vllm-benchmark.yml +++ b/.github/workflows/vllm-benchmark.yml @@ -103,7 +103,7 @@ jobs: - name: Get workflow job id id: get-job-id - uses: pytorch/test-infra/.github/actions/get-workflow-job-id@main + uses: pytorch/test-infra/.github/actions/get-workflow-job-id@36562d6c43fa914f7bdef67ce23e5c31f1387b2e with: github-token: ${{ secrets.GITHUB_TOKEN }} From c5b7a25b2e53a2dcdd39df44e2c7642a25f7e5ea Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 10:06:49 +0900 Subject: [PATCH 08/11] hidemetadata --- .github/scripts/upload_benchmark_results.py | 8 +++++--- .github/workflows/vllm-benchmark.yml | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index 7d04b7c..b90ee9b 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -151,16 +151,19 @@ def get_git_metadata(repo_dir: str) -> Tuple[str, str]: # We couldn't find the branch name return repo_name, "main", hexsha, committed_date - def get_benchmark_metadata( repo_name: str, head_branch: str, head_sha: str, timestamp: int, benchmark_name ) -> Dict[str, Any]: + local_branch = os.getenv("RUN_LOCAL_BRANCH","") + branch_name = head_branch + if local_branch and local_branch != "main": + branch_name = f"{head_branch}({local_branch})" return { "timestamp": timestamp, "schema_version": "v3", "name": benchmark_name, "repo": repo_name, - "head_branch": head_branch, + "head_branch": branch_name, "head_sha": head_sha, "workflow_id": os.getenv("WORKFLOW_RUN_ID", timestamp), "run_attempt": os.getenv("RUN_ATTEMPT", 1), @@ -325,7 +328,6 @@ def upload( def main() -> None: args = parse_args() - if args.repo: if args.head_branch or args.head_sha: warning("No need to set --head-branch and --head-sha when using --repo") diff --git a/.github/workflows/vllm-benchmark.yml b/.github/workflows/vllm-benchmark.yml index 8c1a048..126424a 100644 --- a/.github/workflows/vllm-benchmark.yml +++ b/.github/workflows/vllm-benchmark.yml @@ -357,6 +357,7 @@ jobs: WORKFLOW_RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} JOB_ID: ${{ steps.get-job-id.outputs.job-id }} + RUN_LOCAL_BRANCH: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} run: | set -eux From 5e3cdcc600b0ee13e1da544a294ab76b8a928f7a Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 10:10:50 +0900 Subject: [PATCH 09/11] hidemetadata --- .github/scripts/upload_benchmark_results.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index b90ee9b..ebe23fd 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -154,6 +154,8 @@ def get_git_metadata(repo_dir: str) -> Tuple[str, str]: def get_benchmark_metadata( repo_name: str, head_branch: str, head_sha: str, timestamp: int, benchmark_name ) -> Dict[str, Any]: + + # combine run branch name with the foreign branch name if it's not main or local_branch = os.getenv("RUN_LOCAL_BRANCH","") branch_name = head_branch if local_branch and local_branch != "main": From a02529d18202d491b7026543b150a9030b5f2ab0 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 10:11:14 +0900 Subject: [PATCH 10/11] hidemetadata --- .github/scripts/upload_benchmark_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/upload_benchmark_results.py b/.github/scripts/upload_benchmark_results.py index ebe23fd..33ecbd6 100755 --- a/.github/scripts/upload_benchmark_results.py +++ b/.github/scripts/upload_benchmark_results.py @@ -155,7 +155,7 @@ def get_benchmark_metadata( repo_name: str, head_branch: str, head_sha: str, timestamp: int, benchmark_name ) -> Dict[str, Any]: - # combine run branch name with the foreign branch name if it's not main or + # combine run branch name with the foreign branch name if it's not main local_branch = os.getenv("RUN_LOCAL_BRANCH","") branch_name = head_branch if local_branch and local_branch != "main": From a3db7b577950a19035fccdc58fb0ced6ffacb151 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 3 Dec 2025 10:54:50 +0900 Subject: [PATCH 11/11] hidemetadata --- .github/workflows/sglang-benchmark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sglang-benchmark.yml b/.github/workflows/sglang-benchmark.yml index b680b96..99e6e96 100644 --- a/.github/workflows/sglang-benchmark.yml +++ b/.github/workflows/sglang-benchmark.yml @@ -288,6 +288,7 @@ jobs: WORKFLOW_RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} JOB_ID: ${{ steps.get-job-id.outputs.job-id }} + RUN_LOCAL_BRANCH: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.ref }} run: | set -eux