From 77425a56685298fbcbe3abc87ed9ce316ee2c985 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 25 Apr 2025 16:03:50 -0700 Subject: [PATCH 1/7] Add a notation for private device And also enable Apple benchmark there. This depends on https://github.com/pytorch/test-infra/pull/6579 Signed-off-by: Huy Do --- .github/scripts/extract_benchmark_results.py | 3 ++- .../apple-perf-private-device-experiment.yml | 16 +++++++--------- .github/workflows/apple-perf.yml | 6 ++++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/scripts/extract_benchmark_results.py b/.github/scripts/extract_benchmark_results.py index bbfb4ba6015..6137a452d5f 100755 --- a/.github/scripts/extract_benchmark_results.py +++ b/.github/scripts/extract_benchmark_results.py @@ -349,7 +349,8 @@ def transform( # Overwrite the device name here with the job name as it has more information about # the device, i.e. Samsung Galaxy S22 5G instead of just Samsung for r in benchmark_results: - r["deviceInfo"]["device"] = job_name + is_private_device = job_report.get("is_private_instance", False) + r["deviceInfo"]["device"] = f"{job_name} (private)" if is_private_device else job_name # From https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database return [ diff --git a/.github/workflows/apple-perf-private-device-experiment.yml b/.github/workflows/apple-perf-private-device-experiment.yml index 2e9e444bada..4bb6e8f15ec 100644 --- a/.github/workflows/apple-perf-private-device-experiment.yml +++ b/.github/workflows/apple-perf-private-device-experiment.yml @@ -1,18 +1,16 @@ name: apple-perf (private devices) on: - # TODO (huydhn): Disable the schedule run until we land the change to add device pool and device name - # to separate between public and private iOS devices - # schedule: - # - cron: 0 0,4,8,12,16,20 * * * + schedule: + - cron: 0 0,4,8,12,16,20 * * * pull_request: paths: - .github/workflows/apple-perf-private-device-experiment.yml - # push: - # branches: - # - main - # paths: - # - .github/workflows/apple-perf-private-device-experiment.yml + push: + branches: + - main + paths: + - .github/workflows/apple-perf-private-device-experiment.yml # Note: GitHub has an upper limit of 10 inputs workflow_dispatch: inputs: diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml index 1af1dfb24d3..d66b276483a 100644 --- a/.github/workflows/apple-perf.yml +++ b/.github/workflows/apple-perf.yml @@ -436,7 +436,8 @@ jobs: permissions: id-token: write contents: read - uses: pytorch/test-infra/.github/workflows/mobile_job.yml@main + # TESTING + uses: pytorch/test-infra/.github/workflows/mobile_job.yml@add-device-type-and-id strategy: matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }} fail-fast: false @@ -446,7 +447,8 @@ jobs: device-type: ios # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS runner: linux.2xlarge - test-infra-ref: '' + # TESTING + test-infra-ref: add-device-type-and-id # This is the ARN of ExecuTorch project on AWS project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:02a2cf0f-6d9b-45ee-ba1a-a086587469e6 device-pool-arn: ${{ matrix.device_arn }} From 95387cb0e2475787e4fb137ec72f41c6ee174c87 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 25 Apr 2025 16:27:54 -0700 Subject: [PATCH 2/7] Also add instance_arn just in case Signed-off-by: Huy Do --- .github/scripts/extract_benchmark_results.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/extract_benchmark_results.py b/.github/scripts/extract_benchmark_results.py index 6137a452d5f..deb6f642611 100755 --- a/.github/scripts/extract_benchmark_results.py +++ b/.github/scripts/extract_benchmark_results.py @@ -364,6 +364,7 @@ def transform( "benchmark_config": json.dumps(benchmark_config), "job_conclusion": "SUCCESS", "job_arn": job_report.get("arn", ""), + "instance_arn": job_request.get("instance_arn", ""), }, }, "model": { From aa07b2d1ab837ecc7960d9660ac75e0d4e0ab741 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 25 Apr 2025 17:42:31 -0700 Subject: [PATCH 3/7] Revert "Bump tokenizers dep (#10332)" This reverts commit b415af07be46569903d4d03bd66195b9afa85a96. --- extension/llm/tokenizers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index 3f9c458586e..295ee78e4b0 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit 3f9c458586ee576a7ddafb48eb491f117187e178 +Subproject commit 295ee78e4b0d99d4527bbe81bc3156341366de11 From cfc543278a939c34e83b02c6292078e8377bdb77 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 25 Apr 2025 18:02:11 -0700 Subject: [PATCH 4/7] Fix lint Signed-off-by: Huy Do --- .github/scripts/extract_benchmark_results.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/extract_benchmark_results.py b/.github/scripts/extract_benchmark_results.py index deb6f642611..b40e73f225d 100755 --- a/.github/scripts/extract_benchmark_results.py +++ b/.github/scripts/extract_benchmark_results.py @@ -350,7 +350,9 @@ def transform( # the device, i.e. Samsung Galaxy S22 5G instead of just Samsung for r in benchmark_results: is_private_device = job_report.get("is_private_instance", False) - r["deviceInfo"]["device"] = f"{job_name} (private)" if is_private_device else job_name + r["deviceInfo"]["device"] = ( + f"{job_name} (private)" if is_private_device else job_name + ) # From https://github.com/pytorch/pytorch/wiki/How-to-integrate-with-PyTorch-OSS-benchmark-database return [ From 46b6eab37da51c83304353df4d403ccbb9991ec4 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 25 Apr 2025 20:47:01 -0700 Subject: [PATCH 5/7] Fix typo Signed-off-by: Huy Do --- .github/scripts/extract_benchmark_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/extract_benchmark_results.py b/.github/scripts/extract_benchmark_results.py index b40e73f225d..f646ceaced7 100755 --- a/.github/scripts/extract_benchmark_results.py +++ b/.github/scripts/extract_benchmark_results.py @@ -366,7 +366,7 @@ def transform( "benchmark_config": json.dumps(benchmark_config), "job_conclusion": "SUCCESS", "job_arn": job_report.get("arn", ""), - "instance_arn": job_request.get("instance_arn", ""), + "instance_arn": job_report.get("instance_arn", ""), }, }, "model": { From 6c6ceaf029a210ea284cf1b8bbbf45caa0ad4867 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 28 Apr 2025 11:07:45 -0700 Subject: [PATCH 6/7] Revert "Revert "Bump tokenizers dep (#10332)"" This reverts commit aa07b2d1ab837ecc7960d9660ac75e0d4e0ab741. --- extension/llm/tokenizers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index 295ee78e4b0..3f9c458586e 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit 295ee78e4b0d99d4527bbe81bc3156341366de11 +Subproject commit 3f9c458586ee576a7ddafb48eb491f117187e178 From f37e51c19b47874332000bf3c566a7c5c39285bd Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 28 Apr 2025 19:59:14 -0700 Subject: [PATCH 7/7] Ready to land Signed-off-by: Huy Do --- .github/workflows/apple-perf.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/apple-perf.yml b/.github/workflows/apple-perf.yml index d66b276483a..1af1dfb24d3 100644 --- a/.github/workflows/apple-perf.yml +++ b/.github/workflows/apple-perf.yml @@ -436,8 +436,7 @@ jobs: permissions: id-token: write contents: read - # TESTING - uses: pytorch/test-infra/.github/workflows/mobile_job.yml@add-device-type-and-id + uses: pytorch/test-infra/.github/workflows/mobile_job.yml@main strategy: matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }} fail-fast: false @@ -447,8 +446,7 @@ jobs: device-type: ios # For iOS testing, the runner just needs to call AWS Device Farm, so there is no need to run this on macOS runner: linux.2xlarge - # TESTING - test-infra-ref: add-device-type-and-id + test-infra-ref: '' # This is the ARN of ExecuTorch project on AWS project-arn: arn:aws:devicefarm:us-west-2:308535385114:project:02a2cf0f-6d9b-45ee-ba1a-a086587469e6 device-pool-arn: ${{ matrix.device_arn }}