From a92d926211e2c686ab7d8dc3247f68241213e5b1 Mon Sep 17 00:00:00 2001 From: Egor Date: Fri, 9 Feb 2024 15:21:12 +0100 Subject: [PATCH 1/5] Fixed GCP-specific upload issue --- .github/workflows/execute-test-script.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/execute-test-script.yml b/.github/workflows/execute-test-script.yml index 422b8ef..12e8a30 100644 --- a/.github/workflows/execute-test-script.yml +++ b/.github/workflows/execute-test-script.yml @@ -31,7 +31,7 @@ on: default: spr type: string shutdown_cloud_runner: - description: Whether to shutdown AWS cloud runner + description: Whether to shutdown cloud runner required: false default: false type: boolean @@ -57,7 +57,7 @@ jobs: runner_labels: ${{ steps.set_up_vars.outputs.runner_labels }} results_name: ${{ steps.set_up_vars.outputs.results_name }} steps: - - name: Get conda env and results name for AWS runners + - name: Get conda env and results name for cloud runners id: set_up_vars shell: bash -el {0} run: | @@ -139,15 +139,15 @@ jobs: fi - - name: Upload results.db to artifacts when running in AWS - if: contains(fromJSON(needs.set_up_vars.outputs.runner_labels), 'aws') + - name: Upload results.db to artifacts when running in the cloud + if: ${{ runner_type != 'spr' }} uses: actions/upload-artifact@v3 with: name: ${{ needs.set_up_vars.outputs.results_name }} path: results.db - - name: Shutdown AWS cloud runner - if: contains(fromJSON(needs.set_up_vars.outputs.runner_labels), 'aws') && inputs.shutdown_cloud_runner + - name: Shutdown cloud runner + if: ${{ runner_type != 'spr' }} && inputs.shutdown_cloud_runner shell: bash -el {0} run: sudo shutdown -h +2 @@ -156,7 +156,7 @@ jobs: needs: - set_up_vars - mlp_test - if: contains(fromJSON(needs.set_up_vars.outputs.runner_labels), 'aws') + if: ${{ runner_type != 'spr' }} steps: - uses: actions/checkout@v4 - name: Download results.db from artifacts From ec259da9dcdbcb65ca7775c99ee758e869b5982f Mon Sep 17 00:00:00 2001 From: Egor Date: Fri, 9 Feb 2024 15:23:15 +0100 Subject: [PATCH 2/5] Update execute-test-script.yml --- .github/workflows/execute-test-script.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/execute-test-script.yml b/.github/workflows/execute-test-script.yml index 12e8a30..a4f116c 100644 --- a/.github/workflows/execute-test-script.yml +++ b/.github/workflows/execute-test-script.yml @@ -140,14 +140,14 @@ jobs: - name: Upload results.db to artifacts when running in the cloud - if: ${{ runner_type != 'spr' }} + if: ${{ inputs.runner_type != 'spr' }} uses: actions/upload-artifact@v3 with: name: ${{ needs.set_up_vars.outputs.results_name }} path: results.db - name: Shutdown cloud runner - if: ${{ runner_type != 'spr' }} && inputs.shutdown_cloud_runner + if: ${{ inputs.runner_type != 'spr' }} && inputs.shutdown_cloud_runner shell: bash -el {0} run: sudo shutdown -h +2 @@ -156,7 +156,7 @@ jobs: needs: - set_up_vars - mlp_test - if: ${{ runner_type != 'spr' }} + if: ${{ inputs.runner_type != 'spr' }} steps: - uses: actions/checkout@v4 - name: Download results.db from artifacts From 295ef933a873d3d26e800aa9b952d095afc2a987 Mon Sep 17 00:00:00 2001 From: Egor Krivov Date: Mon, 12 Feb 2024 12:00:07 +0100 Subject: [PATCH 3/5] fixed gcp deps --- .github/workflows/execute-test-script.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/execute-test-script.yml b/.github/workflows/execute-test-script.yml index a4f116c..2a4a984 100644 --- a/.github/workflows/execute-test-script.yml +++ b/.github/workflows/execute-test-script.yml @@ -170,7 +170,7 @@ jobs: run: | ls -l - sudo apt update && sudo apt install sqlite3 + sudo apt update && sudo apt install -y sqlite3 ./db_tools/export_sqlite2csv.sh source ${CONDA}/bin/activate From 1cc2e6beaa030c03f85bd579e4a0c7a802014189 Mon Sep 17 00:00:00 2001 From: Egor Krivov Date: Mon, 12 Feb 2024 12:39:13 +0100 Subject: [PATCH 4/5] removed turn off --- .github/workflows/execute-test-script.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/execute-test-script.yml b/.github/workflows/execute-test-script.yml index 2a4a984..20ac472 100644 --- a/.github/workflows/execute-test-script.yml +++ b/.github/workflows/execute-test-script.yml @@ -30,11 +30,6 @@ on: required: true default: spr type: string - shutdown_cloud_runner: - description: Whether to shutdown cloud runner - required: false - default: false - type: boolean test_script: description: Test script to run required: false @@ -146,11 +141,6 @@ jobs: name: ${{ needs.set_up_vars.outputs.results_name }} path: results.db - - name: Shutdown cloud runner - if: ${{ inputs.runner_type != 'spr' }} && inputs.shutdown_cloud_runner - shell: bash -el {0} - run: sudo shutdown -h +2 - parse_results: runs-on: [self-hosted, glados, spr] needs: From 30fef2656eb4d38aa0f786cce4cca23ca13699b2 Mon Sep 17 00:00:00 2001 From: Egor Krivov Date: Mon, 12 Feb 2024 17:52:42 +0100 Subject: [PATCH 5/5] added sync --- dl_bench/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dl_bench/utils.py b/dl_bench/utils.py index 9300fa2..4d5f673 100644 --- a/dl_bench/utils.py +++ b/dl_bench/utils.py @@ -152,6 +152,10 @@ def to_device(self, x: torch.Tensor): else: raise ValueError("Unknown device") + def sync(self): + if self.device_name == 'cuda': + torch.cuda.synchronize() + def prepare_eval_transformer(self, model): model = model.to(memory_format=torch.channels_last) @@ -390,6 +394,7 @@ def inference(self, backend: Backend): # Duration is inconsistent now with tm.timeit("duration_s"): for i, x in enumerate(test_loader): + backend.sync() s = get_time() x = backend.to_device(x) if backend.dtype != torch.float32: @@ -405,6 +410,7 @@ def inference(self, backend: Backend): start = time.perf_counter() continue + backend.sync() fw_times.append(get_time() - s) n_items += len(x) outputs.append(y)