Add the runscript of PR test

.github/scripts/run.sh

@@ -0,0 +1,52 @@
+#!/bin/sh
+# This script runs TorchBench without installing the pytorch and torch dep packages
+# It assumes pytorch, torchtext, and torchvision have already been installed
+# Usage:
+# run.sh RESULT_DIR [BENCHMARK_FILTER]
+# The RESULT_DIR is required, BENCHMARK_FILTER is optional
+
+set -eo pipefail
+
+# Version of the config
+CONFIG_VER=v0
+CONFIG_DIR=${PWD}/torchbenchmark/score/configs/${CONFIG_VER}
+CONFIG_ENV=${CONFIG_DIR}/config-${CONFIG_VER}.env
+DATA_JSON_PREFIX=$(date +"%Y%m%d_%H%M%S")
+if [ -n "$1" ]; then
+    echo "You must specify output data dir"
+    exit 1
+fi
+DATA_DIR=$1
+
+# Load environment variables
+set -a;
+source ${CONFIG_ENV}
+set +a;
+# Must read BENCHMARK_FILTER after loading the config
+# Because config has a preset BENCHMARK_FILTER
+if [ -n "$2" ]; then
+    BENCHMARK_FILTER="$2"
+fi
+
+sudo nvidia-smi -ac ${GPU_FREQUENCY}
+export CUDA_VISIBLE_DEVICES="${GPU_LIST}"
+export GOMP_CPU_AFFINITY="${CORE_LIST}"
+
+echo "Running benchmark with filter: \"${BENCHMARK_FILTER}\""
+
+# Install benchmark dependencies
+python install.py
+
+# Run the benchmark
+for c in $(seq 1 $NUM_ITER); do
+    taskset -c "${CORE_LIST}" pytest test_bench.py -k "${BENCHMARK_FILTER}" \
+            --benchmark-min-rounds "${NUM_ROUNDS}" \
+            --benchmark-json ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json
+    # Fill in circle_build_num and circle_project_reponame
+    jq --arg run_id "${GITHUB_RUN_ID}" --arg config_version "githubactions-benchmark-${CONFIG_VER}-metal-fullname" \
+       '.machine_info.circle_project_name=$config_version | .machine_info.circle_build_num=$run_id' \
+       ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json > ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json.tmp
+    mv ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json.tmp ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json
+done
+
+echo "Benchmark finished successfully. Output data dir is ${DATA_DIR}."

bisection.py

@@ -1,7 +1,7 @@
 """bisection.py
 Runs bisection to determine PRs that cause performance change.
-It assumes that the pytorch, torchbench, torchtext, torchvision, and torchaudio repositories provided are all clean with the latest code.
-By default, the torchvision, torchaudio, and torchtext package version will be fixed to the latest commit on the pytorch commit date.
+It assumes that the pytorch, torchbench, torchtext and torchvision repositories provided are all clean with the latest code.
+By default, the torchvision and torchtext package version will be fixed to the latest commit on the pytorch commit date.
 
 Usage:
   python bisection.py --work-dir <WORK-DIR> \
@@ -28,7 +28,6 @@
 TORCHBENCH_DEPS = {
     "torchtext": os.path.expandvars("${HOME}/text"),
     "torchvision": os.path.expandvars("${HOME}/vision"),
-    "torchaudio": os.path.expandvars("${HOME}/audio"),
 }
 
 def exist_dir_path(string):
@@ -138,7 +137,7 @@ def prep(self) -> bool:
         self.update_repos()
         return True
 
-    # Update pytorch, torchtext, torchvision, and torchaudio repo
+    # Update pytorch, torchtext, and torchvision repo
     def update_repos(self):
         repos = [self.srcpath]
         repos.extend(TORCHBENCH_DEPS.values())
@@ -188,18 +187,13 @@ def checkout_deps(self, cdate: datetime):
             assert gitutils.checkout_git_commit(TORCHBENCH_DEPS[pkg], dep_commit), "Failed to checkout commit {commit} of {pkg}"
             print("done.")
 
-    # Install dependencies such as torchaudio, torchtext and torchvision
+    # Install dependencies such as torchtext and torchvision
     def build_install_deps(self, build_env):
         # Build torchvision
         print(f"Building torchvision ...", end="", flush=True)
         command = "python setup.py install &> /dev/null"
         subprocess.check_call(command, cwd=TORCHBENCH_DEPS["torchvision"], env=build_env, shell=True)
         print("done")
-        # Build torchaudio
-        print(f"Building torchaudio ...", end="", flush=True)
-        command = "python setup.py install &> /dev/null"
-        subprocess.check_call(command, cwd=TORCHBENCH_DEPS["torchaudio"], env=build_env, shell=True)
-        print("done")
         # Build torchtext
         print(f"Building torchtext ...", end="", flush=True)
         command = "python setup.py clean install &> /dev/null"
@@ -228,7 +222,7 @@ def build(self, commit: Commit):
 
     def cleanup(self, commit: Commit):
         print(f"Cleaning up packages from commit {commit.sha} ...", end="", flush=True)
-        packages = ["torch", "torchtext", "torchvision", "torchaudio"]
+        packages = ["torch", "torchtext", "torchvision"]
         command = "pip uninstall -y " + " ".join(packages) + " &> /dev/null "
         subprocess.check_call(command, shell=True)
         print("done")