Add v1 nightly CI workflow

.github/scripts/run.sh

@@ -50,11 +50,6 @@ for c in $(seq 1 $NUM_ITER); do
     taskset -c "${CORE_LIST}" pytest test_bench.py -k "${BENCHMARK_FILTER}" \
             --benchmark-min-rounds "${NUM_ROUNDS}" \
             --benchmark-json ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json
-    # Fill in circle_build_num and circle_project_reponame
-    jq --arg run_id "${GITHUB_RUN_ID}" --arg config_version "githubactions-benchmark-${CONFIG_VER}-metal-fullname" \
-       '.machine_info.circle_project_name=$config_version | .machine_info.circle_build_num=$run_id' \
-       ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json > ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json.tmp
-    mv ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json.tmp ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json
 done
 
 echo "Benchmark finished successfully. Output data dir is ${DATA_DIR}."

.github/workflows/v1-nightly.yml

@@ -0,0 +1,64 @@
+name: TorchBench nightly ci v1.0
+on:
+  workflow_dispatch:
+
+jobs:
+  run-benchmark:
+    env:
+      # Set to "v1-alpha" for testing. Will set it to "v1" in release.
+      TORCHBENCH_VER: "v1-alpha"
+      CONFIG_VER: "v1"
+      CONDA_ENV_NAME:  "v1-nightly-ci"
+      OUTPUT_DIR: $(echo ${HOME})/.torchbench/v1-nighlty-ci
+    if: ${{ github.repository_owner == 'pytorch' }}
+    runs-on: [self-hosted, bm-runner]
+    env:
+      CONDA_ENV_NAME: torchbench-v1-nightly-ci
+      SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: v1.0
+      - name: Create conda env
+        run: |
+          conda create -y -q --name "${CONDA_ENV_NAME}" python=3.7
+      - name: Install PyTorch nightly
+        run: |
+          . activate "$CONDA_ENV_NAME"
+          # Check if nightly builds are available
+          NIGHTLIES=$(python torchbenchmark/util/torch_nightly.py --packages torch)
+          # If failed, the script will generate empty result
+          if [ -z $NIGHTLIES ]; then
+              echo "Torch nightly build failed. Cancel the workflow."
+              exit 1
+          fi
+          # Install PyTorch nightly from pip
+          pip install --pre torch \
+          -f https://download.pytorch.org/whl/nightly/${CUDA_VERSION}/torch_nightly.html
+      - name: Run benchmark
+        run: |
+          . activate "$CONDA_ENV_NAME"
+          bash ./.github/scripts/run.sh "${OUTPUT_DIR}"
+      - name: Copy artifact and upload to scribe
+        run: |
+          LATEST_RESULT=$(find ${OUTPUT_DIR}/gh${GITHUB_RUN_ID}/ -name "*.json" | sort -r | head -1)
+          echo "Benchmark result file: $LATEST_RESULT"
+          TODAY=$(date "+%Y%m%d%H%M%S")
+          CONFIG_NORM_FILE=${CONFIG_DIR}/${CONFIG_FILE}
+          SCORE_FILE="./benchmark-result-v1-score-${TODAY}.json"
+          # Generate score
+          python compute_score.py --score_version v1 --benchmark_data_file $LATEST_RESULT > $SCORE_FILE
+          # Upload result to Scribe
+          python scripts/upload_scribe.py --pytest_bench_json $LATEST_RESULT --torchbench_score_file $SCORE_FILE 
+          cp ${LATEST_RESULT} ./benchmark-result-v1-${TODAY}.json
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: Benchmark result
+          path: benchmark-result-v1-*.json
+      - name: Remove conda env
+        run: |
+          conda env remove --name "${CONDA_ENV_NAME}"

conftest.py

@@ -73,8 +73,8 @@ def pytest_benchmark_update_machine_info(config, machine_info):
     except ImportError:
         machine_info['torchvision_version'] = '*not-installed*'
 
-    machine_info['circle_build_num'] = os.environ.get("CIRCLE_BUILD_NUM")
-    machine_info['circle_project_name'] = os.environ.get("CIRCLE_PROJECT_REPONAME")
+    machine_info['github_run_id'] = os.environ.get("GITHUB_RUN_ID")
+    machine_info['torchbench_score_version'] = os.environ.get("TORCHBENCH_VER")
 
     try:
         # if running on unexpected machine/os, get_machine_config _may_ not work

scripts/upload_scribe.py

@@ -82,10 +82,16 @@ def __init__(self):
                 'pytorch_version', 'python_version',
                 'torchtext_version', 'torchvision_version',
                 'machine_kernel', 'machine_processor', 'machine_hostname',
-                'circle_build_num', 'circle_project_reponame',
+                'github_run_id', 'torchbench_score_version',
             ],
             'float': [
-                'stddev', 'min', 'median', 'max', 'mean', 'runtime', 'torchbench_score',
+                'stddev', 'min', 'median', 'max', 'mean', 'runtime',
+                'torchbench_score',
+                'torchbench_score_jit_speedup',
+                'torchbench_subscore_cpu_train', 
+                'torchbench_subscore_cpu_infer',
+                'torchbench_subscore_gpu_train',
+                'torchbench_subscore_gpu_infer',
             ]
         }
 
@@ -113,8 +119,8 @@ def post_pytest_benchmarks(self, pytest_json, max_data_upload=100):
                 "machine_kernel": machine_info['release'],
                 "machine_processor": machine_info['processor'],
                 "machine_hostname": machine_info['node'],
-                "circle_build_num": machine_info.get('circle_build_num', None),
-                "circle_project_reponame": machine_info.get('circle_project_name', None),
+                "github_run_id": machine_info.get('github_run_id', None),
+                "torchbench_score_version": machine_info.get('torchbench_score_version', None),
             }
 
             stats_msg = {"stddev": b['stats']['stddev'],
@@ -139,7 +145,7 @@ def post_torchbench_score(self, pytest_json, score):
         machine_info = pytest_json['machine_info']
         commit_info = pytest_json['commit_info']
         upload_time = int(time.time())
-        m = self.format_message({
+        scribe_message = {
             "time": upload_time,
             "benchmark_time": pytest_json['datetime'],
             "git_repo": commit_info['project'],
@@ -154,10 +160,16 @@ def post_torchbench_score(self, pytest_json, score):
             "machine_kernel": machine_info['release'],
             "machine_processor": machine_info['processor'],
             "machine_hostname": machine_info['node'],
-            "circle_build_num": machine_info.get('circle_build_num', None),
-            "circle_project_reponame": machine_info.get('circle_project_name', None),
-            "torchbench_score": score,
-        })
+            "github_run_id": machine_info.get('github_run_id', None),
+            "torchbench_score_version": machine_info.get('torchbench_score_version', None),
+            "torchbench_score": score["score"]["total"],
+            "torchbench_score_jit_speedup": score["score"]["jit-speedup"],
+            "torchbench_subscore_cpu_train": score["score"]["subscore-cpu-train"],
+            "torchbench_subscore_cpu_infer": score["score"]["subscore-cpu-eval"],
+            "torchbench_subscore_gpu_train": score["score"]["subscore-cuda-train"],
+            "torchbench_subscore_gpu_infer": score["score"]["subscore-cuda-eval"],
+        }
+        m = self.format_message(scribe_message)
         self.upload([m])
 
 
@@ -166,13 +178,20 @@ def post_torchbench_score(self, pytest_json, score):
     parser.add_argument("--pytest_bench_json", required=True,
                         type=argparse.FileType('r'),
                         help='Upload json data formatted by pytest-benchmark module')
-    parser.add_argument("--torchbench_score", type=float,
-                        help="optional torchbench score to include")
+    parser.add_argument("--torchbench_score_file", required=True,
+                        type=argparse.FileType('r'),
+                        help="torchbench score file to include")
     args = parser.parse_args()
 
+    # Result sanity check
+    json_name = os.path.basename(args.pytest_bench_json.name)
+    json_score = json.load(args.torchbench_score_file)
+    score_data = None
+    for data in json_score:
+        if os.path.basename(data["file"]) == json_name:
+            score_data = data
+    assert score_data, f"Can't find {json_name} score in {args.torchbench_score_file}. Stop."
     benchmark_uploader = PytorchBenchmarkUploader()
     json_data = json.load(args.pytest_bench_json)
     benchmark_uploader.post_pytest_benchmarks(json_data)
-
-    if args.torchbench_score is not None:
-        benchmark_uploader.post_torchbench_score(json_data, args.torchbench_score)
+    benchmark_uploader.post_torchbench_score(json_data, score_data)

torchbenchmark/score/compute_score_v1.py

@@ -231,12 +231,12 @@ def compute_score(self, data):
         assert not diff_set, f"The request benchmark json doesn't have v1 test: {diff_set}"
 
         summary = {}
-        summary["subscore[jit]"] = self.compute_jit_speedup_score(data)
+        summary["jit-speedup"] = self.compute_jit_speedup_score(data)
         devices = ["cpu", "cuda"]
         tests = ["train", "eval"]
         filters = [(a, b) for a in devices for b in tests]
         for f in filters:
-            key = f"subscore[{f[0]}-{f[1]}]"
+            key = f"subscore-{f[0]}-{f[1]}"
             summary[key] = self._get_subscore(data_norm, self.norm, self.norm_weights, f) * self.target
         summary["total"] = self._get_score(data_norm, self.norm, self.norm_weights) * self.target
         return summary