diff --git a/.github/scripts/run-sweep.sh b/.github/scripts/run-sweep.sh new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/scripts/run.sh b/.github/scripts/run.sh index 97723740ad..ea816a20bf 100644 --- a/.github/scripts/run.sh +++ b/.github/scripts/run.sh @@ -50,11 +50,6 @@ for c in $(seq 1 $NUM_ITER); do taskset -c "${CORE_LIST}" pytest test_bench.py -k "${BENCHMARK_FILTER}" \ --benchmark-min-rounds "${NUM_ROUNDS}" \ --benchmark-json ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json - # Fill in circle_build_num and circle_project_reponame - jq --arg run_id "${GITHUB_RUN_ID}" --arg config_version "githubactions-benchmark-${CONFIG_VER}-metal-fullname" \ - '.machine_info.circle_project_name=$config_version | .machine_info.circle_build_num=$run_id' \ - ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json > ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json.tmp - mv ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json.tmp ${DATA_DIR}/${DATA_JSON_PREFIX}_${c}.json done echo "Benchmark finished successfully. Output data dir is ${DATA_DIR}." diff --git a/.github/workflows/main.yml b/.github/workflows/v0-nightly.yml similarity index 100% rename from .github/workflows/main.yml rename to .github/workflows/v0-nightly.yml diff --git a/.github/workflows/sweep.yml b/.github/workflows/v0-sweep.yml similarity index 100% rename from .github/workflows/sweep.yml rename to .github/workflows/v0-sweep.yml diff --git a/.github/workflows/v1-nightly.yml b/.github/workflows/v1-nightly.yml new file mode 100644 index 0000000000..c4509d6f3c --- /dev/null +++ b/.github/workflows/v1-nightly.yml @@ -0,0 +1,64 @@ +name: TorchBench nightly ci v1.0 +on: + workflow_dispatch: + +jobs: + run-benchmark: + env: + # Set to "v1-alpha" for testing. Will set it to "v1" in release. + TORCHBENCH_VER: "v1-alpha" + CONFIG_VER: "v1" + CONDA_ENV_NAME: "v1-nightly-ci" + OUTPUT_DIR: $(echo ${HOME})/.torchbench/v1-nighlty-ci + if: ${{ github.repository_owner == 'pytorch' }} + runs-on: [self-hosted, bm-runner] + env: + CONDA_ENV_NAME: torchbench-v1-nightly-ci + SCRIBE_GRAPHQL_ACCESS_TOKEN: ${{ secrets.SCRIBE_GRAPHQL_ACCESS_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: v1.0 + - name: Create conda env + run: | + conda create -y -q --name "${CONDA_ENV_NAME}" python=3.7 + - name: Install PyTorch nightly + run: | + . activate "$CONDA_ENV_NAME" + # Check if nightly builds are available + NIGHTLIES=$(python torchbenchmark/util/torch_nightly.py --packages torch) + # If failed, the script will generate empty result + if [ -z $NIGHTLIES ]; then + echo "Torch nightly build failed. Cancel the workflow." + exit 1 + fi + # Install PyTorch nightly from pip + pip install --pre torch \ + -f https://download.pytorch.org/whl/nightly/${CUDA_VERSION}/torch_nightly.html + - name: Run benchmark + run: | + . activate "$CONDA_ENV_NAME" + bash ./.github/scripts/run.sh "${OUTPUT_DIR}" + - name: Copy artifact and upload to scribe + run: | + LATEST_RESULT=$(find ${OUTPUT_DIR}/gh${GITHUB_RUN_ID}/ -name "*.json" | sort -r | head -1) + echo "Benchmark result file: $LATEST_RESULT" + TODAY=$(date "+%Y%m%d%H%M%S") + CONFIG_NORM_FILE=${CONFIG_DIR}/${CONFIG_FILE} + SCORE_FILE="./benchmark-result-v1-score-${TODAY}.json" + # Generate score + python compute_score.py --score_version v1 --benchmark_data_file $LATEST_RESULT > $SCORE_FILE + # Upload result to Scribe + python scripts/upload_scribe.py --pytest_bench_json $LATEST_RESULT --torchbench_score_file $SCORE_FILE + cp ${LATEST_RESULT} ./benchmark-result-v1-${TODAY}.json + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: Benchmark result + path: benchmark-result-v1-*.json + - name: Remove conda env + run: | + conda env remove --name "${CONDA_ENV_NAME}" diff --git a/conftest.py b/conftest.py index e3f62e9bd4..5cd239f929 100644 --- a/conftest.py +++ b/conftest.py @@ -73,8 +73,8 @@ def pytest_benchmark_update_machine_info(config, machine_info): except ImportError: machine_info['torchvision_version'] = '*not-installed*' - machine_info['circle_build_num'] = os.environ.get("CIRCLE_BUILD_NUM") - machine_info['circle_project_name'] = os.environ.get("CIRCLE_PROJECT_REPONAME") + machine_info['github_run_id'] = os.environ.get("GITHUB_RUN_ID") + machine_info['torchbench_score_version'] = os.environ.get("TORCHBENCH_VER") try: # if running on unexpected machine/os, get_machine_config _may_ not work diff --git a/scripts/upload_scribe.py b/scripts/upload_scribe.py index f02a0ac81a..803b65a18f 100644 --- a/scripts/upload_scribe.py +++ b/scripts/upload_scribe.py @@ -82,10 +82,16 @@ def __init__(self): 'pytorch_version', 'python_version', 'torchtext_version', 'torchvision_version', 'machine_kernel', 'machine_processor', 'machine_hostname', - 'circle_build_num', 'circle_project_reponame', + 'github_run_id', 'torchbench_score_version', ], 'float': [ - 'stddev', 'min', 'median', 'max', 'mean', 'runtime', 'torchbench_score', + 'stddev', 'min', 'median', 'max', 'mean', 'runtime', + 'torchbench_score', + 'torchbench_score_jit_speedup', + 'torchbench_subscore_cpu_train', + 'torchbench_subscore_cpu_infer', + 'torchbench_subscore_gpu_train', + 'torchbench_subscore_gpu_infer', ] } @@ -113,8 +119,8 @@ def post_pytest_benchmarks(self, pytest_json, max_data_upload=100): "machine_kernel": machine_info['release'], "machine_processor": machine_info['processor'], "machine_hostname": machine_info['node'], - "circle_build_num": machine_info.get('circle_build_num', None), - "circle_project_reponame": machine_info.get('circle_project_name', None), + "github_run_id": machine_info.get('github_run_id', None), + "torchbench_score_version": machine_info.get('torchbench_score_version', None), } stats_msg = {"stddev": b['stats']['stddev'], @@ -139,7 +145,7 @@ def post_torchbench_score(self, pytest_json, score): machine_info = pytest_json['machine_info'] commit_info = pytest_json['commit_info'] upload_time = int(time.time()) - m = self.format_message({ + scribe_message = { "time": upload_time, "benchmark_time": pytest_json['datetime'], "git_repo": commit_info['project'], @@ -154,10 +160,16 @@ def post_torchbench_score(self, pytest_json, score): "machine_kernel": machine_info['release'], "machine_processor": machine_info['processor'], "machine_hostname": machine_info['node'], - "circle_build_num": machine_info.get('circle_build_num', None), - "circle_project_reponame": machine_info.get('circle_project_name', None), - "torchbench_score": score, - }) + "github_run_id": machine_info.get('github_run_id', None), + "torchbench_score_version": machine_info.get('torchbench_score_version', None), + "torchbench_score": score["score"]["total"], + "torchbench_score_jit_speedup": score["score"]["jit-speedup"], + "torchbench_subscore_cpu_train": score["score"]["subscore-cpu-train"], + "torchbench_subscore_cpu_infer": score["score"]["subscore-cpu-eval"], + "torchbench_subscore_gpu_train": score["score"]["subscore-cuda-train"], + "torchbench_subscore_gpu_infer": score["score"]["subscore-cuda-eval"], + } + m = self.format_message(scribe_message) self.upload([m]) @@ -166,13 +178,20 @@ def post_torchbench_score(self, pytest_json, score): parser.add_argument("--pytest_bench_json", required=True, type=argparse.FileType('r'), help='Upload json data formatted by pytest-benchmark module') - parser.add_argument("--torchbench_score", type=float, - help="optional torchbench score to include") + parser.add_argument("--torchbench_score_file", required=True, + type=argparse.FileType('r'), + help="torchbench score file to include") args = parser.parse_args() + # Result sanity check + json_name = os.path.basename(args.pytest_bench_json.name) + json_score = json.load(args.torchbench_score_file) + score_data = None + for data in json_score: + if os.path.basename(data["file"]) == json_name: + score_data = data + assert score_data, f"Can't find {json_name} score in {args.torchbench_score_file}. Stop." benchmark_uploader = PytorchBenchmarkUploader() json_data = json.load(args.pytest_bench_json) benchmark_uploader.post_pytest_benchmarks(json_data) - - if args.torchbench_score is not None: - benchmark_uploader.post_torchbench_score(json_data, args.torchbench_score) + benchmark_uploader.post_torchbench_score(json_data, score_data) diff --git a/torchbenchmark/score/compute_score_v1.py b/torchbenchmark/score/compute_score_v1.py index e1dcad6796..cc6824b8c7 100755 --- a/torchbenchmark/score/compute_score_v1.py +++ b/torchbenchmark/score/compute_score_v1.py @@ -231,12 +231,12 @@ def compute_score(self, data): assert not diff_set, f"The request benchmark json doesn't have v1 test: {diff_set}" summary = {} - summary["subscore[jit]"] = self.compute_jit_speedup_score(data) + summary["jit-speedup"] = self.compute_jit_speedup_score(data) devices = ["cpu", "cuda"] tests = ["train", "eval"] filters = [(a, b) for a in devices for b in tests] for f in filters: - key = f"subscore[{f[0]}-{f[1]}]" + key = f"subscore-{f[0]}-{f[1]}" summary[key] = self._get_subscore(data_norm, self.norm, self.norm_weights, f) * self.target summary["total"] = self._get_score(data_norm, self.norm, self.norm_weights) * self.target return summary