From 93060bef823e8387b3acf67a142f7d773fb15986 Mon Sep 17 00:00:00 2001 From: Will Constable Date: Tue, 15 Sep 2020 15:06:54 -0700 Subject: [PATCH] Move torch version check to test json and update repo fields - change incorrect/confusing 'pytorch_commit_*' vars to git_* vars since the vars are used in repos such as benchmark - record torch pkg versions explicitly regardless of repo used for benchmark runs --- conftest.py | 18 ++++++++++++++++++ scripts/upload_scribe.py | 27 +++++++++++++++++---------- test_bench.py | 9 +++++---- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/conftest.py b/conftest.py index acaaa4ee80..3e832e680b 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,4 @@ +import os import pytest import torch @@ -21,3 +22,20 @@ def set_fuser(fuser): def pytest_configure(config): set_fuser(config.getoption("fuser")) + +def pytest_benchmark_update_machine_info(config, machine_info): + machine_info['pytorch_version'] = torch.__version__ + try: + import torchtext + machine_info['torchtext_version'] = torchtext.__version__ + except ImportError: + machine_info['torchtext_version'] = '*not-installed*' + + try: + import torchvision + machine_info['torchvision_version'] = torchvision.__version__ + except ImportError: + machine_info['torchvision_version'] = '*not-installed*' + + machine_info['circle_build_num'] = os.environ.get("CIRCLE_BUILD_NUM") + machine_info['circle_project_name'] = os.environ.get("CIRCLE_PROJECT_REPONAME") diff --git a/scripts/upload_scribe.py b/scripts/upload_scribe.py index 785bd55a76..4192642271 100644 --- a/scripts/upload_scribe.py +++ b/scripts/upload_scribe.py @@ -75,9 +75,12 @@ def __init__(self): 'time', 'rounds', ], 'normal': [ - 'benchmark_group', 'benchmark_name', 'benchmark_class', 'benchmark_time', - 'pytorch_commit_id', 'pytorch_branch', 'pytorch_commit_time', 'pytorch_version', - 'pytorch_git_dirty', + 'benchmark_group', 'benchmark_name', + 'benchmark_class', 'benchmark_time', + 'git_repo', 'git_commit_id', 'git_branch', + 'git_commit_time', 'git_dirty', + 'pytorch_version', 'python_version', + 'torchtext_version', 'torchvision_version', 'machine_kernel', 'machine_processor', 'machine_hostname', 'circle_build_num', 'circle_project_reponame', ], @@ -98,16 +101,20 @@ def post_pytest_benchmarks(self, pytest_json): "benchmark_name": b['name'], "benchmark_class": b['fullname'], "benchmark_time": pytest_json['datetime'], - "pytorch_commit_id": commit_info['id'], - "pytorch_branch": commit_info['branch'], - "pytorch_commit_time": commit_info['time'], - "pytorch_version": None, - "pytorch_git_dirty": commit_info['dirty'], + "git_repo": commit_info['project'], + "git_commit_id": commit_info['id'], + "git_branch": commit_info['branch'], + "git_commit_time": commit_info['time'], + "git_dirty": commit_info['dirty'], + "pytorch_version": machine_info.get('pytorch_version', None), + "torchtext_version": machine_info.get('torchtext_version', None), + "torchvision_version": machine_info.get('torchvision_version', None), + "python_version": machine_info['python_implementation_version'], "machine_kernel": machine_info['release'], "machine_processor": machine_info['processor'], "machine_hostname": machine_info['node'], - "circle_build_num": os.environ.get("CIRCLE_BUILD_NUM"), - "circle_project_reponame": os.environ.get("CIRCLE_PROJECT_REPONAME"), + "circle_build_num": machine_info.get('circle_build_num', None), + "circle_project_reponame": machine_info.get('circle_project_name', None), "stddev": b['stats']['stddev'], "rounds": b['stats']['rounds'], "min": b['stats']['min'], diff --git a/test_bench.py b/test_bench.py index 4d4c1ecf3f..769d40d198 100644 --- a/test_bench.py +++ b/test_bench.py @@ -29,10 +29,11 @@ def pytest_generate_tests(metafunc, display_len=24): short = short[:display_len] + "..." short_names.append(short) - metafunc.parametrize('model', all_models, - ids=short_names, scope="class") - metafunc.parametrize('device', ['cpu', 'cuda'], scope='class') - metafunc.parametrize('compiler', ['jit', 'eager'], scope='class') + if metafunc.cls and metafunc.cls.__name__ == "TestBenchNetwork": + metafunc.parametrize('model', all_models, + ids=short_names, scope="class") + metafunc.parametrize('device', ['cpu', 'cuda'], scope='class') + metafunc.parametrize('compiler', ['jit', 'eager'], scope='class') @pytest.fixture(scope='class')