From a5edd8581466a801745512ccc8109dda130b453e Mon Sep 17 00:00:00 2001 From: Pablo Gonzalez Date: Mon, 7 Nov 2022 18:44:01 -0500 Subject: [PATCH] Remove submission_runner folder (#1261) --- tools/submission_runner/README.md | 5 - tools/submission_runner/config.json | 150 ------------------- tools/submission_runner/run_mlperf.py | 204 -------------------------- 3 files changed, 359 deletions(-) delete mode 100644 tools/submission_runner/README.md delete mode 100644 tools/submission_runner/config.json delete mode 100644 tools/submission_runner/run_mlperf.py diff --git a/tools/submission_runner/README.md b/tools/submission_runner/README.md deleted file mode 100644 index 6e019c8d5..000000000 --- a/tools/submission_runner/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# mlperf_runner - -Tools to automate MLPerf logs collection for submission. Please check config.json for info. - -Tested only for Offline scenario. \ No newline at end of file diff --git a/tools/submission_runner/config.json b/tools/submission_runner/config.json deleted file mode 100644 index 39efc2115..000000000 --- a/tools/submission_runner/config.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "comment": "Copy JSON file with system description and add required config section, see below", - - "division": "closed", - "submitter": "Intel", - "status": "preview", - "system_type":"datacenter", - "system_name": "1-node-2S-ICX-OpenVINO-INT8", - - "number_of_nodes": "1", - "host_processor_model_name": "TBD", - "host_processors_per_node": "2", - "host_processor_core_count": "40", - "host_processor_frequency": "", - "host_processor_caches": "", - "host_memory_configuration": "8 slots / 32GB each / 3200 MT/s per socket", - "host_memory_capacity": "??GB", - "host_storage_capacity": "", - "host_storage_type": "", - "host_processor_interconnect": "", - "host_networking": "", - "host_networking_topology": "", - - "accelerators_per_node": "0", - "accelerator_model_name": "N/A", - "accelerator_frequency": "", - "accelerator_host_interconnect": "", - "accelerator_interconnect": "", - "accelerator_interconnect_topology": "", - "accelerator_memory_capacity": "N/A", - "accelerator_memory_configuration": "", - "accelerator_on-chip_memories": "", - "cooling": "", - "hw_notes": "", - - "framework": "OpenVINO", - "operating_system": "Ubuntu 20.04.1 LTS", - "other_software_stack": "5.4.0-45-generic", - "sw_notes": "", - - "config": { - "comment": [ - "Those parameters needed to provide paths to", - "logs_dir - path to output log files (Required)", - "compliance_output_dir - path to output directory for compliance verification (Required in case you going to run compliance verification)", - "dtype - data type (int8, bfloat16, float32, etc.) (Required)", - "accuracy - model accuracy (Optional)", - "scenario - benchmar scenarion (Server, Offline) (Required)", - "system - system name (Required)", - "topology - topology name (3d-unet, bert, dlrm, resnet50, rnnt, ssd-mobilenet, ssd-resnet34) (Required)", - "export - add environment variables (Optional)", - "mlperf_path - path to clonned mlcommons/inference git (Required)", - "redirect_output - if True redirect to console all output, if False - only errors (Optional)" - ], - - "logs_dir": "/root/mlperf_logs", - "dtype": "int8", - "accuracy": "99.9", - "scenario": "Offline", - "system": "1-node-2S-ICX", - "topology": "3d-unet", - "env": [ { "LD_LIBRARY_PATH": "$LD_LIBRARY_PATH:/root/git/mlperf_ext_ov_cpp_v1.0/lib_3d_unet" } ], - "mlperf_path": "/root/git/inference", - "redirect_output": "False", - - "comment_exec": [ - "name - string to display while running script (Required)", - "cmd - command to execute (Required)", - "dir - current directory (Optional)", - "skip - 1,yes,True - skip execution (Optional)" - ], - - "exec": [ - { - "name": "Performace benchmark", - "cmd": "$EXEC -m $DATA/model/3d_unet_model_i8.xml -data $DATA/preprocessed_data/preprocessed_files.pkl -mlperf_conf $DATA/mlperf.conf -user_conf $DATA/user.conf -scenario Offline -streams 8", - "dir": "$PERFORMANCE/RUN_1_DIR" - }, - - { - "name": "Accuracy benchmark", - "cmd": "$EXEC -m $DATA/model/3d_unet_model_i8.xml -data $DATA/preprocessed_data/preprocessed_files.pkl -mlperf_conf $DATA/mlperf.conf -user_conf $DATA/user.conf -scenario Offline -streams 8 -mode Accuracy", - "dir": "$ACCURACY_DIR" - }, - - { - "name": "Accuracy check", - "cmd": "$PYTHON $MLPERF/vision/medical_imaging/3d-unet/accuracy-brats.py --log_file $ACCURACY_DIR/mlperf_log_accuracy.json --preprocessed_data_dir $DATA/preprocessed_data --postprocessed_data_dir $DATA/postprocessed_data --label_data_dir $DATA/nnUNet_raw_data/Task043_BraTS2019/labelsTr --output_dtype float32", - "dir": "$ACCURACY_DIR", - "save_output": "accuracy.txt" - }, - - { - "name": "Compliance TEST01", - "dir": "$TEST01_TMP_DIR", - "cmd": "$EXEC -m $DATA/model/3d_unet_model_i8.xml -data $DATA/preprocessed_data/preprocessed_files.pkl -mlperf_conf $DATA/mlperf.conf -user_conf $DATA/user.conf -scenario Offline -streams 8" - }, - - { - "name": "Compliance TEST04-A", - "dir": "$TEST04-A_TMP_DIR", - "cmd": "$EXEC -m $DATA/model/3d_unet_model_i8.xml -data $DATA/preprocessed_data/preprocessed_files.pkl -mlperf_conf $DATA/mlperf.conf -user_conf $DATA/user.conf -scenario Offline -streams 8" - }, - - { - "name": "Compliance TEST04-B", - "dir": "$TEST04-B_TMP_DIR", - "cmd": "$EXEC -m $DATA/model/3d_unet_model_i8.xml -data $DATA/preprocessed_data/preprocessed_files.pkl -mlperf_conf $DATA/mlperf.conf -user_conf $DATA/user.conf -scenario Offline -streams 8" - }, - - { - "name": "Compliance TEST05", - "dir": "$TEST05_TMP_DIR", - "cmd": "$EXEC -m $DATA/model/3d_unet_model_i8.xml -data $DATA/preprocessed_data/preprocessed_files.pkl -mlperf_conf $DATA/mlperf.conf -user_conf $DATA/user.conf -scenario Offline -streams 8" - }, - - { - "name": "Verification TEST01", - "cmd": "$PYTHON $MLPERF_COMPL/TEST01/run_verification.py -r $RESULTS_DIR -c $TEST01_TMP_DIR -o $COMPLIANCE_DIR --dtype float32" - }, - - { - "name": "Verification TEST04-A, TEST04-B", - "cmd": "$PYTHON $MLPERF_COMPL/TEST04-A/run_verification.py -a $TEST04-A_TMP_DIR -b $TEST04-B_TMP_DIR -o $COMPLIANCE_DIR" - }, - - { - "name": "Verification TEST05", - "cmd": "$PYTHON $MLPERF_COMPL/TEST05/run_verification.py -r $RESULTS_DIR -c $TEST05_TMP_DIR -o $COMPLIANCE_DIR" - }, - - { - "name": "Verification TEST05", - "cmd": "$PYTHON $MLPERF_COMPL/TEST05/run_verification.py -r $RESULTS_DIR -c $TEST05_TMP_DIR -o $COMPLIANCE_DIR" - } - ], - - "comment_aliases": [ - "Aliases to replace in cmd, dir string from exec block" - ], - - "aliases": { - "$PYTHON": "/usr/bin/python3", - "$MLPERF_COMPL": "/root/git/inference/compliance/nvidia", - "$MLPERF": "/root/git/inference", - "$EXEC": "/root/git/mlperf_ext_ov_cpp_v1.0/Release/ov_mlperf_3d_unet", - "$DATA": "/root/data/3d-unet" - } - } -} \ No newline at end of file diff --git a/tools/submission_runner/run_mlperf.py b/tools/submission_runner/run_mlperf.py deleted file mode 100644 index f68f7d38a..000000000 --- a/tools/submission_runner/run_mlperf.py +++ /dev/null @@ -1,204 +0,0 @@ -# !/usr/bin/env python -""" - Copyright (c) 2021 Intel Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -""" - -import os -import argparse -from shutil import copyfile -import json -import shlex -import subprocess - - -def getInputParameters(): - parser = argparse.ArgumentParser() - parser.add_argument('--config', '-c', required=True, type=str, help='Path to config file.') - return parser - - -def run_process(args, dir, env, redirect_output): - output = "" - process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - bufsize=1, universal_newlines=True, cwd=dir, env=env) - while True: - data = process.stdout.readline() - if data == '' and process.poll() is not None : - break - if data: - output += data - if redirect_output: - print(data, end="") - rc = process.poll() - return rc, output - - -def expand_paths(cmd, dct): - for val in dct: - cmd = cmd.replace(val, dct[val]) - return cmd - - -def str2bool(v): - return v.lower() in [ 'yes', 'true', 't', '1' ] - - -def create_env(config): - run_env = os.environ.copy() - if 'env' in config['config']: - export = config['config']['env'] - for item in export: - key = list(item.keys())[0] - run_env[key] = item[key] - return run_env - - -def create_folder(path, name): - if not os.path.isdir(path): - print('Creating {} folder: {}'.format(name, path)) - os.makedirs(path) - else: - print('{} folder exists: {}'.format(name, path)) - - -def main(): - args = getInputParameters().parse_args() - - with open(args.config, 'r') as f: - config_dict = json.load(f) - - run_env = create_env(config_dict) - - logs_path = config_dict['config']['logs_dir'] - - division = os.path.join(logs_path, config_dict['division']) - org = os.path.join(division, config_dict['submitter']) - - code = os.path.join(org, 'code') - results = os.path.join(org, 'results') - compliance = os.path.join(org, 'compliance') - measurements = os.path.join(org, 'measurements') - - redirect_output = str2bool(config_dict['config']['redirect_output']) - - net = config_dict['config']['topology'] - net_name = net - - if 'accuracy' in config_dict['config']: - net = '{}-{}'.format(net, config_dict['config']['accuracy']) - - framework = config_dict['framework'] - dtype = config_dict['config']['dtype'].upper() - scenario = config_dict['config']['scenario'] - mlperf_path = config_dict['config']['mlperf_path'] - - system = '{}-{}-{}'.format(config_dict['config']['system'], framework, dtype) - - mlperf_conf = os.path.join(mlperf_path, 'mlperf.conf') - - aliases = {} - - if 'aliases' in config_dict['config']: - aliases = config_dict['config']['aliases'] - - aliases['$LOGS_DIR'] = logs_path - - code_path = os.path.join(code, framework, net) - - create_folder(org, 'org') - create_folder(code_path, 'code') - - compliance_tests = [ 'TEST01', 'TEST04-A', 'TEST04-B', 'TEST05' ] - compliance_config_path = os.path.join(mlperf_path, 'compliance', 'nvidia') - compliance_paths = {} - - for test in compliance_tests: - compliance_path = os.path.join(compliance, system, net, scenario, test) - compliance_tmp_path = os.path.join(compliance, system, net, scenario, 'tmp', test) - compliance_paths[test] = compliance_path - aliases['${}_DIR'.format(test)] = compliance_path - aliases['${}_TMP_DIR'.format(test)] = compliance_tmp_path - - create_folder(compliance_path, test) - create_folder(compliance_tmp_path, test) - - audit_dst = os.path.join(compliance_tmp_path, 'audit.config') - - if test == 'TEST01': - audit_conf = os.path.join(compliance_config_path, test, net_name, 'audit.config') - else: - audit_conf = os.path.join(compliance_config_path, test, 'audit.config') - - if not os.path.isfile(audit_dst): - print('Copying audit.config file: {}'.format(audit_dst)) - copyfile(audit_conf, audit_dst) - else: - print('audit.config file exists: {}'.format(audit_dst)) - - aliases['$COMPLIANCE_DIR'] = os.path.join(compliance, system, net, scenario) - - measurements_path = os.path.join(measurements, system, net, scenario) - - create_folder(measurements_path, 'measurements') - - mlperf_dst_conf = os.path.join(measurements_path, 'mlperf.conf') - if not os.path.isfile(mlperf_dst_conf): - print('Copying mlperf.conf file: {}'.format(mlperf_dst_conf)) - copyfile(mlperf_conf, mlperf_dst_conf) - else: - print('mlperf.conf file exists: {}'.format(mlperf_dst_conf)) - - results_path = os.path.join(results, system, net, scenario) - - create_folder(results_path, 'results') - - aliases['$RESULTS_DIR'] = results_path - - results_type = [ 'performance/run_1', 'accuracy' ] - for typ in results_type: - result_path = os.path.join(results_path, typ) - create_folder(result_path, typ) - aliases['${}_DIR'.format(typ.upper())] = result_path - - systems_path = os.path.join(org, 'systems') - create_folder(systems_path, 'systems') - - if 'exec' in config_dict['config']: - for exec in config_dict['config']['exec']: - print('Running {}'.format(exec['name'])) - if 'dir' in exec: - dir = expand_paths(exec['dir'], aliases) - else: - dir = None - cmd = expand_paths(exec['cmd'], aliases) - if 'skip' in exec: - if str2bool(exec['skip']): - print('Skipping...') - continue - args = shlex.split(cmd) - code, output = run_process(args, dir, run_env, redirect_output) - if 'save_output' in exec: - path = os.path.join(dir, exec['save_output']) - f = open(path, 'w+') - f.write(output) - f.close() - - if not code == 0: - print('Error returned, exiting...') - break - - -if __name__ == '__main__': - main() \ No newline at end of file