diff --git a/benchmark/.flake8 b/benchmark/.flake8 new file mode 100644 index 000000000..6deafc261 --- /dev/null +++ b/benchmark/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 120 diff --git a/benchmark/.pylintrc b/benchmark/.pylintrc new file mode 100644 index 000000000..08d432d8b --- /dev/null +++ b/benchmark/.pylintrc @@ -0,0 +1,21 @@ +[MASTER] +jobs=0 + +[MESSAGES CONTROL] +disable = fixme, + no-else-return, + too-many-arguments, + too-few-public-methods, + too-many-locals, + too-many-instance-attributes, + no-member, + unnecessary-pass + +[FORMAT] +max-line-length = 120 + +[BASIC] +good-names = i, + j, + k, + o diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 000000000..118fabb72 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,84 @@ +# About the benchmark +The benchmark is used to test the adlik serving performance of different models. Before using the benchmark to test the +performance of the runtime, you need to build the client, the binary, and compile the model. + +## Installing prerequisites + +- python3 +- pip3 + +## Build and install packages + +1. Build clients and serving binary and make client pip packages (see [README.md](../../README.md)). + +2. Install clients pip package: + + ```sh + pip3 install {dir_of_pip_package}/adlik_serving_api-0.0.0-py2.py3-none-any.whl + ``` + +3. Install model_compiler: + + ```sh + cd {Adlik_root_dir}/model_compiler + pip3 install . + ``` + +## Compile the test models + +1. Prepare model code and serving_model.json (If you don't know how to write, you can refer to the existing serving_model.json). + + ```sh + cd {Adlik_root_dir}/benchmark/test + mkdir model_name + cd model_name + ``` + + Then put your prepared model and serving_model.json in the directory model_name. + +2. Run the model code, and save the model in {Adlik_root_dir}/benchmark/test/model_name/model. + + ```sh + cd {Adlik_root_dir}/benchmark/test/model_name + python3 model.py + ``` + +3. Compile the model and save the serving model. + + ```sh + cd {Adlik_root_dir}/benchmark/src + python3 compile_model.py + ``` + + In the compile_model.py you can also specify the files that need to be compiled. + +## Test the serving performance + +1. Deploy a serving service: + + ```sh + cd {dir_of_adlik_serving_binary} + ./adlik_serving --model_base_path={model_serving_dir} --grpc_port={grpc_port} --http_port={http_port} + ``` + + Usually the adlik serving binary is in the directory {Adlik_root_dir}/bazel-bin/adlik_serving, the grpc_port can + be set to 8500 and the http_port can be set to 8501. And It should be noted that the type of the compiled model is + the same as the type of the serving service + +2. Run a client and do inference: + + ```sh + cd {Adlik_root_dir}/benchmark/test/client + python3 xxx_client.py --batch-size=128 path_image + ``` + + The log of serving and client will be saved in time_log.log. + +3. Analyze inference results + + ```sh + cd {Adlik_root_dir}/benchmark/src + python3 test_result.py path_client_log path_serving_log batch_size model_name runtime + ``` + + Then you can get the performance analysis results of the serving. \ No newline at end of file diff --git a/benchmark/bandit.yaml b/benchmark/bandit.yaml new file mode 100644 index 000000000..dd822cb3c --- /dev/null +++ b/benchmark/bandit.yaml @@ -0,0 +1,4 @@ +include: + - '*.py' + +skips: [B404,B603] \ No newline at end of file diff --git a/benchmark/setup.py b/benchmark/setup.py new file mode 100644 index 000000000..d70e3ca73 --- /dev/null +++ b/benchmark/setup.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +# Copyright 2019 ZTE corporation. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Benchmark test. +""" + +from setuptools import find_packages, setup + +_VERSION = '0.0.0' + +_REQUIRED_PACKAGES = [ + 'keras==2.2.4', + 'onnx==1.5.0', + 'protobuf==3.6.1', + 'torch==1.3.0', + 'torchvision==0.4.0', + 'requests', + 'tensorflow==1.14.0', + 'jsonschema==3.1.1', + 'networkx==2.3', + 'defusedxml==0.5.0' +] + +_TEST_REQUIRES = [ + 'bandit==1.6.0', + 'flake8==3.7.7', + 'pylint==2.3.1' +] + +setup( + name="benchmark", + version=_VERSION.replace('-', ''), + author='ZTE', + author_email='ai@zte.com.cn', + packages=find_packages('src'), + package_dir={'': 'src'}, + description=__doc__, + license='Apache 2.0', + keywords='Test serving-lite performance', + install_requires=_REQUIRED_PACKAGES, + extras_require={'test': _TEST_REQUIRES} + +) diff --git a/benchmark/src/automatic_test.py b/benchmark/src/automatic_test.py new file mode 100644 index 000000000..91294fa7d --- /dev/null +++ b/benchmark/src/automatic_test.py @@ -0,0 +1,104 @@ +import subprocess +import argparse +import os +import time + + +def _parse_arguments(): + args_parser = argparse.ArgumentParser() + args_parser.add_argument("-d", "--docker-file-path", type=str, help="The docker file path of the test serving type") + args_parser.add_argument("-s", "--serving-type", type=str, help="The test serving type") + args_parser.add_argument("-b", "--build-directory", type=str, help="The directory which to build the docker") + args_parser.add_argument("-a", "--adlik-directory", type=str, default="Adlik-master", help="The adlik directory") + args_parser.add_argument("-m", "--model-name", type=str, help="The path of model used for test") + args_parser.add_argument("-c", "--client-script", type=str, default="client_script.sh", + help="The script used to infer") + args_parser.add_argument("-ss", "--serving-script", type=str, default="serving_script.sh", + help="The serving script") + args_parser.add_argument("-ov", "--openvino-version", type=str, default="2019.3.344", + help="The version of the OpenVINO") + args_parser.add_argument("-tt", "--tensorrt-tar", type=str, + default="TensorRT-7.0.0.11.Ubuntu-18.04.x86_64-gnu.cuda-10.0.cudnn7.6.tar.gz", + help="The tar version of the TensorRT") + args_parser.add_argument("-tv", "--tensorrt-version", type=str, default="7.0.0.11", help="The version of TensorRT") + args_parser.add_argument("-l", "--log-path", type=str, default="log", help="The path of log directory") + args_parser.add_argument('-tm', '--test-model-path', type=str, help="The path of test model") + args_parser.add_argument("-sj", "--serving-json", type=str, default="serving_model.json", help="The json of model") + args_parser.add_argument("-cis", "--client-inference-script", type=str, required=True, help="The inference script") + args_parser.add_argument("-i", "--image-filename", type=str, required=True, nargs="?", help="Input image.") + args_parser.add_argument("-gl", "--gpu-label", type=int, default=None, help="The GPU label") + args_parser.add_argument("-cs", "--compile-script", type=str, default="compile_script.sh", + help="Compile the model script") + return args_parser.parse_args() + + +def _close_docker(): + close_docker_command = ['sh', '-c', + 'docker rm -f adlik-test'] + subprocess.run(close_docker_command) + + +def _get_result(log_path, model_name): + calculate_command = ['python3', os.path.join(os.path.dirname(__file__), 'test_result.py'), + '-c', os.path.join(log_path, 'client_time.log'), + '-s', os.path.join(log_path, 'serving_time.log'), + '-m', model_name] + with subprocess.Popen(calculate_command) as result_process: + print(result_process.stdout) + + +def _get_log(log_path): + if os.path.exists(os.path.join(log_path, 'client_time.log')): + return False + else: + return True + + +def _docker_build_command(args): + build_arg = f'--build-arg SERVING_SCRIPT={args.serving_script} ' \ + f'--build-arg CLIENT_SCRIPT={args.client_script} ' \ + f'--build-arg TEST_MODEL_PATH={args.test_model_path} ' \ + f'--build-arg SERVING_JSON={args.serving_json} ' \ + f'--build-arg CLIENT_INFERENCE_SCRIPT={args.client_inference_script} ' \ + f'--build-arg IMAGE_FILENAME={args.image_filename} ' \ + f'--build-arg COMPILE_SCRIPT={args.compile_script} ' + + if args.serving_type == 'openvino': + build_arg = build_arg + f'--build-arg OPENVINO_VERSION={args.openvino_version} ' + elif args.serving_type == 'tensorrt': + build_arg = build_arg + f'--build-arg TENSORRT_VERSION={args.tensorrt_version} ' \ + f'--build-arg TENSORRT_TAR={args.tensorrt_tar} ' + else: + build_arg = build_arg + + build_command = f'docker build --build-arg ADLIK_DIRECTORY={args.adlik_directory} ' + build_arg + \ + f' -f {args.docker_file_path} -t adlik-test:{args.serving_type} {args.build_directory}' + return build_command + + +def main(args): + try: + _close_docker() + except Exception: + pass + finally: + docker_build_command = _docker_build_command(args) + + if not args.gpu_label: + docker_run_command = f'docker run -d --name adlik-test -v {args.log_path}:/home/john/log ' \ + f'adlik-test:{args.serving_type}' + else: + docker_run_command = f'NV_GPU={args.gpu_label} nvidia-docker run -d --name adlik-test ' \ + f'-v {args.log_path}:/home/john/log adlik-test:{args.serving_type}' + + test_command = ['sh', '-c', docker_build_command + ' && ' + docker_run_command] + + with subprocess.Popen(test_command): + while _get_log(args.log_path): + time.sleep(10) + _get_result(args.log_path, args.model_name) + _close_docker() + + +if __name__ == '__main__': + main(_parse_arguments()) diff --git a/benchmark/src/compile_model.py b/benchmark/src/compile_model.py new file mode 100644 index 000000000..89be631ac --- /dev/null +++ b/benchmark/src/compile_model.py @@ -0,0 +1,33 @@ +import os +import json +import model_compiler +import argparse + + +def _get_request(request_file, test_model_dir): + request = json.load(request_file) + model_dir = request["input_model"] + request["input_model"] = os.path.join(test_model_dir, model_dir) + export_dir = request["export_path"] + request["export_path"] = os.path.join(test_model_dir, export_dir) + return request + + +def compile_model(args): + request_dir = os.path.join(args.test_model_path, args.serving_model_json) + try: + with open(request_dir, 'r') as request_file: + test_model_dir = args.test_model_path + request = _get_request(request_file, test_model_dir) + result = model_compiler.compile_model(request) + print(result) + except FileNotFoundError: + print(f"Can not compile the model in {os.path.join(test_model_dir, args.model_path)}") + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-t', '--test-model-path', type=str, required=True, help='The path of test model') + parser.add_argument('-s', '--serving-model-json', type=str, default='serving_model.json', help='The json of model') + args = parser.parse_args() + compile_model(args) diff --git a/benchmark/src/supervisord.conf b/benchmark/src/supervisord.conf new file mode 100644 index 000000000..8469277cf --- /dev/null +++ b/benchmark/src/supervisord.conf @@ -0,0 +1,18 @@ +[supervisord] +nodaemon=true + +[program:serving] +command=/home/john/serving_script.sh +priority=1 +autostart=true +autorestart=unexpected +stdout_logfile=/home/john/log/serving.stdout.log +stderr_logfile=/home/john/log/serving.stderr.log + +[program:client] +command=/home/john/client_script.sh && exit +priority=2 +autostart=true +autorestart=unexpected +stdout_logfile=/home/john/log/client.stdout.log +stderr_logfile=/home/john/log/client.stderr.log diff --git a/benchmark/src/test_result.py b/benchmark/src/test_result.py new file mode 100644 index 000000000..27d1993e0 --- /dev/null +++ b/benchmark/src/test_result.py @@ -0,0 +1,72 @@ +""" +The test result of adlik performance +""" +import argparse + + +def _speed_of_client(client_log_path, batch_size): + with open(client_log_path, 'r') as file: + lines = file.readlines() + sum_time = [] + for line in lines: + line = line.strip('\n') + time = line.split('predict:')[-1] + time = float(time.strip(' ')) + sum_time.append(time) + sum_time.pop(0) + batch_num = len(sum_time) + speed_processing_picture = (batch_num * batch_size) / sum(sum_time) + return speed_processing_picture, batch_num + + +def _speed_of_serving(serving_log_path, batch_size): + with open(serving_log_path, 'r') as file: + lines = file.readlines() + runtime = lines[0].partition('found runtime ')[-1] + lines = [line.partition('PredictServiceImpl')[-1] for line in lines] + sum_time = [] + for line in lines: + if line: + line = line.strip('\n') + time = line.partition('time (milliseconds):')[-1] + time = float(time.strip(' ')) + sum_time.append(time) + sum_time.pop(0) + batch_num = len(sum_time) + speed_processing_picture = (batch_num * batch_size) / sum(sum_time) * 1000 + return speed_processing_picture, batch_num, runtime + + +def main(args): + """ + Analyze inference results + """ + speed_processing_picture_client, batch_num = _speed_of_client(args.client_log_path, args.batch_size) + speed_processing_picture_serving, batch_num1, serving_runtime = _speed_of_serving(args.serving_log_path, + args.batch_size) + assert batch_num == batch_num1 + if args.runtime: + serving_runtime = args.runtime + else: + serving_runtime = serving_runtime + tail_latency = 1 / speed_processing_picture_client - 1 / speed_processing_picture_serving + print(f'Model: {args.model_name}, Runtime: {serving_runtime}') + print(f'The speed of processing picture in the client is : {speed_processing_picture_client}') + print(f'The speed of processing picture in the serving is : {speed_processing_picture_serving}') + print(f'The tail latency of one picture is : {tail_latency}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-c', '--client-log-path', type=str, required=True, + help='The path of client log') + parser.add_argument('-s', '--serving-log-path', type=str, required=True, + help='The path of serving log') + parser.add_argument('-b', '--batch-size', type=int, required=False, default=128, + help='Batch size. Default is 128.') + parser.add_argument('-m', '--model-name', type=str, required=True, + help='The name of model') + parser.add_argument('-r', '--runtime', type=str, required=False, default=None, + help='The serving type') + args = parser.parse_args() + main(args) diff --git a/benchmark/test/client/imagenet_client.py b/benchmark/test/client/imagenet_client.py new file mode 100644 index 000000000..d39f37fba --- /dev/null +++ b/benchmark/test/client/imagenet_client.py @@ -0,0 +1,169 @@ +# Copyright 2019 ZTE corporation. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +This is a sample for Adlik_serving prediction +""" + +import argparse +import os +import time +import logging + +from PIL import Image +from adlik_serving import PredictContext, model_config_pb2, tensor_dtype_to_np_dtype +import numpy as np + +FLAGS = None + +ISOTIMEFORMAT = '%Y-%m-%d %H:%M:%S,%f' + + +def _parse_model(config, model_name, batch_size): + if config.max_batch_size == 0: + if batch_size != 1: + raise Exception("batching not supported for model '" + model_name + "'") + else: # max_batch_size > 0 + if batch_size > config.max_batch_size: + raise Exception( + "expecting batch size <= {} for model '{}'".format(config.max_batch_size, model_name)) + + input_ = config.input[0] + output = config.output[0] + if input_.format == model_config_pb2.ModelInput.FORMAT_NHWC: + h = input_.dims[0] + w = input_.dims[1] + c = input_.dims[2] + else: + c = input_.dims[0] + h = input_.dims[1] + w = input_.dims[2] + + return input_.name, output.name, c, h, w, input_.format, tensor_dtype_to_np_dtype(input_.data_type) + + +def _gen_input_data(data_format, dtype, c, h, w): + if os.path.isdir(FLAGS.image_filename): + file_names = [os.path.join(FLAGS.image_filename, f) + for f in os.listdir(FLAGS.image_filename) + if os.path.isfile(os.path.join(FLAGS.image_filename, f))] + else: + file_names = [FLAGS.image_filename] + + file_names.sort() + + image_data = [] + for filename in file_names: + img = Image.open(filename) + array = _preprocess(img, data_format, dtype, c, h, w) + image_data.append(array) + return file_names, image_data + + +def _preprocess(img, data_format, dtype, c, h, w): + if c == 3: + half_the_width = img.size[0] / 2 + half_the_height = img.size[1] / 2 + img4 = img.crop( + ( + half_the_width - 112, + half_the_height - 112, + half_the_width + 112, + half_the_height + 112 + ) + ) + sample_img = img4.convert('RGB') + else: + raise Exception('Imagenet image channel must be 3, bug not {}'.format(c)) + + resized_img = sample_img.resize((h, w), Image.BILINEAR) + resized = np.array(resized_img).reshape((h, w, 3)) + + scaled = resized.astype(dtype) / 255.0 + + # Swap to CHW if necessary + if data_format == model_config_pb2.ModelInput.FORMAT_NCHW: + ordered = np.transpose(scaled, (2, 0, 1)) + else: + ordered = scaled + return ordered + + +def _postprocess(results, file_names, batch_size): + if len(results.tensor) != len(file_names): + raise Exception("expected {} results, got {}".format(batch_size, len(results))) + if len(file_names) != batch_size: + raise Exception("expected {} file names, got {}".format(batch_size, len(file_names))) + + if results.batch_classes: + for i in range(batch_size): + print("Image: '{}', result: {}".format(file_names[i], + results.batch_classes[i])) + else: + print("response doesn't contain 'batch classes' field, get class information from 'tensor' field!") + for i in range(batch_size): + print("Image: '{}', result: {}".format(file_names[i], np.argmax(results.tensor[i]))) + + +def _main(): + context = PredictContext(FLAGS.model_name, url=FLAGS.url, protocol=FLAGS.protocol, verbose=True) + model_config = context.model_config + + input_name, output_name, c, h, w, data_format, dtype = _parse_model( + model_config, FLAGS.model_name, FLAGS.batch_size) + + file_names, image_data = _gen_input_data(data_format, dtype, c, h, w) + + cur_idx = 0 + num_of_images = len(image_data) + + def _next_batch(batch_size): + nonlocal cur_idx + if cur_idx + batch_size <= num_of_images: + inputs = image_data[cur_idx:cur_idx + batch_size] + outputs = file_names[cur_idx:cur_idx + batch_size] + cur_idx = (cur_idx + batch_size) % num_of_images + else: + image_idx = cur_idx + cur_idx = 0 + next_inputs, next_outputs = _next_batch(batch_size - (num_of_images - image_idx)) + inputs = image_data[image_idx:] + next_inputs + outputs = file_names[image_idx:] + next_outputs + + return inputs, outputs + + num_of_batches = 99 + if num_of_images % FLAGS.batch_size != 0: + num_of_batches += 1 + logging.basicConfig(level=logging.DEBUG, + filename=os.path.join(FLAGS.log_path, 'client_time.log'), + filemode='a', + format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s') + for _ in range(num_of_batches): + i_inputs, i_outputs = _next_batch(FLAGS.batch_size) + time1 = time.time() + context.run(inputs={input_name: i_inputs}, + outputs={output_name: FLAGS.classes}, + batch_size=FLAGS.batch_size) + logging.info(f'The time of predict: {time.time() - time1}') + print(f'{_} / {num_of_batches}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-m', '--model-name', type=str, required=False, default='resnet50', + help='Name of model') + parser.add_argument('-b', '--batch-size', type=int, required=False, default=1, + help='Batch size. Default is 1.') + parser.add_argument('-c', '--classes', type=int, required=False, default=1, + help='Number of class results to report. Default is 1.') + parser.add_argument('-u', '--url', type=str, required=False, default='localhost:8500', + help='Server URL. Default is localhost:8500.') + parser.add_argument('-i', '--protocol', type=str, required=False, default='grpc', + help='Protocol ("http"/"grpc") used to ' + + 'communicate with service. Default is "grpc".') + parser.add_argument('image_filename', type=str, nargs='?', + help='Input image.') + parser.add_argument('-l', '--log-path', default='/home/john/Adlik', type=str, help='Log path') + FLAGS = parser.parse_args() + _main() diff --git a/benchmark/test/client/mnist_client.py b/benchmark/test/client/mnist_client.py new file mode 100644 index 000000000..157930a58 --- /dev/null +++ b/benchmark/test/client/mnist_client.py @@ -0,0 +1,159 @@ +# Copyright 2019 ZTE corporation. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +This is a sample for Adlik_serving prediction +""" + +import argparse +import os +import time +import logging + +from PIL import Image +from adlik_serving import PredictContext, model_config_pb2, tensor_dtype_to_np_dtype +import numpy as np + +FLAGS = None + +ISOTIMEFORMAT = '%Y-%m-%d %H:%M:%S,%f' + + +def _parse_model(config, model_name, batch_size): + if config.max_batch_size == 0: + if batch_size != 1: + raise Exception("batching not supported for model '" + model_name + "'") + else: # max_batch_size > 0 + if batch_size > config.max_batch_size: + raise Exception( + "expecting batch size <= {} for model '{}'".format(config.max_batch_size, model_name)) + + input_ = config.input[0] + output = config.output[0] + if input_.format == model_config_pb2.ModelInput.FORMAT_NHWC: + h = input_.dims[0] + w = input_.dims[1] + c = input_.dims[2] + else: + c = input_.dims[0] + h = input_.dims[1] + w = input_.dims[2] + + return input_.name, output.name, c, h, w, input_.format, tensor_dtype_to_np_dtype(input_.data_type) + + +def _gen_input_data(data_format, dtype, c, h, w): + if os.path.isdir(FLAGS.image_filename): + file_names = [os.path.join(FLAGS.image_filename, f) + for f in os.listdir(FLAGS.image_filename) + if os.path.isfile(os.path.join(FLAGS.image_filename, f))] + else: + file_names = [FLAGS.image_filename] + + file_names.sort() + + image_data = [] + for filename in file_names: + img = Image.open(filename) + array = _preprocess(img, data_format, dtype, c, h, w) + image_data.append(array) + return file_names, image_data + + +def _preprocess(img, data_format, dtype, c, h, w): + if c == 1: + sample_img = img.convert('L') + else: + raise Exception('MNIST image channel must be 1, bug not {}'.format(c)) + + resized_img = sample_img.resize((h, w), Image.BILINEAR) + resized = np.array(resized_img).reshape((h, w, 1)) + + scaled = resized.astype(dtype) / 255.0 + + # Swap to CHW if necessary + if data_format == model_config_pb2.ModelInput.FORMAT_NCHW: + ordered = np.transpose(scaled, (2, 0, 1)) + else: + ordered = scaled + return ordered + + +def _postprocess(results, file_names, batch_size): + if len(results.tensor) != len(file_names): + raise Exception("expected {} results, got {}".format(batch_size, len(results))) + if len(file_names) != batch_size: + raise Exception("expected {} file names, got {}".format(batch_size, len(file_names))) + + if results.batch_classes: + for i in range(batch_size): + print("Image: '{}', result: {}".format(file_names[i], + results.batch_classes[i])) + else: + print("response doesn't contain 'batch classes' field, get class information from 'tensor' field!") + for i in range(batch_size): + print("Image: '{}', result: {}".format(file_names[i], np.argmax(results.tensor[i]))) + + +def _main(): + context = PredictContext(FLAGS.model_name, url=FLAGS.url, protocol=FLAGS.protocol, verbose=True) + model_config = context.model_config + + input_name, output_name, c, h, w, data_format, dtype = _parse_model( + model_config, FLAGS.model_name, FLAGS.batch_size) + + file_names, image_data = _gen_input_data(data_format, dtype, c, h, w) + + cur_idx = 0 + num_of_images = len(image_data) + + def _next_batch(batch_size): + nonlocal cur_idx + if cur_idx + batch_size <= num_of_images: + inputs = image_data[cur_idx:cur_idx + batch_size] + outputs = file_names[cur_idx:cur_idx + batch_size] + cur_idx = (cur_idx + batch_size) % num_of_images + else: + image_idx = cur_idx + cur_idx = 0 + next_inputs, next_outputs = _next_batch(batch_size - (num_of_images - image_idx)) + inputs = image_data[image_idx:] + next_inputs + outputs = file_names[image_idx:] + next_outputs + + return inputs, outputs + + num_of_batches = 99 + if num_of_images % FLAGS.batch_size != 0: + num_of_batches += 1 + logging.basicConfig(level=logging.DEBUG, + filename=os.path.join(FLAGS.log_path, 'client_time.log'), + filemode='a', + format='%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s') + for _ in range(num_of_batches): + i_inputs, i_outputs = _next_batch(FLAGS.batch_size) + time1 = time.time() + context.run(inputs={input_name: i_inputs}, + outputs={output_name: FLAGS.classes}, + batch_size=FLAGS.batch_size) + logging.info(f'The time of predict: {time.time() - time1}') + print(f'{_} / {num_of_batches}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('-m', '--model-name', type=str, required=False, default='mnist', + help='Name of model') + parser.add_argument('-b', '--batch-size', type=int, required=False, default=1, + help='Batch size. Default is 1.') + parser.add_argument('-c', '--classes', type=int, required=False, default=1, + help='Number of class results to report. Default is 1.') + parser.add_argument('-u', '--url', type=str, required=False, default='localhost:8500', + help='Server URL. Default is localhost:8500.') + parser.add_argument('-i', '--protocol', type=str, required=False, default='grpc', + help='Protocol ("http"/"grpc") used to ' + + 'communicate with service. Default is "grpc".') + parser.add_argument('image_filename', type=str, nargs='?', + help='Input image.') + parser.add_argument('-l', '--log-path', default='/home/john/Adlik', type=str, help='Log path') + FLAGS = parser.parse_args() + _main() diff --git a/benchmark/test/client_script/client_script.sh b/benchmark/test/client_script/client_script.sh new file mode 100644 index 000000000..8abe1025b --- /dev/null +++ b/benchmark/test/client_script/client_script.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +python3 /home/john/Adlik/benchmark/test/client/$CLIENT_INFERENCE_SCRIPT --batch-size=128 /home/john/Adlik/benchmark/test/data/$IMAGE_FILENAME && \ +mv /home/john/Adlik/client_time.log /home/john/log/client_time.log && \ +mv /home/john/Adlik/serving_time.log /home/john/log/serving_time.log \ No newline at end of file diff --git a/benchmark/test/compile_script/compile_script.sh b/benchmark/test/compile_script/compile_script.sh new file mode 100644 index 000000000..45c61c8c9 --- /dev/null +++ b/benchmark/test/compile_script/compile_script.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +cd /home/john/Adlik/benchmark/src &&\ +python3 compile_model.py -t /home/john/Adlik/model -s $SERVING_JSON \ No newline at end of file diff --git a/benchmark/test/compile_script/openvino_compile_script.sh b/benchmark/test/compile_script/openvino_compile_script.sh new file mode 100644 index 000000000..9615c9324 --- /dev/null +++ b/benchmark/test/compile_script/openvino_compile_script.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +INSTALL_DIR=/opt/intel/openvino_$OPENVINO_VERSION +source $INSTALL_DIR/bin/setupvars.sh +cd /home/john/Adlik/benchmark/src &&\ +python3 compile_model.py -t /home/john/Adlik/model -s $SERVING_JSON \ No newline at end of file diff --git a/benchmark/test/data/imagenet.JPEG b/benchmark/test/data/imagenet.JPEG new file mode 100644 index 000000000..3e5c05202 Binary files /dev/null and b/benchmark/test/data/imagenet.JPEG differ diff --git a/benchmark/test/data/mnist.png b/benchmark/test/data/mnist.png new file mode 100644 index 000000000..a499ffb1b Binary files /dev/null and b/benchmark/test/data/mnist.png differ diff --git a/benchmark/test/docker_build/openvino.Dockerfile b/benchmark/test/docker_build/openvino.Dockerfile new file mode 100644 index 000000000..87e766d9e --- /dev/null +++ b/benchmark/test/docker_build/openvino.Dockerfile @@ -0,0 +1,49 @@ +FROM ubuntu:bionic +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y gnupg && \ + apt-key adv --fetch-keys \ + https://apt.repos.intel.com/openvino/2019/GPG-PUB-KEY-INTEL-OPENVINO-2019 \ + https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg && \ + apt-get autoremove --purge -y gnupg && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + echo "deb https://apt.repos.intel.com/openvino/2019 all main\n\ +deb https://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + automake \ + bazel \ + patch \ + git \ + make \ + intel-openvino-runtime-ubuntu18-2019.3.344 \ + intel-openvino-dev-ubuntu18-2019.3.344 \ + libtbb2 \ + libtool \ + python3-setuptools \ + python3-wheel \ + python3.7-dev \ + python3-six \ + python3-pip && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + + +RUN useradd -m john + +USER john + +WORKDIR /home/john + +RUN bazel version + diff --git a/benchmark/test/docker_build/tensorflow.Dockerfile b/benchmark/test/docker_build/tensorflow.Dockerfile new file mode 100644 index 000000000..3c85c764b --- /dev/null +++ b/benchmark/test/docker_build/tensorflow.Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:bionic +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y gnupg && \ + apt-key adv --fetch-keys \ + https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg && \ + apt-get autoremove --purge -y gnupg && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + echo "deb https://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + automake \ + bazel \ + make \ + patch \ + git \ + libtbb2 \ + libtool \ + python \ + python3-setuptools \ + python3-wheel \ + python3.7-dev \ + python3-six \ + python3-pip && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN useradd -m john + +USER john + +WORKDIR /home/john + +RUN bazel version + diff --git a/benchmark/test/docker_build/tensorflow_gpu.Dockerfile b/benchmark/test/docker_build/tensorflow_gpu.Dockerfile new file mode 100644 index 000000000..cc097627b --- /dev/null +++ b/benchmark/test/docker_build/tensorflow_gpu.Dockerfile @@ -0,0 +1,65 @@ +FROM ubuntu:bionic +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y gnupg && \ + apt-key adv --fetch-keys \ + "https://developer.download.nvidia.com/compute/cuda/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64/7fa2af80.pub" \ + https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg && \ + apt-get autoremove --purge -y gnupg && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + echo "deb https://developer.download.nvidia.com/compute/cuda/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64 /\n\ +deb https://developer.download.nvidia.com/compute/machine-learning/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64 /\n\ +deb https://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + automake \ + bazel \ + make \ + patch \ + cuda-cublas-dev-10-0 \ + cuda-cufft-dev-10-0 \ + cuda-cupti-10-0 \ + cuda-curand-dev-10-0 \ + cuda-cusolver-dev-10-0 \ + cuda-cusparse-dev-10-0 \ + cuda-nvml-dev-10-0 \ + cuda-nvrtc-10-0 \ + git \ + libtbb2 \ + 'libcudnn7=*+cuda10.0' \ + 'libcudnn7-dev=*+cuda10.0' \ + libtool \ + openssh-client \ + rsync \ + python3-setuptools \ + python \ + python3-wheel \ + python3.7-dev \ + python3-six \ + python3-pip && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN apt-mark hold libcudnn7 libcudnn7-dev + +RUN useradd -m john + +USER john + +WORKDIR /home/john + +RUN bazel version + +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + diff --git a/benchmark/test/docker_build/tensorrt.Dockerfile b/benchmark/test/docker_build/tensorrt.Dockerfile new file mode 100644 index 000000000..5e50d151b --- /dev/null +++ b/benchmark/test/docker_build/tensorrt.Dockerfile @@ -0,0 +1,67 @@ +FROM ubuntu:bionic +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y gnupg && \ + apt-key adv --fetch-keys \ + "https://developer.download.nvidia.com/compute/cuda/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64/7fa2af80.pub" \ + https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg && \ + apt-get autoremove --purge -y gnupg && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + echo "deb https://developer.download.nvidia.com/compute/cuda/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64 /\n\ +deb https://developer.download.nvidia.com/compute/machine-learning/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64 /\n\ +deb https://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + automake \ + bazel \ + make \ + patch \ + cuda-cublas-dev-10-0 \ + cuda-cufft-dev-10-0 \ + cuda-cupti-10-0 \ + cuda-curand-dev-10-0 \ + cuda-cusolver-dev-10-0 \ + cuda-cusparse-dev-10-0 \ + cuda-nvml-dev-10-0 \ + cuda-nvrtc-10-0 \ + git \ + libtbb2 \ + 'libcudnn7=*+cuda10.0' \ + 'libcudnn7-dev=*+cuda10.0' \ + 'libnvinfer7=*+cuda10.0' \ + 'libnvinfer-dev=*+cuda10.0' \ + 'libnvonnxparsers7=*+cuda10.0' \ + 'libnvonnxparsers-dev=*+cuda10.0' \ + libtool \ + python \ + python3-setuptools \ + python3-wheel \ + python3.7-dev \ + python3-six \ + python3-pip && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN apt-mark hold libcudnn7 libcudnn7-dev libnvinfer7 libnvinfer-dev libnvonnxparsers7 libnvonnxparsers-dev + +RUN useradd -m john + +USER john + +WORKDIR /home/john + +RUN bazel version + +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + diff --git a/benchmark/test/docker_build/tflite.Dockerfile b/benchmark/test/docker_build/tflite.Dockerfile new file mode 100644 index 000000000..8a2e8fed1 --- /dev/null +++ b/benchmark/test/docker_build/tflite.Dockerfile @@ -0,0 +1,45 @@ +FROM ubuntu:bionic +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y gnupg && \ + apt-key adv --fetch-keys \ + https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg && \ + apt-get autoremove --purge -y gnupg && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + echo "deb https://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + automake \ + bazel \ + make \ + patch \ + python \ + git \ + libtbb2 \ + libtool \ + python3-setuptools \ + python3-wheel \ + python3.7-dev \ + python3-six \ + python3-pip && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN useradd -m john + +USER john + +WORKDIR /home/john + +RUN bazel version + diff --git a/benchmark/test/docker_test/openvino.Dockerfile b/benchmark/test/docker_test/openvino.Dockerfile new file mode 100644 index 000000000..f094fbca8 --- /dev/null +++ b/benchmark/test/docker_test/openvino.Dockerfile @@ -0,0 +1,88 @@ +FROM ubuntu:bionic +ARG ADLIK_DIRECTORY +ARG OPENVINO_VERSION +ARG SERVING_SCRIPT +ARG CLIENT_SCRIPT +ARG TEST_MODEL_PATH +ARG SERVING_JSON +ARG CLIENT_INFERENCE_SCRIPT +ARG IMAGE_FILENAME +ARG COMPILE_SCRIPT +ENV COMPILE_SCRIPT=${COMPILE_SCRIPT} +ENV SERVING_JSON=${SERVING_JSON} +ENV OPENVINO_VERSION=${OPENVINO_VERSION} +ENV CLIENT_INFERENCE_SCRIPT=${CLIENT_INFERENCE_SCRIPT} +ENV IMAGE_FILENAME=${IMAGE_FILENAME} +COPY ${ADLIK_DIRECTORY} /home/john/Adlik +COPY ${TEST_MODEL_PATH} /home/john/Adlik/model +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN apt-get update && \ + apt-get install -y supervisor + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y gnupg && \ + apt-key adv --fetch-keys \ + https://apt.repos.intel.com/openvino/2019/GPG-PUB-KEY-INTEL-OPENVINO-2019 \ + https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg && \ + apt-get autoremove --purge -y gnupg && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + echo "deb https://apt.repos.intel.com/openvino/2019 all main\n\ +deb https://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + automake \ + bazel \ + patch \ + git \ + make \ + intel-openvino-runtime-ubuntu18-$OPENVINO_VERSION \ + intel-openvino-dev-ubuntu18-$OPENVINO_VERSION \ + libtbb2 \ + libtool \ + python3-setuptools \ + python3-wheel \ + python3.7-dev \ + python3-six \ + python3-pip && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +WORKDIR /home/john + +RUN bazel version + +COPY benchmark/src/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +RUN cd /home/john/Adlik &&\ + bazel build //adlik_serving/clients/python:build_pip_package -c opt --distdir=/home/john/Adlik/archives &&\ + mkdir /tmp/pip-packages && bazel-bin/adlik_serving/clients/python/build_pip_package /tmp/pip-packages &&\ + export INTEL_CVSDK_DIR=/opt/intel/openvino_${OPENVINO_VERSION}/ &&\ + export InferenceEngine_DIR=$INTEL_CVSDK_DIR/deployment_tools/inference_engine/share &&\ + bazel build //adlik_serving \ + --config=openvino \ + -c opt \ + --distdir=/home/john/Adlik/archives \ + --jobs=5 &&\ + pip3 install --upgrade pip &&\ + pip3 install /tmp/pip-packages/adlik_serving_api-0.0.0-py2.py3-none-any.whl &&\ + cd /home/john/Adlik/model_compiler &&\ + pip3 install . &&\ + pip3 install -U tensorflow==1.14 defusedxml==0.5.0 networkx==2.3.0 pillow + +COPY ${SERVING_SCRIPT} /home/john/serving_script.sh +RUN chmod +x /home/john/serving_script.sh +COPY ${CLIENT_SCRIPT} /home/john/client_script.sh +RUN chmod +x /home/john/client_script.sh +COPY ${COMPILE_SCRIPT} /home/john/compile_script.sh +RUN chmod +x /home/john/compile_script.sh +CMD /home/john/compile_script.sh && "/usr/bin/supervisord" diff --git a/benchmark/test/docker_test/tensorflow.Dockerfile b/benchmark/test/docker_test/tensorflow.Dockerfile new file mode 100644 index 000000000..5b95d664a --- /dev/null +++ b/benchmark/test/docker_test/tensorflow.Dockerfile @@ -0,0 +1,82 @@ +FROM ubuntu:bionic +ARG ADLIK_DIRECTORY +ARG SERVING_SCRIPT +ARG CLIENT_SCRIPT +ARG TEST_MODEL_PATH +ARG SERVING_JSON +ARG CLIENT_INFERENCE_SCRIPT +ARG IMAGE_FILENAME +ARG COMPILE_SCRIPT +ENV COMPILE_SCRIPT=${COMPILE_SCRIPT} +ENV SERVING_JSON=${SERVING_JSON} +ENV CLIENT_INFERENCE_SCRIPT=${CLIENT_INFERENCE_SCRIPT} +ENV IMAGE_FILENAME=${IMAGE_FILENAME} +COPY ${ADLIK_DIRECTORY} /home/john/Adlik +COPY ${TEST_MODEL_PATH} /home/john/Adlik/model + +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN apt-get update && \ + apt-get install -y supervisor + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y gnupg && \ + apt-key adv --fetch-keys \ + https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg && \ + apt-get autoremove --purge -y gnupg && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + echo "deb https://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + automake \ + bazel \ + make \ + patch \ + git \ + libtbb2 \ + libtool \ + python \ + python3-setuptools \ + python3-wheel \ + python3.7-dev \ + python3-six \ + python3-pip && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +WORKDIR /home/john + +RUN bazel version + +COPY benchmark/src/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +RUN cd /home/john/Adlik &&\ + bazel build //adlik_serving/clients/python:build_pip_package -c opt --distdir=/home/john/Adlik/archives &&\ + mkdir /tmp/pip-packages && bazel-bin/adlik_serving/clients/python/build_pip_package /tmp/pip-packages &&\ + bazel build //adlik_serving \ + --config=tensorflow-cpu \ + -c opt \ + --distdir=/home/john/Adlik/archives \ + --jobs=3 &&\ + pip3 install --upgrade pip &&\ + pip3 install /tmp/pip-packages/adlik_serving_api-0.0.0-py2.py3-none-any.whl &&\ + cd /home/john/Adlik/model_compiler &&\ + pip3 install . &&\ + pip3 install -U pillow + +COPY ${SERVING_SCRIPT} /home/john/serving_script.sh +RUN chmod +x /home/john/serving_script.sh +COPY ${CLIENT_SCRIPT} /home/john/client_script.sh +RUN chmod +x /home/john/client_script.sh +COPY ${COMPILE_SCRIPT} /home/john/compile_script.sh +RUN chmod +x /home/john/compile_script.sh +CMD /home/john/compile_script.sh && "/usr/bin/supervisord" diff --git a/benchmark/test/docker_test/tensorflow_gpu.Dockerfile b/benchmark/test/docker_test/tensorflow_gpu.Dockerfile new file mode 100644 index 000000000..a67c36c82 --- /dev/null +++ b/benchmark/test/docker_test/tensorflow_gpu.Dockerfile @@ -0,0 +1,104 @@ +FROM ubuntu:bionic +ARG ADLIK_DIRECTORY +ARG SERVING_SCRIPT +ARG CLIENT_SCRIPT +ARG TEST_MODEL_PATH +ARG SERVING_JSON +ARG CLIENT_INFERENCE_SCRIPT +ARG IMAGE_FILENAME +ARG COMPILE_SCRIPT +ENV COMPILE_SCRIPT=${COMPILE_SCRIPT} +ENV SERVING_JSON=${SERVING_JSON} +ENV CLIENT_INFERENCE_SCRIPT=${CLIENT_INFERENCE_SCRIPT} +ENV IMAGE_FILENAME=${IMAGE_FILENAME} +COPY ${ADLIK_DIRECTORY} /home/john/Adlik +COPY ${TEST_MODEL_PATH} /home/john/Adlik/model + +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN apt-get update && \ + apt-get install -y supervisor + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y gnupg && \ + apt-key adv --fetch-keys \ + "https://developer.download.nvidia.com/compute/cuda/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64/7fa2af80.pub" \ + https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg && \ + apt-get autoremove --purge -y gnupg && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + echo "deb https://developer.download.nvidia.com/compute/cuda/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64 /\n\ +deb https://developer.download.nvidia.com/compute/machine-learning/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64 /\n\ +deb https://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + automake \ + bazel \ + make \ + patch \ + cuda-cublas-dev-10-0 \ + cuda-cufft-dev-10-0 \ + cuda-cupti-10-0 \ + cuda-curand-dev-10-0 \ + cuda-cusolver-dev-10-0 \ + cuda-cusparse-dev-10-0 \ + cuda-nvml-dev-10-0 \ + cuda-nvrtc-10-0 \ + git \ + libtbb2 \ + 'libcudnn7=*+cuda10.0' \ + 'libcudnn7-dev=*+cuda10.0' \ + libtool \ + openssh-client \ + rsync \ + python3-setuptools \ + python \ + python3-wheel \ + python3.7-dev \ + python3-six \ + python3-pip && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN apt-mark hold libcudnn7 libcudnn7-dev + +WORKDIR /home/john + +RUN bazel version + +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + +COPY benchmark/src/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +RUN cd /home/john/Adlik &&\ + bazel build //adlik_serving/clients/python:build_pip_package -c opt --distdir=/home/john/Adlik/archives &&\ + mkdir /tmp/pip-packages && bazel-bin/adlik_serving/clients/python/build_pip_package /tmp/pip-packages &&\ + env TF_CUDA_VERSION=10.0 \ + bazel build //adlik_serving \ + --config=tensorflow-gpu \ + -c opt \ + --incompatible_use_specific_tool_files=false \ + --distdir=/home/john/Adlik/archives \ + --jobs=50 &&\ + pip3 install --upgrade pip &&\ + pip3 install /tmp/pip-packages/adlik_serving_api-0.0.0-py2.py3-none-any.whl &&\ + cd /home/john/Adlik/model_compiler &&\ + pip3 install . &&\ + pip3 install -U pillow + +COPY ${SERVING_SCRIPT} /home/john/serving_script.sh +RUN chmod +x /home/john/serving_script.sh +COPY ${CLIENT_SCRIPT} /home/john/client_script.sh +RUN chmod +x /home/john/client_script.sh +COPY ${COMPILE_SCRIPT} /home/john/compile_script.sh +RUN chmod +x /home/john/compile_script.sh +CMD /home/john/compile_script.sh && "/usr/bin/supervisord" diff --git a/benchmark/test/docker_test/tensorrt.Dockerfile b/benchmark/test/docker_test/tensorrt.Dockerfile new file mode 100644 index 000000000..434a3452b --- /dev/null +++ b/benchmark/test/docker_test/tensorrt.Dockerfile @@ -0,0 +1,124 @@ +FROM ubuntu:bionic +ARG ADLIK_DIRECTORY +ARG SERVING_SCRIPT +ARG CLIENT_SCRIPT +ARG TEST_MODEL_PATH +ARG SERVING_JSON +ARG CLIENT_INFERENCE_SCRIPT +ARG IMAGE_FILENAME +ARG TENSORRT_VERSION +ARG TENSORRT_TAR +ARG COMPILE_SCRIPT +ENV COMPILE_SCRIPT=${COMPILE_SCRIPT} +ENV CLIENT_INFERENCE_SCRIPT=${CLIENT_INFERENCE_SCRIPT} +ENV IMAGE_FILENAME=${IMAGE_FILENAME} +ENV TENSORRT_VERSION=${TENSORRT_VERSION} +ENV SERVING_JSON=${SERVING_JSON} +COPY ${ADLIK_DIRECTORY} /home/john/Adlik +COPY ${TEST_MODEL_PATH} /home/john/Adlik/model + +RUN mkdir /home/john/tensorrt +COPY ${TENSORRT_TAR} /home/john/tensorrt + +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN apt-get update && \ + apt-get install -y supervisor + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y gnupg && \ + apt-key adv --fetch-keys \ + "https://developer.download.nvidia.com/compute/cuda/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64/7fa2af80.pub" \ + https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg && \ + apt-get autoremove --purge -y gnupg && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + echo "deb https://developer.download.nvidia.com/compute/cuda/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64 /\n\ +deb https://developer.download.nvidia.com/compute/machine-learning/repos/$ID$(echo $VERSION_ID | tr -d .)/x86_64 /\n\ +deb https://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + automake \ + bazel \ + make \ + patch \ + cuda-cublas-dev-10-0 \ + cuda-cufft-dev-10-0 \ + cuda-cupti-10-0 \ + cuda-curand-dev-10-0 \ + cuda-cusolver-dev-10-0 \ + cuda-cusparse-dev-10-0 \ + cuda-nvml-dev-10-0 \ + cuda-nvrtc-10-0 \ + git \ + libtbb2 \ + 'libcudnn7=*+cuda10.0' \ + 'libcudnn7-dev=*+cuda10.0' \ + 'libnvinfer7=*+cuda10.0' \ + 'libnvinfer-dev=*+cuda10.0' \ + 'libnvonnxparsers7=*+cuda10.0' \ + 'libnvonnxparsers-dev=*+cuda10.0' \ + libtool \ + python \ + python3-setuptools \ + python3-wheel \ + python3.7-dev \ + python3-six \ + python3-pip && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN apt-mark hold libcudnn7 libcudnn7-dev libnvinfer7 libnvinfer-dev libnvonnxparsers7 libnvonnxparsers-dev + +WORKDIR /home/john + +RUN bazel version + +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + +COPY benchmark/src/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +RUN cd /home/john/tensorrt &&\ + tar xzvf ${TENSORRT_TAR} &&\ + cd TensorRT-${TENSORRT_VERSION}/python &&\ + pip3 install tensorrt-${TENSORRT_VERSION}-cp36-none-linux_x86_64.whl &&\ + cd /home/john/tensorrt/TensorRT-${TENSORRT_VERSION}/uff &&\ + pip3 install uff-*.whl &&\ + cd /home/john/tensorrt/TensorRT-${TENSORRT_VERSION}/graphsurgeon &&\ + pip3 install graphsurgeon-*.whl + +ENV LD_LIBRARY_PATH=/home/john/tensorrt/TensorRT-${TENSORRT_VERSION}/lib + +RUN cd /home/john/Adlik &&\ + bazel build //adlik_serving/clients/python:build_pip_package -c opt --distdir=/home/john/Adlik/archives &&\ + mkdir /tmp/pip-packages && bazel-bin/adlik_serving/clients/python/build_pip_package /tmp/pip-packages &&\ + env TF_CUDA_VERSION=10.0 \ + bazel build //adlik_serving \ + --config=tensorrt \ + -c opt \ + --action_env=LIBRARY_PATH=/usr/local/cuda-10.0/lib64/stubs \ + --incompatible_use_specific_tool_files=false \ + --distdir=/home/john/Adlik/archives \ + --jobs=50 &&\ + pip3 install --upgrade pip &&\ + pip3 install /tmp/pip-packages/adlik_serving_api-0.0.0-py2.py3-none-any.whl &&\ + cd /home/john/Adlik/model_compiler &&\ + pip3 install . &&\ + pip3 install -U tensorflow==1.14 pillow + +COPY ${SERVING_SCRIPT} /home/john/serving_script.sh +RUN chmod +x /home/john/serving_script.sh +COPY ${CLIENT_SCRIPT} /home/john/client_script.sh +RUN chmod +x /home/john/client_script.sh +COPY ${COMPILE_SCRIPT} /home/john/compile_script.sh +RUN chmod +x /home/john/compile_script.sh +CMD /home/john/compile_script.sh && "/usr/bin/supervisord" diff --git a/benchmark/test/docker_test/tflite.Dockerfile b/benchmark/test/docker_test/tflite.Dockerfile new file mode 100644 index 000000000..20f468d29 --- /dev/null +++ b/benchmark/test/docker_test/tflite.Dockerfile @@ -0,0 +1,87 @@ +FROM ubuntu:bionic +COPY sources.list /etc/apt/sources.list +COPY pip.conf /root/.pip/pip.conf +ARG ADLIK_DIRECTORY +ARG SERVING_SCRIPT +ARG CLIENT_SCRIPT +ARG TEST_MODEL_PATH +ARG SERVING_JSON +ARG CLIENT_INFERENCE_SCRIPT +ARG IMAGE_FILENAME +ARG COMPILE_SCRIPT +ENV COMPILE_SCRIPT=${COMPILE_SCRIPT} +ENV SERVING_JSON=${SERVING_JSON} +ENV CLIENT_INFERENCE_SCRIPT=${CLIENT_INFERENCE_SCRIPT} +ENV IMAGE_FILENAME=${IMAGE_FILENAME} +COPY ${ADLIK_DIRECTORY} /home/john/Adlik +COPY ${TEST_MODEL_PATH} /home/john/Adlik/model +RUN apt-get update && \ + apt-get install --no-install-recommends -y ca-certificates && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN apt-get update && \ + apt-get install -y supervisor + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y gnupg && \ + apt-key adv --fetch-keys \ + https://storage.googleapis.com/bazel-apt/doc/apt-key.pub.gpg && \ + apt-get autoremove --purge -y gnupg && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +RUN . /etc/os-release && \ + echo "deb https://storage.googleapis.com/bazel-apt stable jdk1.8" >> /etc/apt/sources.list + +RUN . /etc/os-release && \ + apt-get update && \ + apt-get install --no-install-recommends -y \ + automake \ + bazel \ + make \ + patch \ + python \ + git \ + libtbb2 \ + libtool \ + python3-setuptools \ + python3-wheel \ + python3.7-dev \ + python3-six \ + python3-pip && \ + apt-get clean && \ + find /var/lib/apt/lists -delete + +WORKDIR /home/john + +RUN bazel version + +COPY benchmark/src/supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +RUN cd /home/john/Adlik &&\ + bazel build //adlik_serving/clients/python:build_pip_package -c opt --distdir=/home/john/Adlik/archives &&\ + mkdir /tmp/pip-packages && bazel-bin/adlik_serving/clients/python/build_pip_package /tmp/pip-packages &&\ + export INTEL_CVSDK_DIR=/opt/intel/openvino_${OPENVINO_VERSION}/ &&\ + export InferenceEngine_DIR=$INTEL_CVSDK_DIR/deployment_tools/inference_engine/share &&\ + bazel build //adlik_serving \ + --config=tensorflow-lite-cpu \ + -c opt \ + --distdir=/home/john/Adlik/archives \ + --jobs=3 &&\ + pip3 install --upgrade pip &&\ + pip3 install /tmp/pip-packages/adlik_serving_api-0.0.0-py2.py3-none-any.whl &&\ + cd /home/john/Adlik/model_compiler &&\ + pip3 install . &&\ + pip3 install -U tensorflow==1.14 defusedxml==0.5.0 networkx==2.3.0 pillow &&\ + cd /home/john/Adlik/benchmark/src &&\ + python3 compile_model.py -t /home/john/Adlik/model -s ${SERVING_JSON} + +COPY ${SERVING_SCRIPT} /home/john/serving_script.sh +RUN chmod +x /home/john/serving_script.sh +COPY ${CLIENT_SCRIPT} /home/john/client_script.sh +RUN chmod +x /home/john/client_script.sh +COPY ${COMPILE_SCRIPT} /home/john/compile_script.sh +RUN chmod +x /home/john/compile_script.sh +CMD /home/john/compile_script.sh && "/usr/bin/supervisord" diff --git a/benchmark/test/serving_script/openvino_serving_script.sh b/benchmark/test/serving_script/openvino_serving_script.sh new file mode 100644 index 000000000..0ba2bb269 --- /dev/null +++ b/benchmark/test/serving_script/openvino_serving_script.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +INSTALL_DIR=/opt/intel/openvino_$OPENVINO_VERSION +source $INSTALL_DIR/bin/setupvars.sh +cd /home/john/Adlik/bazel-bin/adlik_serving && \ +./adlik_serving --model_base_path=/home/john/Adlik/model/model_repos --grpc_port=8500 --http_port=8501 diff --git a/benchmark/test/serving_script/serving_script.sh b/benchmark/test/serving_script/serving_script.sh new file mode 100644 index 000000000..f0b9aa27d --- /dev/null +++ b/benchmark/test/serving_script/serving_script.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +cd /home/john/Adlik/bazel-bin/adlik_serving && \ +./adlik_serving --model_base_path=/home/john/Adlik/model/model_repos --grpc_port=8500 --http_port=8501 \ No newline at end of file diff --git a/benchmark/test/test_automatic_test.py b/benchmark/test/test_automatic_test.py new file mode 100644 index 000000000..fa70d1640 --- /dev/null +++ b/benchmark/test/test_automatic_test.py @@ -0,0 +1,33 @@ +""" +The test of automatic test. +""" + +import unittest +import subprocess +import os + + +class TestAutomaticTest(unittest.TestCase): + """ + The test of automatic test + """ + + @staticmethod + def test_automatic_test(): + base_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + model_command = ['python3', 'benchmark/test/test_model/mnist_keras/mnist_keras.py'] + subprocess.run(args=model_command, cwd=base_dir, check=True) + command = ['python3', 'benchmark/src/automatic_test.py', + '-d', 'benchmark/test/docker_test/openvino.Dockerfile' + '-s', 'openvino', + '-b', '/media/A/work/adlik-test/src/Adlik-master', + ' -a', '.', + '-m', 'mnist', + '-c', 'benchmark/test/client_script/client_script.sh', + '-ss', 'benchmark/test/serving_script/openvino_serving_script.sh', + '-l', '/media/A/work/adlik-test/src/log', + '-tm', 'benchmark/test/test_model/mnist_keras', + '-cis', 'mnist_client.py', + '-i', 'mnist.png', + '-cs', 'benchmark/test/compile_script/openvino_compile_script.sh'] + subprocess.run(args=command, cwd=base_dir, check=True) diff --git a/benchmark/test/test_model/mnist_keras/mnist_keras.py b/benchmark/test/test_model/mnist_keras/mnist_keras.py new file mode 100644 index 000000000..125e2e15d --- /dev/null +++ b/benchmark/test/test_model/mnist_keras/mnist_keras.py @@ -0,0 +1,50 @@ +""" +This is a script for training mnist model. +""" +import os + +import keras +import numpy as np + + +def process_dataset(): + # Import the data + (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data() + + x_train, x_test = x_train / 255.0, x_test / 255.0 + + # Reshape the data + x_train = np.reshape(x_train, (60000, 28, 28, 1)) + x_test = np.reshape(x_test, (10000, 28, 28, 1)) + return x_train, y_train, x_test, y_test + + +def create_model(): + model = keras.models.Sequential() + model.add(keras.layers.Conv2D(32, kernel_size=(3, 3), + activation='relu', + input_shape=(28, 28, 1))) + model.add(keras.layers.Conv2D(64, (3, 3), activation='relu')) + model.add(keras.layers.MaxPooling2D(pool_size=(2, 2))) + model.add(keras.layers.Dropout(0.25)) + model.add(keras.layers.Reshape((9216,))) + model.add(keras.layers.Dense(128, activation='relu')) + model.add(keras.layers.Dropout(0.5)) + model.add(keras.layers.Dense(10, activation='softmax')) + model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy']) + return model + + +def _train(): + x_train, y_train, x_test, y_test = process_dataset() + model = create_model() + model.fit(x_train, y_train, epochs=2, verbose=1) + model.evaluate(x_test, y_test) + save_path = os.path.join(os.path.dirname(__file__), 'model', 'mnist.h5') + dir_name = os.path.dirname(save_path) + os.makedirs(dir_name, exist_ok=True) + model.save(save_path) + + +if __name__ == '__main__': + _train() diff --git a/benchmark/test/test_model/mnist_keras/serving_model.json b/benchmark/test/test_model/mnist_keras/serving_model.json new file mode 100644 index 000000000..d492d5ae6 --- /dev/null +++ b/benchmark/test/test_model/mnist_keras/serving_model.json @@ -0,0 +1,22 @@ +{ + "serving_type": "openvino", + "input_model": "model/mnist.h5", + "export_path": "model_repos", + "input_layer_names": [ + "conv2d_1" + ], + "output_layer_names": [ + "dense_2" + ], + "input_formats": [ + "channels_last" + ], + "input_signatures": [ + "image" + ], + "output_signatures": [ + "label" + ], + "model_name": "mnist", + "max_batch_size": 128 +} \ No newline at end of file diff --git a/benchmark/test/test_model/mnist_pytorch/mnist_pytorch.py b/benchmark/test/test_model/mnist_pytorch/mnist_pytorch.py new file mode 100644 index 000000000..2a79e251d --- /dev/null +++ b/benchmark/test/test_model/mnist_pytorch/mnist_pytorch.py @@ -0,0 +1,93 @@ +""" +This is a script for training mnist model. +""" + +from __future__ import print_function +import os +import torch.nn as nn +import torch.onnx +import torch.nn.functional as F +import torch.optim as optim +from torchvision import datasets, transforms +import torch.utils.data.distributed +import time +import torch.nn.utils +from torch.autograd import Variable + + +def dataset(): + data_dir = os.path.join(os.path.dirname(__file__), 'data') + train_loader = torch.utils.data.DataLoader( + datasets.MNIST(data_dir, train=True, download=True, + transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=50, shuffle=True) + test_loader = torch.utils.data.DataLoader( + datasets.MNIST(data_dir, train=False, transform=transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.1307,), (0.3081,)) + ])), + batch_size=50, shuffle=True) + return train_loader, test_loader + + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.conv1 = nn.Conv2d(1, 20, 5, 1) + self.conv2 = nn.Conv2d(20, 50, 5, 1) + self.fc1 = nn.Linear(4 * 4 * 50, 500) + self.fc2 = nn.Linear(500, 10) + + def forward(self, x): + x = F.relu(self.conv1(x)) + x = F.max_pool2d(x, 2, 2) + x = F.relu(self.conv2(x)) + x = F.max_pool2d(x, 2, 2) + x = x.view(-1, 4 * 4 * 50) + x = F.relu(self.fc1(x)) + x = self.fc2(x) + return F.softmax(x, dim=-1) + + +def _train(model, loss_fn, optimizer, data_loader, epochs): + total_samples = 0 + total_time = 0 + + for epoch in range(epochs): + for step, (x, target) in enumerate(data_loader, 0): + start_time = time.time() + + output = model(x) + loss = loss_fn(output, target) + optimizer.zero_grad() + loss.backward() + # optimizer.step() + + end_time = time.time() + total_time += end_time - start_time + total_samples += len(x) + + print('Epoch {} Step {}: speed = {}.'.format(epoch, step, total_samples / total_time)) + + +def main(): + device = torch.device("cpu") + train_loader, test_loader = dataset() + model = Net().to(device) + loss_fn = torch.nn.CrossEntropyLoss() + optimizer = optim.SGD(model.parameters(), lr=0.01) + + _train(model=model, loss_fn=loss_fn, optimizer=optimizer, data_loader=train_loader, epochs=1) + dummy_input = Variable(torch.randn(1, 1, 28, 28)) + dummy_input = dummy_input.to(device) + save_path = os.path.join(os.path.dirname(__file__), 'model', 'mnist.onnx') + dir_name = os.path.dirname(save_path) + os.makedirs(dir_name, exist_ok=True) + torch.onnx.export(model, dummy_input, save_path, verbose=True, keep_initializers_as_inputs=True) + + +if __name__ == '__main__': + main() diff --git a/benchmark/test/test_model/mnist_pytorch/serving_model.json b/benchmark/test/test_model/mnist_pytorch/serving_model.json new file mode 100644 index 000000000..a5eda3c41 --- /dev/null +++ b/benchmark/test/test_model/mnist_pytorch/serving_model.json @@ -0,0 +1,24 @@ +{ + "serving_type": "openvino", + "input_model": "model/mnist_softmax.onnx", + "export_path": "model_repos", + "input_names": [ + "input.1" + ], + "input_formats": [ + "channels_first" + ], + "output_names": [ + "20" + ], + "input_signatures": [ + "image" + ], + "output_signatures": [ + "label" + ], + "model_name": "mnist", + "job_id": "mnist_pytorch", + "callback": "", + "max_batch_size": 128 +} \ No newline at end of file diff --git a/benchmark/test/test_model/mnist_tensorflow/mnist_tensorflow.py b/benchmark/test/test_model/mnist_tensorflow/mnist_tensorflow.py new file mode 100644 index 000000000..2bc739f18 --- /dev/null +++ b/benchmark/test/test_model/mnist_tensorflow/mnist_tensorflow.py @@ -0,0 +1,134 @@ +""" +CNN-MNIST +""" + +from __future__ import absolute_import, division, print_function, unicode_literals +import tensorflow as tf +from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2 +from tensorflow.keras.layers import Dense, Flatten, Conv2D +from tensorflow.keras import Model +import os + + +def load_dataset(): + mnist = tf.keras.datasets.mnist + (x_train, y_train), (x_test, y_test) = mnist.load_data() + x_train, x_test = x_train / 255.0, x_test / 255.0 + + x_train = x_train[..., tf.newaxis] + x_test = x_test[..., tf.newaxis] + + train_ds = tf.data.Dataset.from_tensor_slices( + (x_train, y_train)).shuffle(10000).batch(32) + test_ds = tf.data.Dataset.from_tensor_slices((x_test, y_test)).batch(32) + return train_ds, test_ds + + +class MyModel(Model): + def __init__(self): + super(MyModel, self).__init__() + self.conv1 = Conv2D(32, 3, activation='relu') + self.flatten = Flatten() + self.d1 = Dense(128, activation='relu') + self.d2 = Dense(10, activation='softmax') + + def call(self, x): + x = self.conv1(x) + x = self.flatten(x) + x = self.d1(x) + return self.d2(x) + + +def get_loss(): + return tf.keras.losses.SparseCategoricalCrossentropy() + + +def get_optimizer(): + return tf.keras.optimizers.Adam() + + +@tf.function +def train_step(images, labels, model, loss_object, optimizer, train_loss, train_accuracy): + with tf.GradientTape() as tape: + predictions = model(images) + print(model.input_names, model.output_names) + loss = loss_object(labels, predictions) + gradients = tape.gradient(loss, model.trainable_variables) + optimizer.apply_gradients(zip(gradients, model.trainable_variables)) + + train_loss(loss) + train_accuracy(labels, predictions) + + +@tf.function +def test_step(images, labels, model, loss_object, test_loss, test_accuracy): + predictions = model(images) + t_loss = loss_object(labels, predictions) + + test_loss(t_loss) + test_accuracy(labels, predictions) + + +def main(): + model = MyModel() + epochs = 2 + train_dataset, test_dataset = load_dataset() + loss_object = get_loss() + optimizer = get_optimizer() + train_loss = tf.keras.metrics.Mean(name='train_loss') + train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='train_accuracy') + test_loss = tf.keras.metrics.Mean(name='test_loss') + test_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(name='test_accuracy') + for epoch in range(epochs): + train_loss.reset_states() + train_accuracy.reset_states() + test_loss.reset_states() + test_accuracy.reset_states() + + for images, labels in train_dataset: + train_step(images, labels, model, loss_object, optimizer, train_loss, train_accuracy) + + for test_images, test_labels in test_dataset: + test_step(test_images, test_labels, model, loss_object, train_loss, train_accuracy) + + template = 'Epoch {}, Loss: {}, Accuracy: {}, Test Loss: {}, Test Accuracy: {}' + print(template.format(epoch + 1, + train_loss.result(), + train_accuracy.result() * 100, + test_loss.result(), + test_accuracy.result() * 100)) + save_path = os.path.join(os.path.dirname(__file__), 'model') + dir_name = os.path.dirname(save_path) + os.makedirs(dir_name, exist_ok=True) + tf.saved_model.save(model, save_path) + + # Convert Keras model to ConcreteFunction + full_model = tf.function(lambda x: model(x)) + full_model = full_model.get_concrete_function( + tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype)) + + # Get frozen ConcreteFunction + frozen_func = convert_variables_to_constants_v2(full_model) + frozen_func.graph.as_graph_def() + + layers = [op.name for op in frozen_func.graph.get_operations()] + print("-" * 50) + print("Frozen model layers: ") + for layer in layers: + print(layer) + + print("-" * 50) + print("Frozen model inputs: ") + print(frozen_func.inputs) + print("Frozen model outputs: ") + print(frozen_func.outputs) + + # Save frozen graph from frozen ConcreteFunction to hard drive + tf.io.write_graph(graph_or_graph_def=frozen_func.graph, + logdir="./frozen_models", + name=os.path.join(save_path, "frozen_graph.pb"), + as_text=False) + + +if __name__ == '__main__': + main() diff --git a/benchmark/test/test_model/mnist_tensorflow/serving_model.json b/benchmark/test/test_model/mnist_tensorflow/serving_model.json new file mode 100644 index 000000000..88a594097 --- /dev/null +++ b/benchmark/test/test_model/mnist_tensorflow/serving_model.json @@ -0,0 +1,22 @@ +{ + "serving_type": "tf", + "input_model": "model/frozen_graph.pb", + "export_path": "model_repos_tf2", + "input_names": [ + "x:0" + ], + "output_names": [ + "Identity:0" + ], + "input_formats": [ + "channels_last" + ], + "input_signatures": [ + "image" + ], + "output_signatures": [ + "label" + ], + "model_name": "mnist", + "max_batch_size": 128 +} \ No newline at end of file diff --git a/benchmark/test/test_model/resnet50_keras/resnet50_keras.py b/benchmark/test/test_model/resnet50_keras/resnet50_keras.py new file mode 100644 index 000000000..7303b998d --- /dev/null +++ b/benchmark/test/test_model/resnet50_keras/resnet50_keras.py @@ -0,0 +1,22 @@ +""" +Resnet50 model +""" + +from keras.applications.resnet50 import ResNet50 +import os + + +def get_model(): + return ResNet50(weights='imagenet') + + +def main(): + model = get_model() + save_path = os.path.join(os.path.dirname(__file__), 'model', 'resnet50.h5') + dir_name = os.path.dirname(save_path) + os.makedirs(dir_name, exist_ok=True) + model.save(save_path) + + +if __name__ == '__main__': + main() diff --git a/benchmark/test/test_model/resnet50_keras/serving_model.json b/benchmark/test/test_model/resnet50_keras/serving_model.json new file mode 100644 index 000000000..5e5290fee --- /dev/null +++ b/benchmark/test/test_model/resnet50_keras/serving_model.json @@ -0,0 +1,22 @@ +{ + "serving_type": "tf", + "input_model": "model/resnet50.h5", + "export_path": "model_repos", + "input_layer_names": [ + "input_1" + ], + "output_layer_names": [ + "fc1000" + ], + "input_formats": [ + "channels_last" + ], + "input_signatures": [ + "image" + ], + "output_signatures": [ + "label" + ], + "model_name": "resnet50", + "max_batch_size": 128 +} \ No newline at end of file diff --git a/benchmark/test/test_model/resnet50_pytorch/resnet50_pytorch.py b/benchmark/test/test_model/resnet50_pytorch/resnet50_pytorch.py new file mode 100644 index 000000000..063714c20 --- /dev/null +++ b/benchmark/test/test_model/resnet50_pytorch/resnet50_pytorch.py @@ -0,0 +1,28 @@ +""" +Resnet50 model +""" + +from torch.autograd import Variable +import torchvision +import torch.onnx +import torch +import os + + +def get_model(): + return torchvision.models.resnet50(pretrained=True) + + +def main(): + device = torch.device('cpu') + dummy_input = Variable(torch.randn(1, 3, 224, 224)) + dummy_input = dummy_input.to(device) + model = get_model() + save_path = os.path.join(os.path.dirname(__file__), 'model', 'resnet50.onnx') + dir_name = os.path.dirname(save_path) + os.makedirs(dir_name, exist_ok=True) + torch.onnx.export(model, dummy_input, save_path, verbose=True, keep_initializers_as_inputs=True) + + +if __name__ == '__main__': + main() diff --git a/benchmark/test/test_model/resnet50_pytorch/serving_model.json b/benchmark/test/test_model/resnet50_pytorch/serving_model.json new file mode 100644 index 000000000..a9f8bb501 --- /dev/null +++ b/benchmark/test/test_model/resnet50_pytorch/serving_model.json @@ -0,0 +1,25 @@ +{ + "serving_type": "openvino", + "input_model": "./model/resnet50.onnx", + "export_path": "model_repos", + "script_path": "./model/mnist_pytorch.py", + "input_names": [ + "input.1" + ], + "input_formats": [ + "channels_first" + ], + "output_names": [ + "495" + ], + "input_signatures": [ + "image" + ], + "output_signatures": [ + "label" + ], + "model_name": "resnet50", + "job_id": "resnst50_pytorch", + "callback": "", + "max_batch_size": 128 +} \ No newline at end of file diff --git a/benchmark/test/test_model/resnet50_tensorflow/resnet50_ckpt.py b/benchmark/test/test_model/resnet50_tensorflow/resnet50_ckpt.py new file mode 100644 index 000000000..8d5c54d08 --- /dev/null +++ b/benchmark/test/test_model/resnet50_tensorflow/resnet50_ckpt.py @@ -0,0 +1,25 @@ +""" +Resnet50 model +""" + +import tensorflow.compat.v1 as tf +import os + + +def _get_model(): + return tf.keras.applications.resnet50.ResNet50(weights='imagenet') + + +def main(): + with tf.Session() as sess: + model = _get_model() + print(model.output_names) + sess.run(tf.global_variables_initializer()) + save_path = os.path.join(os.path.dirname(__file__), 'model', 'resnet50.ckpt') + dir_name = os.path.dirname(save_path) + os.makedirs(dir_name, exist_ok=True) + tf.train.Saver().save(sess, save_path) + + +if __name__ == '__main__': + main() diff --git a/benchmark/test/test_model/resnet50_tensorflow/resnet50_tensorflow.py b/benchmark/test/test_model/resnet50_tensorflow/resnet50_tensorflow.py new file mode 100644 index 000000000..f46e19c3d --- /dev/null +++ b/benchmark/test/test_model/resnet50_tensorflow/resnet50_tensorflow.py @@ -0,0 +1,55 @@ +""" +Resnet50 model +""" + +import tensorflow as tf +from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2 +import os + + +def _get_model(): + return tf.keras.applications.resnet50.ResNet50(include_top=True, weights='imagenet', + input_tensor=None, input_shape=None, + pooling=None, + classes=1000) + + +def main(): + model = _get_model() + print(model.output_names) + save_path = os.path.join(os.path.dirname(__file__), 'model') + dir_name = os.path.dirname(save_path) + os.makedirs(dir_name, exist_ok=True) + # saved model to SavedModel format + tf.saved_model.save(model, save_path) + + # Convert Keras model to ConcreteFunction + full_model = tf.function(lambda x: model(x)) + full_model = full_model.get_concrete_function( + tf.TensorSpec(model.inputs[0].shape, model.inputs[0].dtype)) + + # Get frozen ConcreteFunction + frozen_func = convert_variables_to_constants_v2(full_model) + frozen_func.graph.as_graph_def() + + layers = [op.name for op in frozen_func.graph.get_operations()] + print("-" * 50) + print("Frozen model layers: ") + for layer in layers: + print(layer) + + print("-" * 50) + print("Frozen model inputs: ") + print(frozen_func.inputs) + print("Frozen model outputs: ") + print(frozen_func.outputs) + + # Save frozen graph from frozen ConcreteFunction to hard drive + tf.io.write_graph(graph_or_graph_def=frozen_func.graph, + logdir="./frozen_models", + name=os.path.join(save_path, "frozen_graph.pb"), + as_text=False) + + +if __name__ == '__main__': + main() diff --git a/benchmark/test/test_model/resnet50_tensorflow/serving_model.json b/benchmark/test/test_model/resnet50_tensorflow/serving_model.json new file mode 100644 index 000000000..865f74265 --- /dev/null +++ b/benchmark/test/test_model/resnet50_tensorflow/serving_model.json @@ -0,0 +1,22 @@ +{ + "serving_type": "tf", + "input_model": "model/resnet50.ckpt", + "export_path": "model_repos", + "input_names": [ + "input_1:0" + ], + "output_names": [ + "fc1000/Softmax:0" + ], + "input_formats": [ + "channels_last" + ], + "input_signatures": [ + "image" + ], + "output_signatures": [ + "label" + ], + "model_name": "resnet50", + "max_batch_size": 128 +} \ No newline at end of file diff --git a/benchmark/test/test_model/resnet50_tensorflow/serving_model_pb.json b/benchmark/test/test_model/resnet50_tensorflow/serving_model_pb.json new file mode 100644 index 000000000..f7f91ab1d --- /dev/null +++ b/benchmark/test/test_model/resnet50_tensorflow/serving_model_pb.json @@ -0,0 +1,22 @@ +{ + "serving_type": "tf", + "input_model": "model/frozen_graph.pb", + "export_path": "model_repos_tf2", + "input_names": [ + "x:0" + ], + "output_names": [ + "Identity:0" + ], + "input_formats": [ + "channels_last" + ], + "input_signatures": [ + "image" + ], + "output_signatures": [ + "label" + ], + "model_name": "resnet50", + "max_batch_size": 128 +} \ No newline at end of file diff --git a/benchmark/tox.ini b/benchmark/tox.ini new file mode 100644 index 000000000..a6f3fea8b --- /dev/null +++ b/benchmark/tox.ini @@ -0,0 +1,9 @@ +[tox] +envlist = py37-dev + +[testenv] +commands = pylint src + bandit -c bandit.yaml -r src + flake8 src + +deps = .[test]