Testing adlik performance (#79)

Closes #79
Signed-off-by: zhangkaili <zhang.kaili@zte.com.cn>

benchmark/.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 120

benchmark/.pylintrc

@@ -0,0 +1,21 @@
+[MASTER]
+jobs=0
+
+[MESSAGES CONTROL]
+disable = fixme,
+          no-else-return,
+          too-many-arguments,
+          too-few-public-methods,
+          too-many-locals,
+          too-many-instance-attributes,
+          no-member,
+          unnecessary-pass
+
+[FORMAT]
+max-line-length = 120
+
+[BASIC]
+good-names = i,
+             j,
+             k,
+             o

benchmark/README.md

@@ -0,0 +1,84 @@
+# About the benchmark
+The benchmark is used to test the adlik serving performance of different models. Before using the benchmark to test the 
+performance of the runtime, you need to build the client, the binary, and compile the model.
+
+## Installing prerequisites
+
+- python3
+- pip3
+
+## Build and install packages
+
+1. Build clients and serving binary and make client pip packages (see [README.md](../../README.md)).
+
+2. Install clients pip package:
+
+   ```sh
+   pip3 install {dir_of_pip_package}/adlik_serving_api-0.0.0-py2.py3-none-any.whl
+   ```
+
+3. Install model_compiler:
+
+   ```sh
+   cd {Adlik_root_dir}/model_compiler
+   pip3 install .
+   ```
+
+##  Compile the test models
+
+1. Prepare model code and serving_model.json (If you don't know how to write, you can refer to the existing serving_model.json).
+
+   ```sh
+   cd {Adlik_root_dir}/benchmark/test
+   mkdir model_name
+   cd model_name
+   ```
+
+   Then put your prepared model and serving_model.json in the directory model_name.
+
+2. Run the model code, and save the model in {Adlik_root_dir}/benchmark/test/model_name/model.
+
+   ```sh
+   cd {Adlik_root_dir}/benchmark/test/model_name
+   python3 model.py
+   ```
+
+3. Compile the model and save the serving model.
+
+   ```sh
+   cd {Adlik_root_dir}/benchmark/src
+   python3 compile_model.py
+   ```
+
+   In the compile_model.py you can also specify the files that need to be compiled.
+
+##  Test the serving performance
+
+1. Deploy a serving service:
+
+   ```sh
+   cd {dir_of_adlik_serving_binary}
+   ./adlik_serving --model_base_path={model_serving_dir} --grpc_port={grpc_port} --http_port={http_port}
+   ```
+
+   Usually the adlik serving binary is in the directory {Adlik_root_dir}/bazel-bin/adlik_serving, the grpc_port can
+   be set to 8500 and the http_port can be set to 8501. And It should be noted that the type of the compiled model is 
+   the same as the type of the serving service
+
+2. Run a client and do inference:
+
+   ```sh
+   cd {Adlik_root_dir}/benchmark/test/client
+   python3 xxx_client.py --batch-size=128 path_image
+   ``` 
+
+   The log of serving and client will be saved in time_log.log.
+
+3. Analyze inference results
+
+   ```sh
+   cd {Adlik_root_dir}/benchmark/src
+   python3 test_result.py path_client_log path_serving_log batch_size model_name runtime
+   ```    
+
+   Then you can get the performance analysis results of the serving.

benchmark/bandit.yaml

@@ -0,0 +1,4 @@
+include:
+  - '*.py'
+
+skips: [B404,B603]

benchmark/setup.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+# Copyright 2019 ZTE corporation. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Benchmark test.
+"""
+
+from setuptools import find_packages, setup
+
+_VERSION = '0.0.0'
+
+_REQUIRED_PACKAGES = [
+    'keras==2.2.4',
+    'onnx==1.5.0',
+    'protobuf==3.6.1',
+    'torch==1.3.0',
+    'torchvision==0.4.0',
+    'requests',
+    'tensorflow==1.14.0',
+    'jsonschema==3.1.1',
+    'networkx==2.3',
+    'defusedxml==0.5.0'
+]
+
+_TEST_REQUIRES = [
+    'bandit==1.6.0',
+    'flake8==3.7.7',
+    'pylint==2.3.1'
+]
+
+setup(
+    name="benchmark",
+    version=_VERSION.replace('-', ''),
+    author='ZTE',
+    author_email='ai@zte.com.cn',
+    packages=find_packages('src'),
+    package_dir={'': 'src'},
+    description=__doc__,
+    license='Apache 2.0',
+    keywords='Test serving-lite performance',
+    install_requires=_REQUIRED_PACKAGES,
+    extras_require={'test': _TEST_REQUIRES}
+
+)

benchmark/src/automatic_test.py

@@ -0,0 +1,104 @@
+import subprocess
+import argparse
+import os
+import time
+
+
+def _parse_arguments():
+    args_parser = argparse.ArgumentParser()
+    args_parser.add_argument("-d", "--docker-file-path", type=str, help="The docker file path of the test serving type")
+    args_parser.add_argument("-s", "--serving-type", type=str, help="The test serving type")
+    args_parser.add_argument("-b", "--build-directory", type=str, help="The directory which to build the docker")
+    args_parser.add_argument("-a", "--adlik-directory", type=str, default="Adlik-master", help="The adlik directory")
+    args_parser.add_argument("-m", "--model-name", type=str, help="The path of model used for test")
+    args_parser.add_argument("-c", "--client-script", type=str, default="client_script.sh",
+                             help="The script used to infer")
+    args_parser.add_argument("-ss", "--serving-script", type=str, default="serving_script.sh",
+                             help="The serving script")
+    args_parser.add_argument("-ov", "--openvino-version", type=str, default="2019.3.344",
+                             help="The version of the OpenVINO")
+    args_parser.add_argument("-tt", "--tensorrt-tar", type=str,
+                             default="TensorRT-7.0.0.11.Ubuntu-18.04.x86_64-gnu.cuda-10.0.cudnn7.6.tar.gz",
+                             help="The tar version of the TensorRT")
+    args_parser.add_argument("-tv", "--tensorrt-version", type=str, default="7.0.0.11", help="The version of TensorRT")
+    args_parser.add_argument("-l", "--log-path", type=str, default="log", help="The path of log directory")
+    args_parser.add_argument('-tm', '--test-model-path', type=str, help="The path of test model")
+    args_parser.add_argument("-sj", "--serving-json", type=str, default="serving_model.json", help="The json of model")
+    args_parser.add_argument("-cis", "--client-inference-script", type=str, required=True, help="The inference script")
+    args_parser.add_argument("-i", "--image-filename", type=str, required=True, nargs="?", help="Input image.")
+    args_parser.add_argument("-gl", "--gpu-label", type=int, default=None, help="The GPU label")
+    args_parser.add_argument("-cs", "--compile-script", type=str, default="compile_script.sh",
+                             help="Compile the model script")
+    return args_parser.parse_args()
+
+
+def _close_docker():
+    close_docker_command = ['sh', '-c',
+                            'docker rm -f adlik-test']
+    subprocess.run(close_docker_command)
+
+
+def _get_result(log_path, model_name):
+    calculate_command = ['python3', os.path.join(os.path.dirname(__file__), 'test_result.py'),
+                         '-c', os.path.join(log_path, 'client_time.log'),
+                         '-s', os.path.join(log_path, 'serving_time.log'),
+                         '-m', model_name]
+    with subprocess.Popen(calculate_command) as result_process:
+        print(result_process.stdout)
+
+
+def _get_log(log_path):
+    if os.path.exists(os.path.join(log_path, 'client_time.log')):
+        return False
+    else:
+        return True
+
+
+def _docker_build_command(args):
+    build_arg = f'--build-arg SERVING_SCRIPT={args.serving_script} ' \
+                f'--build-arg CLIENT_SCRIPT={args.client_script} ' \
+                f'--build-arg TEST_MODEL_PATH={args.test_model_path} ' \
+                f'--build-arg SERVING_JSON={args.serving_json} ' \
+                f'--build-arg CLIENT_INFERENCE_SCRIPT={args.client_inference_script} ' \
+                f'--build-arg IMAGE_FILENAME={args.image_filename} ' \
+                f'--build-arg COMPILE_SCRIPT={args.compile_script} '
+
+    if args.serving_type == 'openvino':
+        build_arg = build_arg + f'--build-arg OPENVINO_VERSION={args.openvino_version} '
+    elif args.serving_type == 'tensorrt':
+        build_arg = build_arg + f'--build-arg TENSORRT_VERSION={args.tensorrt_version} ' \
+                                f'--build-arg TENSORRT_TAR={args.tensorrt_tar} '
+    else:
+        build_arg = build_arg
+
+    build_command = f'docker build --build-arg ADLIK_DIRECTORY={args.adlik_directory} ' + build_arg + \
+                    f' -f {args.docker_file_path} -t adlik-test:{args.serving_type} {args.build_directory}'
+    return build_command
+
+
+def main(args):
+    try:
+        _close_docker()
+    except Exception:
+        pass
+    finally:
+        docker_build_command = _docker_build_command(args)
+
+    if not args.gpu_label:
+        docker_run_command = f'docker run -d --name adlik-test -v {args.log_path}:/home/john/log ' \
+                             f'adlik-test:{args.serving_type}'
+    else:
+        docker_run_command = f'NV_GPU={args.gpu_label} nvidia-docker run -d --name adlik-test ' \
+                             f'-v {args.log_path}:/home/john/log adlik-test:{args.serving_type}'
+
+    test_command = ['sh', '-c', docker_build_command + ' && ' + docker_run_command]
+
+    with subprocess.Popen(test_command):
+        while _get_log(args.log_path):
+            time.sleep(10)
+    _get_result(args.log_path, args.model_name)
+    _close_docker()
+
+
+if __name__ == '__main__':
+    main(_parse_arguments())

benchmark/src/compile_model.py

@@ -0,0 +1,33 @@
+import os
+import json
+import model_compiler
+import argparse
+
+
+def _get_request(request_file, test_model_dir):
+    request = json.load(request_file)
+    model_dir = request["input_model"]
+    request["input_model"] = os.path.join(test_model_dir, model_dir)
+    export_dir = request["export_path"]
+    request["export_path"] = os.path.join(test_model_dir, export_dir)
+    return request
+
+
+def compile_model(args):
+    request_dir = os.path.join(args.test_model_path, args.serving_model_json)
+    try:
+        with open(request_dir, 'r') as request_file:
+            test_model_dir = args.test_model_path
+            request = _get_request(request_file, test_model_dir)
+            result = model_compiler.compile_model(request)
+            print(result)
+    except FileNotFoundError:
+        print(f"Can not compile the model in {os.path.join(test_model_dir, args.model_path)}")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-t', '--test-model-path', type=str, required=True, help='The path of test model')
+    parser.add_argument('-s', '--serving-model-json', type=str, default='serving_model.json', help='The json of model')
+    args = parser.parse_args()
+    compile_model(args)

benchmark/src/supervisord.conf

@@ -0,0 +1,18 @@
+[supervisord]
+nodaemon=true
+
+[program:serving]
+command=/home/john/serving_script.sh
+priority=1
+autostart=true
+autorestart=unexpected
+stdout_logfile=/home/john/log/serving.stdout.log
+stderr_logfile=/home/john/log/serving.stderr.log
+
+[program:client]
+command=/home/john/client_script.sh && exit
+priority=2
+autostart=true
+autorestart=unexpected
+stdout_logfile=/home/john/log/client.stdout.log
+stderr_logfile=/home/john/log/client.stderr.log

benchmark/src/test_result.py

@@ -0,0 +1,72 @@
+"""
+The test result of adlik performance
+"""
+import argparse
+
+
+def _speed_of_client(client_log_path, batch_size):
+    with open(client_log_path, 'r') as file:
+        lines = file.readlines()
+        sum_time = []
+        for line in lines:
+            line = line.strip('\n')
+            time = line.split('predict:')[-1]
+            time = float(time.strip(' '))
+            sum_time.append(time)
+        sum_time.pop(0)
+        batch_num = len(sum_time)
+        speed_processing_picture = (batch_num * batch_size) / sum(sum_time)
+    return speed_processing_picture, batch_num
+
+
+def _speed_of_serving(serving_log_path, batch_size):
+    with open(serving_log_path, 'r') as file:
+        lines = file.readlines()
+        runtime = lines[0].partition('found runtime ')[-1]
+        lines = [line.partition('PredictServiceImpl')[-1] for line in lines]
+        sum_time = []
+        for line in lines:
+            if line:
+                line = line.strip('\n')
+                time = line.partition('time (milliseconds):')[-1]
+                time = float(time.strip(' '))
+                sum_time.append(time)
+        sum_time.pop(0)
+        batch_num = len(sum_time)
+        speed_processing_picture = (batch_num * batch_size) / sum(sum_time) * 1000
+    return speed_processing_picture, batch_num, runtime
+
+
+def main(args):
+    """
+    Analyze inference results
+    """
+    speed_processing_picture_client, batch_num = _speed_of_client(args.client_log_path, args.batch_size)
+    speed_processing_picture_serving, batch_num1, serving_runtime = _speed_of_serving(args.serving_log_path,
+                                                                                      args.batch_size)
+    assert batch_num == batch_num1
+    if args.runtime:
+        serving_runtime = args.runtime
+    else:
+        serving_runtime = serving_runtime
+    tail_latency = 1 / speed_processing_picture_client - 1 / speed_processing_picture_serving
+    print(f'Model: {args.model_name}, Runtime: {serving_runtime}')
+    print(f'The speed of processing picture in the client is : {speed_processing_picture_client}')
+    print(f'The speed of processing picture in the serving is : {speed_processing_picture_serving}')
+    print(f'The tail latency of one picture is : {tail_latency}')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', '--client-log-path', type=str, required=True,
+                        help='The path of client log')
+    parser.add_argument('-s', '--serving-log-path', type=str, required=True,
+                        help='The path of serving log')
+    parser.add_argument('-b', '--batch-size', type=int, required=False, default=128,
+                        help='Batch size. Default is 128.')
+    parser.add_argument('-m', '--model-name', type=str, required=True,
+                        help='The name of model')
+    parser.add_argument('-r', '--runtime', type=str, required=False, default=None,
+                        help='The serving type')
+    args = parser.parse_args()
+    main(args)