Testing adlik performance (#79)

Signed-off-by: zhangkaili <zhang.kaili@zte.com.cn>

benchmark/.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 120

benchmark/.pylintrc

@@ -0,0 +1,21 @@
+[MASTER]
+jobs=0
+
+[MESSAGES CONTROL]
+disable = fixme,
+          no-else-return,
+          too-many-arguments,
+          too-few-public-methods,
+          too-many-locals,
+          too-many-instance-attributes,
+          no-member,
+          unnecessary-pass
+
+[FORMAT]
+max-line-length = 120
+
+[BASIC]
+good-names = i,
+             j,
+             k,
+             o

benchmark/README.md

@@ -0,0 +1,84 @@
+# About the benchmark
+The benchmark is used to test the adlik serving performance of different models. Before using the benchmark to test the 
+performance of the runtime, you need to build the client, the binary, and compile the model.
+
+## Installing prerequisites
+
+- python3
+- pip3
+
+## Build and install packages
+
+1. Build clients and serving binary and make client pip packages (see [README.md](../../README.md)).
+
+2. Install clients pip package:
+
+   ```sh
+   pip3 install {dir_of_pip_package}/adlik_serving_api-0.0.0-py2.py3-none-any.whl
+   ```
+
+3. Install model_compiler:
+
+   ```sh
+   cd {Adlik_root_dir}/model_compiler
+   pip3 install .
+   ```
+
+##  Compile the test models
+
+1. Prepare model code and serving_model.json (If you don't know how to write, you can refer to the existing serving_model.json).
+
+   ```sh
+   cd {Adlik_root_dir}/benchmark/test
+   mkdir model_name
+   cd model_name
+   ```
+
+   Then put your prepared model and serving_model.json in the directory model_name.
+
+2. Run the model code, and save the model in {Adlik_root_dir}/benchmark/test/model_name/model.
+
+   ```sh
+   cd {Adlik_root_dir}/benchmark/test/model_name
+   python3 model.py
+   ```
+
+3. Compile the model and save the serving model.
+
+   ```sh
+   cd {Adlik_root_dir}/benchmark/src
+   python3 compile_model.py
+   ```
+
+   In the compile_model.py you can also specify the files that need to be compiled.
+
+##  Test the serving performance
+
+1. Deploy a serving service:
+
+   ```sh
+   cd {dir_of_adlik_serving_binary}
+   ./adlik_serving --model_base_path={model_serving_dir} --grpc_port={grpc_port} --http_port={http_port}
+   ```
+
+   Usually the adlik serving binary is in the directory {Adlik_root_dir}/bazel-bin/adlik_serving, the grpc_port can
+   be set to 8500 and the http_port can be set to 8501. And It should be noted that the type of the compiled model is 
+   the same as the type of the serving service
+
+2. Run a client and do inference:
+
+   ```sh
+   cd {Adlik_root_dir}/benchmark/test/client
+   python3 xxx_client.py --batch-size=128 path_image
+   ``` 
+
+   The log of serving and client will be saved in time_log.log.
+
+3. Analyze inference results
+
+   ```sh
+   cd {Adlik_root_dir}/benchmark/src
+   python3 test_result.py path_client_log path_serving_log batch_size model_name runtime
+   ```    
+
+   Then you can get the performance analysis results of the serving.

benchmark/bandit.yaml

@@ -0,0 +1,4 @@
+include:
+  - '*.py'
+
+skips: [B404,B603]

benchmark/setup.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+
+# Copyright 2019 ZTE corporation. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Benchmark test.
+"""
+
+from setuptools import find_packages, setup
+
+_VERSION = '0.0.0'
+
+_REQUIRED_PACKAGES = [
+    'keras==2.2.4',
+    'onnx==1.5.0',
+    'protobuf==3.6.1',
+    'torch==1.3.0',
+    'torchvision==0.4.0',
+    'requests',
+    'tensorflow==1.14.0',
+    'jsonschema==3.1.1',
+    'networkx==2.3',
+    'defusedxml==0.5.0'
+]
+
+_TEST_REQUIRES = [
+    'bandit==1.6.0',
+    'flake8==3.7.7',
+    'pylint==2.3.1'
+]
+
+setup(
+    name="benchmark",
+    version=_VERSION.replace('-', ''),
+    author='ZTE',
+    author_email='ai@zte.com.cn',
+    packages=find_packages('src'),
+    package_dir={'': 'src'},
+    description=__doc__,
+    license='Apache 2.0',
+    keywords='Test serving-lite performance',
+    install_requires=_REQUIRED_PACKAGES,
+    extras_require={'test': _TEST_REQUIRES}
+
+)

benchmark/src/compile_model.py

@@ -0,0 +1,25 @@
+import os
+import json
+import model_compiler
+
+
+def compile_model():
+    base_dir = os.path.dirname(os.path.dirname(__file__))
+    test_model_dir = os.path.join(base_dir, "test", "test_model")
+    for file in os.listdir(test_model_dir):
+        request_dir = os.path.join(test_model_dir, file, "serving_model.json")
+        try:
+            with open(request_dir, 'r') as request_file:
+                request = json.load(request_file)
+                model_dir = request["input_model"]
+                request["input_model"] = os.path.join(test_model_dir, file, model_dir)
+                export_dir = request["export_path"]
+                request["export_path"] = os.path.join(test_model_dir, file, export_dir)
+                result = model_compiler.compile_model(request)
+                print(result)
+        except FileNotFoundError:
+            print(f"Can not compile the model in {os.path.join(test_model_dir, file)}")
+
+
+if __name__ == '__main__':
+    compile_model()

benchmark/src/test_result.py

@@ -0,0 +1,56 @@
+"""
+The test result of adlik performance
+"""
+
+
+def the_speed_of_client(client_log_path, batch_size):
+    with open(client_log_path, 'r') as file:
+        lines = file.readlines()
+        sum_time = 0
+        for line in lines:
+            line = line.strip('\n')
+            time = line.split('predict:')[-1]
+            time = float(time.strip(' '))
+            sum_time = sum_time + time
+        speed_processing_picture = (len(lines) * batch_size) / sum_time
+    return speed_processing_picture, len(lines)
+
+
+def the_speed_of_serving(serving_log_path, batch_size):
+    with open(serving_log_path, 'r') as file:
+        lines = file.readlines()
+        runtime = lines[0].split('found runtime ')[-1]
+        lines = [line.partition('PredictServiceImpl')[-1] for line in lines]
+        sum_time = 0
+        batch_num = 0
+        for line in lines:
+            if line:
+                line = line.strip('\n')
+                time = line.partition('time (milliseconds):')[-1]
+                batch_num = batch_num + 1
+                time = float(time.strip(' '))
+                sum_time = sum_time + time
+        speed_processing_picture = (batch_num * batch_size) / sum_time * 1000
+    return speed_processing_picture, batch_num, runtime
+
+
+def main(client_log_path, serving_log_path, batch_size, model, runtime=None):
+    speed_processing_picture_client, batch_num = the_speed_of_client(client_log_path, batch_size)
+    speed_processing_picture_serving, batch_num1, serving_runtime = the_speed_of_serving(serving_log_path, batch_size)
+    assert batch_num == batch_num1
+    if runtime:
+        serving_runtime = runtime
+    else:
+        serving_runtime = serving_runtime
+    tail_latency = 1 / speed_processing_picture_client - 1 / speed_processing_picture_serving
+    print(f'Model: {model}, Runtime: {serving_runtime}')
+    print(f'The speed of processing picture in the client is : {speed_processing_picture_client}')
+    print(f'The speed of processing picture in the serving is : {speed_processing_picture_serving}')
+    print(f'The tail latency of one picture is : {tail_latency}')
+
+
+if __name__ == '__main__':
+    main('/media/A/work/adlik-test/adlik/log/resnet50_pytorch_tensorrt_client.log',
+         '/media/A/work/adlik-test/adlik/log/resnet50_pytorch_tensorrt_serving.log',
+         128,
+         'resnet50_pytorch')