Add accuracy test. (#1140)

* Add accuracy test.

* Move if_quant/w_quaint into remark.

* enlarge qsize for CI server and reset postprocess result for nncase.

* Add '\n' for each item in remark.

tests/accuracy_test/compare_util.py

@@ -0,0 +1,175 @@
+# Copyright 2019-2021 Canaan Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# pylint: disable=invalid-name, unused-argument, import-outside-toplevel
+
+import math
+import os
+import re
+import struct
+import numpy as np
+from typing import List, Tuple
+from pathlib import Path
+from operator import le, lt, ge, gt, eq
+import test_utils
+
+
+def cosine(gt: np.ndarray, pred: np.ndarray, *args):
+    # remove the NaN values in the same location.
+    if np.isnan(gt).any() and np.isnan(pred).any():
+        gt_mask = np.isnan(gt)
+        pred_mask = np.isnan(pred)
+        mask = gt_mask & pred_mask
+        gt = gt[~mask]
+        pred = pred[~mask]
+
+    # remove the INF values in the same location.
+    if np.isinf(gt).any() and np.isinf(pred).any():
+        gt_mask = np.isinf(gt)
+        pred_mask = np.isinf(pred)
+        mask = gt_mask & pred_mask
+        gt = gt[~mask]
+        pred = pred[~mask]
+
+    # return -1 if the nan/inf value is still in the array.
+    if np.isnan(gt).any() or np.isnan(pred).any() or np.isinf(gt).any() or np.isinf(pred).any():
+        return -1
+
+    # exclude the situation of all zeros in array.
+    if compare_arrays(gt, pred):
+        return 1
+
+    result = (gt @ pred) / (np.linalg.norm(gt, 2) * np.linalg.norm(pred, 2))
+
+    return -1 if math.isnan(result) else result
+
+
+def compare_arrays(gt: np.ndarray, pred: np.ndarray):
+    return np.array_equal(gt, pred)
+
+
+def euclidean(gt: np.ndarray, pred: np.ndarray, *args):
+    return np.linalg.norm(gt.reshape(-1) - pred.reshape(-1))
+
+
+# def mse(gt: np.ndarray, pred: np.ndarray, *args):
+#     return np.mean((gt - pred) ** 2)
+
+def divide(gt: np.ndarray, pred: np.ndarray):
+
+    # remove the zero values in the same location.
+    gt_mask = np.equal(gt, 0)
+    pred_mask = np.equal(pred, 0)
+    mask = gt_mask & pred_mask
+    gt = gt[~mask]
+    pred = pred[~mask]
+
+    # to avoid divide zero.
+    pred = np.where(np.equal(pred, 0), 1e-7, pred)
+
+    result = np.divide(gt, pred)
+    return result
+
+
+def mean(gt: np.ndarray):
+    return np.mean(gt)
+
+
+def allclose(gt: np.ndarray, pred: np.ndarray, thresh: float):
+    return np.allclose(gt, pred, atol=thresh)
+
+
+def segment_close(gt: np.ndarray, pred: np.ndarray):
+    bucket = np.digitize(gt, [0, 64, 128, 10 ** 18])
+    seg_1 = (bucket == 1)
+    seg_2 = (bucket == 2)
+    seg_3 = (bucket == 3)
+    ret = True
+    if seg_1.size:
+        ret &= np.allclose(gt[seg_1], pred[seg_1], atol=0.6)
+    if seg_2.size:
+        ret &= np.allclose(gt[seg_2], pred[seg_2], atol=2)
+    if seg_3.size:
+        ret &= np.allclose(gt[seg_3], pred[seg_3], rtol=8 / 128)
+    return ret
+
+
+similarity_func = {
+    'cosine': cosine,
+    'euclidean': euclidean,
+    'allclose': np.allclose,
+    'segment': segment_close
+}
+
+
+def compare_binfile(result_path: Tuple[str, str],
+                    ground_truth_path: Tuple[str, str],
+                    dtype,
+                    similarity_name: str = 'cosine',
+                    threshold: float = 0.99,
+                    hist: bool = True) -> bool:
+    # NOTE the result_path is Tuple[ bin_path, txt_path ]
+    ground_truth_path_bin, ground_truth_path_txt = result_path
+    result_path_bin, result_path_txt = ground_truth_path
+    if 'npy' in ground_truth_path_bin:  # bfloat16
+        # gt, pred = bytes.fromhex(gt.strip()), bytes.fromhex(pred.strip())
+        # gt, pred = struct.unpack('>H', gt)[0], struct.unpack('>H', pred)[0]
+        raise NotImplemented("need support bfloat16 judge!")
+    else:
+        gt_arr = np.fromfile(ground_truth_path_bin, dtype).astype(np.float32)
+        pred_arr = np.fromfile(result_path_bin, dtype).astype(np.float32)
+        if gt_arr.size == pred_arr.size:
+            similarity = similarity_func[similarity_name](gt_arr, pred_arr)
+        else:
+            raise ValueError("The number of elements in gt and result not match\n")
+        if hist and not test_utils.in_ci:
+            y, x = np.histogram(gt_arr - pred_arr, 100)
+            p = Path(result_path_bin)
+            np.savetxt(str(p.parent / (p.stem + '_hist.csv')),
+                       np.stack((x[:-1], y)).T, fmt='%f', delimiter=',')
+        similarity_info = f"\n{similarity_name} similarity = {similarity}, threshold = {threshold}\n"
+    if similarity_name in ['cosine', 'segment']:
+        compare_op = lt
+    else:
+        compare_op = gt
+    if compare_op(similarity, threshold):
+        return False, similarity_info
+    if (mean(divide(gt_arr, pred_arr)) > 1.5 or mean(divide(gt_arr, pred_arr)) < 0.6):
+        return False, similarity_info, f"\nmaybe a case of multiples"
+    return True, similarity_info
+
+
+def compare_ndarray(expected: np.ndarray,
+                    actual: np.ndarray,
+                    similarity_name: str = 'cosine',
+                    threshold: float = 0.99,
+                    dump_hist: bool = True,
+                    dump_file: str = 'hist.csv') -> bool:
+    if expected.size == actual.size:
+        similarity = similarity_func[similarity_name](expected.flatten(), actual.flatten())
+    else:
+        return False, f"The numbers of elements in gt({expected.size}) and result({actual.size}) are not match.\n"
+
+    if dump_hist:
+        y, x = np.histogram(expected - actual, 100)
+        np.savetxt(dump_file, np.stack((x[:-1], y)).T, fmt='%f', delimiter=',')
+    similarity_info = f"{similarity_name} similarity = {similarity}, threshold = {threshold}\n"
+
+    if similarity_name in ['cosine', 'segment']:
+        compare_op = lt
+    else:
+        compare_op = gt
+
+    if compare_op(similarity, threshold):
+        return False, similarity_info
+    return True, similarity_info

tests/accuracy_test/config.toml

@@ -0,0 +1,133 @@
+name = 'default_config'
+root = 'tests_output'
+kmodel_name = 'test.kmodel'
+desc_name = 'kmodel.desc'
+
+[compile_opt]
+preprocess = false
+swapRB = false
+input_type = 'uint8'
+input_shape = [1, 224, 224, 3]
+input_range = [0, 255]
+input_file = ""
+mean = [0, 0, 0]
+std = [1, 1, 1]
+input_layout = 'NHWC'
+output_layout = 'NHWC'
+model_layout = 'NHWC'
+letterbox_value = 0
+dump_asm = true
+dump_ir = false
+shape_bucket_enable = false
+shape_bucket_range_info = { }
+shape_bucket_segments_count = 4
+shape_bucket_fix_var_map = { }
+
+[ptq_opt]
+use_mix_quant = false
+use_mse_quant_w = true
+export_quant_scheme = false
+export_weight_range_by_channel = true
+dump_quant_error = false
+dump_quant_error_symmetric_for_signed = true
+quant_type = "uint8"
+w_quant_type = "uint8"
+# ['NoClip', 'Kld']
+calibrate_method = 'NoClip'
+# ['NoFineTuneWeights', 'UseSquant']
+finetune_weights_method = 'NoFineTuneWeights'
+input_mean = 0.5
+input_std = 0.5
+quant_scheme = ""
+quant_scheme_strict_mode = false
+
+[infer_report_opt]
+enabled = false
+priority = 100
+kind = 'N/A'
+model_name = 'N/A'
+report_name = 'infer_report.json'
+
+[generator]
+[generator.inputs]
+# ['random', 'bin', 'image', 'constant_of_shape']
+method = 'random'
+number = 1
+batch = 1
+
+[generator.inputs.random]
+args = false
+
+[generator.inputs.bin]
+# /path/to/bin directory
+roofline_args = ''
+nncase_args = ''
+
+[generator.inputs.image]
+# /path/to/image directory
+args = ''
+
+[generator.inputs.constant_of_shape]
+# shape
+args = []
+
+[generator.calibs]
+method = 'random'
+number = 5
+batch = 1
+
+[generator.calibs.random]
+args = false
+
+[generator.calibs.bin]
+# /path/to/bin directory
+args = ''
+
+[generator.calibs.image]
+# /path/to/image directory
+args = ''
+
+[generator.calibs.constant_of_shape]
+# shape
+args = []
+
+[target]
+
+[target.cpu]
+eval = false
+infer = false
+similarity_name = 'cosine'
+
+[target.cpu.mode.noptq]
+enabled = false
+threshold = 0.999
+
+[target.cpu.mode.ptq]
+enabled = true
+threshold = 0.98
+
+[target.k510]
+eval = true
+infer = true
+similarity_name = 'cosine'
+
+[target.k510.mode.noptq]
+enabled = false
+threshold = 0.99
+
+[target.k510.mode.ptq]
+enabled = true
+threshold = 0.98
+
+[target.k230]
+eval = false
+infer = true
+similarity_name = 'cosine'
+
+[target.k230.mode.noptq]
+enabled = false
+threshold = 0.999
+
+[target.k230.mode.ptq]
+enabled = true
+threshold = 0.96

tests/accuracy_test/evaluator.py

@@ -0,0 +1,72 @@
+# Copyright 2019-2021 Canaan Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# pylint: disable=invalid-name, unused-argument, import-outside-toplevel
+
+from typing import List, Dict, Union, Tuple
+import os
+import nncase
+import numpy as np
+import test_utils
+import preprocess_utils
+from test_utils import *
+
+
+class Evaluator:
+    def run_evaluator(self, compiler, dump_dir):
+        evaluator = compiler.create_evaluator(3)
+
+        # transform input
+        new_inputs = []
+        for idx, value in enumerate(self.inputs):
+            new_value = self.transform_input(
+                value['data'], self.cfg['compile_opt']['input_type'], "infer")
+            new_inputs.append(new_value)
+
+        outputs = []
+        number = self.cfg['generator']['inputs']['number']
+        for n in range(number):
+            # set input
+            for idx, value in enumerate(self.inputs):
+                evaluator.set_input_tensor(idx, nncase.RuntimeTensor.from_numpy(new_inputs[idx][n]))
+
+            # run
+            evaluator.run()
+
+            # get output
+            output = self.dump_outputs(evaluator, dump_dir)
+
+            outputs.append(output)
+        return outputs
+
+    def dump_outputs(self, evaluator, eval_dir):
+        results = []
+        compile_opt = self.cfg['compile_opt']
+        for i in range(evaluator.outputs_size):
+            result = evaluator.get_output_tensor(i).to_numpy()
+            if compile_opt['preprocess']:
+                if(compile_opt['output_layout'] == 'NHWC' and self.model_type in ['caffe', 'onnx']):
+                    result = np.transpose(result, [0, 3, 1, 2])
+                elif (compile_opt['output_layout'] == 'NCHW' and self.model_type in ['tflite']):
+                    result = np.transpose(result, [0, 2, 3, 1])
+                elif compile_opt['output_layout'] not in ["NCHW", "NHWC"] and compile_opt['output_layout'] != "":
+                    tmp_perm = [int(idx) for idx in compile_opt['output_layout'].split(",")]
+                    result = np.transpose(
+                        result, preprocess_utils.get_source_transpose_index(tmp_perm))
+            os.makedirs(eval_dir, exist_ok=True)
+            if not test_utils.in_ci():
+                dump_bin_file(os.path.join(eval_dir, f'nncase_result_{i}.bin'), result)
+                dump_txt_file(os.path.join(eval_dir, f'nncase_result_{i}.txt'), result)
+
+            results.append(result)
+        return results