From a91ef69c515e44055641b00973b622ac3fad4c21 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Fri, 9 Dec 2022 12:25:16 +0800 Subject: [PATCH 01/13] update example with new API Signed-off-by: Mengni Wang --- .../emotion_ferplus/quantization/ptq/main.py | 143 ++++++++++--- .../quantization/ptq/run_benchmark.sh | 4 - .../quantization/ptq/run_tuning.sh | 4 - .../ultraface/quantization/ptq/main.py | 57 +++--- .../quantization/ptq/run_benchmark.sh | 4 - .../ultraface/quantization/ptq/run_tuning.sh | 4 - .../mobilenet_v2/quantization/ptq/main.py | 13 +- .../mobilenet_v3/quantization/ptq/main.py | 11 +- .../alexnet/quantization/ptq/README.md | 6 +- .../alexnet/quantization/ptq/alexnet.yaml | 40 ---- .../alexnet/quantization/ptq/alexnet_qdq.yaml | 40 ---- .../alexnet/quantization/ptq/main.py | 171 ++++++++++++---- .../alexnet/quantization/ptq/run_benchmark.sh | 4 - .../alexnet/quantization/ptq/run_tuning.sh | 8 +- .../arcface/quantization/ptq/README.md | 2 - .../arcface/quantization/ptq/arcface.yaml | 40 ---- .../arcface/quantization/ptq/main.py | 73 ++++--- .../arcface/quantization/ptq/run_benchmark.sh | 4 - .../arcface/quantization/ptq/run_tuning.sh | 4 - .../caffenet/quantization/ptq/README.md | 6 +- .../caffenet/quantization/ptq/caffenet.yaml | 40 ---- .../quantization/ptq/caffenet_qdq.yaml | 40 ---- .../caffenet/quantization/ptq/main.py | 172 ++++++++++++---- .../quantization/ptq/run_benchmark.sh | 4 - .../caffenet/quantization/ptq/run_tuning.sh | 8 +- .../densenet/quantization/ptq/README.md | 9 +- .../densenet/quantization/ptq/densenet.yaml | 100 --------- .../densenet/quantization/ptq/main.py | 192 ++++++++++++++++-- .../quantization/ptq/run_benchmark.sh | 12 +- .../densenet/quantization/ptq/run_tuning.sh | 12 +- .../efficientnet/quantization/ptq/main.py | 16 +- .../fcn/quantization/ptq/README.md | 6 +- .../fcn/quantization/ptq/fcn_rn50.yaml | 46 ----- .../fcn/quantization/ptq/fcn_rn50_qdq.yaml | 46 ----- .../fcn/quantization/ptq/main.py | 69 +++---- .../fcn/quantization/ptq/run_benchmark.sh | 4 - .../fcn/quantization/ptq/run_tuning.sh | 8 +- .../googlenet/quantization/ptq/README.md | 6 +- .../googlenet/quantization/ptq/googlenet.yaml | 40 ---- .../quantization/ptq/googlenet_qdq.yaml | 40 ---- .../googlenet/quantization/ptq/main.py | 175 ++++++++++++---- .../quantization/ptq/run_benchmark.sh | 4 - .../googlenet/quantization/ptq/run_tuning.sh | 8 +- .../inception/quantization/ptq/README.md | 6 +- .../quantization/ptq/inception_v1.yaml | 40 ---- .../quantization/ptq/inception_v1_qdq.yaml | 40 ---- .../inception/quantization/ptq/main.py | 172 ++++++++++++---- .../quantization/ptq/run_benchmark.sh | 4 - .../inception/quantization/ptq/run_tuning.sh | 8 +- .../mnist/quantization/ptq/main.py | 15 +- .../mobilenet/quantization/ptq/main.py | 16 +- .../resnet50/quantization/ptq/main.py | 18 +- .../shufflenet/quantization/ptq/main.py | 17 +- .../squeezenet/quantization/ptq/README.md | 6 +- .../squeezenet/quantization/ptq/main.py | 181 +++++++++++++---- .../quantization/ptq/run_benchmark.sh | 4 - .../squeezenet/quantization/ptq/run_tuning.sh | 8 +- .../quantization/ptq/squeezenet.yaml | 40 ---- .../quantization/ptq/squeezenet_qdq.yaml | 40 ---- .../vgg16/quantization/ptq/main.py | 17 +- .../zfnet/quantization/ptq/README.md | 6 +- .../zfnet/quantization/ptq/main.py | 176 ++++++++++++---- .../zfnet/quantization/ptq/run_benchmark.sh | 4 - .../zfnet/quantization/ptq/run_tuning.sh | 8 +- .../zfnet/quantization/ptq/zfnet512.yaml | 40 ---- .../zfnet/quantization/ptq/zfnet512_qdq.yaml | 40 ---- .../resnet50/quantization/ptq/main.py | 13 +- .../vgg16/quantization/ptq/main.py | 12 +- 68 files changed, 1327 insertions(+), 1299 deletions(-) delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/arcface.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/densenet.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512_qdq.yaml diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py index 188041d91ad..7e24edac314 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py @@ -23,6 +23,7 @@ import pandas as pd from PIL import Image +import onnxruntime as ort logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', @@ -52,11 +53,6 @@ default=False, help="whether quantize the model" ) -parser.add_argument( - '--config', - type=str, - help="config yaml path" -) parser.add_argument( '--output_model', type=str, @@ -69,6 +65,92 @@ ) args = parser.parse_args() +def _topk_shape_validate(preds, labels): + # preds shape can be Nxclass_num or class_num(N=1 by default) + # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax + if isinstance(preds, int): + preds = [preds] + preds = np.array(preds) + elif isinstance(preds, np.ndarray): + preds = np.array(preds) + elif isinstance(preds, list): + preds = np.array(preds) + preds = preds.reshape((-1, preds.shape[-1])) + + # consider labels just int value 1x1 + if isinstance(labels, int): + labels = [labels] + labels = np.array(labels) + elif isinstance(labels, tuple): + labels = np.array([labels]) + labels = labels.reshape((labels.shape[-1], -1)) + elif isinstance(labels, list): + if isinstance(labels[0], int): + labels = np.array(labels) + labels = labels.reshape((labels.shape[0], 1)) + elif isinstance(labels[0], tuple): + labels = np.array(labels) + labels = labels.reshape((labels.shape[-1], -1)) + else: + labels = np.array(labels) + # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) + # only support 2 dimension one-shot labels + # or 1 dimension one-hot class_num will confuse with N + + if len(preds.shape) == 1: + N = 1 + class_num = preds.shape[0] + preds = preds.reshape([-1, class_num]) + elif len(preds.shape) >= 2: + N = preds.shape[0] + preds = preds.reshape([N, -1]) + class_num = preds.shape[1] + + label_N = labels.shape[0] + assert label_N == N, 'labels batch size should same with preds' + labels = labels.reshape([N, -1]) + # one-hot labels will have 2 dimension not equal 1 + if labels.shape[1] != 1: + labels = labels.argsort()[..., -1:] + return preds, labels + +class TopK: + def __init__(self, k=1): + self.k = k + self.num_correct = 0 + self.num_sample = 0 + + def update(self, preds, labels, sample_weight=None): + preds, labels = _topk_shape_validate(preds, labels) + preds = preds.argsort()[..., -self.k:] + if self.k == 1: + correct = accuracy_score(preds, labels, normalize=False) + self.num_correct += correct + + else: + for p, l in zip(preds, labels): + # get top-k labels with np.argpartition + # p = np.argpartition(p, -self.k)[-self.k:] + l = l.astype('int32') + if l in p: + self.num_correct += 1 + + self.num_sample += len(labels) + + def reset(self): + self.num_correct = 0 + self.num_sample = 0 + + def result(self): + if self.num_sample == 0: + logger.warning("Sample num during evaluation is 0.") + return 0 + elif getattr(self, '_hvd', None) is not None: + allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) + allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) + return allgather_num_correct / allgather_num_sample + return self.num_correct / self.num_sample + class Dataloader: def __init__(self, data_path): df = pd.read_csv(data_path) @@ -92,30 +174,41 @@ def preprocess(self, image): img_data = np.array(img) img_data = np.resize(img_data, input_shape) return img_data.astype('float32') + +def eval_func(model, dataloader, metric): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + metric.update(output, label) + return metric.result() if __name__ == "__main__": model = onnx.load(args.model_path) dataloader = Dataloader(args.data_path) + top1 = TopK() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, top1) + if args.benchmark: - from neural_compressor.experimental import Benchmark, common - if args.mode == 'accuracy': - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = dataloader - evaluator(args.mode) - else: - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig() + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor import options - from neural_compressor.experimental import Quantization, common - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.eval_dataloader = dataloader - quantize.calib_dataloader = dataloader - q_model = quantize() - q_model.save(args.output_model) \ No newline at end of file + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig() + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + + q_model.save(args.output_model) + diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh index f609b2ecbc1..5528ec80db6 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -35,7 +32,6 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --config ${config} \ --mode ${mode} \ --data_path ${data_path} \ --benchmark diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh index cd41deda04e..9a3d8b07a29 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -36,7 +33,6 @@ function run_tuning { --model_path ${input_model} \ --output_model ${output_model} \ --data_path ${data_path} \ - --config ${config} \ --tune } diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py index 4dba5a1f2e8..a2237fbe327 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py @@ -23,6 +23,7 @@ import onnx import os from scipy.io import loadmat +import onnxruntime as ort class Dataloader: def __init__(self, data_path, size=[320,240]): @@ -356,6 +357,17 @@ def voc_ap(rec, prec): ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) return ap +def eval_func(model, dataloader, metric, postprocess): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + output, label = postprocess((output, label)) + metric.update(output, label) + return metric.result() + logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', @@ -398,11 +410,6 @@ def voc_ap(rec, prec): default=False, help="whether quantize the model" ) -parser.add_argument( - '--config', - type=str, - help="config yaml path" -) parser.add_argument( '--output_model', type=str, @@ -418,27 +425,27 @@ def voc_ap(rec, prec): if __name__ == "__main__": model = onnx.load(args.model_path) dataloader = Dataloader(args.data_path, size=args.input_size) + metric = AP(args.label_path) + postprocess = Post() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, metric, postprocess) + if args.benchmark: - from neural_compressor.experimental import Benchmark, common - if args.mode == 'accuracy': - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = dataloader - evaluator.postprocess = common.Postprocess(Post) - evaluator.metric = AP(args.label_path) - evaluator(args.mode) - else: - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig() + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.eval_dataloader = dataloader - quantize.calib_dataloader = dataloader - quantize.postprocess = common.Postprocess(Post) - quantize.metric = AP(args.label_path) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig() + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh index f609b2ecbc1..5528ec80db6 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -35,7 +32,6 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --config ${config} \ --mode ${mode} \ --data_path ${data_path} \ --benchmark diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh index cd41deda04e..9a3d8b07a29 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -36,7 +33,6 @@ function run_tuning { --model_path ${input_model} \ --output_model ${output_model} \ --data_path ${data_path} \ - --config ${config} \ --tune } diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py index 89b10b4a5c4..89b774ff75c 100644 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py @@ -70,18 +70,13 @@ model = onnx.load(args.model_path) if args.benchmark: - from neural_compressor.experimental import Benchmark, common + from neural_compressor.experimental import Benchmark evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) + evaluator.model = model evaluator(args.mode) if args.tune: - from neural_compressor.experimental import Quantization, common - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() + from neural_compressor import quantization + q_model = quantization.fit(model, args.config) q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py index 272f8ad89f8..5bd68eea74d 100644 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py @@ -66,14 +66,10 @@ parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' args = parser.parse_args() - model = onnx.load(args.model_path) if args.benchmark: @@ -83,10 +79,7 @@ evaluator(args.mode) if args.tune: - from neural_compressor.experimental import Quantization, common - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() + from neural_compressor import quantization + q_model = quantization.fit(model, args.config) q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md index dcf1c03a698..8363a10866a 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md @@ -20,7 +20,6 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=alexnet.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --output_model=path/to/save @@ -30,17 +29,16 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=alexnet_qdq.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ - --output_model=path/to/save + --output_model=path/to/save \ + --quant_format=QDQ ``` ### Benchmark ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=alexnet.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --mode=performance # or accuracy diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet.yaml deleted file mode 100644 index 34112f2ab99..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: alexnet - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet_qdq.yaml deleted file mode 100644 index 96d334b096d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: alexnet - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py index 9b542c6cb02..0fb1da396ac 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py @@ -25,14 +25,103 @@ import re import os from PIL import Image +import onnxruntime as ort +from sklearn.metrics import accuracy_score logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', level = logging.WARN) -class dataset: +def _topk_shape_validate(preds, labels): + # preds shape can be Nxclass_num or class_num(N=1 by default) + # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax + if isinstance(preds, int): + preds = [preds] + preds = np.array(preds) + elif isinstance(preds, np.ndarray): + preds = np.array(preds) + elif isinstance(preds, list): + preds = np.array(preds) + preds = preds.reshape((-1, preds.shape[-1])) + + # consider labels just int value 1x1 + if isinstance(labels, int): + labels = [labels] + labels = np.array(labels) + elif isinstance(labels, tuple): + labels = np.array([labels]) + labels = labels.reshape((labels.shape[-1], -1)) + elif isinstance(labels, list): + if isinstance(labels[0], int): + labels = np.array(labels) + labels = labels.reshape((labels.shape[0], 1)) + elif isinstance(labels[0], tuple): + labels = np.array(labels) + labels = labels.reshape((labels.shape[-1], -1)) + else: + labels = np.array(labels) + # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) + # only support 2 dimension one-shot labels + # or 1 dimension one-hot class_num will confuse with N + + if len(preds.shape) == 1: + N = 1 + class_num = preds.shape[0] + preds = preds.reshape([-1, class_num]) + elif len(preds.shape) >= 2: + N = preds.shape[0] + preds = preds.reshape([N, -1]) + class_num = preds.shape[1] + + label_N = labels.shape[0] + assert label_N == N, 'labels batch size should same with preds' + labels = labels.reshape([N, -1]) + # one-hot labels will have 2 dimension not equal 1 + if labels.shape[1] != 1: + labels = labels.argsort()[..., -1:] + return preds, labels + +class TopK: + def __init__(self, k=1): + self.k = k + self.num_correct = 0 + self.num_sample = 0 + + def update(self, preds, labels, sample_weight=None): + preds, labels = _topk_shape_validate(preds, labels) + preds = preds.argsort()[..., -self.k:] + if self.k == 1: + correct = accuracy_score(preds, labels, normalize=False) + self.num_correct += correct + + else: + for p, l in zip(preds, labels): + # get top-k labels with np.argpartition + # p = np.argpartition(p, -self.k)[-self.k:] + l = l.astype('int32') + if l in p: + self.num_correct += 1 + + self.num_sample += len(labels) + + def reset(self): + self.num_correct = 0 + self.num_sample = 0 + + def result(self): + if self.num_sample == 0: + logger.warning("Sample num during evaluation is 0.") + return 0 + elif getattr(self, '_hvd', None) is not None: + allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) + allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) + return allgather_num_correct / allgather_num_sample + return self.num_correct / self.num_sample + +class Dataloader: def __init__(self, data_path, image_list): + self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: @@ -41,22 +130,30 @@ def __init__(self, data_path, image_list): src = os.path.join(data_path, image_name) if not os.path.exists(src): continue - self.image_list.append(src) + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) + self.image_list.append(image) self.label_list.append(int(label)) - def __len__(self): - return len(self.image_list) + def __iter__(self): + for image, label in zip(self.image_list, self.label_list): + yield image, label - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label +def eval_func(model, dataloader, metric): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + metric.update(output, label) + return metric.result() if __name__ == "__main__": logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") @@ -90,44 +187,48 @@ def __getitem__(self, index): default=False, help="whether quantize the model" ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) parser.add_argument( '--output_model', type=str, help="output model path" ) + parser.add_argument('--quant_format', + type=str, + default='Default', + choices=['Default', 'QDQ'], + help="quantization format" + ) parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) args = parser.parse_args() model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' + dataloader = Dataloader(args.data_path, args.label_path) + top1 = TopK() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, top1) if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator(args.mode) + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig(quant_format=args.quant_format) + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh index ba9efa687b5..4901e12c6c5 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh @@ -12,9 +12,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -39,7 +36,6 @@ function run_benchmark { --model_path ${input_model} \ --data_path ${data_path} \ --label_path ${label_path} \ - --config ${config} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh index 0f1c6136330..55563483e4e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -28,6 +25,9 @@ function init_params { --label_path=*) label_path=$(echo $var |cut -f2 -d=) ;; + --quant_format=*) + quant_format=$(echo $var |cut -f2 -d=) + ;; esac done @@ -40,7 +40,7 @@ function run_tuning { --data_path ${data_path} \ --label_path ${label_path} \ --output_model ${output_model} \ - --config ${config} \ + --quant_format ${quant_format} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md index dffed180aaa..defb66ade01 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md @@ -18,7 +18,6 @@ wget https://github.com/onnx/models/raw/main/vision/body_analysis/arcface/model/ ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=arcface.yaml \ --data_path=/path/to/faces_ms1m_112x112/task.bin \ --nfolds=nfolds_num \ # optional, data type is int, default is 1 --output_model=path/to/save @@ -28,7 +27,6 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=arcface.yaml \ --data_path=/path/to/faces_ms1m_112x112/task.bin \ --nfolds=nfolds_num \ # optional, data type is int, default is 1 --mode=performance # or accuracy diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/arcface.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/arcface.yaml deleted file mode 100644 index 3a69ee97203..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/arcface.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: arcface - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - configs: - num_of_instance: 1 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py index 76f77322732..1b3d372a3bf 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py @@ -7,6 +7,7 @@ import pickle import logging import argparse +import onnxruntime as ort logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', @@ -110,8 +111,9 @@ def result(self): np.asarray(self.actual_issame), nrof_folds=self.nfolds) return np.mean(accuracy) -class Dataset: +class Dataloader: def __init__(self, data_dir): + self.batch_size = 1 path = os.path.join(data_dir) # Load data if os.path.exists(path): @@ -119,11 +121,19 @@ def __init__(self, data_dir): self.data_list = data_set[0] self.issame_list = data_set[1] - def __getitem__(self, index): - return self.data_list[index], self.issame_list + def __iter__(self): + for data in self.data_list: + yield data, self.issame_list - def __len__(self): - return len(self.data_list) +def eval_func(model, dataloader, metric): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + metric.update(output, label) + return metric.result() if __name__ == '__main__': logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") @@ -152,48 +162,49 @@ def __len__(self): default=False, help="whether quantize the model" ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) parser.add_argument( '--output_model', type=str, help="output model path" ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) parser.add_argument( '--nfolds', type=int, default=1, ) - + parser.add_argument( + '--mode', + type=str, + help="benchmark mode of performance or accuracy" + ) + args = parser.parse_args() # Load image size image_size = load_property(args.data_path) print('image_size', image_size) - ds = Dataset(args.data_path) + + dataloader = Dataloader(args.data_path) model = onnx.load(args.model_path) + metric = Metric(args.nfolds) + def eval(onnx_model): + return eval_func(onnx_model, dataloader, metric) + if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator.metric = Metric(args.nfolds) - evaluator(args.mode) + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - quantize.metric = Metric(args.nfolds) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig(approach='static') + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh index df920d5b232..b7afc9f4177 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -39,7 +36,6 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ --data_path ${data_path} \ - --config ${config} \ --mode ${mode} \ --nfolds ${nfolds} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh index 7428fb058f6..0a228d3f587 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -39,7 +36,6 @@ function run_tuning { --model_path ${input_model} \ --data_path ${data_path} \ --output_model ${output_model} \ - --config ${config} \ --nfolds ${nfolds} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md index 5fbf0c393f4..89e58a07e89 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md @@ -21,7 +21,6 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=caffenet.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --output_model=path/to/save @@ -31,17 +30,16 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=caffenet_qdq.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ - --output_model=path/to/save + --output_model=path/to/save \ + --quant_format=QDQ ``` ### Benchmark ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=caffenet.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --mode=performance # or accuracy diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet.yaml deleted file mode 100644 index 4592b284bca..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: caffenet - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet_qdq.yaml deleted file mode 100644 index 55d3e157956..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: caffenet - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py index 6fab92150ea..bee7b211379 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py @@ -25,14 +25,103 @@ import re import os from PIL import Image +import onnxruntime as ort +from sklearn.metrics import accuracy_score logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', level = logging.WARN) -class dataset: +def _topk_shape_validate(preds, labels): + # preds shape can be Nxclass_num or class_num(N=1 by default) + # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax + if isinstance(preds, int): + preds = [preds] + preds = np.array(preds) + elif isinstance(preds, np.ndarray): + preds = np.array(preds) + elif isinstance(preds, list): + preds = np.array(preds) + preds = preds.reshape((-1, preds.shape[-1])) + + # consider labels just int value 1x1 + if isinstance(labels, int): + labels = [labels] + labels = np.array(labels) + elif isinstance(labels, tuple): + labels = np.array([labels]) + labels = labels.reshape((labels.shape[-1], -1)) + elif isinstance(labels, list): + if isinstance(labels[0], int): + labels = np.array(labels) + labels = labels.reshape((labels.shape[0], 1)) + elif isinstance(labels[0], tuple): + labels = np.array(labels) + labels = labels.reshape((labels.shape[-1], -1)) + else: + labels = np.array(labels) + # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) + # only support 2 dimension one-shot labels + # or 1 dimension one-hot class_num will confuse with N + + if len(preds.shape) == 1: + N = 1 + class_num = preds.shape[0] + preds = preds.reshape([-1, class_num]) + elif len(preds.shape) >= 2: + N = preds.shape[0] + preds = preds.reshape([N, -1]) + class_num = preds.shape[1] + + label_N = labels.shape[0] + assert label_N == N, 'labels batch size should same with preds' + labels = labels.reshape([N, -1]) + # one-hot labels will have 2 dimension not equal 1 + if labels.shape[1] != 1: + labels = labels.argsort()[..., -1:] + return preds, labels + +class TopK: + def __init__(self, k=1): + self.k = k + self.num_correct = 0 + self.num_sample = 0 + + def update(self, preds, labels, sample_weight=None): + preds, labels = _topk_shape_validate(preds, labels) + preds = preds.argsort()[..., -self.k:] + if self.k == 1: + correct = accuracy_score(preds, labels, normalize=False) + self.num_correct += correct + + else: + for p, l in zip(preds, labels): + # get top-k labels with np.argpartition + # p = np.argpartition(p, -self.k)[-self.k:] + l = l.astype('int32') + if l in p: + self.num_correct += 1 + + self.num_sample += len(labels) + + def reset(self): + self.num_correct = 0 + self.num_sample = 0 + + def result(self): + if self.num_sample == 0: + logger.warning("Sample num during evaluation is 0.") + return 0 + elif getattr(self, '_hvd', None) is not None: + allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) + allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) + return allgather_num_correct / allgather_num_sample + return self.num_correct / self.num_sample + +class Dataloader: def __init__(self, data_path, image_list): + self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: @@ -41,22 +130,30 @@ def __init__(self, data_path, image_list): src = os.path.join(data_path, image_name) if not os.path.exists(src): continue - self.image_list.append(src) + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) + self.image_list.append(image) self.label_list.append(int(label)) - def __len__(self): - return len(self.image_list) + def __iter__(self): + for image, label in zip(self.image_list, self.label_list): + yield image, label - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label +def eval_func(model, dataloader, metric): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + metric.update(output, label) + return metric.result() if __name__ == "__main__": logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") @@ -90,11 +187,6 @@ def __getitem__(self, index): default=False, help="whether quantize the model" ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) parser.add_argument( '--output_model', type=str, @@ -103,31 +195,39 @@ def __getitem__(self, index): parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - + parser.add_argument('--quant_format', + type=str, + default='Default', + choices=['Default', 'QDQ'], + help="quantization format" + ) args = parser.parse_args() model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' + dataloader = Dataloader(args.data_path, args.label_path) + top1 = TopK() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, top1) if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator(args.mode) + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig(quant_format=args.quant_format) + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh index ba9efa687b5..4901e12c6c5 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh @@ -12,9 +12,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -39,7 +36,6 @@ function run_benchmark { --model_path ${input_model} \ --data_path ${data_path} \ --label_path ${label_path} \ - --config ${config} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh index 0f1c6136330..55563483e4e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -28,6 +25,9 @@ function init_params { --label_path=*) label_path=$(echo $var |cut -f2 -d=) ;; + --quant_format=*) + quant_format=$(echo $var |cut -f2 -d=) + ;; esac done @@ -40,7 +40,7 @@ function run_tuning { --data_path ${data_path} \ --label_path ${label_path} \ --output_model ${output_model} \ - --config ${config} \ + --quant_format ${quant_format} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md index ed3ef1f3cd3..3ab61d5db27 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md @@ -18,7 +18,8 @@ wget https://github.com/onnx/models/raw/main/vision/classification/densenet-121/ ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=densenet.yaml \ + --data_path=path/to/dataset \ + --label_path=path/to/label \ --output_model=path/to/save ``` @@ -26,7 +27,9 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=densenet.yaml \ - --mode=performance + --data_path=path/to/dataset \ + --label_path=path/to/label \ + --mode=performance # or accuracy + ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/densenet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/densenet.yaml deleted file mode 100644 index aa3d0fa5bc4..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/densenet.yaml +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: densenet - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - op_wise: { - 'Conv_nc_rename_0': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}, - 'Relu_nc_rename_1': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}} - } - calibration: - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.02 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py index ad74bd25887..aff131c766a 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py @@ -22,13 +22,150 @@ import numpy as np import onnx +import onnxruntime as ort +from sklearn.metrics import accuracy_score +import cv2 +from PIL import Image logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', level = logging.WARN) -class squeeze: +def _topk_shape_validate(preds, labels): + # preds shape can be Nxclass_num or class_num(N=1 by default) + # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax + if isinstance(preds, int): + preds = [preds] + preds = np.array(preds) + elif isinstance(preds, np.ndarray): + preds = np.array(preds) + elif isinstance(preds, list): + preds = np.array(preds) + preds = preds.reshape((-1, preds.shape[-1])) + + # consider labels just int value 1x1 + if isinstance(labels, int): + labels = [labels] + labels = np.array(labels) + elif isinstance(labels, tuple): + labels = np.array([labels]) + labels = labels.reshape((labels.shape[-1], -1)) + elif isinstance(labels, list): + if isinstance(labels[0], int): + labels = np.array(labels) + labels = labels.reshape((labels.shape[0], 1)) + elif isinstance(labels[0], tuple): + labels = np.array(labels) + labels = labels.reshape((labels.shape[-1], -1)) + else: + labels = np.array(labels) + # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) + # only support 2 dimension one-shot labels + # or 1 dimension one-hot class_num will confuse with N + + if len(preds.shape) == 1: + N = 1 + class_num = preds.shape[0] + preds = preds.reshape([-1, class_num]) + elif len(preds.shape) >= 2: + N = preds.shape[0] + preds = preds.reshape([N, -1]) + class_num = preds.shape[1] + + label_N = labels.shape[0] + assert label_N == N, 'labels batch size should same with preds' + labels = labels.reshape([N, -1]) + # one-hot labels will have 2 dimension not equal 1 + if labels.shape[1] != 1: + labels = labels.argsort()[..., -1:] + return preds, labels + +class TopK: + def __init__(self, k=1): + self.k = k + self.num_correct = 0 + self.num_sample = 0 + + def update(self, preds, labels, sample_weight=None): + preds, labels = _topk_shape_validate(preds, labels) + preds = preds.argsort()[..., -self.k:] + if self.k == 1: + correct = accuracy_score(preds, labels, normalize=False) + self.num_correct += correct + + else: + for p, l in zip(preds, labels): + # get top-k labels with np.argpartition + # p = np.argpartition(p, -self.k)[-self.k:] + l = l.astype('int32') + if l in p: + self.num_correct += 1 + + self.num_sample += len(labels) + + def reset(self): + self.num_correct = 0 + self.num_sample = 0 + + def result(self): + if self.num_sample == 0: + logger.warning("Sample num during evaluation is 0.") + return 0 + elif getattr(self, '_hvd', None) is not None: + allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) + allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) + return allgather_num_correct / allgather_num_sample + return self.num_correct / self.num_sample + +class Dataloader: + def __init__(self, data_path, image_list): + self.batch_size = 1 + self.image_list = [] + self.label_list = [] + with open(image_list, 'r') as f: + for s in f: + image_name, label = re.split(r"\s+", s.strip()) + src = os.path.join(data_path, image_name) + if not os.path.exists(src): + continue + with Image.open(src) as image: + image = np.array(image.convert('RGB')).astype(np.float32) + image = image / 255. + image = cv2.resize(image, (256,256)) + h, w = image.shape[0], image.shape[1] + if h + 1 < 224 or w + 1 < 224: + raise ValueError( + "Required crop size {} is larger then input image size {}".format( + (224, 224), (h, w))) + + if 224 != h or 224 != w: + y0 = (h - 224) // 2 + x0 = (w - 224) // 2 + image = image[y0:y0 + 224, x0:x0 + 224, :] + + image = (image - [0.485, 0.456, 0.406]) / [0.229, 0.224, 0.225] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) + self.image_list.append(image.astype(np.float32)) + self.label_list.append(int(label)) + + def __iter__(self): + for image, label in zip(self.image_list, self.label_list): + yield image, label + +def eval_func(model, dataloader, metric, postprocess): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + output, label = postprocess((output, label)) + metric.update(output, label) + return metric.result() + +class Squeeze: def __call__(self, sample): preds, labels = sample return np.squeeze(preds), labels @@ -56,9 +193,14 @@ def __call__(self, sample): help="whether quantize the model" ) parser.add_argument( - '--config', + '--data_path', + type=str, + help="Imagenet data path" + ) + parser.add_argument( + '--label_path', type=str, - help="config yaml path" + help="Imagenet label path" ) parser.add_argument( '--output_model', @@ -68,27 +210,37 @@ def __call__(self, sample): parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - args = parser.parse_args() - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.postprocess = common.Postprocess(squeeze) - evaluator(args.mode) + dataloader = Dataloader(args.data_path, args.label_path) + top1 = TopK() + postprocess = Squeeze() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, top1, postprocess) + if args.benchmark: + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.postprocess = common.Postprocess(squeeze) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + from neural_compressor.config import AccuracyCriterion + accuracy_criterion = AccuracyCriterion() + accuracy_criterion.relative = 0.02 + config = PostTrainingQuantConfig( + accuracy_criterion=accuracy_criterion, + op_name_list={'Conv_nc_rename_0': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}, + 'Relu_nc_rename_1': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}}) + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh index 2b7d99703d3..80b6743fec2 100755 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh @@ -13,12 +13,15 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; + --data_path=*) + data_path=$(echo $var |cut -f2 -d=) + ;; + --label_path=*) + label_path=$(echo $var |cut -f2 -d=) + ;; --mode=*) mode=$(echo $var |cut -f2 -d=) ;; @@ -32,7 +35,8 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --config ${config} \ + --data_path ${data_path} \ + --label_path ${label_path} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh index 97d06dab599..25d2cd1b4a6 100755 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh @@ -13,12 +13,15 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; + --data_path=*) + data_path=$(echo $var |cut -f2 -d=) + ;; + --label_path=*) + label_path=$(echo $var |cut -f2 -d=) + ;; --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; @@ -31,8 +34,9 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ + --data_path ${data_path} \ + --label_path ${label_path} \ --output_model ${output_model} \ - --config ${config} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py index 65d04524c17..3d6c89290a4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py @@ -62,26 +62,20 @@ parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) args = parser.parse_args() - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' model = onnx.load(args.model_path) if args.benchmark: - from neural_compressor.experimental import Benchmark, common + from neural_compressor.experimental import Benchmark evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) + evaluator.model = model evaluator(args.mode) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) - + from neural_compressor import quantization + q_model = quantization.fit(model, args.config) + q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md index aa078e0adce..a865e3dec1c 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md @@ -20,7 +20,6 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=fcn_rn50.yaml \ --data_path=path/to/coco/val2017 \ --label_path=path/to/coco/annotations/instances_val2017.json \ --output_model=path/to/save @@ -30,17 +29,16 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=fcn_rn50_qdq.yaml \ --data_path=path/to/coco/val2017 \ --label_path=path/to/coco/annotations/instances_val2017.json \ - --output_model=path/to/save + --output_model=path/to/save \ + --quant_format=QDQ ``` ### Benchmark ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=fcn_rn50.yaml \ --data_path=path/to/coco/val2017 \ --label_path=path/to/coco/annotations/instances_val2017.json \ --mode=performance # or accuracy diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50.yaml deleted file mode 100644 index 78025519822..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: fcn - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, or mxnet; allow new framework backend extension. - inputs: image - outputs: out,aux - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant - -evaluation: # optional. used to config evaluation process. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - configs: # optional. if not specified, use all cores in 1 socket. - cores_per_instance: 28 - num_of_instance: 1 - kmp_blocktime: 1 - performance: - iteration: 100 - configs: - cores_per_instance: 28 - num_of_instance: 1 - kmp_blocktime: 1 - -tuning: - accuracy_criterion: - absolute: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 2%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50_qdq.yaml deleted file mode 100644 index b7ffcd1c42d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50_qdq.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: fcn - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, or mxnet; allow new framework backend extension. - inputs: image - outputs: out,aux - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant - -evaluation: # optional. used to config evaluation process. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - configs: # optional. if not specified, use all cores in 1 socket. - cores_per_instance: 28 - num_of_instance: 1 - kmp_blocktime: 1 - performance: - iteration: 100 - configs: - cores_per_instance: 28 - num_of_instance: 1 - kmp_blocktime: 1 - -tuning: - accuracy_criterion: - absolute: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 2%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py index ab970a0f572..3e6976ba390 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py @@ -66,11 +66,6 @@ default=False, help="whether quantize the model" ) -parser.add_argument( - '--config', - type=str, - help="config yaml path" -) parser.add_argument( '--output_model', type=str, @@ -81,6 +76,12 @@ type=str, help="benchmark mode of performance or accuracy" ) +parser.add_argument('--quant_format', + type=str, + default='Default', + choices=['Default', 'QDQ'], + help="quantization format" +) args = parser.parse_args() # key = COCO id, value = Pascal VOC id @@ -114,8 +115,9 @@ transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) -class Dataset: +class Dataloader: def __init__(self): + self.batch_size = 1 imgIds = self.getImgIdsUnion(cocoGt, VOC_CAT_IDS) self.data = [] for imgId in imgIds: @@ -135,11 +137,9 @@ def __init__(self): output_tensor[0] = 1 - np.max(output_tensor, axis=0) self.data.append((input_tensor, output_tensor)) - def __len__(self): - return len(self.data) - - def __getitem__(self, index): - return self.data[index] + def __iter__(self, index): + for data in self.data: + yield data def getImgIdsUnion(self, gt, catIds): """ @@ -199,31 +199,32 @@ def evaluate(model, dataloader): if __name__ == "__main__": model = onnx.load(args.model_path) - ds = Dataset() + dataloader = Dataloader() def eval(model): - return evaluate(model, ds) - - if args.benchmark and args.mode == "accuracy": - results = eval(model) - print("Batch size = 1") - print("Accuracy: %.5f" % results) - - if args.benchmark and args.mode == "performance": - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator(args.mode) + return evaluate(model, dataloader) + + if args.benchmark: + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(iteration=100, cores_per_instance=28, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_func = eval - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + from neural_compressor.config import AccuracyCriterion + accuracy_criterion = AccuracyCriterion() + accuracy_criterion.absolute = 0.01 + config = PostTrainingQuantConfig( + accuracy_criterion=accuracy_criterion, + quant_format=args.quant_format) + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh index 00d1543581e..bb1127ede51 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -38,7 +35,6 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --config ${config} \ --mode ${mode} \ --data_path ${data_path} \ --label_path ${label_path} \ diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh index 80daa37897d..ca052c688dc 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -28,6 +25,9 @@ function init_params { --label_path=*) label_path=$(echo $var |cut -f2 -d=) ;; + --quant_format=*) + quant_format=$(echo $var |cut -f2 -d=) + ;; esac done @@ -38,9 +38,9 @@ function run_tuning { python main.py \ --model_path ${input_model} \ --output_model ${output_model} \ - --config ${config} \ --data_path ${data_path} \ --label_path ${label_path} \ + --quant_format ${quant_format} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md index 47d19e4e9a1..86f6c971c26 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md @@ -20,7 +20,6 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=googlenet.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --output_model=path/to/save @@ -30,17 +29,16 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=googlenet_qdq.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ - --output_model=path/to/save + --output_model=path/to/save \ + --quant_format=QDQ ``` ### Benchmark ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=googlenet.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --mode=performance # or accuracy diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet.yaml deleted file mode 100644 index f7adc0c8a63..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: googlenet - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet_qdq.yaml deleted file mode 100644 index 66a1fe6389d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: googlenet - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py index 26e54f4a1fe..6858c304215 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py @@ -25,14 +25,103 @@ import re import os from PIL import Image +import onnxruntime as ort +from sklearn.metrics import accuracy_score logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', level = logging.WARN) -class dataset: +def _topk_shape_validate(preds, labels): + # preds shape can be Nxclass_num or class_num(N=1 by default) + # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax + if isinstance(preds, int): + preds = [preds] + preds = np.array(preds) + elif isinstance(preds, np.ndarray): + preds = np.array(preds) + elif isinstance(preds, list): + preds = np.array(preds) + preds = preds.reshape((-1, preds.shape[-1])) + + # consider labels just int value 1x1 + if isinstance(labels, int): + labels = [labels] + labels = np.array(labels) + elif isinstance(labels, tuple): + labels = np.array([labels]) + labels = labels.reshape((labels.shape[-1], -1)) + elif isinstance(labels, list): + if isinstance(labels[0], int): + labels = np.array(labels) + labels = labels.reshape((labels.shape[0], 1)) + elif isinstance(labels[0], tuple): + labels = np.array(labels) + labels = labels.reshape((labels.shape[-1], -1)) + else: + labels = np.array(labels) + # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) + # only support 2 dimension one-shot labels + # or 1 dimension one-hot class_num will confuse with N + + if len(preds.shape) == 1: + N = 1 + class_num = preds.shape[0] + preds = preds.reshape([-1, class_num]) + elif len(preds.shape) >= 2: + N = preds.shape[0] + preds = preds.reshape([N, -1]) + class_num = preds.shape[1] + + label_N = labels.shape[0] + assert label_N == N, 'labels batch size should same with preds' + labels = labels.reshape([N, -1]) + # one-hot labels will have 2 dimension not equal 1 + if labels.shape[1] != 1: + labels = labels.argsort()[..., -1:] + return preds, labels + +class TopK: + def __init__(self, k=1): + self.k = k + self.num_correct = 0 + self.num_sample = 0 + + def update(self, preds, labels, sample_weight=None): + preds, labels = _topk_shape_validate(preds, labels) + preds = preds.argsort()[..., -self.k:] + if self.k == 1: + correct = accuracy_score(preds, labels, normalize=False) + self.num_correct += correct + + else: + for p, l in zip(preds, labels): + # get top-k labels with np.argpartition + # p = np.argpartition(p, -self.k)[-self.k:] + l = l.astype('int32') + if l in p: + self.num_correct += 1 + + self.num_sample += len(labels) + + def reset(self): + self.num_correct = 0 + self.num_sample = 0 + + def result(self): + if self.num_sample == 0: + logger.warning("Sample num during evaluation is 0.") + return 0 + elif getattr(self, '_hvd', None) is not None: + allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) + allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) + return allgather_num_correct / allgather_num_sample + return self.num_correct / self.num_sample + +class Dataloader: def __init__(self, data_path, image_list): + self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: @@ -41,22 +130,30 @@ def __init__(self, data_path, image_list): src = os.path.join(data_path, image_name) if not os.path.exists(src): continue - self.image_list.append(src) + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) + self.image_list.append(image) self.label_list.append(int(label)) - def __len__(self): - return len(self.image_list) + def __iter__(self): + for image, label in zip(self.image_list, self.label_list): + yield image, label - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label +def eval_func(model, dataloader, metric): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + metric.update(output, label) + return metric.result() if __name__ == "__main__": logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") @@ -67,7 +164,7 @@ def __getitem__(self, index): parser.add_argument( '--model_path', type=str, - help="Pre-trained googlenet model on onnx file" + help="Pre-trained model on onnx file" ) parser.add_argument( '--data_path', @@ -90,11 +187,6 @@ def __getitem__(self, index): default=False, help="whether quantize the model" ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) parser.add_argument( '--output_model', type=str, @@ -103,29 +195,38 @@ def __getitem__(self, index): parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) + parser.add_argument('--quant_format', + type=str, + default='Default', + choices=['Default', 'QDQ'], + help="quantization format" + ) args = parser.parse_args() model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' + dataloader = Dataloader(args.data_path, args.label_path) + top1 = TopK() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, top1) if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator.model = common.Model(model) - evaluator(args.mode) - + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig(quant_format=args.quant_format) + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) + diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh index ba9efa687b5..4901e12c6c5 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh @@ -12,9 +12,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -39,7 +36,6 @@ function run_benchmark { --model_path ${input_model} \ --data_path ${data_path} \ --label_path ${label_path} \ - --config ${config} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh index 0f1c6136330..55563483e4e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -28,6 +25,9 @@ function init_params { --label_path=*) label_path=$(echo $var |cut -f2 -d=) ;; + --quant_format=*) + quant_format=$(echo $var |cut -f2 -d=) + ;; esac done @@ -40,7 +40,7 @@ function run_tuning { --data_path ${data_path} \ --label_path ${label_path} \ --output_model ${output_model} \ - --config ${config} \ + --quant_format ${quant_format} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md index 39d8cc361cb..6be0ff773b8 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md @@ -20,7 +20,6 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=inception_v1.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --output_model=path/to/save @@ -30,17 +29,16 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=inception_v1_qdq.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ - --output_model=path/to/save + --output_model=path/to/save \ + --quant_format=QDQ ``` ### Benchmark ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=inception_v1.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --mode=performance # or accuracy diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1.yaml deleted file mode 100644 index 9e8fd3ad38e..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: inception_v1 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1_qdq.yaml deleted file mode 100644 index f30162cebdc..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: inception_v1 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py index 9b8c684466b..60c770fcad5 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py @@ -25,14 +25,103 @@ import re import os from PIL import Image +import onnxruntime as ort +from sklearn.metrics import accuracy_score logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', level = logging.WARN) -class dataset: +def _topk_shape_validate(preds, labels): + # preds shape can be Nxclass_num or class_num(N=1 by default) + # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax + if isinstance(preds, int): + preds = [preds] + preds = np.array(preds) + elif isinstance(preds, np.ndarray): + preds = np.array(preds) + elif isinstance(preds, list): + preds = np.array(preds) + preds = preds.reshape((-1, preds.shape[-1])) + + # consider labels just int value 1x1 + if isinstance(labels, int): + labels = [labels] + labels = np.array(labels) + elif isinstance(labels, tuple): + labels = np.array([labels]) + labels = labels.reshape((labels.shape[-1], -1)) + elif isinstance(labels, list): + if isinstance(labels[0], int): + labels = np.array(labels) + labels = labels.reshape((labels.shape[0], 1)) + elif isinstance(labels[0], tuple): + labels = np.array(labels) + labels = labels.reshape((labels.shape[-1], -1)) + else: + labels = np.array(labels) + # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) + # only support 2 dimension one-shot labels + # or 1 dimension one-hot class_num will confuse with N + + if len(preds.shape) == 1: + N = 1 + class_num = preds.shape[0] + preds = preds.reshape([-1, class_num]) + elif len(preds.shape) >= 2: + N = preds.shape[0] + preds = preds.reshape([N, -1]) + class_num = preds.shape[1] + + label_N = labels.shape[0] + assert label_N == N, 'labels batch size should same with preds' + labels = labels.reshape([N, -1]) + # one-hot labels will have 2 dimension not equal 1 + if labels.shape[1] != 1: + labels = labels.argsort()[..., -1:] + return preds, labels + +class TopK: + def __init__(self, k=1): + self.k = k + self.num_correct = 0 + self.num_sample = 0 + + def update(self, preds, labels, sample_weight=None): + preds, labels = _topk_shape_validate(preds, labels) + preds = preds.argsort()[..., -self.k:] + if self.k == 1: + correct = accuracy_score(preds, labels, normalize=False) + self.num_correct += correct + + else: + for p, l in zip(preds, labels): + # get top-k labels with np.argpartition + # p = np.argpartition(p, -self.k)[-self.k:] + l = l.astype('int32') + if l in p: + self.num_correct += 1 + + self.num_sample += len(labels) + + def reset(self): + self.num_correct = 0 + self.num_sample = 0 + + def result(self): + if self.num_sample == 0: + logger.warning("Sample num during evaluation is 0.") + return 0 + elif getattr(self, '_hvd', None) is not None: + allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) + allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) + return allgather_num_correct / allgather_num_sample + return self.num_correct / self.num_sample + +class Dataloader: def __init__(self, data_path, image_list): + self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: @@ -41,26 +130,35 @@ def __init__(self, data_path, image_list): src = os.path.join(data_path, image_name) if not os.path.exists(src): continue - self.image_list.append(src) + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) + self.image_list.append(image) self.label_list.append(int(label)) - def __len__(self): - return len(self.image_list) + def __iter__(self): + for image, label in zip(self.image_list, self.label_list): + yield image, label - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label +def eval_func(model, dataloader, metric): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + metric.update(output, label) + return metric.result() if __name__ == "__main__": logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") parser = argparse.ArgumentParser( + description="Fine-tune examples for image classification tasks.", formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( @@ -89,11 +187,6 @@ def __getitem__(self, index): default=False, help="whether quantize the model" ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) parser.add_argument( '--output_model', type=str, @@ -102,32 +195,41 @@ def __getitem__(self, index): parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - + parser.add_argument('--quant_format', + type=str, + default='Default', + choices=['Default', 'QDQ'], + help="quantization format" + ) args = parser.parse_args() model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) + dataloader = Dataloader(args.data_path, args.label_path) + top1 = TopK() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, top1) from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' options.onnxrt.graph_optimization.gemm2matmul = False if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator(args.mode) - + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig(quant_format=args.quant_format) + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh index ba9efa687b5..4901e12c6c5 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh @@ -12,9 +12,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -39,7 +36,6 @@ function run_benchmark { --model_path ${input_model} \ --data_path ${data_path} \ --label_path ${label_path} \ - --config ${config} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh index 0f1c6136330..55563483e4e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -28,6 +25,9 @@ function init_params { --label_path=*) label_path=$(echo $var |cut -f2 -d=) ;; + --quant_format=*) + quant_format=$(echo $var |cut -f2 -d=) + ;; esac done @@ -40,7 +40,7 @@ function run_tuning { --data_path ${data_path} \ --label_path ${label_path} \ --output_model ${output_model} \ - --config ${config} \ + --quant_format ${quant_format} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py index af9a7bf08ad..52ecd4759cb 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py @@ -59,26 +59,21 @@ type=str, help="output model path" ) - parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - args = parser.parse_args() model = onnx.load(args.model_path) if args.benchmark: - from neural_compressor.experimental import Benchmark, common + from neural_compressor.experimental import Benchmark evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) + evaluator.model = model evaluator(args.mode) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) + from neural_compressor import quantization + q_model = quantization.fit(model, args.config) + q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py index 3e89f75de62..f8d82593378 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py @@ -63,7 +63,6 @@ parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) @@ -71,17 +70,12 @@ model = onnx.load(args.model_path) if args.benchmark: - from neural_compressor.experimental import Benchmark, common + from neural_compressor.experimental import Benchmark evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) + evaluator.model = model evaluator(args.mode) if args.tune: - from neural_compressor import options - from neural_compressor.experimental import Quantization, common - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) + from neural_compressor import quantization + q_model = quantization.fit(model, args.config) + q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py index 38c506733ff..069a2f74b8a 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py @@ -63,26 +63,18 @@ parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - args = parser.parse_args() model = onnx.load(args.model_path) if args.benchmark: - from neural_compressor.experimental import Benchmark, common + from neural_compressor.experimental import Benchmark evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) + evaluator.model = model evaluator(args.mode) if args.tune: - from neural_compressor.experimental import Quantization, common - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) - + from neural_compressor import quantization + q_model = quantization.fit(model, args.config) + q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py index 3e35879cc58..3f0adff31d5 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py @@ -63,25 +63,18 @@ parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - args = parser.parse_args() model = onnx.load(args.model_path) if args.benchmark: - from neural_compressor.experimental import Benchmark, common + from neural_compressor.experimental import Benchmark evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) + evaluator.model = model evaluator(args.mode) if args.tune: - from neural_compressor.experimental import Quantization, common - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) + from neural_compressor import quantization + q_model = quantization.fit(model, args.config) + q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md index c9aa5439d9c..7931b53de79 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md @@ -20,7 +20,6 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=squeezenet.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --output_model=path/to/save @@ -30,17 +29,16 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=squeezenet_qdq.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ - --output_model=path/to/save + --output_model=path/to/save \ + --quant_format=QDQ ``` ### Benchmark ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=squeezenet.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --mode=performance # or accuracy diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py index 6234719d1e6..76864674b5d 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py @@ -25,19 +25,108 @@ import re import os from PIL import Image +import onnxruntime as ort +from sklearn.metrics import accuracy_score logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', level = logging.WARN) -class squeeze: +class Squeeze: def __call__(self, sample): preds, labels = sample return np.squeeze(preds), labels + +def _topk_shape_validate(preds, labels): + # preds shape can be Nxclass_num or class_num(N=1 by default) + # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax + if isinstance(preds, int): + preds = [preds] + preds = np.array(preds) + elif isinstance(preds, np.ndarray): + preds = np.array(preds) + elif isinstance(preds, list): + preds = np.array(preds) + preds = preds.reshape((-1, preds.shape[-1])) -class dataset: + # consider labels just int value 1x1 + if isinstance(labels, int): + labels = [labels] + labels = np.array(labels) + elif isinstance(labels, tuple): + labels = np.array([labels]) + labels = labels.reshape((labels.shape[-1], -1)) + elif isinstance(labels, list): + if isinstance(labels[0], int): + labels = np.array(labels) + labels = labels.reshape((labels.shape[0], 1)) + elif isinstance(labels[0], tuple): + labels = np.array(labels) + labels = labels.reshape((labels.shape[-1], -1)) + else: + labels = np.array(labels) + # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) + # only support 2 dimension one-shot labels + # or 1 dimension one-hot class_num will confuse with N + + if len(preds.shape) == 1: + N = 1 + class_num = preds.shape[0] + preds = preds.reshape([-1, class_num]) + elif len(preds.shape) >= 2: + N = preds.shape[0] + preds = preds.reshape([N, -1]) + class_num = preds.shape[1] + + label_N = labels.shape[0] + assert label_N == N, 'labels batch size should same with preds' + labels = labels.reshape([N, -1]) + # one-hot labels will have 2 dimension not equal 1 + if labels.shape[1] != 1: + labels = labels.argsort()[..., -1:] + return preds, labels + +class TopK: + def __init__(self, k=1): + self.k = k + self.num_correct = 0 + self.num_sample = 0 + + def update(self, preds, labels, sample_weight=None): + preds, labels = _topk_shape_validate(preds, labels) + preds = preds.argsort()[..., -self.k:] + if self.k == 1: + correct = accuracy_score(preds, labels, normalize=False) + self.num_correct += correct + + else: + for p, l in zip(preds, labels): + # get top-k labels with np.argpartition + # p = np.argpartition(p, -self.k)[-self.k:] + l = l.astype('int32') + if l in p: + self.num_correct += 1 + + self.num_sample += len(labels) + + def reset(self): + self.num_correct = 0 + self.num_sample = 0 + + def result(self): + if self.num_sample == 0: + logger.warning("Sample num during evaluation is 0.") + return 0 + elif getattr(self, '_hvd', None) is not None: + allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) + allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) + return allgather_num_correct / allgather_num_sample + return self.num_correct / self.num_sample + +class Dataloader: def __init__(self, data_path, image_list): + self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: @@ -46,27 +135,36 @@ def __init__(self, data_path, image_list): src = os.path.join(data_path, image_name) if not os.path.exists(src): continue - self.image_list.append(src) + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) + self.image_list.append(image) self.label_list.append(int(label)) - def __len__(self): - return len(self.image_list) + def __iter__(self): + for image, label in zip(self.image_list, self.label_list): + yield image, label - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label +def eval_func(model, dataloader, metric, postprocess): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + output, label = postprocess((output, label)) + metric.update(output, label) + return metric.result() if __name__ == "__main__": logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") parser = argparse.ArgumentParser( - description="Squeezenet fine-tune examples for image classification tasks.", + description="Googlenet fine-tune examples for image classification tasks.", formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( @@ -95,11 +193,6 @@ def __getitem__(self, index): default=False, help="whether quantize the model" ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) parser.add_argument( '--output_model', type=str, @@ -108,32 +201,38 @@ def __getitem__(self, index): parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - + parser.add_argument('--quant_format', + type=str, + default='Default', + choices=['Default', 'QDQ'], + help="quantization format" + ) args = parser.parse_args() model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) + dataloader = Dataloader(args.data_path, args.label_path) + top1 = TopK() + postprocess = Squeeze() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, top1, postprocess) if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator.postprocess = common.Postprocess(squeeze) - evaluator(args.mode) - + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - quantize.postprocess = common.Postprocess(squeeze) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig(quant_format=args.quant_format) + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh index ba9efa687b5..4901e12c6c5 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh @@ -12,9 +12,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -39,7 +36,6 @@ function run_benchmark { --model_path ${input_model} \ --data_path ${data_path} \ --label_path ${label_path} \ - --config ${config} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh index 0f1c6136330..55563483e4e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -28,6 +25,9 @@ function init_params { --label_path=*) label_path=$(echo $var |cut -f2 -d=) ;; + --quant_format=*) + quant_format=$(echo $var |cut -f2 -d=) + ;; esac done @@ -40,7 +40,7 @@ function run_tuning { --data_path ${data_path} \ --label_path ${label_path} \ --output_model ${output_model} \ - --config ${config} \ + --quant_format ${quant_format} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet.yaml deleted file mode 100644 index 3ef03492941..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: squeezenet - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet_qdq.yaml deleted file mode 100644 index c352f6756ea..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: squeezenet - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py index 53e55f3d406..600bb88a6da 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py @@ -59,30 +59,21 @@ type=str, help="output model path" ) - parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - args = parser.parse_args() model = onnx.load(args.model_path) - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - if args.benchmark: - from neural_compressor.experimental import Benchmark, common + from neural_compressor.experimental import Benchmark evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) + evaluator.model = model evaluator(args.mode) if args.tune: - from neural_compressor.experimental import Quantization, common - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() + from neural_compressor import quantization + q_model = quantization.fit(model, args.config) q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md index a72b369444a..c1a90e84463 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md @@ -20,7 +20,6 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=zfnet512.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --output_model=path/to/save @@ -30,17 +29,16 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=zfnet512_qdq.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ - --output_model=path/to/save + --output_model=path/to/save \ + --quant_format=QDQ ``` ### Benchmark ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=zfnet512.yaml \ --data_path=/path/to/imagenet \ --label_path=/path/to/imagenet/label \ --mode=performance # or accuracy diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py index 7da9d06052b..28da787c065 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py @@ -25,14 +25,103 @@ import re import os from PIL import Image +import onnxruntime as ort +from sklearn.metrics import accuracy_score logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', level = logging.WARN) -class dataset: +def _topk_shape_validate(preds, labels): + # preds shape can be Nxclass_num or class_num(N=1 by default) + # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax + if isinstance(preds, int): + preds = [preds] + preds = np.array(preds) + elif isinstance(preds, np.ndarray): + preds = np.array(preds) + elif isinstance(preds, list): + preds = np.array(preds) + preds = preds.reshape((-1, preds.shape[-1])) + + # consider labels just int value 1x1 + if isinstance(labels, int): + labels = [labels] + labels = np.array(labels) + elif isinstance(labels, tuple): + labels = np.array([labels]) + labels = labels.reshape((labels.shape[-1], -1)) + elif isinstance(labels, list): + if isinstance(labels[0], int): + labels = np.array(labels) + labels = labels.reshape((labels.shape[0], 1)) + elif isinstance(labels[0], tuple): + labels = np.array(labels) + labels = labels.reshape((labels.shape[-1], -1)) + else: + labels = np.array(labels) + # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) + # only support 2 dimension one-shot labels + # or 1 dimension one-hot class_num will confuse with N + + if len(preds.shape) == 1: + N = 1 + class_num = preds.shape[0] + preds = preds.reshape([-1, class_num]) + elif len(preds.shape) >= 2: + N = preds.shape[0] + preds = preds.reshape([N, -1]) + class_num = preds.shape[1] + + label_N = labels.shape[0] + assert label_N == N, 'labels batch size should same with preds' + labels = labels.reshape([N, -1]) + # one-hot labels will have 2 dimension not equal 1 + if labels.shape[1] != 1: + labels = labels.argsort()[..., -1:] + return preds, labels + +class TopK: + def __init__(self, k=1): + self.k = k + self.num_correct = 0 + self.num_sample = 0 + + def update(self, preds, labels, sample_weight=None): + preds, labels = _topk_shape_validate(preds, labels) + preds = preds.argsort()[..., -self.k:] + if self.k == 1: + correct = accuracy_score(preds, labels, normalize=False) + self.num_correct += correct + + else: + for p, l in zip(preds, labels): + # get top-k labels with np.argpartition + # p = np.argpartition(p, -self.k)[-self.k:] + l = l.astype('int32') + if l in p: + self.num_correct += 1 + + self.num_sample += len(labels) + + def reset(self): + self.num_correct = 0 + self.num_sample = 0 + + def result(self): + if self.num_sample == 0: + logger.warning("Sample num during evaluation is 0.") + return 0 + elif getattr(self, '_hvd', None) is not None: + allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) + allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) + return allgather_num_correct / allgather_num_sample + return self.num_correct / self.num_sample + +class Dataloader: def __init__(self, data_path, image_list): + self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: @@ -41,27 +130,35 @@ def __init__(self, data_path, image_list): src = os.path.join(data_path, image_name) if not os.path.exists(src): continue - self.image_list.append(src) + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) + self.image_list.append(image) self.label_list.append(int(label)) - def __len__(self): - return len(self.image_list) + def __iter__(self): + for image, label in zip(self.image_list, self.label_list): + yield image, label - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label +def eval_func(model, dataloader, metric): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + metric.update(output, label) + return metric.result() if __name__ == "__main__": logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") parser = argparse.ArgumentParser( - description="ZFNet fine-tune examples for image classification tasks.", + description="Googlenet fine-tune examples for image classification tasks.", formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( @@ -90,11 +187,6 @@ def __getitem__(self, index): default=False, help="whether quantize the model" ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) parser.add_argument( '--output_model', type=str, @@ -103,31 +195,37 @@ def __getitem__(self, index): parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - + parser.add_argument('--quant_format', + type=str, + default='Default', + choices=['Default', 'QDQ'], + help="quantization format" + ) args = parser.parse_args() model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' + dataloader = Dataloader(args.data_path, args.label_path) + top1 = TopK() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, top1) if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator(args.mode) - + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig(quant_format=args.quant_format) + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh index ba9efa687b5..4901e12c6c5 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh @@ -12,9 +12,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -39,7 +36,6 @@ function run_benchmark { --model_path ${input_model} \ --data_path ${data_path} \ --label_path ${label_path} \ - --config ${config} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh index 0f1c6136330..55563483e4e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh @@ -13,9 +13,6 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; @@ -28,6 +25,9 @@ function init_params { --label_path=*) label_path=$(echo $var |cut -f2 -d=) ;; + --quant_format=*) + quant_format=$(echo $var |cut -f2 -d=) + ;; esac done @@ -40,7 +40,7 @@ function run_tuning { --data_path ${data_path} \ --label_path ${label_path} \ --output_model ${output_model} \ - --config ${config} \ + --quant_format ${quant_format} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512.yaml deleted file mode 100644 index 348a646251f..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: zfnet512 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512_qdq.yaml deleted file mode 100644 index 162af6aa4b2..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: zfnet512 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py index 2a80449914a..f2e63a84ffc 100644 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py @@ -65,7 +65,6 @@ type=str, help="benchmark mode of performance or accuracy" ) - args = parser.parse_args() model = onnx.load(args.model_path) @@ -73,15 +72,9 @@ from neural_compressor.experimental import Benchmark, common evaluator = Benchmark(args.config) evaluator.model = common.Model(model) - evaluator(args.mode) + evaluator(args.mode) if args.tune: - from neural_compressor.experimental import Quantization, common - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() + from neural_compressor import quantization + q_model = quantization.fit(model, args.config) q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py index 95fe75ec506..f84c2fa6c2c 100644 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py @@ -60,15 +60,12 @@ type=str, help="output model path" ) - parser.add_argument( '--mode', type=str, help="benchmark mode of performance or accuracy" ) - args = parser.parse_args() - model = onnx.load(args.model_path) if args.benchmark: from neural_compressor.experimental import Benchmark, common @@ -77,11 +74,6 @@ evaluator(args.mode) if args.tune: - from neural_compressor.experimental import Quantization, common - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() + from neural_compressor import quantization + q_model = quantization.fit(model, args.config) q_model.save(args.output_model) From f194b14719d9cfb72ebe5cdfc29976e76c0e5577 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Fri, 9 Dec 2022 21:21:32 +0800 Subject: [PATCH 02/13] update link and script Signed-off-by: Mengni Wang --- examples/.config/model_params_onnxrt.json | 138 +++++++----------- examples/README.md | 26 ++-- .../quantization/ptq/README.md | 4 +- .../quantization/ptq/emotion_ferplus.yaml | 42 ------ .../emotion_ferplus/quantization/ptq/main.py | 9 +- .../quantization/ptq/run_benchmark.sh | 6 +- .../quantization/ptq/run_tuning.sh | 6 +- .../ultraface/quantization/ptq/README.md | 4 +- .../ultraface/quantization/ptq/main.py | 8 +- .../ultraface/quantization/ptq/ultraface.yaml | 42 ------ .../mobilenet_v2/quantization/ptq/README.md | 57 -------- .../mobilenet_v2/quantization/ptq/main.py | 82 ----------- .../quantization/ptq/mobilenet_v2.yaml | 84 ----------- .../quantization/ptq/mobilenet_v2_qdq.yaml | 84 ----------- .../quantization/ptq/requirements.txt | 9 -- .../quantization/ptq/run_benchmark.bat | 9 -- .../quantization/ptq/run_benchmark.sh | 41 ------ .../quantization/ptq/run_tuning.bat | 8 - .../quantization/ptq/run_tuning.sh | 39 ----- .../mobilenet_v3/quantization/ptq/main.py | 85 ----------- .../quantization/ptq/mobilenet_v3.yaml | 77 ---------- .../quantization/ptq/mobilenet_v3_qdq.yaml | 77 ---------- .../mobilenet_v3/quantization/ptq/readme.md | 43 ------ .../quantization/ptq/requirements.txt | 8 - .../quantization/ptq/run_benchmark.sh | 41 ------ .../quantization/ptq/run_tuning.sh | 39 ----- .../alexnet/quantization/ptq/README.md | 9 +- .../alexnet/quantization/ptq/main.py | 22 ++- .../alexnet/quantization/ptq/run_benchmark.sh | 10 +- .../alexnet/quantization/ptq/run_tuning.sh | 10 +- .../arcface/quantization/ptq/README.md | 6 +- .../arcface/quantization/ptq/main.py | 6 +- .../arcface/quantization/ptq/run_benchmark.sh | 10 +- .../arcface/quantization/ptq/run_tuning.sh | 10 +- .../caffenet/quantization/ptq/README.md | 9 +- .../caffenet/quantization/ptq/main.py | 22 ++- .../quantization/ptq/run_benchmark.sh | 10 +- .../caffenet/quantization/ptq/run_tuning.sh | 10 +- .../densenet/quantization/ptq/README.md | 6 +- .../densenet/quantization/ptq/main.py | 15 +- .../quantization/ptq/run_benchmark.sh | 10 +- .../densenet/quantization/ptq/run_tuning.sh | 10 +- .../efficientnet/quantization/ptq/README.md | 42 ------ .../quantization/ptq/efficientnet.yaml | 94 ------------ .../quantization/ptq/efficientnet_qdq.yaml | 94 ------------ .../efficientnet/quantization/ptq/main.py | 81 ---------- .../quantization/ptq/requirements.txt | 6 - .../quantization/ptq/run_benchmark.sh | 40 ----- .../quantization/ptq/run_tuning.sh | 39 ----- .../fcn/quantization/ptq/README.md | 9 +- .../fcn/quantization/ptq/main.py | 18 +-- .../fcn/quantization/ptq/run_benchmark.sh | 10 +- .../fcn/quantization/ptq/run_tuning.sh | 10 +- .../googlenet/quantization/ptq/README.md | 9 +- .../googlenet/quantization/ptq/main.py | 22 ++- .../quantization/ptq/run_benchmark.sh | 10 +- .../googlenet/quantization/ptq/run_tuning.sh | 10 +- .../inception/quantization/ptq/README.md | 9 +- .../inception/quantization/ptq/main.py | 22 ++- .../quantization/ptq/run_benchmark.sh | 10 +- .../inception/quantization/ptq/run_tuning.sh | 10 +- .../mnist/quantization/ptq/README.md | 30 ---- .../mnist/quantization/ptq/main.py | 79 ---------- .../mnist/quantization/ptq/mnist.yaml | 73 --------- .../mnist/quantization/ptq/requirements.txt | 6 - .../mnist/quantization/ptq/run_benchmark.sh | 41 ------ .../mnist/quantization/ptq/run_tuning.sh | 39 ----- .../mobilenet/quantization/ptq/README.md | 42 ------ .../mobilenet/quantization/ptq/main.py | 81 ---------- .../quantization/ptq/mobilenetv2.yaml | 96 ------------ .../quantization/ptq/mobilenetv2_qdq.yaml | 96 ------------ .../quantization/ptq/requirements.txt | 6 - .../quantization/ptq/run_benchmark.sh | 41 ------ .../mobilenet/quantization/ptq/run_tuning.sh | 39 ----- .../resnet50/quantization/ptq/README.md | 41 ------ .../resnet50/quantization/ptq/main.py | 80 ---------- .../quantization/ptq/requirements.txt | 8 - .../quantization/ptq/resnet50_v1_5.yaml | 97 ------------ .../quantization/ptq/resnet50_v1_5_qdq.yaml | 97 ------------ .../quantization/ptq/run_benchmark.sh | 41 ------ .../resnet50/quantization/ptq/run_tuning.sh | 39 ----- .../shufflenet/quantization/ptq/README.md | 42 ------ .../shufflenet/quantization/ptq/main.py | 80 ---------- .../quantization/ptq/requirements.txt | 6 - .../quantization/ptq/run_benchmark.sh | 41 ------ .../shufflenet/quantization/ptq/run_tuning.sh | 39 ----- .../quantization/ptq/shufflenetv2.yaml | 97 ------------ .../quantization/ptq/shufflenetv2_qdq.yaml | 97 ------------ .../squeezenet/quantization/ptq/README.md | 9 +- .../squeezenet/quantization/ptq/main.py | 22 ++- .../quantization/ptq/run_benchmark.sh | 10 +- .../squeezenet/quantization/ptq/run_tuning.sh | 10 +- .../vgg16/quantization/ptq/README.md | 41 ------ .../vgg16/quantization/ptq/main.py | 79 ---------- .../vgg16/quantization/ptq/requirements.txt | 8 - .../vgg16/quantization/ptq/run_benchmark.sh | 41 ------ .../vgg16/quantization/ptq/run_tuning.sh | 39 ----- .../vgg16/quantization/ptq/vgg16.yaml | 101 ------------- .../vgg16/quantization/ptq/vgg16_qdq.yaml | 101 ------------- .../zfnet/quantization/ptq/README.md | 9 +- .../zfnet/quantization/ptq/main.py | 22 ++- .../zfnet/quantization/ptq/run_benchmark.sh | 10 +- .../zfnet/quantization/ptq/run_tuning.sh | 10 +- .../resnet50/quantization/ptq/README.md | 69 --------- .../resnet50/quantization/ptq/main.py | 80 ---------- .../quantization/ptq/requirements.txt | 9 -- .../quantization/ptq/resnet50_v1_5.yaml | 80 ---------- .../ptq/resnet50_v1_5_mlperf.yaml | 105 ------------- .../ptq/resnet50_v1_5_mlperf_qdq.yaml | 105 ------------- .../quantization/ptq/resnet50_v1_5_qdq.yaml | 80 ---------- .../quantization/ptq/run_benchmark.bat | 9 -- .../quantization/ptq/run_benchmark.sh | 41 ------ .../resnet50/quantization/ptq/run_tuning.bat | 8 - .../resnet50/quantization/ptq/run_tuning.sh | 39 ----- .../vgg16/quantization/ptq/README.md | 58 -------- .../vgg16/quantization/ptq/main.py | 79 ---------- .../vgg16/quantization/ptq/requirements.txt | 9 -- .../vgg16/quantization/ptq/run_benchmark.bat | 9 -- .../vgg16/quantization/ptq/run_benchmark.sh | 41 ------ .../vgg16/quantization/ptq/run_tuning.bat | 8 - .../vgg16/quantization/ptq/run_tuning.sh | 39 ----- .../vgg16/quantization/ptq/vgg16.yaml | 80 ---------- .../vgg16/quantization/ptq/vgg16_qdq.yaml | 80 ---------- 123 files changed, 238 insertions(+), 4564 deletions(-) delete mode 100644 examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/emotion_ferplus.yaml delete mode 100644 examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/ultraface.yaml delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2.yaml delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.bat delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.bat delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3.yaml delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/readme.md delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/mnist.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/requirements.txt delete mode 100755 examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_benchmark.sh delete mode 100755 examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/requirements.txt delete mode 100755 examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_benchmark.sh delete mode 100755 examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16.yaml delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5.yaml delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf.yaml delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.bat delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.bat delete mode 100644 examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/vgg16/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/vgg16/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.bat delete mode 100644 examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.bat delete mode 100644 examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16.yaml delete mode 100644 examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16_qdq.yaml diff --git a/examples/.config/model_params_onnxrt.json b/examples/.config/model_params_onnxrt.json index e0b08c699ad..24c659be62f 100644 --- a/examples/.config/model_params_onnxrt.json +++ b/examples/.config/model_params_onnxrt.json @@ -164,12 +164,10 @@ }, "googlenet-12": { "model_src_dir": "image_recognition/onnx_model_zoo/googlenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/googlenet-12/googlenet-12.onnx", - "yaml": "googlenet.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "shufflenet-v2-12": { "model_src_dir": "image_recognition/onnx_model_zoo/shufflenet/quantization/ptq", @@ -182,30 +180,24 @@ }, "squeezenet": { "model_src_dir": "image_recognition/onnx_model_zoo/squeezenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/squeezenet/squeezenet1.0-12.onnx", - "yaml": "squeezenet.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "caffenet": { "model_src_dir": "image_recognition/onnx_model_zoo/caffenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/caffenet/caffenet-12.onnx", - "yaml": "caffenet.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "alexnet": { "model_src_dir": "image_recognition/onnx_model_zoo/alexnet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/alexnet/bvlcalexnet-12.onnx", - "yaml": "alexnet.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "efficientnet": { "model_src_dir": "image_recognition/onnx_model_zoo/efficientnet/quantization/ptq", @@ -218,21 +210,17 @@ }, "zfnet": { "model_src_dir": "image_recognition/onnx_model_zoo/zfnet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/zfnet/zfnet512-12.onnx", - "yaml": "zfnet512.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "inception_v1": { "model_src_dir": "image_recognition/onnx_model_zoo/inception/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/inception_v1/inception-v1-12.onnx", - "yaml": "inception_v1.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "ssd-12": { "model_src_dir": "object_detection/onnx_model_zoo/ssd/quantization/ptq", @@ -245,12 +233,10 @@ }, "fcn": { "model_src_dir": "image_recognition/onnx_model_zoo/fcn/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/coco2017/coco/val2017", + "dataset_location": "/tf_dataset2/datasets/coco2017/coco", "input_model": "/tf_dataset2/models/onnx/fcn/fcn-resnet50-11.onnx", - "yaml": "fcn_rn50.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "ssd_mobilenet_v1-2": { "model_src_dir": "object_detection/onnx_model_zoo/ssd_mobilenet_v1/quantization/ptq", @@ -281,12 +267,10 @@ }, "densenet": { "model_src_dir": "image_recognition/onnx_model_zoo/densenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/densenet/densenet-12.onnx", - "yaml": "densenet.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "mnist": { "model_src_dir": "image_recognition/onnx_model_zoo/mnist/quantization/ptq", @@ -452,12 +436,10 @@ }, "googlenet-12_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/googlenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/googlenet-12/googlenet-13.onnx", - "yaml": "googlenet_qdq.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "shufflenet-v2-12_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/shufflenet/quantization/ptq", @@ -470,30 +452,24 @@ }, "squeezenet_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/squeezenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/squeezenet/squeezenet1.0-13.onnx", - "yaml": "squeezenet_qdq.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "caffenet_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/caffenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/", "input_model": "/tf_dataset2/models/onnx/caffenet/caffenet-13.onnx", - "yaml": "caffenet_qdq.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "alexnet_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/alexnet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/", "input_model": "/tf_dataset2/models/onnx/alexnet/bvlcalexnet-13.onnx", - "yaml": "alexnet_qdq.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "efficientnet_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/efficientnet/quantization/ptq", @@ -506,21 +482,17 @@ }, "zfnet_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/zfnet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/zfnet/zfnet512-13.onnx", - "yaml": "zfnet512_qdq.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "inception_v1_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/inception/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/inception_v1/inception-v1-13.onnx", - "yaml": "inception_v1_qdq.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "ssd-12_qdq": { "model_src_dir": "object_detection/onnx_model_zoo/ssd/quantization/ptq", @@ -533,12 +505,10 @@ }, "fcn_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/fcn/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/coco2017/coco/val2017", + "dataset_location": "/tf_dataset2/datasets/coco2017/coco", "input_model": "/tf_dataset2/models/onnx/fcn/fcn-resnet50-13.onnx", - "yaml": "fcn_rn50_qdq.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "ssd_mobilenet_v1-2_qdq": { "model_src_dir": "object_detection/onnx_model_zoo/ssd_mobilenet_v1/quantization/ptq", @@ -598,28 +568,22 @@ "model_src_dir": "image_recognition/onnx_model_zoo/arcface/quantization/ptq", "dataset_location": "/tf_dataset2/datasets/faces_ms1m_112x112/lfw.bin", "input_model": "/tf_dataset2/models/onnx/arcface/arcfaceresnet100-11.onnx", - "yaml": "arcface.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "emotion_ferplus": { "model_src_dir": "body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq", "dataset_location": "/tf_dataset2/datasets/emotion_data/emotion_ferplus/fer2013/fer2013.csv", "input_model": "/tf_dataset2/models/onnx/emotion_ferplus/emotion-ferplus-12.onnx", - "yaml": "emotion_ferplus.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "ultraface": { "model_src_dir": "body_analysis/onnx_model_zoo/ultraface/quantization/ptq", "dataset_location": "/tf_dataset2/datasets/wider_face_data/WIDER_val", "input_model": "/tf_dataset2/models/onnx/ultraface/version-RFB-320-12.onnx", - "yaml": "ultraface.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "unet": { "model_src_dir": "image_recognition/unet/quantization/ptq", diff --git a/examples/README.md b/examples/README.md index 03cfc8a8f59..79937500628 100644 --- a/examples/README.md +++ b/examples/README.md @@ -774,31 +774,31 @@ Intel® Neural Compressor validated examples with multiple compression technique ResNet50 V1.5 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ResNet50 V1.5 MLPerf Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq VGG16 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq MobileNet V2 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq MobileNet V3 MLPerf Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq AlexNet @@ -822,7 +822,7 @@ Intel® Neural Compressor validated examples with multiple compression technique EfficientNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq FCN @@ -846,37 +846,37 @@ Intel® Neural Compressor validated examples with multiple compression technique MNIST Image Recognition Post-Training Static Quantization - qlinearops + qlinearops MobileNet V2 (ONNX Model Zoo) Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ResNet50 V1.5 (ONNX Model Zoo) Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ShuffleNet V2 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq SqueezeNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq VGG16 (ONNX Model Zoo) Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ZFNet @@ -1068,7 +1068,7 @@ Intel® Neural Compressor validated examples with multiple compression technique Emotion FERPlus Body Analysis Post-Training Static Quantization - qlinearops + qlinearops Ultra Face diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/README.md b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/README.md index 72a77147f6f..76a7fbc031b 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/README.md +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/README.md @@ -20,7 +20,7 @@ wget https://github.com/onnx/models/raw/main/vision/body_analysis/emotion_ferplu ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx --config=emotion_ferplus.yaml \ - --data_path=/path/to/data \ + --dataset_location=/path/to/data \ --output_model=path/to/save ``` @@ -29,6 +29,6 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx --config=emotion_ferplus.yaml \ - --data_path=/path/to/data \ + --dataset_location=/path/to/data \ --mode=performance ``` diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/emotion_ferplus.yaml b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/emotion_ferplus.yaml deleted file mode 100644 index 811c68648b4..00000000000 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/emotion_ferplus.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: emotion_ferplus - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant - -evaluation: - performance: - dataloader: - batch_size: 1 - dataset: - dummy: - shape: [100,1,64,64] - accuracy: - metric: - topk: 1 - -tuning: - accuracy_criterion: - absolute: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 2%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. - diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py index 7e24edac314..3d75f7f4ed2 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py @@ -24,6 +24,7 @@ from PIL import Image import onnxruntime as ort +from sklearn.metrics import accuracy_score logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', @@ -34,7 +35,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, ) parser.add_argument( @@ -152,8 +153,8 @@ def result(self): return self.num_correct / self.num_sample class Dataloader: - def __init__(self, data_path): - df = pd.read_csv(data_path) + def __init__(self, dataset_location): + df = pd.read_csv(dataset_location) df = df[df['Usage']=='PublicTest'] images = [np.reshape(np.fromstring(image, dtype=np.uint8, sep=' '), (48, 48)) for image in df['pixels']] labels = np.array(list(map(int, df['emotion']))) @@ -187,7 +188,7 @@ def eval_func(model, dataloader, metric): if __name__ == "__main__": model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path) + dataloader = Dataloader(args.dataset_location) top1 = TopK() def eval(onnx_model): return eval_func(onnx_model, dataloader, top1) diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh index 5528ec80db6..2d211d2bbff 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh @@ -16,8 +16,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -33,7 +33,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ --mode ${mode} \ - --data_path ${data_path} \ + --dataset_location ${dataset_location} \ --benchmark } diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh index 9a3d8b07a29..9f907bfa0eb 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh @@ -16,8 +16,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --output_model=*) output_model=$(echo $var |cut -f2 -d=) @@ -32,7 +32,7 @@ function run_tuning { python main.py \ --model_path ${input_model} \ --output_model ${output_model} \ - --data_path ${data_path} \ + --dataset_location ${dataset_location} \ --tune } diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/README.md b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/README.md index edd5a07875f..38e1880112e 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/README.md +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/README.md @@ -20,7 +20,7 @@ wget https://github.com/onnx/models/raw/main/vision/body_analysis/ultraface/mode ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx --config=ultraface.yaml \ - --data_path=/path/to/data \ + --dataset_location=/path/to/data \ --output_model=path/to/save ``` @@ -29,6 +29,6 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx --config=ultraface.yaml \ - --data_path=/path/to/data \ + --dataset_location=/path/to/data \ --mode=performance ``` diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py index a2237fbe327..dca3b5fc626 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py @@ -26,12 +26,12 @@ import onnxruntime as ort class Dataloader: - def __init__(self, data_path, size=[320,240]): + def __init__(self, dataset_location, size=[320,240]): self.batch_size = 1 image_mean=np.array([127, 127, 127], dtype=np.float32) image_std = 128.0 self.data = [] - for parent, dir_names, file_names in os.walk(data_path): + for parent, dir_names, file_names in os.walk(dataset_location): for file_name in file_names: if not file_name.lower().endswith('jpg'): continue @@ -377,7 +377,7 @@ def eval_func(model, dataloader, metric, postprocess): formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Path of wider face validation dataset." ) @@ -424,7 +424,7 @@ def eval_func(model, dataloader, metric, postprocess): if __name__ == "__main__": model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path, size=args.input_size) + dataloader = Dataloader(args.dataset_location, size=args.input_size) metric = AP(args.label_path) postprocess = Post() def eval(onnx_model): diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/ultraface.yaml b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/ultraface.yaml deleted file mode 100644 index e1662d26f3d..00000000000 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/ultraface.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: ultraface - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant - -evaluation: - performance: - dataloader: - batch_size: 1 - dataset: - dummy: - shape: [100,3,240,320] - accuracy: - metric: - topk: 1 - -tuning: - accuracy_criterion: - absolute: 0.02 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 2%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. - diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/README.md b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/README.md deleted file mode 100644 index eba8d257b4d..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Evaluate performance of ONNX Runtime(Mobilenet v2) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model exported from PyTorch and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Please refer to [pytorch official guide](https://pytorch.org/docs/stable/onnx.html) for detailed model export. The following is a simple example: - -```python -import torch -import torchvision -batch_size = 1 -model = torchvision.models.mobilenet_v2(pretrained=True) -x = torch.randn(batch_size, 3, 224, 224) - -# Export the model -torch.onnx.export(model, # model being run - x, # model input (or a tuple for multiple inputs) - "mobilenet_v2.onnx", # where to save the model (can be a file or file-like object) - export_params=True, # store the trained parameter weights inside the model file - opset_version=11, # the ONNX version to export the model to, please ensure at least 11. - do_constant_folding=True, # whether to execute constant folding for optimization - input_names = ['input'], # the model's input names - output_names = ['output'], # the model's output names - dynamic_axes={'input' : {0 : 'batch_size'}, # variable length axes - 'output' : {0 : 'batch_size'}}) -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v2.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v2_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v2.yaml \ - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py deleted file mode 100644 index 89b774ff75c..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py +++ /dev/null @@ -1,82 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Mobilenet_v2 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained mobilenet_v2 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark - evaluator = Benchmark(args.config) - evaluator.model = model - evaluator(args.mode) - - if args.tune: - from neural_compressor import quantization - q_model = quantization.fit(model, args.config) - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2.yaml b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2.yaml deleted file mode 100644 index ea4d0d2f6e1..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenet_v2 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gather_.*?': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - }, - 'Gemm_174_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.03 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2_qdq.yaml b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2_qdq.yaml deleted file mode 100644 index 59963333698..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2_qdq.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenet_v2 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gather_.*?': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - }, - 'Gemm_174_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.03 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/requirements.txt deleted file mode 100644 index 63d774f50ec..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -torch -torchvision -onnxruntime-extensions; python_version < '3.10' - -pillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.bat b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.bat deleted file mode 100644 index 8da25b85410..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.bat +++ /dev/null @@ -1,9 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -set mode=%4 -shift -shift -python main.py --model_path %input_model% --config %config% --benchmark --mode %mode%--output_model %output_model% diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.bat b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.bat deleted file mode 100644 index 56fa6ccab82..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.bat +++ /dev/null @@ -1,8 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -shift -shift -python main.py --model_path %input_model% --config %config% --tune --output_model %output_model% \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py deleted file mode 100644 index 5bd68eea74d..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py +++ /dev/null @@ -1,85 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Mobilenet_v3 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - default='mobilenet_v3.onnx', - help="Pre-trained mobilenet_v3 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - default='mobilenet_v3.yaml', - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - default='mobilenet_v3_tune.onnx', - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - model = onnx.load(args.model_path) - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor import quantization - q_model = quantization.fit(model, args.config) - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3.yaml b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3.yaml deleted file mode 100644 index c5e2bc1b83b..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenet_v3 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - BilinearImagenet: - height: 224 - width: 224 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - BilinearImagenet: - height: 224 - width: 224 - postprocess: - transform: - LabelShift: -1 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 500 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - BilinearImagenet: - height: 224 - width: 224 - postprocess: - transform: - LabelShift: -1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3_qdq.yaml b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3_qdq.yaml deleted file mode 100644 index bba292a4414..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3_qdq.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenet_v3 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - BilinearImagenet: - height: 224 - width: 224 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - BilinearImagenet: - height: 224 - width: 224 - postprocess: - transform: - LabelShift: -1 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 500 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - BilinearImagenet: - height: 224 - width: 224 - postprocess: - transform: - LabelShift: -1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/readme.md b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/readme.md deleted file mode 100644 index 57f86f9a6f9..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/readme.md +++ /dev/null @@ -1,43 +0,0 @@ -# Evaluate performance of ONNX Runtime(Mobilenet v3) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model exported from PyTorch and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Use [tf2onnx tool](https://github.com/onnx/tensorflow-onnx) to convert tflite to onnx model. - -```bash -wget https://github.com/mlcommons/mobile_models/blob/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_float.tflite - -python -m tf2onnx.convert --opset 11 --tflite mobilenet_edgetpu_224_1.0_float.tflite --output mobilenet_v3.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v3.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v3_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v3.yaml \ - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/requirements.txt deleted file mode 100644 index eb22b0efa75..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -tf2onnx -onnxruntime-extensions; python_version < '3.10' - -pillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md index 8363a10866a..1e631fa47ca 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md @@ -20,8 +20,7 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save ``` @@ -29,8 +28,7 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save \ --quant_format=QDQ ``` @@ -39,8 +37,7 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py index 0fb1da396ac..4a91f52d1ed 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py @@ -120,14 +120,14 @@ def result(self): return self.num_correct / self.num_sample class Dataloader: - def __init__(self, data_path, image_list): + def __init__(self, dataset_location, image_list): self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: for s in f: image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) + src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue with Image.open(src) as image: @@ -167,15 +167,10 @@ def eval_func(model, dataloader, metric): help="Pre-trained model on onnx file" ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Imagenet data path" ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) parser.add_argument( '--benchmark', action='store_true', \ @@ -192,10 +187,11 @@ def eval_func(model, dataloader, metric): type=str, help="output model path" ) - parser.add_argument('--quant_format', + parser.add_argument( + '--quant_format', type=str, - default='Default', - choices=['Default', 'QDQ'], + default='default', + choices=['default', 'QDQ', 'QOperator'], help="quantization format" ) parser.add_argument( @@ -207,7 +203,9 @@ def eval_func(model, dataloader, metric): args = parser.parse_args() model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path, args.label_path) + data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') + label_path = os.path.join(args.dataset_location, 'val.txt') + dataloader = Dataloader(data_path, label_path) top1 = TopK() def eval(onnx_model): return eval_func(onnx_model, dataloader, top1) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh index 4901e12c6c5..7ec6f7ace0e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh @@ -15,11 +15,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -34,8 +31,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh index 55563483e4e..29ed086faf4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh @@ -19,11 +19,8 @@ function init_params { --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --quant_format=*) quant_format=$(echo $var |cut -f2 -d=) @@ -37,8 +34,7 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --output_model ${output_model} \ --quant_format ${quant_format} \ --tune diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md index defb66ade01..98391e58e86 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md @@ -18,8 +18,7 @@ wget https://github.com/onnx/models/raw/main/vision/body_analysis/arcface/model/ ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/faces_ms1m_112x112/task.bin \ - --nfolds=nfolds_num \ # optional, data type is int, default is 1 + --dataset_location=/path/to/faces_ms1m_112x112/task.bin \ --output_model=path/to/save ``` @@ -27,8 +26,7 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/faces_ms1m_112x112/task.bin \ - --nfolds=nfolds_num \ # optional, data type is int, default is 1 + --dataset_location=/path/to/faces_ms1m_112x112/task.bin \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py index 1b3d372a3bf..c2c4798f288 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py @@ -147,7 +147,7 @@ def eval_func(model, dataloader, metric): help="Pre-trained model on onnx file" ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Imagenet data path" ) @@ -180,10 +180,10 @@ def eval_func(model, dataloader, metric): args = parser.parse_args() # Load image size - image_size = load_property(args.data_path) + image_size = load_property(args.dataset_location) print('image_size', image_size) - dataloader = Dataloader(args.data_path) + dataloader = Dataloader(args.dataset_location) model = onnx.load(args.model_path) metric = Metric(args.nfolds) def eval(onnx_model): diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh index b7afc9f4177..628585aa638 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh @@ -16,15 +16,12 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) ;; - --nfolds=*) - nfolds=$(echo $var |cut -f2 -d=) - ;; esac done @@ -35,9 +32,8 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ + --dataset_location ${dataset_location} \ --mode ${mode} \ - --nfolds ${nfolds} \ --benchmark } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh index 0a228d3f587..40fa42e9525 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh @@ -19,11 +19,8 @@ function init_params { --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --nfolds=*) - nfolds=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; esac done @@ -34,9 +31,8 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ + --dataset_location ${dataset_location} \ --output_model ${output_model} \ - --nfolds ${nfolds} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md index 89e58a07e89..4c03fdda60a 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md @@ -21,8 +21,7 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save ``` @@ -30,8 +29,7 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save \ --quant_format=QDQ ``` @@ -40,8 +38,7 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py index bee7b211379..edeea5712fa 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py @@ -120,14 +120,14 @@ def result(self): return self.num_correct / self.num_sample class Dataloader: - def __init__(self, data_path, image_list): + def __init__(self, dataset_location, image_list): self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: for s in f: image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) + src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue with Image.open(src) as image: @@ -167,15 +167,10 @@ def eval_func(model, dataloader, metric): help="Pre-trained model on onnx file" ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Imagenet data path" ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) parser.add_argument( '--benchmark', action='store_true', \ @@ -197,16 +192,19 @@ def eval_func(model, dataloader, metric): type=str, help="benchmark mode of performance or accuracy" ) - parser.add_argument('--quant_format', + parser.add_argument( + '--quant_format', type=str, - default='Default', - choices=['Default', 'QDQ'], + default='default', + choices=['default', 'QDQ', 'QOperator'], help="quantization format" ) args = parser.parse_args() model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path, args.label_path) + data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') + label_path = os.path.join(args.dataset_location, 'val.txt') + dataloader = Dataloader(data_path, label_path) top1 = TopK() def eval(onnx_model): return eval_func(onnx_model, dataloader, top1) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh index 4901e12c6c5..7ec6f7ace0e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh @@ -15,11 +15,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -34,8 +31,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh index 55563483e4e..29ed086faf4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh @@ -19,11 +19,8 @@ function init_params { --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --quant_format=*) quant_format=$(echo $var |cut -f2 -d=) @@ -37,8 +34,7 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --output_model ${output_model} \ --quant_format ${quant_format} \ --tune diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md index 3ab61d5db27..41249cf360b 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md @@ -18,8 +18,7 @@ wget https://github.com/onnx/models/raw/main/vision/classification/densenet-121/ ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=path/to/dataset \ - --label_path=path/to/label \ + --dataset_location=path/to/dataset \ --output_model=path/to/save ``` @@ -27,8 +26,7 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=path/to/dataset \ - --label_path=path/to/label \ + --dataset_location=path/to/dataset \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py index aff131c766a..e0b51434e1a 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py @@ -119,14 +119,14 @@ def result(self): return self.num_correct / self.num_sample class Dataloader: - def __init__(self, data_path, image_list): + def __init__(self, dataset_location, image_list): self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: for s in f: image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) + src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue with Image.open(src) as image: @@ -193,15 +193,10 @@ def __call__(self, sample): help="whether quantize the model" ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Imagenet data path" ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) parser.add_argument( '--output_model', type=str, @@ -214,7 +209,9 @@ def __call__(self, sample): ) args = parser.parse_args() model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path, args.label_path) + data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') + label_path = os.path.join(args.dataset_location, 'val.txt') + dataloader = Dataloader(data_path, label_path) top1 = TopK() postprocess = Squeeze() def eval(onnx_model): diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh index 80b6743fec2..340848c36ad 100755 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh @@ -16,11 +16,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -35,8 +32,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh index 25d2cd1b4a6..e4eb6389c94 100755 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh @@ -16,11 +16,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --output_model=*) output_model=$(echo $var |cut -f2 -d=) @@ -34,8 +31,7 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --output_model ${output_model} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/README.md deleted file mode 100644 index edd1310d996..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Evaluate performance of ONNX Runtime(EfficientNet-Lite4) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/efficientnet-lite4/model/efficientnet-lite4-11.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=efficientnet.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=efficientnet_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=efficientnet.yaml \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet.yaml deleted file mode 100644 index 4dfc024c1e9..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: EfficientNet-Lite4 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet_qdq.yaml deleted file mode 100644 index cb9f2ff142c..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet_qdq.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: EfficientNet-Lite4 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py deleted file mode 100644 index 3d6c89290a4..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py +++ /dev/null @@ -1,81 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - - if args.benchmark: - from neural_compressor.experimental import Benchmark - evaluator = Benchmark(args.config) - evaluator.model = model - evaluator(args.mode) - - if args.tune: - from neural_compressor import quantization - q_model = quantization.fit(model, args.config) - q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/requirements.txt deleted file mode 100644 index 4b56442f97d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 416dde8f2c0..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md index a865e3dec1c..52dedf3882a 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md @@ -20,8 +20,7 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=path/to/coco/val2017 \ - --label_path=path/to/coco/annotations/instances_val2017.json \ + --dataset_location=path/to/coco/val2017 \ --output_model=path/to/save ``` @@ -29,8 +28,7 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=path/to/coco/val2017 \ - --label_path=path/to/coco/annotations/instances_val2017.json \ + --dataset_location=path/to/coco/val2017 \ --output_model=path/to/save \ --quant_format=QDQ ``` @@ -39,7 +37,6 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=path/to/coco/val2017 \ - --label_path=path/to/coco/annotations/instances_val2017.json \ + --dataset_location=path/to/coco/val2017 \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py index 3e6976ba390..350a75b5c08 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py @@ -46,12 +46,7 @@ help="Pre-trained model on onnx file" ) parser.add_argument( - '--label_path', - type=str, - help="Annotation file path" -) -parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Path to val2017 of COCO" ) @@ -76,10 +71,11 @@ type=str, help="benchmark mode of performance or accuracy" ) -parser.add_argument('--quant_format', +parser.add_argument( + '--quant_format', type=str, - default='Default', - choices=['Default', 'QDQ'], + default='default', + choices=['default', 'QDQ', 'QOperator'], help="quantization format" ) args = parser.parse_args() @@ -108,7 +104,7 @@ 72: 20, # tv } VOC_CAT_IDS = list(COCO_TO_VOC.keys()) -cocoGt = COCO(str(args.label_path)) +cocoGt = COCO(os.path.join(args.dataset_location, 'annotations/instances_val2017.json')) preprocess = transforms.Compose([ transforms.ToTensor(), @@ -121,7 +117,7 @@ def __init__(self): imgIds = self.getImgIdsUnion(cocoGt, VOC_CAT_IDS) self.data = [] for imgId in imgIds: - img_path = os.path.join(args.data_path, cocoGt.imgs[imgId]['file_name']) + img_path = os.path.join(os.path.join(args.dataset_location, 'val2017'), cocoGt.imgs[imgId]['file_name']) if os.path.exists(img_path): input_tensor = self.load_image(img_path) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh index bb1127ede51..3f1e78f6cb4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh @@ -19,11 +19,8 @@ function init_params { --mode=*) mode=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; esac done @@ -36,8 +33,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ --mode ${mode} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --benchmark } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh index ca052c688dc..5cf411ef94d 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh @@ -19,11 +19,8 @@ function init_params { --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --quant_format=*) quant_format=$(echo $var |cut -f2 -d=) @@ -38,8 +35,7 @@ function run_tuning { python main.py \ --model_path ${input_model} \ --output_model ${output_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --quant_format ${quant_format} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md index 86f6c971c26..3865d94328d 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md @@ -20,8 +20,7 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save ``` @@ -29,8 +28,7 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save \ --quant_format=QDQ ``` @@ -39,8 +37,7 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py index 6858c304215..281bd03b403 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py @@ -120,14 +120,14 @@ def result(self): return self.num_correct / self.num_sample class Dataloader: - def __init__(self, data_path, image_list): + def __init__(self, dataset_location, image_list): self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: for s in f: image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) + src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue with Image.open(src) as image: @@ -167,15 +167,10 @@ def eval_func(model, dataloader, metric): help="Pre-trained model on onnx file" ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Imagenet data path" ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) parser.add_argument( '--benchmark', action='store_true', \ @@ -197,16 +192,19 @@ def eval_func(model, dataloader, metric): type=str, help="benchmark mode of performance or accuracy" ) - parser.add_argument('--quant_format', + parser.add_argument( + '--quant_format', type=str, - default='Default', - choices=['Default', 'QDQ'], + default='default', + choices=['default', 'QDQ', 'QOperator'], help="quantization format" ) args = parser.parse_args() model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path, args.label_path) + data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') + label_path = os.path.join(args.dataset_location, 'val.txt') + dataloader = Dataloader(data_path, label_path) top1 = TopK() def eval(onnx_model): return eval_func(onnx_model, dataloader, top1) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh index 4901e12c6c5..7ec6f7ace0e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh @@ -15,11 +15,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -34,8 +31,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh index 55563483e4e..29ed086faf4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh @@ -19,11 +19,8 @@ function init_params { --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --quant_format=*) quant_format=$(echo $var |cut -f2 -d=) @@ -37,8 +34,7 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --output_model ${output_model} \ --quant_format ${quant_format} \ --tune diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md index 6be0ff773b8..412b1b2322a 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md @@ -20,8 +20,7 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save ``` @@ -29,8 +28,7 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save \ --quant_format=QDQ ``` @@ -39,8 +37,7 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py index 60c770fcad5..19edd9b6b70 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py @@ -120,14 +120,14 @@ def result(self): return self.num_correct / self.num_sample class Dataloader: - def __init__(self, data_path, image_list): + def __init__(self, dataset_location, image_list): self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: for s in f: image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) + src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue with Image.open(src) as image: @@ -167,15 +167,10 @@ def eval_func(model, dataloader, metric): help="Pre-trained model on onnx file" ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Imagenet data path" ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) parser.add_argument( '--benchmark', action='store_true', \ @@ -197,16 +192,19 @@ def eval_func(model, dataloader, metric): type=str, help="benchmark mode of performance or accuracy" ) - parser.add_argument('--quant_format', + parser.add_argument( + '--quant_format', type=str, - default='Default', - choices=['Default', 'QDQ'], + default='default', + choices=['default', 'QDQ', 'QOperator'], help="quantization format" ) args = parser.parse_args() model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path, args.label_path) + data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') + label_path = os.path.join(args.dataset_location, 'val.txt') + dataloader = Dataloader(data_path, label_path) top1 = TopK() def eval(onnx_model): return eval_func(onnx_model, dataloader, top1) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh index 4901e12c6c5..7ec6f7ace0e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh @@ -15,11 +15,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -34,8 +31,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh index 55563483e4e..29ed086faf4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh @@ -19,11 +19,8 @@ function init_params { --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --quant_format=*) quant_format=$(echo $var |cut -f2 -d=) @@ -37,8 +34,7 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --output_model ${output_model} \ --quant_format ${quant_format} \ --tune diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/README.md deleted file mode 100644 index 2f35d299226..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Evaluate performance of ONNX Runtime(MNIST) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.8.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/mnist/model/mnist-12.onnx -``` - -### Quantization -To quantize the model, run `main.py` with the path to the model: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mnist.yaml \ - --output_model=path/to/save -``` - -### Performance -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mnist.yaml \ - --output_model=path/to/save -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py deleted file mode 100644 index 52ecd4759cb..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="MNIST - Handwritten Digit Recognition quantization example.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark - evaluator = Benchmark(args.config) - evaluator.model = model - evaluator(args.mode) - - if args.tune: - from neural_compressor import quantization - q_model = quantization.fit(model, args.config) - q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/mnist.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/mnist.yaml deleted file mode 100644 index 38b9bb1c791..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/mnist.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: mnist - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - dataloader: - batch_size: 1 - dataset: - MNIST: - root: /path/to/calibration/dataset - transform: - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - MNIST: - root: /path/to/evaluation/dataset - transform: - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 28 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - MNIST: - root: /path/to/evaluation/dataset - transform: - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/requirements.txt deleted file mode 100644 index a67636f2d5a..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.8.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_benchmark.sh deleted file mode 100755 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_tuning.sh deleted file mode 100755 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/README.md deleted file mode 100644 index 61a3c6f68e1..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Evaluate performance of ONNX Runtime(Mobilenet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/mobilenet/model/mobilenetv2-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenetv2.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenetv2_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenetv2.yaml \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py deleted file mode 100644 index f8d82593378..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py +++ /dev/null @@ -1,81 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Mobilenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained mobilenet model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark - evaluator = Benchmark(args.config) - evaluator.model = model - evaluator(args.mode) - - if args.tune: - from neural_compressor import quantization - q_model = quantization.fit(model, args.config) - q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2.yaml deleted file mode 100644 index 204897a4a39..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenetv2 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.02 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2_qdq.yaml deleted file mode 100644 index 3d2b9c10ca1..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2_qdq.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenetv2 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.02 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/requirements.txt deleted file mode 100644 index f62a897bffa..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_benchmark.sh deleted file mode 100755 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_tuning.sh deleted file mode 100755 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/README.md deleted file mode 100644 index f239f3181c5..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Evaluate performance of ONNX Runtime(ResNet 50) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/resnet/model/resnet50-v1-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5.yaml \ - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py deleted file mode 100644 index 069a2f74b8a..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py +++ /dev/null @@ -1,80 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Resnet50 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained resnet50 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark - evaluator = Benchmark(args.config) - evaluator.model = model - evaluator(args.mode) - - if args.tune: - from neural_compressor import quantization - q_model = quantization.fit(model, args.config) - q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/requirements.txt deleted file mode 100644 index 494373e0bf3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -torch -torchvision -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5.yaml deleted file mode 100644 index 966dabfbd9c..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml deleted file mode 100644 index 8e4cec4d888..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/README.md deleted file mode 100644 index abd6ce83fcd..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Evaluate performance of ONNX Runtime(Shufflenet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/shufflenet/model/shufflenet-v2-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=shufflenetv2.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=shufflenetv2_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=shufflenetv2.yaml \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py deleted file mode 100644 index 3f0adff31d5..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py +++ /dev/null @@ -1,80 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Shufflenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark - evaluator = Benchmark(args.config) - evaluator.model = model - evaluator(args.mode) - - if args.tune: - from neural_compressor import quantization - q_model = quantization.fit(model, args.config) - q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/requirements.txt deleted file mode 100644 index f62a897bffa..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2.yaml deleted file mode 100644 index 8d7cab0cb4e..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: shufflenetv2 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2_qdq.yaml deleted file mode 100644 index 6c37dfd56a6..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2_qdq.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: shufflenetv2 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md index 7931b53de79..6a61b3816c6 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md @@ -20,8 +20,7 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save ``` @@ -29,8 +28,7 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save \ --quant_format=QDQ ``` @@ -39,8 +37,7 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py index 76864674b5d..91766e27d55 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py @@ -125,14 +125,14 @@ def result(self): return self.num_correct / self.num_sample class Dataloader: - def __init__(self, data_path, image_list): + def __init__(self, dataset_location, image_list): self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: for s in f: image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) + src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue with Image.open(src) as image: @@ -173,15 +173,10 @@ def eval_func(model, dataloader, metric, postprocess): help="Pre-trained model on onnx file" ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Imagenet data path" ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) parser.add_argument( '--benchmark', action='store_true', \ @@ -203,16 +198,19 @@ def eval_func(model, dataloader, metric, postprocess): type=str, help="benchmark mode of performance or accuracy" ) - parser.add_argument('--quant_format', + parser.add_argument( + '--quant_format', type=str, - default='Default', - choices=['Default', 'QDQ'], + default='default', + choices=['default', 'QDQ', 'QOperator'], help="quantization format" ) args = parser.parse_args() model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path, args.label_path) + data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') + label_path = os.path.join(args.dataset_location, 'val.txt') + dataloader = Dataloader(data_path, label_path) top1 = TopK() postprocess = Squeeze() def eval(onnx_model): diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh index 4901e12c6c5..7ec6f7ace0e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh @@ -15,11 +15,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -34,8 +31,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh index 55563483e4e..29ed086faf4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh @@ -19,11 +19,8 @@ function init_params { --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --quant_format=*) quant_format=$(echo $var |cut -f2 -d=) @@ -37,8 +34,7 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --output_model ${output_model} \ --quant_format ${quant_format} \ --tune diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/README.md deleted file mode 100644 index 80de9fff000..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Evaluate performance of ONNX Runtime(VGG16) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/vgg/model/vgg16-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16.yaml \ - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py deleted file mode 100644 index 600bb88a6da..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="VGG16 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained vgg16 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - args = parser.parse_args() - model = onnx.load(args.model_path) - - if args.benchmark: - from neural_compressor.experimental import Benchmark - evaluator = Benchmark(args.config) - evaluator.model = model - evaluator(args.mode) - - if args.tune: - from neural_compressor import quantization - q_model = quantization.fit(model, args.config) - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/requirements.txt deleted file mode 100644 index 494373e0bf3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -torch -torchvision -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16.yaml deleted file mode 100644 index 5b1b165fdb7..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: vgg16 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16_qdq.yaml deleted file mode 100644 index fb647b11f6a..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16_qdq.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: vgg16 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md index c1a90e84463..dbf3a738d90 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md @@ -20,8 +20,7 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save ``` @@ -29,8 +28,7 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save \ --quant_format=QDQ ``` @@ -39,8 +37,7 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py index 28da787c065..a33ec28f01b 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py @@ -120,14 +120,14 @@ def result(self): return self.num_correct / self.num_sample class Dataloader: - def __init__(self, data_path, image_list): + def __init__(self, dataset_location, image_list): self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: for s in f: image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) + src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue with Image.open(src) as image: @@ -167,15 +167,10 @@ def eval_func(model, dataloader, metric): help="Pre-trained model on onnx file" ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Imagenet data path" ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) parser.add_argument( '--benchmark', action='store_true', \ @@ -197,16 +192,19 @@ def eval_func(model, dataloader, metric): type=str, help="benchmark mode of performance or accuracy" ) - parser.add_argument('--quant_format', + parser.add_argument( + '--quant_format', type=str, - default='Default', - choices=['Default', 'QDQ'], + default='default', + choices=['default', 'QDQ', 'QOperator'], help="quantization format" ) args = parser.parse_args() model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path, args.label_path) + data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') + label_path = os.path.join(args.dataset_location, 'val.txt') + dataloader = Dataloader(data_path, label_path) top1 = TopK() def eval(onnx_model): return eval_func(onnx_model, dataloader, top1) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh index 4901e12c6c5..7ec6f7ace0e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh @@ -15,11 +15,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -34,8 +31,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh index 55563483e4e..29ed086faf4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh @@ -19,11 +19,8 @@ function init_params { --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --quant_format=*) quant_format=$(echo $var |cut -f2 -d=) @@ -37,8 +34,7 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --output_model ${output_model} \ --quant_format ${quant_format} \ --tune diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/README.md b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/README.md deleted file mode 100644 index 985f8284a48..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Evaluate performance of ONNX Runtime(ResNet 50) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model exported from PyTorch and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model - -#### ResNet 50 from torchvision -Please refer to [pytorch official guide](https://pytorch.org/docs/stable/onnx.html) for detailed model export. The following is a simple example: - -```python -import torch -import torchvision -batch_size = 1 -model = torchvision.models.resnet50(pretrained=True) -x = torch.randn(batch_size, 3, 224, 224, requires_grad=True) -torch_out = model(x) - -# Export the model -torch.onnx.export(model, # model being run - x, # model input (or a tuple for multiple inputs) - "resnet50.onnx", # where to save the model (can be a file or file-like object) - export_params=True, # store the trained parameter weights inside the model file - opset_version=11, # the ONNX version to export the model to, please ensure at least 11. - do_constant_folding=True, # whether to execute constant folding for optimization - input_names = ['input'], # the model's input names - output_names = ['output'], # the model's output names - dynamic_axes={'input' : {0 : 'batch_size'}, # variable length axes - 'output' : {0 : 'batch_size'}}) -``` - -#### ResNet 50 from MLPerf -Please refer to [MLPerf Inference Benchmarks for Image Classification and Object Detection Tasks](https://github.com/mlcommons/inference/tree/master/vision/classification_and_detection#mlperf-inference-benchmarks-for-image-classification-and-object-detection-tasks) for model details. Use [tf2onnx tool](https://github.com/onnx/tensorflow-onnx) to convert tensorflow model to onnx model. - -```bash -wget https://zenodo.org/record/2535873/files/resnet50_v1.pb - -python -m tf2onnx.convert --input resnet50_v1.pb --output resnet50_v1.onnx --inputs-as-nchw input_tensor:0 --inputs input_tensor:0 --outputs softmax_tensor:0 --opset 11 -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5.yaml \ # or resnet50_v1_5_mlperf.yaml for ResNet50 from MLPerf - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5_qdq.yaml \ # or resnet50_v1_5_mlperf_qdq.yaml for ResNet50 from MLPerf - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5.yaml \ # or resnet50_v1_5_mlperf.yaml for ResNet50 from MLPerf - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py deleted file mode 100644 index f2e63a84ffc..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py +++ /dev/null @@ -1,80 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Resnet50 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained resnet50 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - args = parser.parse_args() - model = onnx.load(args.model_path) - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor import quantization - q_model = quantization.fit(model, args.config) - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/requirements.txt deleted file mode 100644 index 96b96e53d95..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -torch -torchvision -onnxruntime-extensions; python_version < '3.10' - -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5.yaml b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5.yaml deleted file mode 100644 index 5bb0f619d18..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gemm_174_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf.yaml b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf.yaml deleted file mode 100644 index e7aa83c1c23..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset # NOTE: modify to calibration dataset location if needed - image_list: /path/to/calibration/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - op_wise: { - 'gemm_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset # NOTE: modify to evaluation dataset location if needed - image_list: /path/to/evaluation/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - postprocess: - transform: - LabelShift: -1 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset # NOTE: modify to evaluation dataset location if needed - image_list: /path/to/evaluation/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf_qdq.yaml b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf_qdq.yaml deleted file mode 100644 index 58fd4ff3567..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf_qdq.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset # NOTE: modify to calibration dataset location if needed - image_list: /path/to/calibration/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - op_wise: { - 'gemm_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset # NOTE: modify to evaluation dataset location if needed - image_list: /path/to/evaluation/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - postprocess: - transform: - LabelShift: -1 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset # NOTE: modify to evaluation dataset location if needed - image_list: /path/to/evaluation/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml deleted file mode 100644 index c9269f56815..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gemm_174_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.bat b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.bat deleted file mode 100644 index 8da25b85410..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.bat +++ /dev/null @@ -1,9 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -set mode=%4 -shift -shift -python main.py --model_path %input_model% --config %config% --benchmark --mode %mode%--output_model %output_model% diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.bat b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.bat deleted file mode 100644 index 56fa6ccab82..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.bat +++ /dev/null @@ -1,8 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -shift -shift -python main.py --model_path %input_model% --config %config% --tune --output_model %output_model% \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/README.md b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/README.md deleted file mode 100644 index 7ad081ed162..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Evaluate performance of ONNX Runtime(VGG16) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model exported from PyTorch and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Please refer to [pytorch official guide](https://pytorch.org/docs/stable/onnx.html) for detailed model export. The following is a simple example: - -```python -import torch -import torchvision -batch_size = 1 -model = torchvision.models.vgg16(pretrained=True) -x = torch.randn(batch_size, 3, 224, 224, requires_grad=True) -torch_out = model(x) - -# Export the model -torch.onnx.export(model, # model being run - x, # model input (or a tuple for multiple inputs) - "vgg16.onnx", # where to save the model (can be a file or file-like object) - export_params=True, # store the trained parameter weights inside the model file - opset_version=11, # the ONNX version to export the model to, please ensure at least 11. - do_constant_folding=True, # whether to execute constant folding for optimization - input_names = ['input'], # the model's input names - output_names = ['output'], # the model's output names - dynamic_axes={'input' : {0 : 'batch_size'}, # variable length axes - 'output' : {0 : 'batch_size'}}) -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16.yaml \ - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py deleted file mode 100644 index f84c2fa6c2c..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py +++ /dev/null @@ -1,79 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="VGG16 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained vgg16 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - args = parser.parse_args() - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor import quantization - q_model = quantization.fit(model, args.config) - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/requirements.txt deleted file mode 100644 index 96b96e53d95..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -torch -torchvision -onnxruntime-extensions; python_version < '3.10' - -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.bat b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.bat deleted file mode 100644 index 8da25b85410..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.bat +++ /dev/null @@ -1,9 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -set mode=%4 -shift -shift -python main.py --model_path %input_model% --config %config% --benchmark --mode %mode%--output_model %output_model% diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.bat b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.bat deleted file mode 100644 index 56fa6ccab82..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.bat +++ /dev/null @@ -1,8 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -shift -shift -python main.py --model_path %input_model% --config %config% --tune --output_model %output_model% \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16.yaml b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16.yaml deleted file mode 100644 index 09374ebecbd..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: vgg16 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gemm_37_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16_qdq.yaml b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16_qdq.yaml deleted file mode 100644 index 59ba806aa09..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16_qdq.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: vgg16 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gemm_37_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. From 59380b403c6fdaf986ddba60479d4b6779db5664 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Thu, 15 Dec 2022 17:34:48 +0800 Subject: [PATCH 03/13] add bs Signed-off-by: Mengni Wang --- .../fcn/quantization/ptq/README.md | 1 + .../fcn/quantization/ptq/main.py | 21 ++++++++++++++++--- .../fcn/quantization/ptq/run_benchmark.sh | 4 ++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md index 52dedf3882a..369c6b167df 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md @@ -38,5 +38,6 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx --dataset_location=path/to/coco/val2017 \ + --batch_size=batch_size \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py index 350a75b5c08..f6ab61d9ffd 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py @@ -71,6 +71,11 @@ type=str, help="benchmark mode of performance or accuracy" ) +parser.add_argument( + '--batch_size', + type=int, + default=1, +) parser.add_argument( '--quant_format', type=str, @@ -112,8 +117,8 @@ ]) class Dataloader: - def __init__(self): - self.batch_size = 1 + def __init__(self, batch_size): + self.batch_size = batch_size imgIds = self.getImgIdsUnion(cocoGt, VOC_CAT_IDS) self.data = [] for imgId in imgIds: @@ -134,8 +139,18 @@ def __init__(self): self.data.append((input_tensor, output_tensor)) def __iter__(self, index): + inputs = [] + labels = [] + idx = self.batch_size for data in self.data: - yield data + inputs.append(data[0]) + labels.append(data[1]) + idx -= 1 + if idx < 0: + yield np.array(inputs), labels + inputs = [] + labels = [] + idx = self.batch_size def getImgIdsUnion(self, gt, catIds): """ diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh index 3f1e78f6cb4..da2287b02e4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh @@ -19,6 +19,9 @@ function init_params { --mode=*) mode=$(echo $var |cut -f2 -d=) ;; + --batch_size=*) + mode=$(echo $var |cut -f2 -d=) + ;; --dataset_location=*) dataset_location=$(echo $var |cut -f2 -d=) ;; @@ -34,6 +37,7 @@ function run_benchmark { --model_path ${input_model} \ --mode ${mode} \ --dataset_location ${dataset_location} \ + --batch_size ${batch_size} \ --benchmark } From b1142fc9127ff4b5f81b03c2e44aad34e2ceb59f Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Mon, 19 Dec 2022 09:53:51 +0800 Subject: [PATCH 04/13] rebase and fix bug Signed-off-by: Mengni Wang --- .../onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py | 2 +- .../onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh | 6 +++--- .../onnx_model_zoo/arcface/quantization/ptq/main.py | 2 +- .../onnx_model_zoo/densenet/quantization/ptq/main.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py index 3d75f7f4ed2..03b6ef0cdfa 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py @@ -159,7 +159,7 @@ def __init__(self, dataset_location): images = [np.reshape(np.fromstring(image, dtype=np.uint8, sep=' '), (48, 48)) for image in df['pixels']] labels = np.array(list(map(int, df['emotion']))) self.batch_size = 1 - self.data = [(self.preprocess(image), label) for image, label in zip(images, labels)] + self.data = [(self.preprocess(image), [label]) for image, label in zip(images, labels)] def __len__(self): return len(self.data) diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh index 9a3d8b07a29..9f907bfa0eb 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh @@ -16,8 +16,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --output_model=*) output_model=$(echo $var |cut -f2 -d=) @@ -32,7 +32,7 @@ function run_tuning { python main.py \ --model_path ${input_model} \ --output_model ${output_model} \ - --data_path ${data_path} \ + --dataset_location ${dataset_location} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py index c2c4798f288..c18eae1583f 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py @@ -123,7 +123,7 @@ def __init__(self, data_dir): def __iter__(self): for data in self.data_list: - yield data, self.issame_list + yield np.expand_dims(data, axis=0), self.issame_list def eval_func(model, dataloader, metric): metric.reset() diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py index e0b51434e1a..7ebee28cac2 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py @@ -19,7 +19,7 @@ import logging import argparse - +import os import numpy as np import onnx import onnxruntime as ort From 59d9c85ea6f9eabf35d2e1e6bfac63dd1176e158 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Wed, 21 Dec 2022 11:31:57 +0800 Subject: [PATCH 05/13] fix bug Signed-off-by: Mengni Wang --- .../ultraface/quantization/ptq/run_benchmark.sh | 6 +++--- .../onnx_model_zoo/densenet/quantization/ptq/main.py | 1 + .../onnx_model_zoo/fcn/quantization/ptq/main.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh index 5528ec80db6..2d211d2bbff 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh @@ -16,8 +16,8 @@ function init_params { --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -33,7 +33,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ --mode ${mode} \ - --data_path ${data_path} \ + --dataset_location ${dataset_location} \ --benchmark } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py index 7ebee28cac2..39e2d0d40c9 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py @@ -20,6 +20,7 @@ import logging import argparse import os +import re import numpy as np import onnx import onnxruntime as ort diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py index f6ab61d9ffd..f5789c26b4e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py @@ -210,7 +210,7 @@ def evaluate(model, dataloader): if __name__ == "__main__": model = onnx.load(args.model_path) - dataloader = Dataloader() + dataloader = Dataloader(args.batch_size) def eval(model): return evaluate(model, dataloader) From 4790772b2aff1d8e6c82fe60cc32c35d08c41916 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Fri, 23 Dec 2022 15:16:43 +0800 Subject: [PATCH 06/13] fix config Signed-off-by: Mengni Wang --- .../onnx_model_zoo/densenet/quantization/ptq/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py index 39e2d0d40c9..ddb8d6823b8 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py @@ -235,8 +235,7 @@ def eval(onnx_model): accuracy_criterion.relative = 0.02 config = PostTrainingQuantConfig( accuracy_criterion=accuracy_criterion, - op_name_list={'Conv_nc_rename_0': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}, - 'Relu_nc_rename_1': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}}) + op_name_list={'Conv_nc_rename_431': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}} q_model = quantization.fit(model, config, calib_dataloader=dataloader, eval_func=eval) From 687c3a46fbfdabbae4a106a4606f369baf8c6d47 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Mon, 26 Dec 2022 10:35:26 +0800 Subject: [PATCH 07/13] fix script Signed-off-by: Mengni Wang --- .../onnx_model_zoo/densenet/quantization/ptq/main.py | 2 +- .../onnx_model_zoo/fcn/quantization/ptq/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py index ddb8d6823b8..982eb8ccef5 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py @@ -235,7 +235,7 @@ def eval(onnx_model): accuracy_criterion.relative = 0.02 config = PostTrainingQuantConfig( accuracy_criterion=accuracy_criterion, - op_name_list={'Conv_nc_rename_431': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}} + op_name_list={'Conv_nc_rename_431': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}}) q_model = quantization.fit(model, config, calib_dataloader=dataloader, eval_func=eval) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py index f5789c26b4e..5a38543e561 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py @@ -138,7 +138,7 @@ def __init__(self, batch_size): output_tensor[0] = 1 - np.max(output_tensor, axis=0) self.data.append((input_tensor, output_tensor)) - def __iter__(self, index): + def __iter__(self): inputs = [] labels = [] idx = self.batch_size From d272a50e95ad6f1e24bf5042f10ccae8381d8743 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Mon, 26 Dec 2022 17:23:20 +0800 Subject: [PATCH 08/13] fix fcn Signed-off-by: Mengni Wang --- .../fcn/quantization/ptq/main.py | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py index 5a38543e561..9601d791892 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py @@ -71,11 +71,6 @@ type=str, help="benchmark mode of performance or accuracy" ) -parser.add_argument( - '--batch_size', - type=int, - default=1, -) parser.add_argument( '--quant_format', type=str, @@ -117,8 +112,8 @@ ]) class Dataloader: - def __init__(self, batch_size): - self.batch_size = batch_size + def __init__(self): + self.batch_size = 1 imgIds = self.getImgIdsUnion(cocoGt, VOC_CAT_IDS) self.data = [] for imgId in imgIds: @@ -139,18 +134,8 @@ def __init__(self, batch_size): self.data.append((input_tensor, output_tensor)) def __iter__(self): - inputs = [] - labels = [] - idx = self.batch_size for data in self.data: - inputs.append(data[0]) - labels.append(data[1]) - idx -= 1 - if idx < 0: - yield np.array(inputs), labels - inputs = [] - labels = [] - idx = self.batch_size + yield data def getImgIdsUnion(self, gt, catIds): """ @@ -210,7 +195,7 @@ def evaluate(model, dataloader): if __name__ == "__main__": model = onnx.load(args.model_path) - dataloader = Dataloader(args.batch_size) + dataloader = Dataloader() def eval(model): return evaluate(model, dataloader) From 6899319262e54d9c049323673a0152c251cd1001 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Tue, 27 Dec 2022 10:13:29 +0800 Subject: [PATCH 09/13] fix bug Signed-off-by: Mengni Wang --- .../onnx_model_zoo/fcn/quantization/ptq/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py index 9601d791892..be9f027539f 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py @@ -131,6 +131,7 @@ def __init__(self): # Set everything not labeled to be background output_tensor[0] = 1 - np.max(output_tensor, axis=0) + input_tensor = input_tensor[np.newaxis, ...] self.data.append((input_tensor, output_tensor)) def __iter__(self): @@ -176,7 +177,6 @@ def evaluate(model, dataloader): providers=onnxruntime.get_available_providers()) idx = 1 for input_tensor, target_tensor in dataloader: - input_tensor = input_tensor[np.newaxis, ...] target_tensor = target_tensor[np.newaxis, ...] model_tensor = sess.run(["out"], {"input": input_tensor})[0] From 92deb8dab4c5798e04b9e24c8f553d63c7d4c025 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Tue, 27 Dec 2022 14:21:20 +0800 Subject: [PATCH 10/13] remove paramter Signed-off-by: Mengni Wang --- .../onnx_model_zoo/fcn/quantization/ptq/README.md | 1 - .../onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh | 4 ---- 2 files changed, 5 deletions(-) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md index 369c6b167df..52dedf3882a 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md @@ -38,6 +38,5 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx --dataset_location=path/to/coco/val2017 \ - --batch_size=batch_size \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh index da2287b02e4..3f1e78f6cb4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh @@ -19,9 +19,6 @@ function init_params { --mode=*) mode=$(echo $var |cut -f2 -d=) ;; - --batch_size=*) - mode=$(echo $var |cut -f2 -d=) - ;; --dataset_location=*) dataset_location=$(echo $var |cut -f2 -d=) ;; @@ -37,7 +34,6 @@ function run_benchmark { --model_path ${input_model} \ --mode ${mode} \ --dataset_location ${dataset_location} \ - --batch_size ${batch_size} \ --benchmark } From d1fbf0f2cfffa1ec131049115581611f52d91f99 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Wed, 28 Dec 2022 15:56:23 +0800 Subject: [PATCH 11/13] fix perf Signed-off-by: Mengni Wang --- .../alexnet/quantization/ptq/main.py | 21 +++++---- .../caffenet/quantization/ptq/main.py | 21 +++++---- .../densenet/quantization/ptq/main.py | 46 ++++++++++--------- .../googlenet/quantization/ptq/main.py | 21 +++++---- .../squeezenet/quantization/ptq/main.py | 21 +++++---- .../zfnet/quantization/ptq/main.py | 21 +++++---- 6 files changed, 79 insertions(+), 72 deletions(-) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py index 4a91f52d1ed..0eaee028fc6 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py @@ -130,19 +130,20 @@ def __init__(self, dataset_location, image_list): src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue - with Image.open(src) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - self.image_list.append(image) + + self.image_list.append(src) self.label_list.append(int(label)) def __iter__(self): - for image, label in zip(self.image_list, self.label_list): + for src, label in zip(self.image_list, self.label_list): + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) yield image, label def eval_func(model, dataloader, metric): diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py index edeea5712fa..cffe0bcdc6d 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py @@ -130,19 +130,20 @@ def __init__(self, dataset_location, image_list): src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue - with Image.open(src) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - self.image_list.append(image) + + self.image_list.append(src) self.label_list.append(int(label)) def __iter__(self): - for image, label in zip(self.image_list, self.label_list): + for src, label in zip(self.image_list, self.label_list): + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) yield image, label def eval_func(model, dataloader, metric): diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py index 982eb8ccef5..3607e17d6bf 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py @@ -130,30 +130,31 @@ def __init__(self, dataset_location, image_list): src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue - with Image.open(src) as image: - image = np.array(image.convert('RGB')).astype(np.float32) - image = image / 255. - image = cv2.resize(image, (256,256)) - h, w = image.shape[0], image.shape[1] - if h + 1 < 224 or w + 1 < 224: - raise ValueError( - "Required crop size {} is larger then input image size {}".format( - (224, 224), (h, w))) - - if 224 != h or 224 != w: - y0 = (h - 224) // 2 - x0 = (w - 224) // 2 - image = image[y0:y0 + 224, x0:x0 + 224, :] - - image = (image - [0.485, 0.456, 0.406]) / [0.229, 0.224, 0.225] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - self.image_list.append(image.astype(np.float32)) + + self.image_list.append(src) self.label_list.append(int(label)) def __iter__(self): - for image, label in zip(self.image_list, self.label_list): - yield image, label + for src, label in zip(self.image_list, self.label_list): + with Image.open(src) as image: + image = np.array(image.convert('RGB')).astype(np.float32) + image = image / 255. + image = cv2.resize(image, (256,256)) + h, w = image.shape[0], image.shape[1] + if h + 1 < 224 or w + 1 < 224: + raise ValueError( + "Required crop size {} is larger then input image size {}".format( + (224, 224), (h, w))) + + if 224 != h or 224 != w: + y0 = (h - 224) // 2 + x0 = (w - 224) // 2 + image = image[y0:y0 + 224, x0:x0 + 224, :] + + image = (image - [0.485, 0.456, 0.406]) / [0.229, 0.224, 0.225] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) + yield image.astype(np.float32), label def eval_func(model, dataloader, metric, postprocess): metric.reset() @@ -235,7 +236,8 @@ def eval(onnx_model): accuracy_criterion.relative = 0.02 config = PostTrainingQuantConfig( accuracy_criterion=accuracy_criterion, - op_name_list={'Conv_nc_rename_431': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}}) + op_name_list={'Conv_nc_rename_0': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}, + 'Relu_nc_rename_1': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}}) q_model = quantization.fit(model, config, calib_dataloader=dataloader, eval_func=eval) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py index 281bd03b403..dce8d0437ca 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py @@ -130,19 +130,20 @@ def __init__(self, dataset_location, image_list): src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue - with Image.open(src) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - self.image_list.append(image) + + self.image_list.append(src) self.label_list.append(int(label)) def __iter__(self): - for image, label in zip(self.image_list, self.label_list): + for src, label in zip(self.image_list, self.label_list): + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) yield image, label def eval_func(model, dataloader, metric): diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py index 91766e27d55..0a35f946f4b 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py @@ -135,19 +135,20 @@ def __init__(self, dataset_location, image_list): src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue - with Image.open(src) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - self.image_list.append(image) + + self.image_list.append(src) self.label_list.append(int(label)) def __iter__(self): - for image, label in zip(self.image_list, self.label_list): + for src, label in zip(self.image_list, self.label_list): + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) yield image, label def eval_func(model, dataloader, metric, postprocess): diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py index a33ec28f01b..8dfdfcaa13f 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py @@ -130,19 +130,20 @@ def __init__(self, dataset_location, image_list): src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue - with Image.open(src) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - self.image_list.append(image) + + self.image_list.append(src) self.label_list.append(int(label)) def __iter__(self): - for image, label in zip(self.image_list, self.label_list): + for src, label in zip(self.image_list, self.label_list): + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) yield image, label def eval_func(model, dataloader, metric): From 456072160779213408dd5e4cb0dfc5ae8f325512 Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Thu, 29 Dec 2022 13:38:57 +0800 Subject: [PATCH 12/13] remove example Signed-off-by: Mengni Wang --- examples/.config/model_params_onnxrt.json | 2 +- examples/README.md | 36 +-- .../arcface/quantization/ptq/README.md | 0 .../arcface/quantization/ptq/main.py | 0 .../arcface/quantization/ptq/requirements.txt | 0 .../arcface/quantization/ptq/run_benchmark.sh | 0 .../arcface/quantization/ptq/run_tuning.sh | 0 .../alexnet/quantization/ptq/README.md | 43 --- .../alexnet/quantization/ptq/main.py | 233 ----------------- .../alexnet/quantization/ptq/requirements.txt | 6 - .../alexnet/quantization/ptq/run_benchmark.sh | 40 --- .../alexnet/quantization/ptq/run_tuning.sh | 43 --- .../caffenet/quantization/ptq/README.md | 44 ---- .../caffenet/quantization/ptq/main.py | 232 ----------------- .../quantization/ptq/requirements.txt | 6 - .../quantization/ptq/run_benchmark.sh | 40 --- .../caffenet/quantization/ptq/run_tuning.sh | 43 --- .../densenet/quantization/ptq/README.md | 33 --- .../densenet/quantization/ptq/main.py | 245 ------------------ .../quantization/ptq/requirements.txt | 6 - .../quantization/ptq/run_benchmark.sh | 41 --- .../densenet/quantization/ptq/run_tuning.sh | 39 --- .../googlenet/quantization/ptq/README.md | 43 --- .../googlenet/quantization/ptq/main.py | 231 ----------------- .../quantization/ptq/requirements.txt | 7 - .../quantization/ptq/run_benchmark.sh | 40 --- .../googlenet/quantization/ptq/run_tuning.sh | 43 --- .../inception/quantization/ptq/README.md | 43 --- .../inception/quantization/ptq/main.py | 233 ----------------- .../quantization/ptq/requirements.txt | 6 - .../quantization/ptq/run_benchmark.sh | 40 --- .../inception/quantization/ptq/run_tuning.sh | 43 --- .../zfnet/quantization/ptq/README.md | 43 --- .../zfnet/quantization/ptq/main.py | 230 ---------------- .../zfnet/quantization/ptq/requirements.txt | 6 - .../zfnet/quantization/ptq/run_benchmark.sh | 40 --- .../zfnet/quantization/ptq/run_tuning.sh | 43 --- 37 files changed, 19 insertions(+), 2204 deletions(-) rename examples/onnxrt/{image_recognition => body_analysis}/onnx_model_zoo/arcface/quantization/ptq/README.md (100%) rename examples/onnxrt/{image_recognition => body_analysis}/onnx_model_zoo/arcface/quantization/ptq/main.py (100%) rename examples/onnxrt/{image_recognition => body_analysis}/onnx_model_zoo/arcface/quantization/ptq/requirements.txt (100%) rename examples/onnxrt/{image_recognition => body_analysis}/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh (100%) rename examples/onnxrt/{image_recognition => body_analysis}/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh (100%) delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/requirements.txt delete mode 100755 examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh delete mode 100755 examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/requirements.txt delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh delete mode 100644 examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh diff --git a/examples/.config/model_params_onnxrt.json b/examples/.config/model_params_onnxrt.json index 24c659be62f..58c3802936e 100644 --- a/examples/.config/model_params_onnxrt.json +++ b/examples/.config/model_params_onnxrt.json @@ -565,7 +565,7 @@ "new_benchmark": true }, "arcface": { - "model_src_dir": "image_recognition/onnx_model_zoo/arcface/quantization/ptq", + "model_src_dir": "body_analysis/onnx_model_zoo/arcface/quantization/ptq", "dataset_location": "/tf_dataset2/datasets/faces_ms1m_112x112/lfw.bin", "input_model": "/tf_dataset2/models/onnx/arcface/arcfaceresnet100-11.onnx", "main_script": "main.py", diff --git a/examples/README.md b/examples/README.md index 79937500628..c8adc8f49a5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -774,55 +774,55 @@ Intel® Neural Compressor validated examples with multiple compression technique ResNet50 V1.5 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ResNet50 V1.5 MLPerf Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq VGG16 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq MobileNet V2 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq MobileNet V3 MLPerf Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq AlexNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq CaffeNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq DenseNet Image Recognition Post-Training Static Quantization - qlinearops + qlinearops EfficientNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq FCN @@ -834,13 +834,13 @@ Intel® Neural Compressor validated examples with multiple compression technique GoogleNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq Inception V1 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq MNIST @@ -852,43 +852,43 @@ Intel® Neural Compressor validated examples with multiple compression technique MobileNet V2 (ONNX Model Zoo) Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ResNet50 V1.5 (ONNX Model Zoo) Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ShuffleNet V2 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq SqueezeNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq VGG16 (ONNX Model Zoo) Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ZFNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ArcFace Image Recognition Post-Training Static Quantization - qlinearops + qlinearops BERT base MRPC diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/README.md similarity index 100% rename from examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md rename to examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/README.md diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/main.py similarity index 100% rename from examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py rename to examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/main.py diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/requirements.txt b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/requirements.txt similarity index 100% rename from examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/requirements.txt rename to examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/requirements.txt diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh similarity index 100% rename from examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh rename to examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh similarity index 100% rename from examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh rename to examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md deleted file mode 100644 index 1e631fa47ca..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Evaluate performance of ONNX Runtime(Alexnet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.8.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/alexnet/model/bvlcalexnet-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --output_model=path/to/save \ - --quant_format=QDQ -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py deleted file mode 100644 index 0eaee028fc6..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py +++ /dev/null @@ -1,233 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx -import re -import os -from PIL import Image -import onnxruntime as ort -from sklearn.metrics import accuracy_score - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -def _topk_shape_validate(preds, labels): - # preds shape can be Nxclass_num or class_num(N=1 by default) - # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax - if isinstance(preds, int): - preds = [preds] - preds = np.array(preds) - elif isinstance(preds, np.ndarray): - preds = np.array(preds) - elif isinstance(preds, list): - preds = np.array(preds) - preds = preds.reshape((-1, preds.shape[-1])) - - # consider labels just int value 1x1 - if isinstance(labels, int): - labels = [labels] - labels = np.array(labels) - elif isinstance(labels, tuple): - labels = np.array([labels]) - labels = labels.reshape((labels.shape[-1], -1)) - elif isinstance(labels, list): - if isinstance(labels[0], int): - labels = np.array(labels) - labels = labels.reshape((labels.shape[0], 1)) - elif isinstance(labels[0], tuple): - labels = np.array(labels) - labels = labels.reshape((labels.shape[-1], -1)) - else: - labels = np.array(labels) - # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) - # only support 2 dimension one-shot labels - # or 1 dimension one-hot class_num will confuse with N - - if len(preds.shape) == 1: - N = 1 - class_num = preds.shape[0] - preds = preds.reshape([-1, class_num]) - elif len(preds.shape) >= 2: - N = preds.shape[0] - preds = preds.reshape([N, -1]) - class_num = preds.shape[1] - - label_N = labels.shape[0] - assert label_N == N, 'labels batch size should same with preds' - labels = labels.reshape([N, -1]) - # one-hot labels will have 2 dimension not equal 1 - if labels.shape[1] != 1: - labels = labels.argsort()[..., -1:] - return preds, labels - -class TopK: - def __init__(self, k=1): - self.k = k - self.num_correct = 0 - self.num_sample = 0 - - def update(self, preds, labels, sample_weight=None): - preds, labels = _topk_shape_validate(preds, labels) - preds = preds.argsort()[..., -self.k:] - if self.k == 1: - correct = accuracy_score(preds, labels, normalize=False) - self.num_correct += correct - - else: - for p, l in zip(preds, labels): - # get top-k labels with np.argpartition - # p = np.argpartition(p, -self.k)[-self.k:] - l = l.astype('int32') - if l in p: - self.num_correct += 1 - - self.num_sample += len(labels) - - def reset(self): - self.num_correct = 0 - self.num_sample = 0 - - def result(self): - if self.num_sample == 0: - logger.warning("Sample num during evaluation is 0.") - return 0 - elif getattr(self, '_hvd', None) is not None: - allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) - allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) - return allgather_num_correct / allgather_num_sample - return self.num_correct / self.num_sample - -class Dataloader: - def __init__(self, dataset_location, image_list): - self.batch_size = 1 - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(dataset_location, image_name) - if not os.path.exists(src): - continue - - self.image_list.append(src) - self.label_list.append(int(label)) - - def __iter__(self): - for src, label in zip(self.image_list, self.label_list): - with Image.open(src) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - yield image, label - -def eval_func(model, dataloader, metric): - metric.reset() - sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) - ort_inputs = {} - input_names = [i.name for i in sess.get_inputs()] - for input_data, label in dataloader: - output = sess.run(None, dict(zip(input_names, [input_data]))) - metric.update(output, label) - return metric.result() - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Alexnet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--dataset_location', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--quant_format', - type=str, - default='default', - choices=['default', 'QDQ', 'QOperator'], - help="quantization format" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') - label_path = os.path.join(args.dataset_location, 'val.txt') - dataloader = Dataloader(data_path, label_path) - top1 = TopK() - def eval(onnx_model): - return eval_func(onnx_model, dataloader, top1) - - if args.benchmark: - if args.mode == 'performance': - from neural_compressor.benchmark import fit - from neural_compressor.config import BenchmarkConfig - conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) - fit(model, conf, b_dataloader=dataloader) - elif args.mode == 'accuracy': - acc_result = eval(model) - print("Batch size = %d" % dataloader.batch_size) - print("Accuracy: %.5f" % acc_result) - - if args.tune: - from neural_compressor import quantization, PostTrainingQuantConfig - config = PostTrainingQuantConfig(quant_format=args.quant_format) - - q_model = quantization.fit(model, config, calib_dataloader=dataloader, - eval_func=eval) - - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/requirements.txt deleted file mode 100644 index 4b56442f97d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 7ec6f7ace0e..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 29ed086faf4..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --quant_format=*) - quant_format=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --output_model ${output_model} \ - --quant_format ${quant_format} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md deleted file mode 100644 index 4c03fdda60a..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Evaluate performance of ONNX Runtime(Caffenet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/caffenet/model/caffenet-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --output_model=path/to/save \ - --quant_format=QDQ -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py deleted file mode 100644 index cffe0bcdc6d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py +++ /dev/null @@ -1,232 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx -import re -import os -from PIL import Image -import onnxruntime as ort -from sklearn.metrics import accuracy_score - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -def _topk_shape_validate(preds, labels): - # preds shape can be Nxclass_num or class_num(N=1 by default) - # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax - if isinstance(preds, int): - preds = [preds] - preds = np.array(preds) - elif isinstance(preds, np.ndarray): - preds = np.array(preds) - elif isinstance(preds, list): - preds = np.array(preds) - preds = preds.reshape((-1, preds.shape[-1])) - - # consider labels just int value 1x1 - if isinstance(labels, int): - labels = [labels] - labels = np.array(labels) - elif isinstance(labels, tuple): - labels = np.array([labels]) - labels = labels.reshape((labels.shape[-1], -1)) - elif isinstance(labels, list): - if isinstance(labels[0], int): - labels = np.array(labels) - labels = labels.reshape((labels.shape[0], 1)) - elif isinstance(labels[0], tuple): - labels = np.array(labels) - labels = labels.reshape((labels.shape[-1], -1)) - else: - labels = np.array(labels) - # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) - # only support 2 dimension one-shot labels - # or 1 dimension one-hot class_num will confuse with N - - if len(preds.shape) == 1: - N = 1 - class_num = preds.shape[0] - preds = preds.reshape([-1, class_num]) - elif len(preds.shape) >= 2: - N = preds.shape[0] - preds = preds.reshape([N, -1]) - class_num = preds.shape[1] - - label_N = labels.shape[0] - assert label_N == N, 'labels batch size should same with preds' - labels = labels.reshape([N, -1]) - # one-hot labels will have 2 dimension not equal 1 - if labels.shape[1] != 1: - labels = labels.argsort()[..., -1:] - return preds, labels - -class TopK: - def __init__(self, k=1): - self.k = k - self.num_correct = 0 - self.num_sample = 0 - - def update(self, preds, labels, sample_weight=None): - preds, labels = _topk_shape_validate(preds, labels) - preds = preds.argsort()[..., -self.k:] - if self.k == 1: - correct = accuracy_score(preds, labels, normalize=False) - self.num_correct += correct - - else: - for p, l in zip(preds, labels): - # get top-k labels with np.argpartition - # p = np.argpartition(p, -self.k)[-self.k:] - l = l.astype('int32') - if l in p: - self.num_correct += 1 - - self.num_sample += len(labels) - - def reset(self): - self.num_correct = 0 - self.num_sample = 0 - - def result(self): - if self.num_sample == 0: - logger.warning("Sample num during evaluation is 0.") - return 0 - elif getattr(self, '_hvd', None) is not None: - allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) - allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) - return allgather_num_correct / allgather_num_sample - return self.num_correct / self.num_sample - -class Dataloader: - def __init__(self, dataset_location, image_list): - self.batch_size = 1 - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(dataset_location, image_name) - if not os.path.exists(src): - continue - - self.image_list.append(src) - self.label_list.append(int(label)) - - def __iter__(self): - for src, label in zip(self.image_list, self.label_list): - with Image.open(src) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - yield image, label - -def eval_func(model, dataloader, metric): - metric.reset() - sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) - ort_inputs = {} - input_names = [i.name for i in sess.get_inputs()] - for input_data, label in dataloader: - output = sess.run(None, dict(zip(input_names, [input_data]))) - metric.update(output, label) - return metric.result() - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Caffenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--dataset_location', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - parser.add_argument( - '--quant_format', - type=str, - default='default', - choices=['default', 'QDQ', 'QOperator'], - help="quantization format" - ) - args = parser.parse_args() - - model = onnx.load(args.model_path) - data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') - label_path = os.path.join(args.dataset_location, 'val.txt') - dataloader = Dataloader(data_path, label_path) - top1 = TopK() - def eval(onnx_model): - return eval_func(onnx_model, dataloader, top1) - - if args.benchmark: - if args.mode == 'performance': - from neural_compressor.benchmark import fit - from neural_compressor.config import BenchmarkConfig - conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) - fit(model, conf, b_dataloader=dataloader) - elif args.mode == 'accuracy': - acc_result = eval(model) - print("Batch size = %d" % dataloader.batch_size) - print("Accuracy: %.5f" % acc_result) - - if args.tune: - from neural_compressor import quantization, PostTrainingQuantConfig - config = PostTrainingQuantConfig(quant_format=args.quant_format) - - q_model = quantization.fit(model, config, calib_dataloader=dataloader, - eval_func=eval) - - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/requirements.txt deleted file mode 100644 index 4b56442f97d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 7ec6f7ace0e..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 29ed086faf4..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --quant_format=*) - quant_format=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --output_model ${output_model} \ - --quant_format ${quant_format} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md deleted file mode 100644 index 41249cf360b..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Evaluate performance of ONNX Runtime(Densenet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/densenet-121/model/densenet-12.onnx -``` - -### Quantization - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=path/to/dataset \ - --output_model=path/to/save -``` - -### Performance - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=path/to/dataset \ - --mode=performance # or accuracy - -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py deleted file mode 100644 index 3607e17d6bf..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py +++ /dev/null @@ -1,245 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse -import os -import re -import numpy as np -import onnx -import onnxruntime as ort -from sklearn.metrics import accuracy_score -import cv2 -from PIL import Image - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -def _topk_shape_validate(preds, labels): - # preds shape can be Nxclass_num or class_num(N=1 by default) - # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax - if isinstance(preds, int): - preds = [preds] - preds = np.array(preds) - elif isinstance(preds, np.ndarray): - preds = np.array(preds) - elif isinstance(preds, list): - preds = np.array(preds) - preds = preds.reshape((-1, preds.shape[-1])) - - # consider labels just int value 1x1 - if isinstance(labels, int): - labels = [labels] - labels = np.array(labels) - elif isinstance(labels, tuple): - labels = np.array([labels]) - labels = labels.reshape((labels.shape[-1], -1)) - elif isinstance(labels, list): - if isinstance(labels[0], int): - labels = np.array(labels) - labels = labels.reshape((labels.shape[0], 1)) - elif isinstance(labels[0], tuple): - labels = np.array(labels) - labels = labels.reshape((labels.shape[-1], -1)) - else: - labels = np.array(labels) - # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) - # only support 2 dimension one-shot labels - # or 1 dimension one-hot class_num will confuse with N - - if len(preds.shape) == 1: - N = 1 - class_num = preds.shape[0] - preds = preds.reshape([-1, class_num]) - elif len(preds.shape) >= 2: - N = preds.shape[0] - preds = preds.reshape([N, -1]) - class_num = preds.shape[1] - - label_N = labels.shape[0] - assert label_N == N, 'labels batch size should same with preds' - labels = labels.reshape([N, -1]) - # one-hot labels will have 2 dimension not equal 1 - if labels.shape[1] != 1: - labels = labels.argsort()[..., -1:] - return preds, labels - -class TopK: - def __init__(self, k=1): - self.k = k - self.num_correct = 0 - self.num_sample = 0 - - def update(self, preds, labels, sample_weight=None): - preds, labels = _topk_shape_validate(preds, labels) - preds = preds.argsort()[..., -self.k:] - if self.k == 1: - correct = accuracy_score(preds, labels, normalize=False) - self.num_correct += correct - - else: - for p, l in zip(preds, labels): - # get top-k labels with np.argpartition - # p = np.argpartition(p, -self.k)[-self.k:] - l = l.astype('int32') - if l in p: - self.num_correct += 1 - - self.num_sample += len(labels) - - def reset(self): - self.num_correct = 0 - self.num_sample = 0 - - def result(self): - if self.num_sample == 0: - logger.warning("Sample num during evaluation is 0.") - return 0 - elif getattr(self, '_hvd', None) is not None: - allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) - allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) - return allgather_num_correct / allgather_num_sample - return self.num_correct / self.num_sample - -class Dataloader: - def __init__(self, dataset_location, image_list): - self.batch_size = 1 - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(dataset_location, image_name) - if not os.path.exists(src): - continue - - self.image_list.append(src) - self.label_list.append(int(label)) - - def __iter__(self): - for src, label in zip(self.image_list, self.label_list): - with Image.open(src) as image: - image = np.array(image.convert('RGB')).astype(np.float32) - image = image / 255. - image = cv2.resize(image, (256,256)) - h, w = image.shape[0], image.shape[1] - if h + 1 < 224 or w + 1 < 224: - raise ValueError( - "Required crop size {} is larger then input image size {}".format( - (224, 224), (h, w))) - - if 224 != h or 224 != w: - y0 = (h - 224) // 2 - x0 = (w - 224) // 2 - image = image[y0:y0 + 224, x0:x0 + 224, :] - - image = (image - [0.485, 0.456, 0.406]) / [0.229, 0.224, 0.225] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - yield image.astype(np.float32), label - -def eval_func(model, dataloader, metric, postprocess): - metric.reset() - sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) - ort_inputs = {} - input_names = [i.name for i in sess.get_inputs()] - for input_data, label in dataloader: - output = sess.run(None, dict(zip(input_names, [input_data]))) - output, label = postprocess((output, label)) - metric.update(output, label) - return metric.result() - -class Squeeze: - def __call__(self, sample): - preds, labels = sample - return np.squeeze(preds), labels - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Densenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--dataset_location', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - args = parser.parse_args() - model = onnx.load(args.model_path) - data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') - label_path = os.path.join(args.dataset_location, 'val.txt') - dataloader = Dataloader(data_path, label_path) - top1 = TopK() - postprocess = Squeeze() - def eval(onnx_model): - return eval_func(onnx_model, dataloader, top1, postprocess) - - if args.benchmark: - if args.mode == 'performance': - from neural_compressor.benchmark import fit - from neural_compressor.config import BenchmarkConfig - conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) - fit(model, conf, b_dataloader=dataloader) - elif args.mode == 'accuracy': - acc_result = eval(model) - print("Batch size = %d" % dataloader.batch_size) - print("Accuracy: %.5f" % acc_result) - if args.tune: - from neural_compressor import quantization, PostTrainingQuantConfig - from neural_compressor.config import AccuracyCriterion - accuracy_criterion = AccuracyCriterion() - accuracy_criterion.relative = 0.02 - config = PostTrainingQuantConfig( - accuracy_criterion=accuracy_criterion, - op_name_list={'Conv_nc_rename_0': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}, - 'Relu_nc_rename_1': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}}) - - q_model = quantization.fit(model, config, calib_dataloader=dataloader, - eval_func=eval) - - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/requirements.txt deleted file mode 100644 index f62a897bffa..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh deleted file mode 100755 index 340848c36ad..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh deleted file mode 100755 index e4eb6389c94..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --output_model ${output_model} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md deleted file mode 100644 index 3865d94328d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Evaluate performance of ONNX Runtime(Googlenet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/inception_and_googlenet/googlenet/model/googlenet-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --output_model=path/to/save \ - --quant_format=QDQ -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py deleted file mode 100644 index dce8d0437ca..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py +++ /dev/null @@ -1,231 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx -import re -import os -from PIL import Image -import onnxruntime as ort -from sklearn.metrics import accuracy_score - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -def _topk_shape_validate(preds, labels): - # preds shape can be Nxclass_num or class_num(N=1 by default) - # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax - if isinstance(preds, int): - preds = [preds] - preds = np.array(preds) - elif isinstance(preds, np.ndarray): - preds = np.array(preds) - elif isinstance(preds, list): - preds = np.array(preds) - preds = preds.reshape((-1, preds.shape[-1])) - - # consider labels just int value 1x1 - if isinstance(labels, int): - labels = [labels] - labels = np.array(labels) - elif isinstance(labels, tuple): - labels = np.array([labels]) - labels = labels.reshape((labels.shape[-1], -1)) - elif isinstance(labels, list): - if isinstance(labels[0], int): - labels = np.array(labels) - labels = labels.reshape((labels.shape[0], 1)) - elif isinstance(labels[0], tuple): - labels = np.array(labels) - labels = labels.reshape((labels.shape[-1], -1)) - else: - labels = np.array(labels) - # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) - # only support 2 dimension one-shot labels - # or 1 dimension one-hot class_num will confuse with N - - if len(preds.shape) == 1: - N = 1 - class_num = preds.shape[0] - preds = preds.reshape([-1, class_num]) - elif len(preds.shape) >= 2: - N = preds.shape[0] - preds = preds.reshape([N, -1]) - class_num = preds.shape[1] - - label_N = labels.shape[0] - assert label_N == N, 'labels batch size should same with preds' - labels = labels.reshape([N, -1]) - # one-hot labels will have 2 dimension not equal 1 - if labels.shape[1] != 1: - labels = labels.argsort()[..., -1:] - return preds, labels - -class TopK: - def __init__(self, k=1): - self.k = k - self.num_correct = 0 - self.num_sample = 0 - - def update(self, preds, labels, sample_weight=None): - preds, labels = _topk_shape_validate(preds, labels) - preds = preds.argsort()[..., -self.k:] - if self.k == 1: - correct = accuracy_score(preds, labels, normalize=False) - self.num_correct += correct - - else: - for p, l in zip(preds, labels): - # get top-k labels with np.argpartition - # p = np.argpartition(p, -self.k)[-self.k:] - l = l.astype('int32') - if l in p: - self.num_correct += 1 - - self.num_sample += len(labels) - - def reset(self): - self.num_correct = 0 - self.num_sample = 0 - - def result(self): - if self.num_sample == 0: - logger.warning("Sample num during evaluation is 0.") - return 0 - elif getattr(self, '_hvd', None) is not None: - allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) - allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) - return allgather_num_correct / allgather_num_sample - return self.num_correct / self.num_sample - -class Dataloader: - def __init__(self, dataset_location, image_list): - self.batch_size = 1 - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(dataset_location, image_name) - if not os.path.exists(src): - continue - - self.image_list.append(src) - self.label_list.append(int(label)) - - def __iter__(self): - for src, label in zip(self.image_list, self.label_list): - with Image.open(src) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - yield image, label - -def eval_func(model, dataloader, metric): - metric.reset() - sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) - ort_inputs = {} - input_names = [i.name for i in sess.get_inputs()] - for input_data, label in dataloader: - output = sess.run(None, dict(zip(input_names, [input_data]))) - metric.update(output, label) - return metric.result() - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Googlenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--dataset_location', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - parser.add_argument( - '--quant_format', - type=str, - default='default', - choices=['default', 'QDQ', 'QOperator'], - help="quantization format" - ) - args = parser.parse_args() - - model = onnx.load(args.model_path) - data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') - label_path = os.path.join(args.dataset_location, 'val.txt') - dataloader = Dataloader(data_path, label_path) - top1 = TopK() - def eval(onnx_model): - return eval_func(onnx_model, dataloader, top1) - - if args.benchmark: - if args.mode == 'performance': - from neural_compressor.benchmark import fit - from neural_compressor.config import BenchmarkConfig - conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) - fit(model, conf, b_dataloader=dataloader) - elif args.mode == 'accuracy': - acc_result = eval(model) - print("Batch size = %d" % dataloader.batch_size) - print("Accuracy: %.5f" % acc_result) - if args.tune: - from neural_compressor import quantization, PostTrainingQuantConfig - config = PostTrainingQuantConfig(quant_format=args.quant_format) - - q_model = quantization.fit(model, config, calib_dataloader=dataloader, - eval_func=eval) - - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/requirements.txt deleted file mode 100644 index 788d60f9ae8..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -opencv-python -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 7ec6f7ace0e..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 29ed086faf4..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --quant_format=*) - quant_format=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --output_model ${output_model} \ - --quant_format ${quant_format} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md deleted file mode 100644 index 412b1b2322a..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Evaluate performance of ONNX Runtime(Inception) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/inception_and_googlenet/inception_v1/model/inception-v1-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --output_model=path/to/save \ - --quant_format=QDQ -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py deleted file mode 100644 index 19edd9b6b70..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py +++ /dev/null @@ -1,233 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx -import re -import os -from PIL import Image -import onnxruntime as ort -from sklearn.metrics import accuracy_score - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -def _topk_shape_validate(preds, labels): - # preds shape can be Nxclass_num or class_num(N=1 by default) - # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax - if isinstance(preds, int): - preds = [preds] - preds = np.array(preds) - elif isinstance(preds, np.ndarray): - preds = np.array(preds) - elif isinstance(preds, list): - preds = np.array(preds) - preds = preds.reshape((-1, preds.shape[-1])) - - # consider labels just int value 1x1 - if isinstance(labels, int): - labels = [labels] - labels = np.array(labels) - elif isinstance(labels, tuple): - labels = np.array([labels]) - labels = labels.reshape((labels.shape[-1], -1)) - elif isinstance(labels, list): - if isinstance(labels[0], int): - labels = np.array(labels) - labels = labels.reshape((labels.shape[0], 1)) - elif isinstance(labels[0], tuple): - labels = np.array(labels) - labels = labels.reshape((labels.shape[-1], -1)) - else: - labels = np.array(labels) - # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) - # only support 2 dimension one-shot labels - # or 1 dimension one-hot class_num will confuse with N - - if len(preds.shape) == 1: - N = 1 - class_num = preds.shape[0] - preds = preds.reshape([-1, class_num]) - elif len(preds.shape) >= 2: - N = preds.shape[0] - preds = preds.reshape([N, -1]) - class_num = preds.shape[1] - - label_N = labels.shape[0] - assert label_N == N, 'labels batch size should same with preds' - labels = labels.reshape([N, -1]) - # one-hot labels will have 2 dimension not equal 1 - if labels.shape[1] != 1: - labels = labels.argsort()[..., -1:] - return preds, labels - -class TopK: - def __init__(self, k=1): - self.k = k - self.num_correct = 0 - self.num_sample = 0 - - def update(self, preds, labels, sample_weight=None): - preds, labels = _topk_shape_validate(preds, labels) - preds = preds.argsort()[..., -self.k:] - if self.k == 1: - correct = accuracy_score(preds, labels, normalize=False) - self.num_correct += correct - - else: - for p, l in zip(preds, labels): - # get top-k labels with np.argpartition - # p = np.argpartition(p, -self.k)[-self.k:] - l = l.astype('int32') - if l in p: - self.num_correct += 1 - - self.num_sample += len(labels) - - def reset(self): - self.num_correct = 0 - self.num_sample = 0 - - def result(self): - if self.num_sample == 0: - logger.warning("Sample num during evaluation is 0.") - return 0 - elif getattr(self, '_hvd', None) is not None: - allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) - allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) - return allgather_num_correct / allgather_num_sample - return self.num_correct / self.num_sample - -class Dataloader: - def __init__(self, dataset_location, image_list): - self.batch_size = 1 - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(dataset_location, image_name) - if not os.path.exists(src): - continue - with Image.open(src) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - self.image_list.append(image) - self.label_list.append(int(label)) - - def __iter__(self): - for image, label in zip(self.image_list, self.label_list): - yield image, label - -def eval_func(model, dataloader, metric): - metric.reset() - sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) - ort_inputs = {} - input_names = [i.name for i in sess.get_inputs()] - for input_data, label in dataloader: - output = sess.run(None, dict(zip(input_names, [input_data]))) - metric.update(output, label) - return metric.result() - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--dataset_location', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - parser.add_argument( - '--quant_format', - type=str, - default='default', - choices=['default', 'QDQ', 'QOperator'], - help="quantization format" - ) - args = parser.parse_args() - - model = onnx.load(args.model_path) - data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') - label_path = os.path.join(args.dataset_location, 'val.txt') - dataloader = Dataloader(data_path, label_path) - top1 = TopK() - def eval(onnx_model): - return eval_func(onnx_model, dataloader, top1) - - from neural_compressor import options - options.onnxrt.graph_optimization.gemm2matmul = False - - if args.benchmark: - if args.mode == 'performance': - from neural_compressor.benchmark import fit - from neural_compressor.config import BenchmarkConfig - conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) - fit(model, conf, b_dataloader=dataloader) - elif args.mode == 'accuracy': - acc_result = eval(model) - print("Batch size = %d" % dataloader.batch_size) - print("Accuracy: %.5f" % acc_result) - if args.tune: - from neural_compressor import quantization, PostTrainingQuantConfig - config = PostTrainingQuantConfig(quant_format=args.quant_format) - - q_model = quantization.fit(model, config, calib_dataloader=dataloader, - eval_func=eval) - - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/requirements.txt deleted file mode 100644 index 4b56442f97d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 7ec6f7ace0e..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh deleted file mode 100644 index 29ed086faf4..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --quant_format=*) - quant_format=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --output_model ${output_model} \ - --quant_format ${quant_format} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md deleted file mode 100644 index dbf3a738d90..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Evaluate performance of ONNX Runtime(ZFNet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/zfnet-512/model/zfnet512-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --output_model=path/to/save \ - --quant_format=QDQ -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --dataset_location=/path/to/imagenet \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py deleted file mode 100644 index 8dfdfcaa13f..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py +++ /dev/null @@ -1,230 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx -import re -import os -from PIL import Image -import onnxruntime as ort -from sklearn.metrics import accuracy_score - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -def _topk_shape_validate(preds, labels): - # preds shape can be Nxclass_num or class_num(N=1 by default) - # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax - if isinstance(preds, int): - preds = [preds] - preds = np.array(preds) - elif isinstance(preds, np.ndarray): - preds = np.array(preds) - elif isinstance(preds, list): - preds = np.array(preds) - preds = preds.reshape((-1, preds.shape[-1])) - - # consider labels just int value 1x1 - if isinstance(labels, int): - labels = [labels] - labels = np.array(labels) - elif isinstance(labels, tuple): - labels = np.array([labels]) - labels = labels.reshape((labels.shape[-1], -1)) - elif isinstance(labels, list): - if isinstance(labels[0], int): - labels = np.array(labels) - labels = labels.reshape((labels.shape[0], 1)) - elif isinstance(labels[0], tuple): - labels = np.array(labels) - labels = labels.reshape((labels.shape[-1], -1)) - else: - labels = np.array(labels) - # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) - # only support 2 dimension one-shot labels - # or 1 dimension one-hot class_num will confuse with N - - if len(preds.shape) == 1: - N = 1 - class_num = preds.shape[0] - preds = preds.reshape([-1, class_num]) - elif len(preds.shape) >= 2: - N = preds.shape[0] - preds = preds.reshape([N, -1]) - class_num = preds.shape[1] - - label_N = labels.shape[0] - assert label_N == N, 'labels batch size should same with preds' - labels = labels.reshape([N, -1]) - # one-hot labels will have 2 dimension not equal 1 - if labels.shape[1] != 1: - labels = labels.argsort()[..., -1:] - return preds, labels - -class TopK: - def __init__(self, k=1): - self.k = k - self.num_correct = 0 - self.num_sample = 0 - - def update(self, preds, labels, sample_weight=None): - preds, labels = _topk_shape_validate(preds, labels) - preds = preds.argsort()[..., -self.k:] - if self.k == 1: - correct = accuracy_score(preds, labels, normalize=False) - self.num_correct += correct - - else: - for p, l in zip(preds, labels): - # get top-k labels with np.argpartition - # p = np.argpartition(p, -self.k)[-self.k:] - l = l.astype('int32') - if l in p: - self.num_correct += 1 - - self.num_sample += len(labels) - - def reset(self): - self.num_correct = 0 - self.num_sample = 0 - - def result(self): - if self.num_sample == 0: - logger.warning("Sample num during evaluation is 0.") - return 0 - elif getattr(self, '_hvd', None) is not None: - allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) - allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) - return allgather_num_correct / allgather_num_sample - return self.num_correct / self.num_sample - -class Dataloader: - def __init__(self, dataset_location, image_list): - self.batch_size = 1 - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(dataset_location, image_name) - if not os.path.exists(src): - continue - - self.image_list.append(src) - self.label_list.append(int(label)) - - def __iter__(self): - for src, label in zip(self.image_list, self.label_list): - with Image.open(src) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - image = np.expand_dims(image, axis=0) - yield image, label - -def eval_func(model, dataloader, metric): - metric.reset() - sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) - ort_inputs = {} - input_names = [i.name for i in sess.get_inputs()] - for input_data, label in dataloader: - output = sess.run(None, dict(zip(input_names, [input_data]))) - metric.update(output, label) - return metric.result() - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Googlenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--dataset_location', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - parser.add_argument( - '--quant_format', - type=str, - default='default', - choices=['default', 'QDQ', 'QOperator'], - help="quantization format" - ) - args = parser.parse_args() - - model = onnx.load(args.model_path) - data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') - label_path = os.path.join(args.dataset_location, 'val.txt') - dataloader = Dataloader(data_path, label_path) - top1 = TopK() - def eval(onnx_model): - return eval_func(onnx_model, dataloader, top1) - - if args.benchmark: - if args.mode == 'performance': - from neural_compressor.benchmark import fit - from neural_compressor.config import BenchmarkConfig - conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) - fit(model, conf, b_dataloader=dataloader) - elif args.mode == 'accuracy': - acc_result = eval(model) - print("Batch size = %d" % dataloader.batch_size) - print("Accuracy: %.5f" % acc_result) - if args.tune: - from neural_compressor import quantization, PostTrainingQuantConfig - config = PostTrainingQuantConfig(quant_format=args.quant_format) - - q_model = quantization.fit(model, config, calib_dataloader=dataloader, - eval_func=eval) - - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/requirements.txt deleted file mode 100644 index 4b56442f97d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 7ec6f7ace0e..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 29ed086faf4..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --dataset_location=*) - dataset_location=$(echo $var |cut -f2 -d=) - ;; - --quant_format=*) - quant_format=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --dataset_location ${dataset_location} \ - --output_model ${output_model} \ - --quant_format ${quant_format} \ - --tune -} - -main "$@" From 104ce6bda6c1b15ddca78e7a6cee55f2ae57956d Mon Sep 17 00:00:00 2001 From: Mengni Wang Date: Thu, 29 Dec 2022 15:16:57 +0800 Subject: [PATCH 13/13] update config Signed-off-by: Mengni Wang --- examples/.config/model_params_onnxrt.json | 96 ++++++++++++++--------- 1 file changed, 60 insertions(+), 36 deletions(-) diff --git a/examples/.config/model_params_onnxrt.json b/examples/.config/model_params_onnxrt.json index 58c3802936e..1d84f63a00b 100644 --- a/examples/.config/model_params_onnxrt.json +++ b/examples/.config/model_params_onnxrt.json @@ -164,10 +164,12 @@ }, "googlenet-12": { "model_src_dir": "image_recognition/onnx_model_zoo/googlenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/googlenet-12/googlenet-12.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "googlenet.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "shufflenet-v2-12": { "model_src_dir": "image_recognition/onnx_model_zoo/shufflenet/quantization/ptq", @@ -187,17 +189,21 @@ }, "caffenet": { "model_src_dir": "image_recognition/onnx_model_zoo/caffenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/caffenet/caffenet-12.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "caffenet.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "alexnet": { "model_src_dir": "image_recognition/onnx_model_zoo/alexnet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/alexnet/bvlcalexnet-12.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "alexnet.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "efficientnet": { "model_src_dir": "image_recognition/onnx_model_zoo/efficientnet/quantization/ptq", @@ -210,17 +216,21 @@ }, "zfnet": { "model_src_dir": "image_recognition/onnx_model_zoo/zfnet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/zfnet/zfnet512-12.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "zfnet512.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "inception_v1": { "model_src_dir": "image_recognition/onnx_model_zoo/inception/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/inception_v1/inception-v1-12.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "inception_v1.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "ssd-12": { "model_src_dir": "object_detection/onnx_model_zoo/ssd/quantization/ptq", @@ -267,10 +277,12 @@ }, "densenet": { "model_src_dir": "image_recognition/onnx_model_zoo/densenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/densenet/densenet-12.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "densenet.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "mnist": { "model_src_dir": "image_recognition/onnx_model_zoo/mnist/quantization/ptq", @@ -436,10 +448,12 @@ }, "googlenet-12_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/googlenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/googlenet-12/googlenet-13.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "googlenet_qdq.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "shufflenet-v2-12_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/shufflenet/quantization/ptq", @@ -452,24 +466,30 @@ }, "squeezenet_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/squeezenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/squeezenet/squeezenet1.0-13.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "squeezenet_qdq.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "caffenet_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/caffenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/caffenet/caffenet-13.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "caffenet_qdq.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "alexnet_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/alexnet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/alexnet/bvlcalexnet-13.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "alexnet_qdq.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "efficientnet_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/efficientnet/quantization/ptq", @@ -482,17 +502,21 @@ }, "zfnet_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/zfnet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/zfnet/zfnet512-13.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "zfnet512_qdq.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "inception_v1_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/inception/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", "input_model": "/tf_dataset2/models/onnx/inception_v1/inception-v1-13.onnx", - "main_script": "main.py", - "batch_size": 1 + "yaml": "inception_v1_qdq.yaml", + "strategy": "basic", + "batch_size": 1, + "new_benchmark": true }, "ssd-12_qdq": { "model_src_dir": "object_detection/onnx_model_zoo/ssd/quantization/ptq",