diff --git a/examples/.config/model_params_onnxrt.json b/examples/.config/model_params_onnxrt.json index e0b08c699ad..1d84f63a00b 100644 --- a/examples/.config/model_params_onnxrt.json +++ b/examples/.config/model_params_onnxrt.json @@ -182,12 +182,10 @@ }, "squeezenet": { "model_src_dir": "image_recognition/onnx_model_zoo/squeezenet/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw/ILSVRC2012_img_val", + "dataset_location": "/tf_dataset2/datasets/imagenet/ImagenetRaw", "input_model": "/tf_dataset2/models/onnx/squeezenet/squeezenet1.0-12.onnx", - "yaml": "squeezenet.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "caffenet": { "model_src_dir": "image_recognition/onnx_model_zoo/caffenet/quantization/ptq", @@ -245,12 +243,10 @@ }, "fcn": { "model_src_dir": "image_recognition/onnx_model_zoo/fcn/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/coco2017/coco/val2017", + "dataset_location": "/tf_dataset2/datasets/coco2017/coco", "input_model": "/tf_dataset2/models/onnx/fcn/fcn-resnet50-11.onnx", - "yaml": "fcn_rn50.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "ssd_mobilenet_v1-2": { "model_src_dir": "object_detection/onnx_model_zoo/ssd_mobilenet_v1/quantization/ptq", @@ -533,12 +529,10 @@ }, "fcn_qdq": { "model_src_dir": "image_recognition/onnx_model_zoo/fcn/quantization/ptq", - "dataset_location": "/tf_dataset2/datasets/coco2017/coco/val2017", + "dataset_location": "/tf_dataset2/datasets/coco2017/coco", "input_model": "/tf_dataset2/models/onnx/fcn/fcn-resnet50-13.onnx", - "yaml": "fcn_rn50_qdq.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "ssd_mobilenet_v1-2_qdq": { "model_src_dir": "object_detection/onnx_model_zoo/ssd_mobilenet_v1/quantization/ptq", @@ -595,31 +589,25 @@ "new_benchmark": true }, "arcface": { - "model_src_dir": "image_recognition/onnx_model_zoo/arcface/quantization/ptq", + "model_src_dir": "body_analysis/onnx_model_zoo/arcface/quantization/ptq", "dataset_location": "/tf_dataset2/datasets/faces_ms1m_112x112/lfw.bin", "input_model": "/tf_dataset2/models/onnx/arcface/arcfaceresnet100-11.onnx", - "yaml": "arcface.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "emotion_ferplus": { "model_src_dir": "body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq", "dataset_location": "/tf_dataset2/datasets/emotion_data/emotion_ferplus/fer2013/fer2013.csv", "input_model": "/tf_dataset2/models/onnx/emotion_ferplus/emotion-ferplus-12.onnx", - "yaml": "emotion_ferplus.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "ultraface": { "model_src_dir": "body_analysis/onnx_model_zoo/ultraface/quantization/ptq", "dataset_location": "/tf_dataset2/datasets/wider_face_data/WIDER_val", "input_model": "/tf_dataset2/models/onnx/ultraface/version-RFB-320-12.onnx", - "yaml": "ultraface.yaml", - "strategy": "basic", - "batch_size": 1, - "new_benchmark": true + "main_script": "main.py", + "batch_size": 1 }, "unet": { "model_src_dir": "image_recognition/unet/quantization/ptq", diff --git a/examples/README.md b/examples/README.md index 03cfc8a8f59..c8adc8f49a5 100644 --- a/examples/README.md +++ b/examples/README.md @@ -774,55 +774,55 @@ Intel® Neural Compressor validated examples with multiple compression technique ResNet50 V1.5 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ResNet50 V1.5 MLPerf Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq VGG16 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq MobileNet V2 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq MobileNet V3 MLPerf Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq AlexNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq CaffeNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq DenseNet Image Recognition Post-Training Static Quantization - qlinearops + qlinearops EfficientNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq FCN @@ -834,37 +834,37 @@ Intel® Neural Compressor validated examples with multiple compression technique GoogleNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq Inception V1 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq MNIST Image Recognition Post-Training Static Quantization - qlinearops + qlinearops MobileNet V2 (ONNX Model Zoo) Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ResNet50 V1.5 (ONNX Model Zoo) Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ShuffleNet V2 Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq SqueezeNet @@ -876,19 +876,19 @@ Intel® Neural Compressor validated examples with multiple compression technique VGG16 (ONNX Model Zoo) Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ZFNet Image Recognition Post-Training Static Quantization - qlinearops / qdq + qlinearops / qdq ArcFace Image Recognition Post-Training Static Quantization - qlinearops + qlinearops BERT base MRPC @@ -1068,7 +1068,7 @@ Intel® Neural Compressor validated examples with multiple compression technique Emotion FERPlus Body Analysis Post-Training Static Quantization - qlinearops + qlinearops Ultra Face diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/README.md similarity index 71% rename from examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md rename to examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/README.md index dffed180aaa..98391e58e86 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/README.md +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/README.md @@ -18,9 +18,7 @@ wget https://github.com/onnx/models/raw/main/vision/body_analysis/arcface/model/ ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=arcface.yaml \ - --data_path=/path/to/faces_ms1m_112x112/task.bin \ - --nfolds=nfolds_num \ # optional, data type is int, default is 1 + --dataset_location=/path/to/faces_ms1m_112x112/task.bin \ --output_model=path/to/save ``` @@ -28,9 +26,7 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=arcface.yaml \ - --data_path=/path/to/faces_ms1m_112x112/task.bin \ - --nfolds=nfolds_num \ # optional, data type is int, default is 1 + --dataset_location=/path/to/faces_ms1m_112x112/task.bin \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/main.py similarity index 75% rename from examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py rename to examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/main.py index 76f77322732..c18eae1583f 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/main.py +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/main.py @@ -7,6 +7,7 @@ import pickle import logging import argparse +import onnxruntime as ort logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', @@ -110,8 +111,9 @@ def result(self): np.asarray(self.actual_issame), nrof_folds=self.nfolds) return np.mean(accuracy) -class Dataset: +class Dataloader: def __init__(self, data_dir): + self.batch_size = 1 path = os.path.join(data_dir) # Load data if os.path.exists(path): @@ -119,11 +121,19 @@ def __init__(self, data_dir): self.data_list = data_set[0] self.issame_list = data_set[1] - def __getitem__(self, index): - return self.data_list[index], self.issame_list + def __iter__(self): + for data in self.data_list: + yield np.expand_dims(data, axis=0), self.issame_list - def __len__(self): - return len(self.data_list) +def eval_func(model, dataloader, metric): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + metric.update(output, label) + return metric.result() if __name__ == '__main__': logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") @@ -137,7 +147,7 @@ def __len__(self): help="Pre-trained model on onnx file" ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Imagenet data path" ) @@ -152,48 +162,49 @@ def __len__(self): default=False, help="whether quantize the model" ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) parser.add_argument( '--output_model', type=str, help="output model path" ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) parser.add_argument( '--nfolds', type=int, default=1, ) - + parser.add_argument( + '--mode', + type=str, + help="benchmark mode of performance or accuracy" + ) + args = parser.parse_args() # Load image size - image_size = load_property(args.data_path) + image_size = load_property(args.dataset_location) print('image_size', image_size) - ds = Dataset(args.data_path) + + dataloader = Dataloader(args.dataset_location) model = onnx.load(args.model_path) + metric = Metric(args.nfolds) + def eval(onnx_model): + return eval_func(onnx_model, dataloader, metric) + if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator.metric = Metric(args.nfolds) - evaluator(args.mode) + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - quantize.metric = Metric(args.nfolds) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig(approach='static') + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/requirements.txt b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/requirements.txt similarity index 100% rename from examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/requirements.txt rename to examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/requirements.txt diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh similarity index 73% rename from examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.sh rename to examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh index 2b7d99703d3..628585aa638 100644 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh @@ -9,16 +9,16 @@ function main { # init params function init_params { - + nfolds=1 for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) + ;; --mode=*) mode=$(echo $var |cut -f2 -d=) ;; @@ -32,8 +32,8 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ + --dataset_location ${dataset_location} \ + --mode ${mode} \ --benchmark } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh old mode 100755 new mode 100644 similarity index 78% rename from examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh rename to examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh index 97d06dab599..40fa42e9525 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh @@ -9,19 +9,19 @@ function main { # init params function init_params { - + nfolds=1 for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) + ;; esac done @@ -31,8 +31,8 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ + --dataset_location ${dataset_location} \ --output_model ${output_model} \ - --config ${config} \ --tune } diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/README.md b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/README.md index 72a77147f6f..76a7fbc031b 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/README.md +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/README.md @@ -20,7 +20,7 @@ wget https://github.com/onnx/models/raw/main/vision/body_analysis/emotion_ferplu ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx --config=emotion_ferplus.yaml \ - --data_path=/path/to/data \ + --dataset_location=/path/to/data \ --output_model=path/to/save ``` @@ -29,6 +29,6 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx --config=emotion_ferplus.yaml \ - --data_path=/path/to/data \ + --dataset_location=/path/to/data \ --mode=performance ``` diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/emotion_ferplus.yaml b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/emotion_ferplus.yaml deleted file mode 100644 index 811c68648b4..00000000000 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/emotion_ferplus.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: emotion_ferplus - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant - -evaluation: - performance: - dataloader: - batch_size: 1 - dataset: - dummy: - shape: [100,1,64,64] - accuracy: - metric: - topk: 1 - -tuning: - accuracy_criterion: - absolute: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 2%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. - diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py index 188041d91ad..03b6ef0cdfa 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/main.py @@ -23,6 +23,8 @@ import pandas as pd from PIL import Image +import onnxruntime as ort +from sklearn.metrics import accuracy_score logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', @@ -33,7 +35,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, ) parser.add_argument( @@ -52,11 +54,6 @@ default=False, help="whether quantize the model" ) -parser.add_argument( - '--config', - type=str, - help="config yaml path" -) parser.add_argument( '--output_model', type=str, @@ -69,14 +66,100 @@ ) args = parser.parse_args() +def _topk_shape_validate(preds, labels): + # preds shape can be Nxclass_num or class_num(N=1 by default) + # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax + if isinstance(preds, int): + preds = [preds] + preds = np.array(preds) + elif isinstance(preds, np.ndarray): + preds = np.array(preds) + elif isinstance(preds, list): + preds = np.array(preds) + preds = preds.reshape((-1, preds.shape[-1])) + + # consider labels just int value 1x1 + if isinstance(labels, int): + labels = [labels] + labels = np.array(labels) + elif isinstance(labels, tuple): + labels = np.array([labels]) + labels = labels.reshape((labels.shape[-1], -1)) + elif isinstance(labels, list): + if isinstance(labels[0], int): + labels = np.array(labels) + labels = labels.reshape((labels.shape[0], 1)) + elif isinstance(labels[0], tuple): + labels = np.array(labels) + labels = labels.reshape((labels.shape[-1], -1)) + else: + labels = np.array(labels) + # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) + # only support 2 dimension one-shot labels + # or 1 dimension one-hot class_num will confuse with N + + if len(preds.shape) == 1: + N = 1 + class_num = preds.shape[0] + preds = preds.reshape([-1, class_num]) + elif len(preds.shape) >= 2: + N = preds.shape[0] + preds = preds.reshape([N, -1]) + class_num = preds.shape[1] + + label_N = labels.shape[0] + assert label_N == N, 'labels batch size should same with preds' + labels = labels.reshape([N, -1]) + # one-hot labels will have 2 dimension not equal 1 + if labels.shape[1] != 1: + labels = labels.argsort()[..., -1:] + return preds, labels + +class TopK: + def __init__(self, k=1): + self.k = k + self.num_correct = 0 + self.num_sample = 0 + + def update(self, preds, labels, sample_weight=None): + preds, labels = _topk_shape_validate(preds, labels) + preds = preds.argsort()[..., -self.k:] + if self.k == 1: + correct = accuracy_score(preds, labels, normalize=False) + self.num_correct += correct + + else: + for p, l in zip(preds, labels): + # get top-k labels with np.argpartition + # p = np.argpartition(p, -self.k)[-self.k:] + l = l.astype('int32') + if l in p: + self.num_correct += 1 + + self.num_sample += len(labels) + + def reset(self): + self.num_correct = 0 + self.num_sample = 0 + + def result(self): + if self.num_sample == 0: + logger.warning("Sample num during evaluation is 0.") + return 0 + elif getattr(self, '_hvd', None) is not None: + allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) + allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) + return allgather_num_correct / allgather_num_sample + return self.num_correct / self.num_sample + class Dataloader: - def __init__(self, data_path): - df = pd.read_csv(data_path) + def __init__(self, dataset_location): + df = pd.read_csv(dataset_location) df = df[df['Usage']=='PublicTest'] images = [np.reshape(np.fromstring(image, dtype=np.uint8, sep=' '), (48, 48)) for image in df['pixels']] labels = np.array(list(map(int, df['emotion']))) self.batch_size = 1 - self.data = [(self.preprocess(image), label) for image, label in zip(images, labels)] + self.data = [(self.preprocess(image), [label]) for image, label in zip(images, labels)] def __len__(self): return len(self.data) @@ -92,30 +175,41 @@ def preprocess(self, image): img_data = np.array(img) img_data = np.resize(img_data, input_shape) return img_data.astype('float32') + +def eval_func(model, dataloader, metric): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + metric.update(output, label) + return metric.result() if __name__ == "__main__": model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path) + dataloader = Dataloader(args.dataset_location) + top1 = TopK() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, top1) + if args.benchmark: - from neural_compressor.experimental import Benchmark, common - if args.mode == 'accuracy': - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = dataloader - evaluator(args.mode) - else: - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig() + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor import options - from neural_compressor.experimental import Quantization, common - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.eval_dataloader = dataloader - quantize.calib_dataloader = dataloader - q_model = quantize() - q_model.save(args.output_model) \ No newline at end of file + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig() + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + + q_model.save(args.output_model) + diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh index f609b2ecbc1..2d211d2bbff 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_benchmark.sh @@ -13,14 +13,11 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -35,9 +32,8 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --config ${config} \ --mode ${mode} \ - --data_path ${data_path} \ + --dataset_location ${dataset_location} \ --benchmark } diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh index cd41deda04e..9f907bfa0eb 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/emotion_ferplus/quantization/ptq/run_tuning.sh @@ -13,14 +13,11 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --output_model=*) output_model=$(echo $var |cut -f2 -d=) @@ -35,8 +32,7 @@ function run_tuning { python main.py \ --model_path ${input_model} \ --output_model ${output_model} \ - --data_path ${data_path} \ - --config ${config} \ + --dataset_location ${dataset_location} \ --tune } diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/README.md b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/README.md index edd5a07875f..38e1880112e 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/README.md +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/README.md @@ -20,7 +20,7 @@ wget https://github.com/onnx/models/raw/main/vision/body_analysis/ultraface/mode ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx --config=ultraface.yaml \ - --data_path=/path/to/data \ + --dataset_location=/path/to/data \ --output_model=path/to/save ``` @@ -29,6 +29,6 @@ bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx --config=ultraface.yaml \ - --data_path=/path/to/data \ + --dataset_location=/path/to/data \ --mode=performance ``` diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py index 4dba5a1f2e8..dca3b5fc626 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/main.py @@ -23,14 +23,15 @@ import onnx import os from scipy.io import loadmat +import onnxruntime as ort class Dataloader: - def __init__(self, data_path, size=[320,240]): + def __init__(self, dataset_location, size=[320,240]): self.batch_size = 1 image_mean=np.array([127, 127, 127], dtype=np.float32) image_std = 128.0 self.data = [] - for parent, dir_names, file_names in os.walk(data_path): + for parent, dir_names, file_names in os.walk(dataset_location): for file_name in file_names: if not file_name.lower().endswith('jpg'): continue @@ -356,6 +357,17 @@ def voc_ap(rec, prec): ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) return ap +def eval_func(model, dataloader, metric, postprocess): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + output, label = postprocess((output, label)) + metric.update(output, label) + return metric.result() + logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', @@ -365,7 +377,7 @@ def voc_ap(rec, prec): formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Path of wider face validation dataset." ) @@ -398,11 +410,6 @@ def voc_ap(rec, prec): default=False, help="whether quantize the model" ) -parser.add_argument( - '--config', - type=str, - help="config yaml path" -) parser.add_argument( '--output_model', type=str, @@ -417,28 +424,28 @@ def voc_ap(rec, prec): if __name__ == "__main__": model = onnx.load(args.model_path) - dataloader = Dataloader(args.data_path, size=args.input_size) + dataloader = Dataloader(args.dataset_location, size=args.input_size) + metric = AP(args.label_path) + postprocess = Post() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, metric, postprocess) + if args.benchmark: - from neural_compressor.experimental import Benchmark, common - if args.mode == 'accuracy': - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = dataloader - evaluator.postprocess = common.Postprocess(Post) - evaluator.metric = AP(args.label_path) - evaluator(args.mode) - else: - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig() + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.eval_dataloader = dataloader - quantize.calib_dataloader = dataloader - quantize.postprocess = common.Postprocess(Post) - quantize.metric = AP(args.label_path) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig() + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) \ No newline at end of file diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh index f609b2ecbc1..2d211d2bbff 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_benchmark.sh @@ -13,14 +13,11 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -35,9 +32,8 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --config ${config} \ --mode ${mode} \ - --data_path ${data_path} \ + --dataset_location ${dataset_location} \ --benchmark } diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh index cd41deda04e..9f907bfa0eb 100644 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/run_tuning.sh @@ -13,14 +13,11 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --output_model=*) output_model=$(echo $var |cut -f2 -d=) @@ -35,8 +32,7 @@ function run_tuning { python main.py \ --model_path ${input_model} \ --output_model ${output_model} \ - --data_path ${data_path} \ - --config ${config} \ + --dataset_location ${dataset_location} \ --tune } diff --git a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/ultraface.yaml b/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/ultraface.yaml deleted file mode 100644 index e1662d26f3d..00000000000 --- a/examples/onnxrt/body_analysis/onnx_model_zoo/ultraface/quantization/ptq/ultraface.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: ultraface - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant - -evaluation: - performance: - dataloader: - batch_size: 1 - dataset: - dummy: - shape: [100,3,240,320] - accuracy: - metric: - topk: 1 - -tuning: - accuracy_criterion: - absolute: 0.02 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 2%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. - diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/README.md b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/README.md deleted file mode 100644 index eba8d257b4d..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Evaluate performance of ONNX Runtime(Mobilenet v2) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model exported from PyTorch and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Please refer to [pytorch official guide](https://pytorch.org/docs/stable/onnx.html) for detailed model export. The following is a simple example: - -```python -import torch -import torchvision -batch_size = 1 -model = torchvision.models.mobilenet_v2(pretrained=True) -x = torch.randn(batch_size, 3, 224, 224) - -# Export the model -torch.onnx.export(model, # model being run - x, # model input (or a tuple for multiple inputs) - "mobilenet_v2.onnx", # where to save the model (can be a file or file-like object) - export_params=True, # store the trained parameter weights inside the model file - opset_version=11, # the ONNX version to export the model to, please ensure at least 11. - do_constant_folding=True, # whether to execute constant folding for optimization - input_names = ['input'], # the model's input names - output_names = ['output'], # the model's output names - dynamic_axes={'input' : {0 : 'batch_size'}, # variable length axes - 'output' : {0 : 'batch_size'}}) -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v2.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v2_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v2.yaml \ - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py deleted file mode 100644 index 89b10b4a5c4..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/main.py +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Mobilenet_v2 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained mobilenet_v2 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2.yaml b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2.yaml deleted file mode 100644 index ea4d0d2f6e1..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenet_v2 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gather_.*?': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - }, - 'Gemm_174_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.03 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2_qdq.yaml b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2_qdq.yaml deleted file mode 100644 index 59963333698..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/mobilenet_v2_qdq.yaml +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenet_v2 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gather_.*?': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - }, - 'Gemm_174_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.03 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/requirements.txt deleted file mode 100644 index 63d774f50ec..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -torch -torchvision -onnxruntime-extensions; python_version < '3.10' - -pillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.bat b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.bat deleted file mode 100644 index 8da25b85410..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_benchmark.bat +++ /dev/null @@ -1,9 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -set mode=%4 -shift -shift -python main.py --model_path %input_model% --config %config% --benchmark --mode %mode%--output_model %output_model% diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.bat b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.bat deleted file mode 100644 index 56fa6ccab82..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.bat +++ /dev/null @@ -1,8 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -shift -shift -python main.py --model_path %input_model% --config %config% --tune --output_model %output_model% \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v2/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py deleted file mode 100644 index 272f8ad89f8..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/main.py +++ /dev/null @@ -1,92 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Mobilenet_v3 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - default='mobilenet_v3.onnx', - help="Pre-trained mobilenet_v3 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - default='mobilenet_v3.yaml', - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - default='mobilenet_v3_tune.onnx', - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - args = parser.parse_args() - - model = onnx.load(args.model_path) - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3.yaml b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3.yaml deleted file mode 100644 index c5e2bc1b83b..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenet_v3 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - BilinearImagenet: - height: 224 - width: 224 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - BilinearImagenet: - height: 224 - width: 224 - postprocess: - transform: - LabelShift: -1 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 500 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - BilinearImagenet: - height: 224 - width: 224 - postprocess: - transform: - LabelShift: -1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3_qdq.yaml b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3_qdq.yaml deleted file mode 100644 index bba292a4414..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/mobilenet_v3_qdq.yaml +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenet_v3 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - BilinearImagenet: - height: 224 - width: 224 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - BilinearImagenet: - height: 224 - width: 224 - postprocess: - transform: - LabelShift: -1 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 500 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - BilinearImagenet: - height: 224 - width: 224 - postprocess: - transform: - LabelShift: -1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/readme.md b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/readme.md deleted file mode 100644 index 57f86f9a6f9..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/readme.md +++ /dev/null @@ -1,43 +0,0 @@ -# Evaluate performance of ONNX Runtime(Mobilenet v3) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model exported from PyTorch and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Use [tf2onnx tool](https://github.com/onnx/tensorflow-onnx) to convert tflite to onnx model. - -```bash -wget https://github.com/mlcommons/mobile_models/blob/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_float.tflite - -python -m tf2onnx.convert --opset 11 --tflite mobilenet_edgetpu_224_1.0_float.tflite --output mobilenet_v3.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v3.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v3_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenet_v3.yaml \ - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/requirements.txt deleted file mode 100644 index eb22b0efa75..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -tf2onnx -onnxruntime-extensions; python_version < '3.10' - -pillow>=8.1.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/mobilenet_v3/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md deleted file mode 100644 index dcf1c03a698..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Evaluate performance of ONNX Runtime(Alexnet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.8.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/alexnet/model/bvlcalexnet-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=alexnet.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=alexnet_qdq.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=alexnet.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet.yaml deleted file mode 100644 index 34112f2ab99..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: alexnet - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet_qdq.yaml deleted file mode 100644 index 96d334b096d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/alexnet_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: alexnet - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py deleted file mode 100644 index 9b542c6cb02..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/main.py +++ /dev/null @@ -1,133 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx -import re -import os -from PIL import Image - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -class dataset: - def __init__(self, data_path, image_list): - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) - if not os.path.exists(src): - continue - self.image_list.append(src) - self.label_list.append(int(label)) - - def __len__(self): - return len(self.image_list) - - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Alexnet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--data_path', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - q_model = quantize() - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/requirements.txt deleted file mode 100644 index 4b56442f97d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index ba9efa687b5..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 0f1c6136330..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/alexnet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/arcface.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/arcface.yaml deleted file mode 100644 index 3a69ee97203..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/arcface.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: arcface - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - configs: - num_of_instance: 1 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh deleted file mode 100644 index df920d5b232..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - nfolds=1 - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - --nfolds=*) - nfolds=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --config ${config} \ - --mode ${mode} \ - --nfolds ${nfolds} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh deleted file mode 100644 index 7428fb058f6..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/arcface/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - nfolds=1 - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --nfolds=*) - nfolds=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --output_model ${output_model} \ - --config ${config} \ - --nfolds ${nfolds} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md deleted file mode 100644 index 5fbf0c393f4..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/README.md +++ /dev/null @@ -1,49 +0,0 @@ -# Evaluate performance of ONNX Runtime(Caffenet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/caffenet/model/caffenet-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=caffenet.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=caffenet_qdq.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=caffenet.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet.yaml deleted file mode 100644 index 4592b284bca..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: caffenet - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet_qdq.yaml deleted file mode 100644 index 55d3e157956..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/caffenet_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: caffenet - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py deleted file mode 100644 index 6fab92150ea..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/main.py +++ /dev/null @@ -1,133 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx -import re -import os -from PIL import Image - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -class dataset: - def __init__(self, data_path, image_list): - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) - if not os.path.exists(src): - continue - self.image_list.append(src) - self.label_list.append(int(label)) - - def __len__(self): - return len(self.image_list) - - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Caffenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--data_path', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - q_model = quantize() - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/requirements.txt deleted file mode 100644 index 4b56442f97d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index ba9efa687b5..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 0f1c6136330..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/caffenet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md deleted file mode 100644 index ed3ef1f3cd3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# Evaluate performance of ONNX Runtime(Densenet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/densenet-121/model/densenet-12.onnx -``` - -### Quantization - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=densenet.yaml \ - --output_model=path/to/save -``` - -### Performance - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=densenet.yaml \ - --mode=performance -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/densenet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/densenet.yaml deleted file mode 100644 index aa3d0fa5bc4..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/densenet.yaml +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: densenet - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - op_wise: { - 'Conv_nc_rename_0': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}}, - 'Relu_nc_rename_1': {'activation': {'dtype': ['fp32']}, 'weight': {'dtype': ['fp32']}} - } - calibration: - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.02 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py deleted file mode 100644 index ad74bd25887..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/main.py +++ /dev/null @@ -1,94 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -class squeeze: - def __call__(self, sample): - preds, labels = sample - return np.squeeze(preds), labels - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Densenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.postprocess = common.Postprocess(squeeze) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.postprocess = common.Postprocess(squeeze) - q_model = quantize() - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/requirements.txt deleted file mode 100644 index f62a897bffa..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh deleted file mode 100755 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/densenet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/README.md deleted file mode 100644 index edd1310d996..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Evaluate performance of ONNX Runtime(EfficientNet-Lite4) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/efficientnet-lite4/model/efficientnet-lite4-11.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=efficientnet.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=efficientnet_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=efficientnet.yaml \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet.yaml deleted file mode 100644 index 4dfc024c1e9..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: EfficientNet-Lite4 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet_qdq.yaml deleted file mode 100644 index cb9f2ff142c..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/efficientnet_qdq.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: EfficientNet-Lite4 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [127.0, 127.0, 127.0] - std: [128.0, 128.0, 128.0] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py deleted file mode 100644 index 65d04524c17..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/main.py +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - model = onnx.load(args.model_path) - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/requirements.txt deleted file mode 100644 index 4b56442f97d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 416dde8f2c0..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/efficientnet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md index aa078e0adce..52dedf3882a 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/README.md @@ -20,9 +20,7 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=fcn_rn50.yaml \ - --data_path=path/to/coco/val2017 \ - --label_path=path/to/coco/annotations/instances_val2017.json \ + --dataset_location=path/to/coco/val2017 \ --output_model=path/to/save ``` @@ -30,18 +28,15 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=fcn_rn50_qdq.yaml \ - --data_path=path/to/coco/val2017 \ - --label_path=path/to/coco/annotations/instances_val2017.json \ - --output_model=path/to/save + --dataset_location=path/to/coco/val2017 \ + --output_model=path/to/save \ + --quant_format=QDQ ``` ### Benchmark ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=fcn_rn50.yaml \ - --data_path=path/to/coco/val2017 \ - --label_path=path/to/coco/annotations/instances_val2017.json \ + --dataset_location=path/to/coco/val2017 \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50.yaml deleted file mode 100644 index 78025519822..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: fcn - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, or mxnet; allow new framework backend extension. - inputs: image - outputs: out,aux - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant - -evaluation: # optional. used to config evaluation process. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - configs: # optional. if not specified, use all cores in 1 socket. - cores_per_instance: 28 - num_of_instance: 1 - kmp_blocktime: 1 - performance: - iteration: 100 - configs: - cores_per_instance: 28 - num_of_instance: 1 - kmp_blocktime: 1 - -tuning: - accuracy_criterion: - absolute: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 2%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50_qdq.yaml deleted file mode 100644 index b7ffcd1c42d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/fcn_rn50_qdq.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: fcn - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, or mxnet; allow new framework backend extension. - inputs: image - outputs: out,aux - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant - -evaluation: # optional. used to config evaluation process. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - configs: # optional. if not specified, use all cores in 1 socket. - cores_per_instance: 28 - num_of_instance: 1 - kmp_blocktime: 1 - performance: - iteration: 100 - configs: - cores_per_instance: 28 - num_of_instance: 1 - kmp_blocktime: 1 - -tuning: - accuracy_criterion: - absolute: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 2%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py index ab970a0f572..be9f027539f 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/main.py @@ -46,12 +46,7 @@ help="Pre-trained model on onnx file" ) parser.add_argument( - '--label_path', - type=str, - help="Annotation file path" -) -parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Path to val2017 of COCO" ) @@ -66,11 +61,6 @@ default=False, help="whether quantize the model" ) -parser.add_argument( - '--config', - type=str, - help="config yaml path" -) parser.add_argument( '--output_model', type=str, @@ -81,6 +71,13 @@ type=str, help="benchmark mode of performance or accuracy" ) +parser.add_argument( + '--quant_format', + type=str, + default='default', + choices=['default', 'QDQ', 'QOperator'], + help="quantization format" +) args = parser.parse_args() # key = COCO id, value = Pascal VOC id @@ -107,19 +104,20 @@ 72: 20, # tv } VOC_CAT_IDS = list(COCO_TO_VOC.keys()) -cocoGt = COCO(str(args.label_path)) +cocoGt = COCO(os.path.join(args.dataset_location, 'annotations/instances_val2017.json')) preprocess = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ]) -class Dataset: +class Dataloader: def __init__(self): + self.batch_size = 1 imgIds = self.getImgIdsUnion(cocoGt, VOC_CAT_IDS) self.data = [] for imgId in imgIds: - img_path = os.path.join(args.data_path, cocoGt.imgs[imgId]['file_name']) + img_path = os.path.join(os.path.join(args.dataset_location, 'val2017'), cocoGt.imgs[imgId]['file_name']) if os.path.exists(img_path): input_tensor = self.load_image(img_path) @@ -133,13 +131,12 @@ def __init__(self): # Set everything not labeled to be background output_tensor[0] = 1 - np.max(output_tensor, axis=0) + input_tensor = input_tensor[np.newaxis, ...] self.data.append((input_tensor, output_tensor)) - def __len__(self): - return len(self.data) - - def __getitem__(self, index): - return self.data[index] + def __iter__(self): + for data in self.data: + yield data def getImgIdsUnion(self, gt, catIds): """ @@ -180,7 +177,6 @@ def evaluate(model, dataloader): providers=onnxruntime.get_available_providers()) idx = 1 for input_tensor, target_tensor in dataloader: - input_tensor = input_tensor[np.newaxis, ...] target_tensor = target_tensor[np.newaxis, ...] model_tensor = sess.run(["out"], {"input": input_tensor})[0] @@ -199,31 +195,32 @@ def evaluate(model, dataloader): if __name__ == "__main__": model = onnx.load(args.model_path) - ds = Dataset() + dataloader = Dataloader() def eval(model): - return evaluate(model, ds) - - if args.benchmark and args.mode == "accuracy": - results = eval(model) - print("Batch size = 1") - print("Accuracy: %.5f" % results) - - if args.benchmark and args.mode == "performance": - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator(args.mode) + return evaluate(model, dataloader) + + if args.benchmark: + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(iteration=100, cores_per_instance=28, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_func = eval - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + from neural_compressor.config import AccuracyCriterion + accuracy_criterion = AccuracyCriterion() + accuracy_criterion.absolute = 0.01 + config = PostTrainingQuantConfig( + accuracy_criterion=accuracy_criterion, + quant_format=args.quant_format) + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh index 00d1543581e..3f1e78f6cb4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_benchmark.sh @@ -13,20 +13,14 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; esac done @@ -38,10 +32,8 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --config ${config} \ --mode ${mode} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --benchmark } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh index 80daa37897d..5cf411ef94d 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/fcn/quantization/ptq/run_tuning.sh @@ -13,20 +13,17 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --quant_format=*) + quant_format=$(echo $var |cut -f2 -d=) ;; esac done @@ -38,9 +35,8 @@ function run_tuning { python main.py \ --model_path ${input_model} \ --output_model ${output_model} \ - --config ${config} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ + --quant_format ${quant_format} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md deleted file mode 100644 index 47d19e4e9a1..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Evaluate performance of ONNX Runtime(Googlenet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/inception_and_googlenet/googlenet/model/googlenet-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=googlenet.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=googlenet_qdq.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=googlenet.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet.yaml deleted file mode 100644 index f7adc0c8a63..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: googlenet - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet_qdq.yaml deleted file mode 100644 index 66a1fe6389d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/googlenet_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: googlenet - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py deleted file mode 100644 index 26e54f4a1fe..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/main.py +++ /dev/null @@ -1,131 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx -import re -import os -from PIL import Image - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -class dataset: - def __init__(self, data_path, image_list): - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) - if not os.path.exists(src): - continue - self.image_list.append(src) - self.label_list.append(int(label)) - - def __len__(self): - return len(self.image_list) - - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Googlenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained googlenet model on onnx file" - ) - parser.add_argument( - '--data_path', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - args = parser.parse_args() - - model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - q_model = quantize() - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/requirements.txt deleted file mode 100644 index 788d60f9ae8..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -opencv-python -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index ba9efa687b5..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 0f1c6136330..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/googlenet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md deleted file mode 100644 index 39d8cc361cb..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Evaluate performance of ONNX Runtime(Inception) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/inception_and_googlenet/inception_v1/model/inception-v1-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=inception_v1.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=inception_v1_qdq.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=inception_v1.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1.yaml deleted file mode 100644 index 9e8fd3ad38e..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: inception_v1 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1_qdq.yaml deleted file mode 100644 index f30162cebdc..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/inception_v1_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: inception_v1 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py deleted file mode 100644 index 9b8c684466b..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/main.py +++ /dev/null @@ -1,133 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx -import re -import os -from PIL import Image - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -class dataset: - def __init__(self, data_path, image_list): - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) - if not os.path.exists(src): - continue - self.image_list.append(src) - self.label_list.append(int(label)) - - def __len__(self): - return len(self.image_list) - - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--data_path', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - options.onnxrt.graph_optimization.gemm2matmul = False - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - q_model = quantize() - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/requirements.txt deleted file mode 100644 index 4b56442f97d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh deleted file mode 100644 index ba9efa687b5..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh deleted file mode 100644 index 0f1c6136330..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/inception/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/README.md deleted file mode 100644 index 2f35d299226..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Evaluate performance of ONNX Runtime(MNIST) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.8.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/mnist/model/mnist-12.onnx -``` - -### Quantization -To quantize the model, run `main.py` with the path to the model: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mnist.yaml \ - --output_model=path/to/save -``` - -### Performance -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mnist.yaml \ - --output_model=path/to/save -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py deleted file mode 100644 index af9a7bf08ad..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/main.py +++ /dev/null @@ -1,84 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="MNIST - Handwritten Digit Recognition quantization example.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/mnist.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/mnist.yaml deleted file mode 100644 index 38b9bb1c791..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/mnist.yaml +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: mnist - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - dataloader: - batch_size: 1 - dataset: - MNIST: - root: /path/to/calibration/dataset - transform: - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - MNIST: - root: /path/to/evaluation/dataset - transform: - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 28 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - MNIST: - root: /path/to/evaluation/dataset - transform: - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/requirements.txt deleted file mode 100644 index a67636f2d5a..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.8.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_benchmark.sh deleted file mode 100755 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_tuning.sh deleted file mode 100755 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mnist/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/README.md deleted file mode 100644 index 61a3c6f68e1..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Evaluate performance of ONNX Runtime(Mobilenet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/mobilenet/model/mobilenetv2-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenetv2.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenetv2_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=mobilenetv2.yaml \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py deleted file mode 100644 index 3e89f75de62..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/main.py +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Mobilenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained mobilenet model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor import options - from neural_compressor.experimental import Quantization, common - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2.yaml deleted file mode 100644 index 204897a4a39..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenetv2 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.02 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2_qdq.yaml deleted file mode 100644 index 3d2b9c10ca1..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/mobilenetv2_qdq.yaml +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: mobilenetv2 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.02 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/requirements.txt deleted file mode 100644 index f62a897bffa..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_benchmark.sh deleted file mode 100755 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_tuning.sh deleted file mode 100755 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/mobilenet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/README.md deleted file mode 100644 index f239f3181c5..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Evaluate performance of ONNX Runtime(ResNet 50) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/resnet/model/resnet50-v1-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5.yaml \ - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py deleted file mode 100644 index 38c506733ff..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/main.py +++ /dev/null @@ -1,88 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Resnet50 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained resnet50 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/requirements.txt deleted file mode 100644 index 494373e0bf3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -torch -torchvision -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5.yaml deleted file mode 100644 index 966dabfbd9c..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml deleted file mode 100644 index 8e4cec4d888..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/resnet50/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/README.md deleted file mode 100644 index abd6ce83fcd..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Evaluate performance of ONNX Runtime(Shufflenet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/shufflenet/model/shufflenet-v2-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=shufflenetv2.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=shufflenetv2_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=shufflenetv2.yaml \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py deleted file mode 100644 index 3e35879cc58..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/main.py +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Shufflenet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/requirements.txt deleted file mode 100644 index f62a897bffa..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2.yaml deleted file mode 100644 index 8d7cab0cb4e..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: shufflenetv2 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2_qdq.yaml deleted file mode 100644 index 6c37dfd56a6..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/shufflenet/quantization/ptq/shufflenetv2_qdq.yaml +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: shufflenetv2 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md index c9aa5439d9c..6a61b3816c6 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/README.md @@ -20,9 +20,7 @@ Quantize model with QLinearOps: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=squeezenet.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --output_model=path/to/save ``` @@ -30,19 +28,16 @@ Quantize model with QDQ mode: ```bash bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=squeezenet_qdq.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save + --dataset_location=/path/to/imagenet \ + --output_model=path/to/save \ + --quant_format=QDQ ``` ### Benchmark ```bash bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=squeezenet.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ + --dataset_location=/path/to/imagenet \ --mode=performance # or accuracy ``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py index 6234719d1e6..0a35f946f4b 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/main.py @@ -25,48 +25,147 @@ import re import os from PIL import Image +import onnxruntime as ort +from sklearn.metrics import accuracy_score logger = logging.getLogger(__name__) logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', level = logging.WARN) -class squeeze: +class Squeeze: def __call__(self, sample): preds, labels = sample return np.squeeze(preds), labels + +def _topk_shape_validate(preds, labels): + # preds shape can be Nxclass_num or class_num(N=1 by default) + # it's more suitable for 'Accuracy' with preds shape Nx1(or 1) output from argmax + if isinstance(preds, int): + preds = [preds] + preds = np.array(preds) + elif isinstance(preds, np.ndarray): + preds = np.array(preds) + elif isinstance(preds, list): + preds = np.array(preds) + preds = preds.reshape((-1, preds.shape[-1])) -class dataset: - def __init__(self, data_path, image_list): + # consider labels just int value 1x1 + if isinstance(labels, int): + labels = [labels] + labels = np.array(labels) + elif isinstance(labels, tuple): + labels = np.array([labels]) + labels = labels.reshape((labels.shape[-1], -1)) + elif isinstance(labels, list): + if isinstance(labels[0], int): + labels = np.array(labels) + labels = labels.reshape((labels.shape[0], 1)) + elif isinstance(labels[0], tuple): + labels = np.array(labels) + labels = labels.reshape((labels.shape[-1], -1)) + else: + labels = np.array(labels) + # labels most have 2 axis, 2 cases: N(or Nx1 sparse) or Nxclass_num(one-hot) + # only support 2 dimension one-shot labels + # or 1 dimension one-hot class_num will confuse with N + + if len(preds.shape) == 1: + N = 1 + class_num = preds.shape[0] + preds = preds.reshape([-1, class_num]) + elif len(preds.shape) >= 2: + N = preds.shape[0] + preds = preds.reshape([N, -1]) + class_num = preds.shape[1] + + label_N = labels.shape[0] + assert label_N == N, 'labels batch size should same with preds' + labels = labels.reshape([N, -1]) + # one-hot labels will have 2 dimension not equal 1 + if labels.shape[1] != 1: + labels = labels.argsort()[..., -1:] + return preds, labels + +class TopK: + def __init__(self, k=1): + self.k = k + self.num_correct = 0 + self.num_sample = 0 + + def update(self, preds, labels, sample_weight=None): + preds, labels = _topk_shape_validate(preds, labels) + preds = preds.argsort()[..., -self.k:] + if self.k == 1: + correct = accuracy_score(preds, labels, normalize=False) + self.num_correct += correct + + else: + for p, l in zip(preds, labels): + # get top-k labels with np.argpartition + # p = np.argpartition(p, -self.k)[-self.k:] + l = l.astype('int32') + if l in p: + self.num_correct += 1 + + self.num_sample += len(labels) + + def reset(self): + self.num_correct = 0 + self.num_sample = 0 + + def result(self): + if self.num_sample == 0: + logger.warning("Sample num during evaluation is 0.") + return 0 + elif getattr(self, '_hvd', None) is not None: + allgather_num_correct = sum(self._hvd.allgather_object(self.num_correct)) + allgather_num_sample = sum(self._hvd.allgather_object(self.num_sample)) + return allgather_num_correct / allgather_num_sample + return self.num_correct / self.num_sample + +class Dataloader: + def __init__(self, dataset_location, image_list): + self.batch_size = 1 self.image_list = [] self.label_list = [] with open(image_list, 'r') as f: for s in f: image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) + src = os.path.join(dataset_location, image_name) if not os.path.exists(src): continue + self.image_list.append(src) self.label_list.append(int(label)) - def __len__(self): - return len(self.image_list) + def __iter__(self): + for src, label in zip(self.image_list, self.label_list): + with Image.open(src) as image: + image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) + image[:, :, 0] -= 123.68 + image[:, :, 1] -= 116.779 + image[:, :, 2] -= 103.939 + image[:,:,[0,1,2]] = image[:,:,[2,1,0]] + image = image.transpose((2, 0, 1)) + image = np.expand_dims(image, axis=0) + yield image, label - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label +def eval_func(model, dataloader, metric, postprocess): + metric.reset() + sess = ort.InferenceSession(model.SerializeToString(), providers=ort.get_available_providers()) + ort_inputs = {} + input_names = [i.name for i in sess.get_inputs()] + for input_data, label in dataloader: + output = sess.run(None, dict(zip(input_names, [input_data]))) + output, label = postprocess((output, label)) + metric.update(output, label) + return metric.result() if __name__ == "__main__": logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") parser = argparse.ArgumentParser( - description="Squeezenet fine-tune examples for image classification tasks.", + description="Googlenet fine-tune examples for image classification tasks.", formatter_class=argparse.ArgumentDefaultsHelpFormatter ) parser.add_argument( @@ -75,15 +174,10 @@ def __getitem__(self, index): help="Pre-trained model on onnx file" ) parser.add_argument( - '--data_path', + '--dataset_location', type=str, help="Imagenet data path" ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) parser.add_argument( '--benchmark', action='store_true', \ @@ -95,11 +189,6 @@ def __getitem__(self, index): default=False, help="whether quantize the model" ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) parser.add_argument( '--output_model', type=str, @@ -108,32 +197,41 @@ def __getitem__(self, index): parser.add_argument( '--mode', type=str, - default='performance', help="benchmark mode of performance or accuracy" ) - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - + parser.add_argument( + '--quant_format', + type=str, + default='default', + choices=['default', 'QDQ', 'QOperator'], + help="quantization format" + ) args = parser.parse_args() model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) + data_path = os.path.join(args.dataset_location, 'ILSVRC2012_img_val') + label_path = os.path.join(args.dataset_location, 'val.txt') + dataloader = Dataloader(data_path, label_path) + top1 = TopK() + postprocess = Squeeze() + def eval(onnx_model): + return eval_func(onnx_model, dataloader, top1, postprocess) if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator.postprocess = common.Postprocess(squeeze) - evaluator(args.mode) - + if args.mode == 'performance': + from neural_compressor.benchmark import fit + from neural_compressor.config import BenchmarkConfig + conf = BenchmarkConfig(warmup=10, iteration=1000, cores_per_instance=4, num_of_instance=1) + fit(model, conf, b_dataloader=dataloader) + elif args.mode == 'accuracy': + acc_result = eval(model) + print("Batch size = %d" % dataloader.batch_size) + print("Accuracy: %.5f" % acc_result) if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - quantize.postprocess = common.Postprocess(squeeze) - q_model = quantize() + from neural_compressor import quantization, PostTrainingQuantConfig + config = PostTrainingQuantConfig(quant_format=args.quant_format) + + q_model = quantization.fit(model, config, calib_dataloader=dataloader, + eval_func=eval) + q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh index ba9efa687b5..7ec6f7ace0e 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_benchmark.sh @@ -12,17 +12,11 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; --mode=*) mode=$(echo $var |cut -f2 -d=) @@ -37,9 +31,7 @@ function run_benchmark { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --config ${config} \ + --dataset_location ${dataset_location} \ --mode=${mode} \ --benchmark diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh index 0f1c6136330..29ed086faf4 100644 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh +++ b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/run_tuning.sh @@ -13,20 +13,17 @@ function init_params { for var in "$@" do case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; --input_model=*) input_model=$(echo $var |cut -f2 -d=) ;; --output_model=*) output_model=$(echo $var |cut -f2 -d=) ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) + --dataset_location=*) + dataset_location=$(echo $var |cut -f2 -d=) ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) + --quant_format=*) + quant_format=$(echo $var |cut -f2 -d=) ;; esac done @@ -37,10 +34,9 @@ function init_params { function run_tuning { python main.py \ --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ + --dataset_location ${dataset_location} \ --output_model ${output_model} \ - --config ${config} \ + --quant_format ${quant_format} \ --tune } diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet.yaml deleted file mode 100644 index 3ef03492941..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: squeezenet - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet_qdq.yaml deleted file mode 100644 index c352f6756ea..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/squeezenet/quantization/ptq/squeezenet_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: squeezenet - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/README.md deleted file mode 100644 index 80de9fff000..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Evaluate performance of ONNX Runtime(VGG16) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/vgg/model/vgg16-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16.yaml \ - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py deleted file mode 100644 index 53e55f3d406..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/main.py +++ /dev/null @@ -1,88 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="VGG16 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained vgg16 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - model = onnx.load(args.model_path) - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/requirements.txt deleted file mode 100644 index 494373e0bf3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -torch -torchvision -onnxruntime-extensions; python_version < '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16.yaml deleted file mode 100644 index 5b1b165fdb7..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: vgg16 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16_qdq.yaml deleted file mode 100644 index fb647b11f6a..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/vgg16/quantization/ptq/vgg16_qdq.yaml +++ /dev/null @@ -1,101 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: vgg16 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - Rescale: {} - Resize: - size: 256 - CenterCrop: - size: 224 - Normalize: - mean: [0.485, 0.456, 0.406] - std: [0.229, 0.224, 0.225] - Transpose: - perm: [2, 0, 1] - Cast: - dtype: float32 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md deleted file mode 100644 index a72b369444a..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Evaluate performance of ONNX Runtime(ZFNet) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model from [ONNX Model Zoo](https://github.com/onnx/models) and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Download model from [ONNX Model Zoo](https://github.com/onnx/models) - -```shell -wget https://github.com/onnx/models/raw/main/vision/classification/zfnet-512/model/zfnet512-12.onnx -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=zfnet512.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=zfnet512_qdq.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=zfnet512.yaml \ - --data_path=/path/to/imagenet \ - --label_path=/path/to/imagenet/label \ - --mode=performance # or accuracy -``` - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py deleted file mode 100644 index 7da9d06052b..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/main.py +++ /dev/null @@ -1,133 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx -import re -import os -from PIL import Image - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -class dataset: - def __init__(self, data_path, image_list): - self.image_list = [] - self.label_list = [] - with open(image_list, 'r') as f: - for s in f: - image_name, label = re.split(r"\s+", s.strip()) - src = os.path.join(data_path, image_name) - if not os.path.exists(src): - continue - self.image_list.append(src) - self.label_list.append(int(label)) - - def __len__(self): - return len(self.image_list) - - def __getitem__(self, index): - image_path, label = self.image_list[index], self.label_list[index] - with Image.open(image_path) as image: - image = np.array(image.convert('RGB').resize((224, 224))).astype(np.float32) - image[:, :, 0] -= 123.68 - image[:, :, 1] -= 116.779 - image[:, :, 2] -= 103.939 - image[:,:,[0,1,2]] = image[:,:,[2,1,0]] - image = image.transpose((2, 0, 1)) - return image, label - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="ZFNet fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained model on onnx file" - ) - parser.add_argument( - '--data_path', - type=str, - help="Imagenet data path" - ) - parser.add_argument( - '--label_path', - type=str, - help="Imagenet label path" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - default='performance', - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - ds = dataset(args.data_path, args.label_path) - - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator.b_dataloader = common.DataLoader(ds) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - quantize = Quantization(args.config) - quantize.model = common.Model(model) - quantize.calib_dataloader = common.DataLoader(ds) - quantize.eval_dataloader = common.DataLoader(ds) - q_model = quantize() - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/requirements.txt deleted file mode 100644 index 4b56442f97d..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability -onnxruntime-extensions; python_version < '3.10' diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh deleted file mode 100644 index ba9efa687b5..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh deleted file mode 100644 index 0f1c6136330..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - --data_path=*) - data_path=$(echo $var |cut -f2 -d=) - ;; - --label_path=*) - label_path=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --data_path ${data_path} \ - --label_path ${label_path} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512.yaml deleted file mode 100644 index 348a646251f..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: zfnet512 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512_qdq.yaml b/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512_qdq.yaml deleted file mode 100644 index 162af6aa4b2..00000000000 --- a/examples/onnxrt/image_recognition/onnx_model_zoo/zfnet/quantization/ptq/zfnet512_qdq.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 1.0 - -model: # mandatory. used to specify model specific information. - name: zfnet512 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. -evaluation: # optional. required if user doesn't provide eval_func in lpot.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in lpot.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 1000 - configs: - cores_per_instance: 4 - num_of_instance: 1 - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/README.md b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/README.md deleted file mode 100644 index 985f8284a48..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Evaluate performance of ONNX Runtime(ResNet 50) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model exported from PyTorch and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model - -#### ResNet 50 from torchvision -Please refer to [pytorch official guide](https://pytorch.org/docs/stable/onnx.html) for detailed model export. The following is a simple example: - -```python -import torch -import torchvision -batch_size = 1 -model = torchvision.models.resnet50(pretrained=True) -x = torch.randn(batch_size, 3, 224, 224, requires_grad=True) -torch_out = model(x) - -# Export the model -torch.onnx.export(model, # model being run - x, # model input (or a tuple for multiple inputs) - "resnet50.onnx", # where to save the model (can be a file or file-like object) - export_params=True, # store the trained parameter weights inside the model file - opset_version=11, # the ONNX version to export the model to, please ensure at least 11. - do_constant_folding=True, # whether to execute constant folding for optimization - input_names = ['input'], # the model's input names - output_names = ['output'], # the model's output names - dynamic_axes={'input' : {0 : 'batch_size'}, # variable length axes - 'output' : {0 : 'batch_size'}}) -``` - -#### ResNet 50 from MLPerf -Please refer to [MLPerf Inference Benchmarks for Image Classification and Object Detection Tasks](https://github.com/mlcommons/inference/tree/master/vision/classification_and_detection#mlperf-inference-benchmarks-for-image-classification-and-object-detection-tasks) for model details. Use [tf2onnx tool](https://github.com/onnx/tensorflow-onnx) to convert tensorflow model to onnx model. - -```bash -wget https://zenodo.org/record/2535873/files/resnet50_v1.pb - -python -m tf2onnx.convert --input resnet50_v1.pb --output resnet50_v1.onnx --inputs-as-nchw input_tensor:0 --inputs input_tensor:0 --outputs softmax_tensor:0 --opset 11 -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5.yaml \ # or resnet50_v1_5_mlperf.yaml for ResNet50 from MLPerf - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5_qdq.yaml \ # or resnet50_v1_5_mlperf_qdq.yaml for ResNet50 from MLPerf - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=resnet50_v1_5.yaml \ # or resnet50_v1_5_mlperf.yaml for ResNet50 from MLPerf - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py deleted file mode 100644 index 2a80449914a..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/main.py +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="Resnet50 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained resnet50 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - model = onnx.load(args.model_path) - - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) - diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/requirements.txt deleted file mode 100644 index 96b96e53d95..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -torch -torchvision -onnxruntime-extensions; python_version < '3.10' - -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5.yaml b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5.yaml deleted file mode 100644 index 5bb0f619d18..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gemm_174_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf.yaml b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf.yaml deleted file mode 100644 index e7aa83c1c23..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset # NOTE: modify to calibration dataset location if needed - image_list: /path/to/calibration/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - op_wise: { - 'gemm_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset # NOTE: modify to evaluation dataset location if needed - image_list: /path/to/evaluation/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - postprocess: - transform: - LabelShift: -1 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset # NOTE: modify to evaluation dataset location if needed - image_list: /path/to/evaluation/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf_qdq.yaml b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf_qdq.yaml deleted file mode 100644 index 58fd4ff3567..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_mlperf_qdq.yaml +++ /dev/null @@ -1,105 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset # NOTE: modify to calibration dataset location if needed - image_list: /path/to/calibration/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - op_wise: { - 'gemm_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset # NOTE: modify to evaluation dataset location if needed - image_list: /path/to/evaluation/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - postprocess: - transform: - LabelShift: -1 - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset # NOTE: modify to evaluation dataset location if needed - image_list: /path/to/evaluation/label # data file, record image_names and their labels - transform: - ResizeWithAspectRatio: - height: 224 - width: 224 - CenterCrop: - size: 224 - Normalize: - mean: [123.68, 116.78, 103.94] - Cast: - dtype: float32 - Transpose: - perm: [2, 0, 1] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml deleted file mode 100644 index c9269f56815..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/resnet50_v1_5_qdq.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: resnet50_v1_5 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50, 100 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gemm_174_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.bat b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.bat deleted file mode 100644 index 8da25b85410..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.bat +++ /dev/null @@ -1,9 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -set mode=%4 -shift -shift -python main.py --model_path %input_model% --config %config% --benchmark --mode %mode%--output_model %output_model% diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.bat b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.bat deleted file mode 100644 index 56fa6ccab82..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.bat +++ /dev/null @@ -1,8 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -shift -shift -python main.py --model_path %input_model% --config %config% --tune --output_model %output_model% \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/resnet50/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/README.md b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/README.md deleted file mode 100644 index 7ad081ed162..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# Evaluate performance of ONNX Runtime(VGG16) ->ONNX runtime quantization is under active development. please use 1.6.0+ to get more quantization support. - -This example load an image classification model exported from PyTorch and confirm its accuracy and speed based on [ILSVR2012 validation Imagenet dataset](http://www.image-net.org/challenges/LSVRC/2012/downloads). You need to download this dataset yourself. - -### Environment -onnx: 1.9.0 -onnxruntime: 1.10.0 - -### Prepare model -Please refer to [pytorch official guide](https://pytorch.org/docs/stable/onnx.html) for detailed model export. The following is a simple example: - -```python -import torch -import torchvision -batch_size = 1 -model = torchvision.models.vgg16(pretrained=True) -x = torch.randn(batch_size, 3, 224, 224, requires_grad=True) -torch_out = model(x) - -# Export the model -torch.onnx.export(model, # model being run - x, # model input (or a tuple for multiple inputs) - "vgg16.onnx", # where to save the model (can be a file or file-like object) - export_params=True, # store the trained parameter weights inside the model file - opset_version=11, # the ONNX version to export the model to, please ensure at least 11. - do_constant_folding=True, # whether to execute constant folding for optimization - input_names = ['input'], # the model's input names - output_names = ['output'], # the model's output names - dynamic_axes={'input' : {0 : 'batch_size'}, # variable length axes - 'output' : {0 : 'batch_size'}}) -``` - -### Quantization - -Quantize model with QLinearOps: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16.yaml \ - --output_model=path/to/save -``` - -Quantize model with QDQ mode: - -```bash -bash run_tuning.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16_qdq.yaml \ - --output_model=path/to/save -``` - -### Benchmark - -```bash -bash run_benchmark.sh --input_model=path/to/model \ # model path as *.onnx - --config=vgg16.yaml \ - --mode=performance # or accuracy -``` diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py deleted file mode 100644 index 95fe75ec506..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/main.py +++ /dev/null @@ -1,87 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# pylint:disable=redefined-outer-name,logging-format-interpolation - - -import logging -import argparse - -import numpy as np -import onnx - -logger = logging.getLogger(__name__) -logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', - datefmt = '%m/%d/%Y %H:%M:%S', - level = logging.WARN) - -if __name__ == "__main__": - logger.info("Evaluating ONNXRuntime full precision accuracy and performance:") - parser = argparse.ArgumentParser( - description="VGG16 fine-tune examples for image classification tasks.", - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model_path', - type=str, - help="Pre-trained vgg16 model on onnx file" - ) - parser.add_argument( - '--benchmark', - action='store_true', \ - default=False - ) - parser.add_argument( - '--tune', - action='store_true', \ - default=False, - help="whether quantize the model" - ) - parser.add_argument( - '--config', - type=str, - help="config yaml path" - ) - parser.add_argument( - '--output_model', - type=str, - help="output model path" - ) - - parser.add_argument( - '--mode', - type=str, - help="benchmark mode of performance or accuracy" - ) - - args = parser.parse_args() - - model = onnx.load(args.model_path) - if args.benchmark: - from neural_compressor.experimental import Benchmark, common - evaluator = Benchmark(args.config) - evaluator.model = common.Model(model) - evaluator(args.mode) - - if args.tune: - from neural_compressor.experimental import Quantization, common - from neural_compressor import options - options.onnxrt.graph_optimization.level = 'ENABLE_BASIC' - - quantize = Quantization(args.config) - quantize.model = common.Model(model) - q_model = quantize() - q_model.save(args.output_model) diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/requirements.txt b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/requirements.txt deleted file mode 100644 index 96b96e53d95..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -onnx==1.9.0; python_version < '3.10' -onnx==1.12.0; python_version == '3.10' -onnxruntime==1.10.0; python_version < '3.10' -onnxruntime==1.12.0; python_version == '3.10' -torch -torchvision -onnxruntime-extensions; python_version < '3.10' - -pillow>=8.2.0 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.bat b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.bat deleted file mode 100644 index 8da25b85410..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.bat +++ /dev/null @@ -1,9 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -set mode=%4 -shift -shift -python main.py --model_path %input_model% --config %config% --benchmark --mode %mode%--output_model %output_model% diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.sh b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.sh deleted file mode 100644 index 2b7d99703d3..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_benchmark.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_benchmark - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --mode=*) - mode=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_benchmark -function run_benchmark { - - python main.py \ - --model_path ${input_model} \ - --config ${config} \ - --mode=${mode} \ - --benchmark - -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.bat b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.bat deleted file mode 100644 index 56fa6ccab82..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.bat +++ /dev/null @@ -1,8 +0,0 @@ -echo off - -set input_model=%1 -set config=%2 -set output_model=%3 -shift -shift -python main.py --model_path %input_model% --config %config% --tune --output_model %output_model% \ No newline at end of file diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.sh b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.sh deleted file mode 100644 index 97d06dab599..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/run_tuning.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -x - -function main { - init_params "$@" - run_tuning - -} - -# init params -function init_params { - - for var in "$@" - do - case $var in - --config=*) - config=$(echo $var |cut -f2 -d=) - ;; - --input_model=*) - input_model=$(echo $var |cut -f2 -d=) - ;; - --output_model=*) - output_model=$(echo $var |cut -f2 -d=) - ;; - esac - done - -} - -# run_tuning -function run_tuning { - python main.py \ - --model_path ${input_model} \ - --output_model ${output_model} \ - --config ${config} \ - --tune -} - -main "$@" diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16.yaml b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16.yaml deleted file mode 100644 index 09374ebecbd..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: vgg16 - framework: onnxrt_qlinearops # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gemm_37_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning. diff --git a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16_qdq.yaml b/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16_qdq.yaml deleted file mode 100644 index 59ba806aa09..00000000000 --- a/examples/onnxrt/image_recognition/vgg16/quantization/ptq/vgg16_qdq.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) 2021 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -model: # mandatory. used to specify model specific information. - name: vgg16 - framework: onnxrt_qdq # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension. - -quantization: # optional. tuning constraints on model-wise for advance user to reduce tuning space. - approach: post_training_static_quant # optional. default value is post_training_static_quant. - calibration: - sampling_size: 50 # optional. default value is 100. used to set how many samples should be used in calibration. - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/calibration/dataset - image_list: /path/to/calibration/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - op_wise: { - 'Gemm_37_MatMul': { - 'activation': {'dtype': ['fp32']}, - 'weight': {'dtype': ['fp32']} - } - } - -evaluation: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - accuracy: # optional. required if user doesn't provide eval_func in neural_compressor.Quantization. - metric: - topk: 1 # built-in metrics are topk, map, f1, allow user to register new metric. - dataloader: - batch_size: 32 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - performance: # optional. used to benchmark performance of passing model. - warmup: 10 - iteration: 100 - configs: - cores_per_instance: 4 - num_of_instance: 7 - dataloader: - batch_size: 1 - dataset: - ImagenetRaw: - data_path: /path/to/evaluation/dataset - image_list: /path/to/evaluation/label - transform: - ResizeCropImagenet: - height: 224 - width: 224 - mean_value: [0.485, 0.456, 0.406] - -tuning: - accuracy_criterion: - relative: 0.01 # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%. - exit_policy: - timeout: 0 # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit. - random_seed: 9527 # optional. random seed for deterministic tuning.