Fix framework name in DataLoader API and refine dataloader & metric d…

…oc (#1048) Signed-off-by: yuwenzho <yuwen.zhou@intel.com>
intel · Jul 3, 2023 · 447cc7f · 447cc7f
1 parent f2fec43
commit 447cc7f
Show file tree

Hide file tree

Showing 22 changed files with 31 additions and 29 deletions.
diff --git a/.azure-pipelines/scripts/codeScan/pyspelling/inc_dict.txt b/.azure-pipelines/scripts/codeScan/pyspelling/inc_dict.txt
@@ -2676,3 +2676,4 @@ fdb
 jJA
 wWLes
 xHKe
+PR
diff --git a/docs/source/dataloader.md b/docs/source/dataloader.md
@@ -5,9 +5,9 @@ DataLoader
 
 2. [Supported Framework Dataloader Matrix](#supported-framework-dataloader-matrix)
 
-3. [Get Started with Dataloader](#get-start-with-dataloader)
+3. [Get Started with Dataloader](#get-started-with-dataloader)
 
-    3.1 [Use Intel® Neural Compressor DataLoader API](#use-intel®-neural-compressor-dataloader-api)
+    3.1 [Use Intel® Neural Compressor DataLoader API](#use-intel-neural-compressor-dataloader-api)
 
     3.2 [Build Custom Dataloader with Python API](#build-custom-dataloader-with-python-api)
 
@@ -44,7 +44,7 @@ Of cause, users can also use frameworks own dataloader in Neural Compressor.
 Acceptable parameters for `DataLoader` API including:
 | Parameter     | Description     |
 |:--------------|:----------|
-|framework (str)| different frameworks, such as `tensorflow`, `keras`, `mxnet`, `pytorch` and `onnxrt`.|
+|framework (str)| different frameworks, such as `tensorflow`, `tensorflow_itex`, `keras`, `mxnet`, `pytorch` and `onnxruntime`.|
 |dataset (object)| A dataset object from which to get data. Dataset must implement __iter__ or __getitem__ method.|
 |batch_size (int, optional)| How many samples per batch to load. Defaults to 1.|
 |collate_fn (Callable, optional)| Callable function that processes the batch you want to return from your dataloader. Defaults to None.|
@@ -66,6 +66,7 @@ dataloader = DataLoader(framework='tensorflow', dataset=dataset)
 config = PostTrainingQuantConfig()
 q_model = quantization.fit(model, config, calib_dataloader=dataloader, eval_func=eval)
 ```
+> Note: `DataLoader(framework='onnxruntime', dataset=dataset)` failed in neural-compressor v2.2. We have fixed it in this [PR](https://github.com/intel/neural-compressor/pull/1048).
 
 ### Build Custom Dataloader with Python API
 

diff --git a/docs/source/metric.md b/docs/source/metric.md
@@ -11,9 +11,9 @@ Metrics
 
     2.4. [ONNXRT](#onnxrt)
 
-3. [Get Started with Metric](#get-start-with-metric)
+3. [Get Started with Metric](#get-started-with-metric)
 
-    3.1. [Use Intel® Neural Compressor Metric API](#use-intel®-neural-compressor-metric-api)
+    3.1. [Use Intel® Neural Compressor Metric API](#use-intel-neural-compressor-metric-api)
 
     3.2. [Build Custom Metric with Python API](#build-custom-metric-with-python-api)
 
@@ -105,7 +105,7 @@ q_model = fit(model, config, calib_dataloader=calib_dataloader, eval_dataloader=
 
 ### Build Custom Metric with Python API
 
-Please refer to [Metrics code](../neural_compressor/metric), users can also register their own metric as follows:
+Please refer to [Metrics code](../../neural_compressor/metric), users can also register their own metric as follows:
 
 ```python
 class NewMetric(object):

diff --git a/examples/onnxrt/nlp/bert/quantization/ptq_dynamic/main.py b/examples/onnxrt/nlp/bert/quantization/ptq_dynamic/main.py
@@ -352,7 +352,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
+    dataloader = DataLoader(framework='onnxruntime', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

diff --git a/examples/onnxrt/nlp/bert/quantization/ptq_static/main.py b/examples/onnxrt/nlp/bert/quantization/ptq_static/main.py
@@ -359,7 +359,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
+    dataloader = DataLoader(framework='onnxruntime', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

diff --git a/examples/onnxrt/nlp/distilbert/quantization/ptq_dynamic/main.py b/examples/onnxrt/nlp/distilbert/quantization/ptq_dynamic/main.py
@@ -345,7 +345,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
+    dataloader = DataLoader(framework='onnxruntime', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

diff --git a/examples/onnxrt/nlp/distilbert/quantization/ptq_static/main.py b/examples/onnxrt/nlp/distilbert/quantization/ptq_static/main.py
@@ -352,7 +352,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
+    dataloader = DataLoader(framework='onnxruntime', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

diff --git a/examples/onnxrt/nlp/huggingface_model/question_answering/quantization/ptq_dynamic/main.py b/examples/onnxrt/nlp/huggingface_model/question_answering/quantization/ptq_dynamic/main.py
@@ -493,7 +493,7 @@ def eval_func(model, *args):
             conf = BenchmarkConfig(iteration=100,
                                    cores_per_instance=28,
                                    num_of_instance=1)
-            b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
+            b_dataloader = DataLoader(framework='onnxruntime', dataset=b_dataset, batch_size=model_args.batch_size)
             fit(model, conf, b_dataloader=b_dataloader)
         elif model_args.mode == 'accuracy':
             eval_f1 = eval_func(model)

diff --git a/examples/onnxrt/nlp/huggingface_model/question_answering/quantization/ptq_static/main.py b/examples/onnxrt/nlp/huggingface_model/question_answering/quantization/ptq_static/main.py
@@ -499,7 +499,7 @@ def eval_func(model, *args):
         q_model = quantization.fit(model, 
                                    config,
                                    eval_func=eval_func,
-                                   calib_dataloader=DataLoader(framework='onnxrt', 
+                                   calib_dataloader=DataLoader(framework='onnxruntime', 
                                                                dataset=calib_dataset, 
                                                                batch_size=model_args.batch_size)
                                    )
@@ -514,7 +514,7 @@ def eval_func(model, *args):
             conf = BenchmarkConfig(iteration=100,
                                    cores_per_instance=28,
                                    num_of_instance=1)
-            b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
+            b_dataloader = DataLoader(framework='onnxruntime', dataset=b_dataset, batch_size=model_args.batch_size)
             fit(model, conf, b_dataloader=b_dataloader)
         elif model_args.mode == 'accuracy':
             eval_f1 = eval_func(model)

diff --git a/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_dynamic/main.py b/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_dynamic/main.py
@@ -347,7 +347,7 @@ def result(self):
                                 data_dir=args.data_path,
                                 model_name_or_path=args.model_name_or_path,
                                 task=args.task)
-    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
+    dataloader = DataLoader(framework='onnxruntime', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model, *args):

diff --git a/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_static/main.py b/examples/onnxrt/nlp/huggingface_model/text_classification/quantization/ptq_static/main.py
@@ -354,7 +354,7 @@ def result(self):
                                 data_dir=args.data_path,
                                 model_name_or_path=args.model_name_or_path,
                                 task=args.task)
-    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
+    dataloader = DataLoader(framework='onnxruntime', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model, *args):

diff --git a/...rt/nlp/huggingface_model/token_classification/layoutlmft/quantization/ptq_dynamic/main.py b/...rt/nlp/huggingface_model/token_classification/layoutlmft/quantization/ptq_dynamic/main.py
@@ -455,7 +455,7 @@ def eval_func(model):
                 conf = BenchmarkConfig(iteration=100,
                                         cores_per_instance=28,
                                         num_of_instance=1)
-                b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
+                b_dataloader = DataLoader(framework='onnxruntime', dataset=b_dataset, batch_size=model_args.batch_size)
                 fit(onnx_model, conf, b_dataloader=b_dataloader)
             elif model_args.mode == 'accuracy':
                 eval_f1 = eval_func(onnx_model)

diff --git a/...xrt/nlp/huggingface_model/token_classification/layoutlmft/quantization/ptq_static/main.py b/...xrt/nlp/huggingface_model/token_classification/layoutlmft/quantization/ptq_static/main.py
@@ -456,7 +456,7 @@ def eval_func(model):
             q_model = quantization.fit(onnx_model, 
                                        config,
                                        eval_func=eval_func,
-                                       calib_dataloader=DataLoader(framework='onnxrt',
+                                       calib_dataloader=DataLoader(framework='onnxruntime',
                                                                    dataset=calib_dataset, 
                                                                    batch_size=1))
             q_model.save(model_args.save_path)
@@ -469,7 +469,7 @@ def eval_func(model):
                 conf = BenchmarkConfig(iteration=100,
                                         cores_per_instance=28,
                                         num_of_instance=1)
-                b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
+                b_dataloader = DataLoader(framework='onnxruntime', dataset=b_dataset, batch_size=model_args.batch_size)
                 fit(onnx_model, conf, b_dataloader=b_dataloader)
             elif model_args.mode == 'accuracy':
                 eval_f1 = eval_func(onnx_model)

diff --git a/...rt/nlp/huggingface_model/token_classification/layoutlmv3/quantization/ptq_dynamic/main.py b/...rt/nlp/huggingface_model/token_classification/layoutlmv3/quantization/ptq_dynamic/main.py
@@ -490,7 +490,7 @@ def eval_func(model):
             conf = BenchmarkConfig(iteration=100,
                                     cores_per_instance=28,
                                     num_of_instance=1)
-            b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
+            b_dataloader = DataLoader(framework='onnxruntime', dataset=b_dataset, batch_size=model_args.batch_size)
             fit(onnx_model, conf, b_dataloader=b_dataloader)
         elif model_args.mode == 'accuracy':
             eval_f1 = eval_func(onnx_model)

diff --git a/...xrt/nlp/huggingface_model/token_classification/layoutlmv3/quantization/ptq_static/main.py b/...xrt/nlp/huggingface_model/token_classification/layoutlmv3/quantization/ptq_static/main.py
@@ -488,7 +488,7 @@ def eval_func(model):
         q_model = quantization.fit(onnx_model, 
                                     config,
                                     eval_func=eval_func,
-                                    calib_dataloader=DataLoader(framework='onnxrt', 
+                                    calib_dataloader=DataLoader(framework='onnxruntime', 
                                                                 dataset=calib_dataset, 
                                                                 batch_size=1))
         q_model.save(model_args.save_path)
@@ -502,7 +502,7 @@ def eval_func(model):
             conf = BenchmarkConfig(iteration=100,
                                     cores_per_instance=28,
                                     num_of_instance=1,)
-            b_dataloader = DataLoader(framework='onnxrt', dataset=b_dataset, batch_size=model_args.batch_size)
+            b_dataloader = DataLoader(framework='onnxruntime', dataset=b_dataset, batch_size=model_args.batch_size)
             fit(onnx_model, conf, b_dataloader=b_dataloader)
         elif model_args.mode == 'accuracy':
             eval_f1 = eval_func(onnx_model)

diff --git a/examples/onnxrt/nlp/mobilebert/quantization/ptq_dynamic/main.py b/examples/onnxrt/nlp/mobilebert/quantization/ptq_dynamic/main.py
@@ -352,7 +352,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
+    dataloader = DataLoader(framework='onnxruntime', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

diff --git a/examples/onnxrt/nlp/mobilebert/quantization/ptq_static/main.py b/examples/onnxrt/nlp/mobilebert/quantization/ptq_static/main.py
@@ -359,7 +359,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
+    dataloader = DataLoader(framework='onnxruntime', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

diff --git a/examples/onnxrt/nlp/onnx_model_zoo/gpt2/quantization/ptq_dynamic/gpt2.py b/examples/onnxrt/nlp/onnx_model_zoo/gpt2/quantization/ptq_dynamic/gpt2.py
@@ -243,7 +243,7 @@ def eval_func(model):
             conf = BenchmarkConfig(iteration=100,
                                    cores_per_instance=4,
                                    num_of_instance=1)
-            b_dataloader = DataLoader(framework='onnxrt', dataset=ds, batch_size=args.eval_batch_size)
+            b_dataloader = DataLoader(framework='onnxruntime', dataset=ds, batch_size=args.eval_batch_size)
             fit(model, conf, b_dataloader=b_dataloader)
         else:
             evaluate(args, model, tokenizer)

diff --git a/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/main.py b/examples/onnxrt/nlp/roberta/quantization/ptq_dynamic/main.py
@@ -352,7 +352,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
+    dataloader = DataLoader(framework='onnxruntime', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

diff --git a/examples/onnxrt/nlp/roberta/quantization/ptq_static/main.py b/examples/onnxrt/nlp/roberta/quantization/ptq_static/main.py
@@ -359,7 +359,7 @@ def result(self):
                                 task=args.task,
                                 model_type=args.model_type,
                                 dynamic_length=args.dynamic_length)
-    dataloader = DataLoader(framework='onnxrt', dataset=dataset, batch_size=args.batch_size)
+    dataloader = DataLoader(framework='onnxruntime', dataset=dataset, batch_size=args.batch_size)
     metric = ONNXRTGLUE(args.task)
 
     def eval_func(model):

diff --git a/neural_compressor/data/dataloaders/dataloader.py b/neural_compressor/data/dataloaders/dataloader.py
@@ -28,7 +28,7 @@
                "pytorch": PyTorchDataLoader,
                "pytorch_ipex": PyTorchDataLoader,
                "pytorch_fx": PyTorchDataLoader,
-               "onnxrt": ONNXRTDataLoader,
+               "onnxruntime": ONNXRTDataLoader,
                "onnxrt_qlinearops": ONNXRTDataLoader,
                "onnxrt_integerops": ONNXRTDataLoader,
                "onnxrt_qdq": ONNXRTDataLoader,
@@ -67,7 +67,7 @@ def __new__(cls, framework, dataset, batch_size=1, collate_fn=None,
                 Defaults to False.
         """
         assert framework in ('tensorflow', 'tensorflow_itex', 'keras',\
-                             'pytorch', 'pytorch_ipex', 'pytorch_fx', 'onnxrt', 'onnxrt_qdqops', \
+                             'pytorch', 'pytorch_ipex', 'pytorch_fx', 'onnxruntime', 'onnxrt_qdqops', \
                              'onnxrt_qlinearops', 'onnxrt_integerops', 'mxnet'), \
                              "framework support tensorflow pytorch mxnet onnxruntime"
         return DATALOADERS[framework](dataset=dataset,

diff --git a/test/adaptor/onnxrt_adaptor/test_adaptor_onnxrt.py b/test/adaptor/onnxrt_adaptor/test_adaptor_onnxrt.py
@@ -1416,7 +1416,7 @@ def test_query_block_info(self):
         self.assertEqual(len(q_capability['block_wise']), 6)
 
     def test_dataloader_input(self):
-        cv_dataloader = DataLoader(framework='onnxrt', dataset=DummyCVDataset_list(shape=(3, 224, 224)))
+        cv_dataloader = DataLoader(framework='onnxruntime', dataset=DummyCVDataset_list(shape=(3, 224, 224)))
         quantizer = Quantization('qlinear.yaml')
         quantizer.calib_dataloader = cv_dataloader
         quantizer.eval_dataloader = cv_dataloader