support sq auto tune (#537)

Signed-off-by: changwangss <chang1.wang@intel.com>
intel · Oct 24, 2023 · 2fde68c · 2fde68c
1 parent 936ada5
commit 2fde68c
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/examples/huggingface/pytorch/text-generation/quantization/run_generation.py b/examples/huggingface/pytorch/text-generation/quantization/run_generation.py
@@ -114,7 +114,7 @@
     excluded_precisions = [] if args.int8_bf16_mixed else ["bf16"]
     quantization_config = SmoothQuantConfig(
                                 tokenizer=tokenizer,  # either two of one, tokenizer or calib_func
-                                alpha=float(args.alpha),    # default is 0.5
+                                alpha="auto" if args.alpha == "auto" else float(args.alpha),    # default is 0.5
                                 op_type_dict=op_type_dict,  # default is {}
                                 excluded_precisions=excluded_precisions,  # default is []
                                )
@@ -134,7 +134,6 @@
     user_model = AutoModelForCausalLM.from_pretrained(args.model,
                                                       quantization_config=quantization_config,
                                                       trust_remote_code=args.trust_remote_code,
-                                                      torchscript=True if args.sq else False,
                                                       use_llm_runtime=False
                                                       )
     if args.sq:

diff --git a/intel_extension_for_transformers/transformers/modeling/modeling_auto.py b/intel_extension_for_transformers/transformers/modeling/modeling_auto.py
@@ -259,7 +259,12 @@ def default_calib_func(model):
             else:
                 calib_func = calib_func
             model.config.torchscript = True
-            model = quantization.fit(model, conf, calib_func=calib_func)
+            model = quantization.fit(
+                                    model, 
+                                    conf,
+                                    calib_func=calib_func,
+                                    calib_dataloader=calib_dataloader if quantization_config.alpha=="auto" else None
+                                    )
             logger.info("SmoothQuant done.")
         return model