Add description of fp8 function support in md file #8610
Conversation
|
Thanks for your contribution! |
Codecov ReportAll modified and coverable lines are covered by tests ✅
Additional details and impacted files@@ Coverage Diff @@
## develop #8610 +/- ##
===========================================
- Coverage 55.80% 54.42% -1.39%
===========================================
Files 620 632 +12
Lines 96642 99475 +2833
===========================================
+ Hits 53928 54135 +207
- Misses 42714 45340 +2626 ☔ View full report in Codecov by Sentry. |
llm/argument.py
Outdated
| @@ -177,8 +177,8 @@ class ModelArgument: | |||
| @dataclass | |||
| class QuantArgument: | |||
| quant_type: str = field( | |||
| default="a8w8", | |||
| metadata={"help": "Quantization type. Supported values: a8w8, weight_only_int4, weight_only_int8"}, | |||
| default="a8w8(int)", | |||
There was a problem hiding this comment.
不要用()表示是int还是float，可以新增afp8wfp8代表fp8量化
llm/quant.py
Outdated
| if quant_args.quant_type == "a8w8": | ||
| activation = AVGObserver(quant_bits=8) | ||
| weight = weight_observer(quant_bits=8) | ||
| if quant_args.quant_type == "a8w8(int)": |
llm/utils/argument.py
Outdated
| @@ -214,8 +221,8 @@ class ModelArgument: | |||
| @dataclass | |||
| class QuantArgument: | |||
| quant_type: str = field( | |||
| default="a8w8", | |||
| metadata={"help": "Quantization type. Supported values: a8w8, weight_only_int4, weight_only_int8"}, | |||
| default=QuantType.WFP8AFP8, | |||
| @@ -77,7 +77,7 @@ python run_finetune.py ./config/llama/awq_argument.json | |||
|
|
|||
| <summary>  量化参数（QuantArgument）</summary><div> | |||
|
|
|||
| - `quant_type`: PTQ,QAT量化类型，默认为A8W8。支持A8W8,WINT4，WINT8：A8W8指对激活（输入）进行INT8量化，对模型权重进行INT8量化；WINT4指仅对模型权重进行INT4量化，后续使用WeightOnly进行推理；WINT8指仅对模型权重进行INT8量化，后续使用WeightOnly进行推理。 | |||
| - `quant_type`: PTQ,QAT量化类型，默认为WINT8AINT8。支持WINT8AINT8,WFP8AFP8,WEIGHT_ONLY_INT4,WEIGHT_ONLY_INT8：WINT8AINT8指对激活（输入）以及模型权重进行INT8量化；WFP8AFP8指对激活（输入）以及模型权重进行FP8量化；WINT4指仅对模型权重进行INT4量化，后续使用WeightOnly进行推理；WINT8指仅对模型权重进行INT8量化，后续使用WeightOnly进行推理。 | |||
|
This Pull Request is stale because it has been open for 60 days with no activity. 当前Pull Request 60天内无活动，被标记为stale。 |
|
|
PR types
New features
PR changes
Docs
Description
目前支持FP8 PTQ功能，添加相关的文档介绍以及FP8 PTQ功能使用案例