From 7d020db2c90cd6bbb13e30f24ac31555d5b91c75 Mon Sep 17 00:00:00 2001 From: wenhuach21 Date: Mon, 18 Mar 2024 17:46:24 +0800 Subject: [PATCH 1/5] fix a bug in example --- examples/language-modeling/main_customized_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/language-modeling/main_customized_data.py b/examples/language-modeling/main_customized_data.py index 608e03a61..798894aa0 100644 --- a/examples/language-modeling/main_customized_data.py +++ b/examples/language-modeling/main_customized_data.py @@ -243,7 +243,7 @@ def get_library_version(library_name): print( f"{n} will not be quantized due to its shape not being divisible by 32, resulting in an exporting issue to autogptq") - data = customized_data(tokenizer, args.seqlen) + data = customized_data() autoround = round(model, tokenizer, args.bits, args.group_size, dataloader=data, sym=args.sym, batch_size=args.train_bs, From aed8be0dac4930c3457bda21fcede4361a3c6813 Mon Sep 17 00:00:00 2001 From: wenhuach21 Date: Mon, 20 May 2024 16:04:06 +0800 Subject: [PATCH 2/5] fix gradient_accmulate bug in lm-head --- auto_round/autoround.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/auto_round/autoround.py b/auto_round/autoround.py index ec277077d..68827f2bf 100644 --- a/auto_round/autoround.py +++ b/auto_round/autoround.py @@ -842,9 +842,6 @@ def cache_inter_data(self, block_names, n_samples, layer_names=[], last_cache_na if last_cache_name is None and len(block_names) + len(layer_names) == 1: self.last_cache_name = block_names[0] if len(block_names) == 1 else layer_names[0] calib_bs = self.train_bs - if not self.low_gpu_mem_usage and len(layer_names) > 1: ## persume has lm-head - calib_bs = 1 - self.hook_handles = [] self._replace_forward() self.calib(n_samples, calib_bs) @@ -1015,7 +1012,7 @@ def quant_layer(self, layer_name, inputs, q_inputs=None, device=torch.device("cp scaler = self.get_scaler() # pylint: disable=assignment-from-none init_loss = None best_v, best_min_scale, best_max_scale = torch.tensor(0), torch.tensor(0), torch.tensor(0) - gradient_accumulate_steps = self.train_bs // train_bs + gradient_accumulate_steps = self.gradient_accumulate_steps for i in range(self.iters): total_loss = 0 for _ in range(gradient_accumulate_steps): From f4814f2128393dacc432e43e5673a576a79034c7 Mon Sep 17 00:00:00 2001 From: wenhuach21 Date: Fri, 24 May 2024 22:19:49 +0800 Subject: [PATCH 3/5] correct the doc --- docs/gemma-2b-acc.md | 4 ++-- docs/gemma-7b-acc.md | 4 ++-- docs/gemma-7b-it-acc.md | 4 ++-- examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh | 2 +- examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh | 2 +- examples/language-modeling/scripts/Yi-6B-Chat.sh | 2 +- examples/language-modeling/scripts/baichuan2-7b-chat.sh | 2 +- examples/language-modeling/scripts/neural-chat-7b-v3-1.sh | 2 +- examples/language-modeling/scripts/neural-chat-7b-v3-3.sh | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/gemma-2b-acc.md b/docs/gemma-2b-acc.md index 0b5c5e918..033d37dca 100644 --- a/docs/gemma-2b-acc.md +++ b/docs/gemma-2b-acc.md @@ -11,8 +11,8 @@ Please note that there is a discrepancy between the baseline result and the offi lm_eval --model hf --model_args pretrained="Intel/gemma-2b-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 16 ``` -| Metric | FP16 | INT4 | -| -------------- | ------ |--------| +| Metric | BF16 | INT4 | +| -------------- |--------|--------| | Avg. | 0.5383 | 0.5338 | | mmlu | 0.3337 | 0.3276 | | lambada_openai | 0.6398 | 0.6319 | diff --git a/docs/gemma-7b-acc.md b/docs/gemma-7b-acc.md index 9f03be4dd..b371eadd9 100644 --- a/docs/gemma-7b-acc.md +++ b/docs/gemma-7b-acc.md @@ -6,8 +6,8 @@ Please note that there is a discrepancy between the baseline result and the offi lm_eval --model hf --model_args pretrained="Intel/gemma-7b-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 32 ``` -| Metric | FP16 | int4 | -| -------------- | ------ | ------ | +| Metric | BF16 | int4 | +| -------------- |--------| ------ | | Avg. | 0.6239 | 0.6307 | | mmlu | 0.6162 | 0.6147 | | lambada_openai | 0.6751 | 0.7204 | diff --git a/docs/gemma-7b-it-acc.md b/docs/gemma-7b-it-acc.md index 1d1f1b52f..1cd2395f2 100644 --- a/docs/gemma-7b-it-acc.md +++ b/docs/gemma-7b-it-acc.md @@ -6,8 +6,8 @@ Please note that there is a discrepancy between the baseline result and the offi lm_eval --model hf --model_args pretrained="Intel/gemma-7b-it-int4-inc",autogptq=True,gptq_use_triton=True --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,rte,arc_easy,arc_challenge,mmlu --batch_size 32 ``` -| Metric | FP16 | int4 | -| -------------- | ------ | ------ | +| Metric | BF16 | int4 | +| -------------- |--------| ------ | | Avg. | 0.6022 | 0.6017 | | mmlu | 0.5029 | 0.4993 | | lambada_openai | 0.6035 | 0.6286 | diff --git a/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh b/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh index a64143260..51ef47e3b 100644 --- a/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh +++ b/examples/language-modeling/scripts/Mixtral-8x7B-v0.1.sh @@ -6,4 +6,4 @@ python3 main.py \ --iters 1000 \ --deployment_device 'gpu' \ --output_dir "./tmp_autoround" \ ---disable_quanted_input \ \ No newline at end of file +--disable_quanted_input \ No newline at end of file diff --git a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh index 3e1f75ffb..84ab60490 100644 --- a/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh +++ b/examples/language-modeling/scripts/Qwen1.5-7B-Chat-asym.sh @@ -5,5 +5,5 @@ python3 main.py \ --bits 4 \ --iters 1000 \ --deployment_device 'gpu' \ ---minmax_lr 2e-3 \ +--minmax_lr 2e-3 diff --git a/examples/language-modeling/scripts/Yi-6B-Chat.sh b/examples/language-modeling/scripts/Yi-6B-Chat.sh index 57fa0727f..ce59f1bbe 100644 --- a/examples/language-modeling/scripts/Yi-6B-Chat.sh +++ b/examples/language-modeling/scripts/Yi-6B-Chat.sh @@ -5,4 +5,4 @@ python3 main.py \ --bits 4 \ --iters 1000 \ --deployment_device 'gpu' \ ---minmax_lr 2e-3 \ +--minmax_lr 2e-3 diff --git a/examples/language-modeling/scripts/baichuan2-7b-chat.sh b/examples/language-modeling/scripts/baichuan2-7b-chat.sh index 7fb02dcfa..51d674509 100644 --- a/examples/language-modeling/scripts/baichuan2-7b-chat.sh +++ b/examples/language-modeling/scripts/baichuan2-7b-chat.sh @@ -5,4 +5,4 @@ python3 main.py \ --bits 4 \ --iters 1000 \ --deployment_device 'gpu' \ ---minmax_lr 2e-3 \ +--minmax_lr 2e-3 diff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh index d98afe8c4..208694e0b 100644 --- a/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh +++ b/examples/language-modeling/scripts/neural-chat-7b-v3-1.sh @@ -8,4 +8,4 @@ python3 main.py \ --minmax_lr 0.0002 \ --deployment_device 'gpu' \ --output_dir "./tmp_autoround" \ ---disable_quanted_input \ \ No newline at end of file +--disable_quanted_input \ No newline at end of file diff --git a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh index 99ea7ccfe..c7db8b1f4 100644 --- a/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh +++ b/examples/language-modeling/scripts/neural-chat-7b-v3-3.sh @@ -7,4 +7,4 @@ python3 main.py \ --minmax_lr 0.0002 \ --deployment_device 'gpu' \ --output_dir "./tmp_autoround" \ ---disable_quanted_input \ \ No newline at end of file +--disable_quanted_input \ No newline at end of file From f1029313837d6a57fda9db51715614b79c44a131 Mon Sep 17 00:00:00 2001 From: wenhuach21 Date: Mon, 27 May 2024 16:50:03 +0800 Subject: [PATCH 4/5] remove fp32 conversion as no need now --- auto_round/export/export_to_autogptq.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/auto_round/export/export_to_autogptq.py b/auto_round/export/export_to_autogptq.py index 73fcc50a9..06f4694b8 100644 --- a/auto_round/export/export_to_autogptq.py +++ b/auto_round/export/export_to_autogptq.py @@ -126,8 +126,7 @@ def save_quantized_as_autogptq(output_dir, use_triton=True, inplace=True, **kwar info = weight_config[key] if not check_to_quantized(info): continue - info["zp"] = info["zp"].to(torch.float32) - quantizers[key] = (None, info["scale"].to(torch.float32), info["zp"], info["g_idx"]) + quantizers[key] = (None, info["scale"], info["zp"], info["g_idx"]) pack_model( compressed_model, quantizers, From d201814759303fb80c9a70e76c622d9e7c190317 Mon Sep 17 00:00:00 2001 From: wenhuach21 Date: Mon, 27 May 2024 17:16:12 +0800 Subject: [PATCH 5/5] update phi2 recipe and remove falcon data as we don't trust the qdq accuracy anymore --- README.md | 3 +-- docs/phi-2-acc.md | 12 ++++++------ examples/language-modeling/scripts/phi-2.sh | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 51c6fbe42..926bf5db5 100644 --- a/README.md +++ b/README.md @@ -158,8 +158,7 @@ print(tokenizer.decode(model.generate(**inputs, max_new_tokens=50)[0])) | Intel/neural-chat-7b-v3-3 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-3-int4-inc), [accuracy](./docs/neural-chat-7b-v3-3-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-3.sh), [example](./examples/language-modeling/) | | Intel/neural-chat-7b-v3-1 | [HF-int4-model](https://huggingface.co/Intel/neural-chat-7b-v3-1-int4-inc), [accuracy](./docs/neural-chat-7b-v3-1-acc.md), [recipe](./examples/language-modeling/scripts/neural-chat-7b-v3-1.sh), [example](./examples/language-modeling/) | | mistralai/Mistral-7B-v0.1 | [HF-int4-model-lmhead](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc-lmhead),[HF-int4-model](https://huggingface.co/Intel/Mistral-7B-v0.1-int4-inc), [accuracy](./docs/Mistral-7B-v0.1-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-v0.1.sh), [example](./examples/language-modeling/) | -| microsoft/phi-2 | [HF-int4-model](https://huggingface.co/Intel/phi-2-int4-inc), [accuracy](./docs/phi-2-acc.md), [recipe](./examples/language-modeling/scripts/phi-2.sh), [example](./examples/language-modeling/) -| tiiuae/falcon-7b | [HF-int4-model](https://huggingface.co/Intel/falcon-7b-int4-inc), [accuracy](./docs/falcon-7b-acc.md), [recipe](./examples/language-modeling/scripts/falcon-7b.sh), [example](./examples/language-modeling/) | +| microsoft/phi-2 | [HF-int4-sym-model](https://huggingface.co/Intel/phi-2-int4-inc), [accuracy](./docs/phi-2-acc.md), [recipe](./examples/language-modeling/scripts/phi-2.sh), [example](./examples/language-modeling/) | google/gemma-2b | [HF-int4-model](https://huggingface.co/Intel/gemma-2b-int4-inc), [accuracy](./docs/gemma-2b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-2b.sh), [example](./examples/language-modeling/) | mistralai/Mistral-7B-Instruct-v0.2 | [HF-int4-model](https://huggingface.co/Intel/Mistral-7B-Instruct-v0.2-int4-inc) (under review), [accuracy](./docs/Mistral-7B-Instruct-v0.2-acc.md), [recipe](./examples/language-modeling/scripts/Mistral-7B-Instruct-v0.2.sh), [example](./examples/language-modeling/) | | google/gemma-7b | [HF-int4-model](https://huggingface.co/Intel/gemma-7b-int4-inc) (under review), [accuracy](./docs/gemma-7b-acc.md), [recipe](./examples/language-modeling/scripts/gemma-7b.sh), [example](./examples/language-modeling/) | diff --git a/docs/phi-2-acc.md b/docs/phi-2-acc.md index ecf765151..0ccdfeb57 100644 --- a/docs/phi-2-acc.md +++ b/docs/phi-2-acc.md @@ -1,12 +1,12 @@ -Install [lm-eval-harness](https://github.com/EleutherAI/lm-evaluation-harness.git) from source, and the git id f3b7917091afba325af3980a35d8a6dcba03dc3f is used +pip install lm-eval==0.4.2 -Download the model from hf(coming soon) or follow examples/language-modeling/scripts/phi-2.sh to generate the model +Due to the significant accuracy drop with the asymmetric kernel for this model, we opted to use symmetric quantization. -Since we encountered an issue evaluating this model with lm-eval, we opted to evaluate the qdq model instead. In our assessment, we found that its accuracy closely matches that of the real quantized model in most cases except for some small models like opt-125m. +```bash +lm_eval --model hf --model_args pretrained="Intel/phi-2-int4-inc" --device cuda:0 --tasks lambada_openai,hellaswag,piqa,winogrande,truthfulqa_mc1,openbookqa,boolq,arc_easy,arc_challenge,mmlu --batch_size 16 +``` - - -| Metric | FP16 | INT4 qdq | +| Metric | FP16 | INT4 | | -------------- | ------ | -------- | | Avg. | 0.6155 | 0.6163 | | mmlu | 0.5448 | 0.5417 | diff --git a/examples/language-modeling/scripts/phi-2.sh b/examples/language-modeling/scripts/phi-2.sh index 79abcbc6a..46e27226f 100644 --- a/examples/language-modeling/scripts/phi-2.sh +++ b/examples/language-modeling/scripts/phi-2.sh @@ -4,7 +4,7 @@ python3 main.py \ --group_size 128 \ --bits 4 \ --iters 1000 \ +--sym \ --deployment_device 'gpu' \ ---disable_trust_remote_code \ --output_dir "./tmp_autoround" \ ---disable_quanted_input \ No newline at end of file +--disable_low_gpu_mem_usage