This document will show some examples to introduce how to correctly use prompt templates in Neural Speed and [ITREX](https://github.com/intel/intel-extension-for-transformers).

For the base model (without SFT for pre-training), prompt can be directly encoded into token ids without adding any special prefix or suffix token. But for the chat model, we need some prompt templates to generate correct and human understandable words. The reason is that these models are usually trained with specific prompt templates.

from transformers import AutoTokenizer, TextStreamer

from neural_speed import Model

prompt = "你好"

tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True)

inputs = tokenizer.build_chat_input(prompt)['input_ids']

model = Model()

model.init_from_bin(args.model_name, gguf_path)

outputs = model.generate(inputs, max_new_tokens=300, do_sample=True)

words = tokenizer.decode(outputs[0])

from transformers import AutoTokenizer, TextStreamer

from intel_extension_for_transformers.transformers import AutoModelForCausalLM, WeightOnlyQuantConfig

model_name = "meta-llama/Llama-2-7b-chat-hf"

woq_config = WeightOnlyQuantConfig(compute_dtype="int8", weight_dtype="int4")

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

streamer = TextStreamer(tokenizer)

model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=woq_config, trust_remote_code=True)

while True:

prompt = input("> ").strip()

if prompt == "quit":

break

b_prompt = "[INST]{}[/INST]".format(prompt) # prompt template for llama2

inputs = tokenizer(b_prompt, return_tensors="pt").input_ids

outputs = model.generate(inputs, streamer=streamer, interactive=True, ignore_prompt=True, do_sample=True)

from transformers import AutoTokenizer, TextStreamer

from intel_extension_for_transformers.transformers import AutoModelForCausalLM, WeightOnlyQuantConfig

model_name = "THUDM/chatglm2-6b" # or local path to model

woq_config = WeightOnlyQuantConfig(compute_dtype="int8", weight_dtype="int4")

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

streamer = TextStreamer(tokenizer)

model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=woq_config, trust_remote_code=True)

while True:

prompt = input("> ").strip()

if prompt == "quit":

break

prompt = tokenizer.build_prompt(prompt) # prompt template for chatglm2

inputs = tokenizer([prompt], return_tensors="pt").input_ids

outputs = model.generate(inputs, streamer=streamer, interactive=True, ignore_prompt=True, do_sample=True, n_keep=2)

from transformers import AutoTokenizer, TextStreamer

from intel_extension_for_transformers.transformers import AutoModelForCausalLM, WeightOnlyQuantConfig

model_name = "Qwen/Qwen-7B-Chat" # or local path to model

woq_config = WeightOnlyQuantConfig(compute_dtype="int8", weight_dtype="int4")

tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)

streamer = TextStreamer(tokenizer)

model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=woq_config, trust_remote_code=True)

while True:

prompt = input("> ").strip()

if prompt == "quit":

break

prompt = "\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n".format(prompt) # prompt template for qwen

inputs = tokenizer([prompt], return_tensors="pt").input_ids

outputs = model.generate(inputs, streamer=streamer, interactive=True, ignore_prompt=True, do_sample=True)

<a href="https://huggingface.co/THUDM/chatglm2-6b" target="_blank" rel="noopener noreferrer">ChatGLM2-6B</a>,

<a href="https://huggingface.co/THUDM/chatglm3-6b" target="_blank" rel="noopener noreferrer">ChatGLM3-6B</a></td>

elif model_type == "chatglm2" or model_type == "chatglm3":

# For ChatGLM3

if model_type == "chatglm" and "chatglm3" in model_config._name_or_path:

# due to the same model architecture.

model_type = "chatglm2"

# chatglm2, bloom, chatglm3

compile_quant(quant_chatglm3 quant_model.cpp chatglm2 chatglm2)

compile_run(run_chatglm3 main_run.cpp main_pybind.cpp chatglm3 chatglm3)

#elif MODEL_NAME_ID == 20

PYBIND11_MODULE(chatglm3_cpp, m)

} else if (params.model_arch == MODEL_CHATGLM || params.model_arch == MODEL_BAICHUAN ||

params.model_arch == MODEL_CHATGLM3) {

params.model_arch == MODEL_BAICHUAN || params.model_arch == MODEL_CHATGLM3) {

def quantize_q8_0(tensor: torch.Tensor) -> torch.Tensor:

# equivalent to ggml_quantize_q8_0 in ggml.c

assert tensor.shape[1] % GGML_QK8_0 == 0

tensor = tensor.view(-1, GGML_QK8_0)

scale = tensor.abs().max(dim=-1, keepdim=True).values / ((1 << 7) - 1)

tensor = (tensor / scale).round().clamp(min=-128, max=127).char()

# add scale into each block

tensor = torch.cat((scale.half().view(torch.int8), tensor), dim=-1)

return tensor

def quantize_q5_0(tensor: torch.Tensor) -> torch.Tensor:

# equivalent to ggml_quantize_q5_0 in ggml.c

assert tensor.shape[1] % GGML_QK5_0 == 0

tensor = tensor.view(-1, GGML_QK5_0)

abs_max_indices = tensor.abs().max(dim=-1, keepdim=True).indices

max_values = torch.take_along_dim(tensor, abs_max_indices, dim=-1)

scale = max_values / -16

tensor = (tensor / scale + 16).round().clamp(min=0, max=31).char()

qs = (tensor[:, :16] & 0x0F) | (tensor[:, 16:] << 4)

qh = torch.zeros(tensor.shape[:-1], dtype=torch.int32)

for i in range(32):

qh |= ((tensor[:, i] & 0x10) >> 4).int() << i

# add scale into each block

tensor = torch.cat((scale.half().view(torch.int8), qh[..., None].view(torch.int8), qs), dim=-1)

return tensor

def quantize_q5_1(tensor: torch.Tensor) -> torch.Tensor:

# equivalent to ggml_quantize_q5_1 in ggml.c

assert tensor.shape[1] % GGML_QK5_1 == 0

tensor = tensor.view(-1, GGML_QK5_1)

min_vals = tensor.min(dim=-1, keepdim=True).values

max_vals = tensor.max(dim=-1, keepdim=True).values

scale = (max_vals - min_vals) / ((1 << 5) - 1)

tensor = ((tensor - min_vals) / scale).round().clamp(min=0, max=31).char()

qs = (tensor[:, :16] & 0x0F) | (tensor[:, 16:] << 4)

qh = torch.zeros(tensor.shape[:-1], dtype=torch.int32)

for i in range(32):

qh |= ((tensor[:, i] & 0x10) >> 4).int() << i

# add scale & min into each block

tensor = torch.cat(

(scale.half().view(torch.int8), min_vals.half().view(torch.int8), qh[..., None].view(torch.int8), qs), dim=-1)

return tensor