[NeuralChat] Support magicoder model (#1067)

* Support magicoder model and refine load model

Signed-off-by: lvliang-intel <liang1.lv@intel.com>

intel_extension_for_transformers/llm/quantization/optimization.py

@@ -52,10 +52,11 @@ def optimize(self, model, use_llm_runtime=False):
             or re.search("bloom", model_name, re.IGNORECASE)
             or re.search("llama", model_name, re.IGNORECASE)
             or re.search("opt", model_name, re.IGNORECASE)
-            or re.search("neural-chat-7b-v1", model_name, re.IGNORECASE)
-            or re.search("neural-chat-7b-v2", model_name, re.IGNORECASE)
-            or re.search("neural-chat-7b-v3", model_name, re.IGNORECASE)
+            or re.search("neural-chat", model_name, re.IGNORECASE)
             or re.search("starcoder", model_name, re.IGNORECASE)
+            or re.search("codegen", model_name, re.IGNORECASE)
+            or re.search("mistral", model_name, re.IGNORECASE)
+            or re.search("magicoder", model_name, re.IGNORECASE)
             or re.search("solar", model_name, re.IGNORECASE)
         ):
             from intel_extension_for_transformers.transformers import AutoModelForCausalLM

intel_extension_for_transformers/neural_chat/chatbot.py

@@ -69,7 +69,7 @@ def build_chatbot(config: PipelineConfig=None):
             return
 
     # create model adapter
-    if "llama" in config.model_name_or_path.lower() or "magicoder" in config.model_name_or_path.lower():
+    if "llama" in config.model_name_or_path.lower():
         from .models.llama_model import LlamaModel
         adapter = LlamaModel()
     elif "mpt" in config.model_name_or_path.lower():
@@ -95,7 +95,8 @@ def build_chatbot(config: PipelineConfig=None):
          "flan-t5" in config.model_name_or_path.lower() or \
          "bloom" in config.model_name_or_path.lower() or \
          "starcoder" in config.model_name_or_path.lower() or \
-         "codegen" in config.model_name_or_path.lower():
+         "codegen" in config.model_name_or_path.lower() or \
+         "magicoder" in config.model_name_or_path.lower():
         from .models.base_model import BaseModel
         adapter = BaseModel()
     else:
@@ -163,6 +164,7 @@ def build_chatbot(config: PipelineConfig=None):
     try:
         adapter.load_model(parameters)
     except RuntimeError as e:
+        logger.error(f"Exception: {e}")
         if "out of memory" in str(e):
             set_latest_error(ErrorCodes.ERROR_OUT_OF_MEMORY)
         elif "devices are busy or unavailable" in str(e):
@@ -173,6 +175,7 @@ def build_chatbot(config: PipelineConfig=None):
             set_latest_error(ErrorCodes.ERROR_GENERIC)
         return
     except ValueError as e:
+        logger.error(f"Exception: {e}")
         if "load_model: unsupported device" in str(e):
             set_latest_error(ErrorCodes.ERROR_DEVICE_NOT_SUPPORTED)
         elif "load_model: unsupported model" in str(e):
@@ -187,6 +190,7 @@ def build_chatbot(config: PipelineConfig=None):
             set_latest_error(ErrorCodes.ERROR_GENERIC)
         return
     except Exception as e:
+        logger.error(f"Exception: {e}")
         set_latest_error(ErrorCodes.ERROR_GENERIC)
         return
     return adapter
@@ -204,14 +208,17 @@ def finetune_model(config: BaseFinetuningConfig):
     try:
         finetuning.finetune()
     except FileNotFoundError as e:
+        logger.error(f"Exception: {e}")
         if "Couldn't find a dataset script" in str(e):
             set_latest_error(ErrorCodes.ERROR_DATASET_NOT_FOUND)
     except ValueError as e:
+        logger.error(f"Exception: {e}")
         if "--do_eval requires a validation dataset" in str(e):
             set_latest_error(ErrorCodes.ERROR_VALIDATION_FILE_NOT_FOUND)
         elif "--do_train requires a train dataset" in str(e):
             set_latest_error(ErrorCodes.ERROR_TRAIN_FILE_NOT_FOUND)
     except Exception as e:
+        logger.error(f"Exception: {e}")
         if config.finetune_args.peft == "lora":
             set_latest_error(ErrorCodes.ERROR_LORA_FINETUNE_FAIL)
         elif config.finetune_args.peft == "llama_adapter":
@@ -237,6 +244,7 @@ def optimize_model(model, config, use_llm_runtime=False):
     try:
         model = optimization.optimize(model, use_llm_runtime)
     except Exception as e:
+        logger.error(f"Exception: {e}")
         from intel_extension_for_transformers.transformers import (
             MixedPrecisionConfig,
             WeightOnlyQuantConfig,

intel_extension_for_transformers/neural_chat/models/base_model.py

@@ -24,6 +24,7 @@
 from ..utils.common import is_audio_file
 from .model_utils import load_model, predict, predict_stream, MODELS
 from ..prompts import PromptTemplate
+from ..prompts.prompt import MAGICODER_PROMPT
 from ..utils.error_utils import set_latest_error
 from ..errorcode import ErrorCodes
 import logging
@@ -163,7 +164,7 @@ def predict_stream(self, query, origin_query="", config=None):
         self.get_conv_template(self.model_name, config.task)
         if (self.conv_template.roles[0] in query and self.conv_template.roles[1] in query) or \
               "starcoder" in self.model_name.lower() or "codellama" in self.model_name.lower() or \
-              "codegen" in self.model_name.lower():
+              "codegen" in self.model_name.lower() or "magicoder" in self.model_name.lower():
             query_include_prompt = True
 
         # plugin pre actions
@@ -207,6 +208,16 @@ def predict_stream(self, query, origin_query="", config=None):
         if not query_include_prompt and not is_plugin_enabled("retrieval"):
             query = self.prepare_prompt(query, self.model_name, config.task)
 
+        # Phind/Phind-CodeLlama-34B-v2 model accpects Alpaca/Vicuna instruction format.
+        if "phind" in self.model_name.lower():
+            conv_template = PromptTemplate(name="phind")
+            conv_template.append_message(conv_template.roles[0], query)
+            conv_template.append_message(conv_template.roles[1], None)
+            query = conv_template.get_prompt()
+
+        if "magicoder" in self.model_name.lower():
+            query = MAGICODER_PROMPT.format(instruction=query)
+
         try:
             response = predict_stream(
                 **construct_parameters(query, self.model_name, self.device, self.assistant_model, config))
@@ -256,7 +267,7 @@ def predict(self, query, origin_query="", config=None):
         self.get_conv_template(self.model_name, config.task)
         if (self.conv_template.roles[0] in query and self.conv_template.roles[1] in query) or \
                "starcoder" in self.model_name.lower() or "codellama" in self.model_name.lower() or \
-               "codegen" in self.model_name.lower():
+               "codegen" in self.model_name.lower() or "magicoder" in self.model_name.lower():
             query_include_prompt = True
 
         # plugin pre actions
@@ -298,6 +309,9 @@ def predict(self, query, origin_query="", config=None):
             conv_template.append_message(conv_template.roles[1], None)
             query = conv_template.get_prompt()
 
+        if "magicoder" in self.model_name.lower():
+            query = MAGICODER_PROMPT.format(instruction=query)
+
         # LLM inference
         try:
             response = predict(

intel_extension_for_transformers/neural_chat/models/model_utils.py

@@ -416,30 +416,45 @@ def load_model(
     else:
         MODELS[model_name]["assistant_model"] = None
 
+    try:
+        config = AutoConfig.from_pretrained(model_name, use_auth_token=hf_access_token, trust_remote_code=True \
+                                            if (re.search("chatglm", model_name, re.IGNORECASE) or \
+                                               re.search("qwen", model_name, re.IGNORECASE)) else False)
+    except ValueError as e:
+        logging.error(f"Exception: {e}")
+        if "Unrecognized model in" in str(e):
+            raise ValueError(f"load_model: model config is not found, {e}")
+        else:
+            raise ValueError(f"load_model: unknown ValueError occurred, {e}")
+    except EnvironmentError as e:
+        logging.error(f"Exception: {e}")
+        if "not a local folder and is not a valid model identifier" in str(e):
+            raise ValueError(f"load_model: model name or path is not found, {e}")
+        else:
+            raise ValueError(f"load_model: unknown EnvironmentError occurred, {e}")
+    except Exception as e:
+        logging.error(f"Exception: {e}")
+        raise ValueError(f"load_model: an unexpected error occurred, {e}")
+
+    MODELS[model_name]["model_type"] = config.model_type
+
     try:
         tokenizer = AutoTokenizer.from_pretrained(
             tokenizer_name,
-            use_fast=False if (re.search("llama", model_name, re.IGNORECASE)
-                or re.search("neural-chat-7b-v2", model_name, re.IGNORECASE)) else True,
+            use_fast=False if config.model_type == "llama" else True,
             use_auth_token=hf_access_token,
             trust_remote_code=True if (re.search("qwen", model_name, re.IGNORECASE) or \
                 re.search("chatglm", model_name, re.IGNORECASE)) else False,
         )
     except EnvironmentError as e:
+        logging.error(f"Exception: {e}")
         if "not a local folder and is not a valid model identifier" in str(e):
-            raise ValueError("load_model: tokenizer is not found")
-        else:
-            raise
-
-    try:
-        config = AutoConfig.from_pretrained(model_name, use_auth_token=hf_access_token, trust_remote_code=True \
-                                            if (re.search("chatglm", model_name, re.IGNORECASE) or \
-                                               re.search("qwen", model_name, re.IGNORECASE)) else False)
-    except ValueError as e:
-        if "Unrecognized model in" in str(e):
-            raise ValueError("load_model: model config is not found")
+            raise ValueError(f"load_model: tokenizer is not found, {e}")
         else:
-            raise
+            raise ValueError(f"load_model: unknown EnvironmentError occurred, {e}")
+    except Exception as e:
+        logging.error(f"Exception: {e}")
+        raise ValueError(f"load_model: an unexpected error occurred, {e}")
 
     load_to_meta = model_on_meta(config)
 
@@ -478,33 +493,26 @@ def load_model(
                     trust_remote_code=True)
         elif ((
             re.search("gpt", model_name, re.IGNORECASE)
-            or re.search("mpt", model_name, re.IGNORECASE)
-            or re.search("bloom", model_name, re.IGNORECASE)
-            or re.search("llama", model_name, re.IGNORECASE)
-            or re.search("magicoder", model_name, re.IGNORECASE)
-            or re.search("neural-chat-7b-v1", model_name, re.IGNORECASE)
-            or re.search("neural-chat-7b-v2", model_name, re.IGNORECASE)
-            or re.search("neural-chat-7b-v3", model_name, re.IGNORECASE)
-            or re.search("qwen", model_name, re.IGNORECASE)
-            or re.search("starcoder", model_name, re.IGNORECASE)
-            or re.search("codellama", model_name, re.IGNORECASE)
-            or re.search("mistral", model_name, re.IGNORECASE)
-            or re.search("codegen", model_name, re.IGNORECASE)
-        ) and not ipex_int8) or re.search("opt", model_name, re.IGNORECASE):
+            or config.model_type == "bloom"
+            or config.model_type == "qwen"
+            or config.model_type == "gpt_bigcode"
+            or config.model_type == "mpt"
+            or config.model_type == "llama"
+            or config.model_type == "mistral"
+        ) and not ipex_int8) or config.model_type == "opt":
             with smart_context_manager(use_deepspeed=use_deepspeed):
                 model = AutoModelForCausalLM.from_pretrained(
                     model_name,
                     use_auth_token=hf_access_token,
                     torch_dtype=torch_dtype,
                     low_cpu_mem_usage=True,
                     quantization_config=bitsandbytes_quant_config,
-                    trust_remote_code=True if (re.search("qwen", model_name, re.IGNORECASE) or \
+                    trust_remote_code=True if (config.model_type == "qwen" or \
                         re.search("codegen", model_name, re.IGNORECASE)) else False
                 )
         elif (
-                (re.search("starcoder", model_name, re.IGNORECASE)
-                or re.search("codellama", model_name, re.IGNORECASE)
-                or re.search("codegen", model_name, re.IGNORECASE)
+                (config.model_type == "gpt_bigcode"
+                 or config.model_type == "llama"
                 ) and ipex_int8
             ):
             with smart_context_manager(use_deepspeed=use_deepspeed):
@@ -520,9 +528,9 @@ def load_model(
                         model_name,
                         file_name="best_model.pt",
                     )
-        elif(
-                (re.search("llama", model_name, re.IGNORECASE)
-                or re.search("opt", model_name, re.IGNORECASE)
+        elif (
+                (config.model_type == "llama"
+                or config.model_type == "opt"
                 or re.search("gpt_neox", model_name, re.IGNORECASE)
                 or re.search("gptj", model_name, re.IGNORECASE)
                 or re.search("falcon", model_name, re.IGNORECASE)
@@ -547,10 +555,14 @@ def load_model(
             raise ValueError(f"unsupported model name or path {model_name}, \
             only supports FLAN-T5/LLAMA/MPT/GPT/BLOOM/OPT/QWEN/NEURAL-CHAT/MISTRAL/CODELLAMA/STARCODER/CODEGEN now.")
     except EnvironmentError as e:
+        logging.error(f"Exception: {e}")
         if "not a local folder and is not a valid model identifier" in str(e):
             raise ValueError("load_model: model name or path is not found")
         else:
-            raise
+            raise ValueError(f"load_model: unknown EnvironmentError occurred, {e}")
+    except Exception as e:
+        logging.error(f"Exception: {e}")
+        raise ValueError(f"load_model: an unexpected error occurred, {e}")
 
     if re.search("llama", model.config.architectures[0], re.IGNORECASE):
         # unwind broken decapoda-research config
@@ -1192,6 +1204,8 @@ def predict(**params):
         output = tokenizer.decode(generation_output.sequences[0], skip_special_tokens=True)
     if "### Response:" in output:
         return output.split("### Response:")[1].strip()
+    if "@@ Response" in output:
+        return output.split("@@ Response")[1].strip()
     if "### Assistant" in output:
         return output.split("### Assistant:")[1].strip()
     if "\nassistant\n" in output:

intel_extension_for_transformers/neural_chat/prompts/prompt.py

@@ -220,4 +220,13 @@ def get_prompt(self) -> str:
         return res
 
     def clear_messages(self) -> str:
-        self.conv.messages = []
+        self.conv.messages = []
+
+# pylint: disable=C0301
+MAGICODER_PROMPT = """You are an exceptionally intelligent coding assistant that consistently delivers accurate and reliable responses to user instructions.
+
+@@ Instruction
+{instruction}
+
+@@ Response
+"""