From 84866fffd21ee829a4c70974cd2a69b774eee893 Mon Sep 17 00:00:00 2001 From: zhangkaihuo Date: Fri, 31 May 2024 16:14:58 +0800 Subject: [PATCH 1/2] support lm_head --- gguf-py/gguf/constants.py | 1 + llama.cpp | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 978fcada3b42c..292345426af91 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -631,6 +631,7 @@ class MODEL_TENSOR(IntEnum): ], MODEL_ARCH.MINICPM: [ MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT, MODEL_TENSOR.OUTPUT_NORM, MODEL_TENSOR.ROPE_FREQS, MODEL_TENSOR.ATTN_NORM, diff --git a/llama.cpp b/llama.cpp index 0ef756a52860c..9a19a736b3328 100644 --- a/llama.cpp +++ b/llama.cpp @@ -4941,7 +4941,8 @@ static bool llm_load_tensors( // output { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - if (model.arch != LLM_ARCH_MINICPM){ + //if (model.arch != LLM_ARCH_MINICPM){ + { model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, false); // if output is NULL, init from the input tok embed if (model.output == NULL) { @@ -10120,7 +10121,7 @@ struct llm_build_context { cb(cur, "lmhead_scaling", -1); // lm_head - cur = ggml_mul_mat(ctx0, model.tok_embd, cur); + cur = ggml_mul_mat(ctx0, model.output, cur); cb(cur, "result_output", -1); ggml_build_forward_expand(gf, cur); From 5d19aa8250bdeec260a412fe8c3ad976c83b656d Mon Sep 17 00:00:00 2001 From: zhangkaihuo Date: Mon, 3 Jun 2024 13:47:08 +0800 Subject: [PATCH 2/2] remove the code block --- llama.cpp | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/llama.cpp b/llama.cpp index cd8c3f8abed70..3861e4a688286 100644 --- a/llama.cpp +++ b/llama.cpp @@ -5129,12 +5129,10 @@ static bool llm_load_tensors( // output { model.output_norm = ml.create_tensor(ctx_output, tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}); - { - model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); - // if output is NULL, init from the input tok embed - if (model.output == NULL) { - model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); - } + model.output = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED); + // if output is NULL, init from the input tok embed + if (model.output == NULL) { + model.output = ml.create_tensor(ctx_output, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_DUPLICATED); } }