From c6bba89ea9cd55d6cb73a3f9c92b1eb867334592 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 27 Nov 2025 16:03:09 +0200 Subject: [PATCH] arch : add description about LLM_TENSOR_INFOS --- src/llama-arch.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 7ef87acf1b3..549fdd02250 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -2487,6 +2487,16 @@ static const std::map> LLM_TENSOR_N }, }; +// declare information about the model weight tensors: +// - the layer in which the tensor is going to be used. this is needed in order to assign the correct buffer type for the weight +// - the operator which is going to use the weight. this is needed to determine if the respective backend supports the operator +// +// for example, input layers are usually assigned to CPU/host buffer types +// +// a mismatch between the declared information and the actual layer/op in which the tensor is used can lead to sub-optimal +// assignment of the buffer types and extra overhead during computation +// example: https://github.com/ggml-org/llama.cpp/pull/17548 +// static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_TOKEN_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}}, {LLM_TENSOR_POS_EMBD, {LLM_TENSOR_LAYER_INPUT, GGML_OP_GET_ROWS}},