Skip to content

Commit

Permalink
llama : fix integer overflow during quantization (ggerganov#4284)
Browse files Browse the repository at this point in the history
happens with multi-threaded quantization of Qwen-72B

ggml-ci
  • Loading branch information
ggerganov committed Dec 1, 2023
1 parent 8d6d9f0 commit 880f579
Showing 1 changed file with 13 additions and 10 deletions.
23 changes: 13 additions & 10 deletions llama.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7655,18 +7655,21 @@ static void llama_convert_tensor_internal(
return;
}

auto block_size = tensor->type == GGML_TYPE_F16 ? 1 : (size_t)ggml_blck_size(tensor->type);
auto block_size_bytes = ggml_type_size(tensor->type);
size_t block_size = tensor->type == GGML_TYPE_F16 ? 1 : (size_t)ggml_blck_size(tensor->type);
size_t block_size_bytes = ggml_type_size(tensor->type);

GGML_ASSERT(nelements % block_size == 0);
auto nblocks = nelements / block_size;
auto blocks_per_thread = nblocks / nthread;
auto spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count

for (auto tnum = 0, in_buff_offs = 0, out_buff_offs = 0; tnum < nthread; tnum++) {
auto thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread
auto thr_elems = thr_blocks * block_size; // number of elements for this thread
auto thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread
size_t nblocks = nelements / block_size;
size_t blocks_per_thread = nblocks / nthread;
size_t spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count

size_t in_buff_offs = 0;
size_t out_buff_offs = 0;

for (int tnum = 0; tnum < nthread; tnum++) {
size_t thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread
size_t thr_elems = thr_blocks * block_size; // number of elements for this thread
size_t thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread

auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) {
if (typ == GGML_TYPE_F16) {
Expand Down

0 comments on commit 880f579

Please sign in to comment.