From f7e46f86c1653623c3f63c247b18e69daf216832 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Sat, 14 Oct 2023 11:15:04 -0700 Subject: [PATCH 1/2] Add validation for tensor_split size exceeding LLAMA_MAX_DEVICES --- llama_cpp/llama.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index fdde7ea01..cf5f5f35d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -308,6 +308,8 @@ def __init__( self.tensor_split = tensor_split self._p_tensor_split = None if self.tensor_split is not None: + if len(self.tensor_split) > llama_cpp.LLAMA_MAX_DEVICES: + raise ValueError(f"Attempt to split tensors above the maximum supported devices. Current LLAMA_MAX_DEVICES={llama_cpp.LLAMA_MAX_DEVICES}") # Type conversion and expand the list to the length of LLAMA_MAX_DEVICES FloatArray = ctypes.c_float * llama_cpp.LLAMA_MAX_DEVICES self._c_tensor_split = FloatArray( From 4b51ca3d414ebe053c81b4bc67fe96c3bbbf0ee8 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Sat, 14 Oct 2023 13:05:51 -0700 Subject: [PATCH 2/2] reword --- llama_cpp/llama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index cf5f5f35d..434d824e8 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -309,7 +309,7 @@ def __init__( self._p_tensor_split = None if self.tensor_split is not None: if len(self.tensor_split) > llama_cpp.LLAMA_MAX_DEVICES: - raise ValueError(f"Attempt to split tensors above the maximum supported devices. Current LLAMA_MAX_DEVICES={llama_cpp.LLAMA_MAX_DEVICES}") + raise ValueError(f"Attempt to split tensors that exceed maximum supported devices. Current LLAMA_MAX_DEVICES={llama_cpp.LLAMA_MAX_DEVICES}") # Type conversion and expand the list to the length of LLAMA_MAX_DEVICES FloatArray = ctypes.c_float * llama_cpp.LLAMA_MAX_DEVICES self._c_tensor_split = FloatArray(