llama : fix Metal KV cache sync (close #1695)

ggerganov · Jun 5, 2023 · d1f563a · d1f563a
1 parent 827f5ed
commit d1f563a
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/llama.cpp b/llama.cpp
@@ -1455,6 +1455,14 @@ static bool llama_eval_internal(
         // When we implement Matrix x Matrix Metal multiplication, we can avoid this branch.
         // But for now, we have focused only on Matrix x Vector Metal multiplication.
         //
+        // TODO: avoid these syncs via shared memory (ref #1696)
+        //
+        if (lctx.ctx_metal) {
+            // We need to sync the GPU KV cache with the CPU KV cache
+            ggml_metal_get_tensor(lctx.ctx_metal, kv_self.k);
+            ggml_metal_get_tensor(lctx.ctx_metal, kv_self.v);
+        }
+
         ggml_graph_compute(ctx0, &gf);
 
         if (lctx.ctx_metal) {