From f823526e8e18d8431398314a6eb2201eef42f235 Mon Sep 17 00:00:00 2001 From: Vaibhavs10 Date: Thu, 22 May 2025 15:28:59 +0200 Subject: [PATCH 1/6] llama-cli -> llama-server. --- packages/tasks/src/local-apps.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index af2f798d12..a92f20a1e4 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -119,12 +119,12 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[] { title: "Install from brew", setup: "brew install llama.cpp", - content: command("llama-cli"), + content: command("llama-server"), }, { title: "Install from WinGet (Windows)", setup: "winget install llama.cpp", - content: command("llama-cli"), + content: command("llama-server"), }, { title: "Use pre-built binary", @@ -133,7 +133,7 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[] "# Download pre-built binary from:", "# https://github.com/ggerganov/llama.cpp/releases", ].join("\n"), - content: command("./llama-cli"), + content: command("./llama-server"), }, { title: "Build from source code", @@ -141,9 +141,9 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[] "git clone https://github.com/ggerganov/llama.cpp.git", "cd llama.cpp", "cmake -B build -DLLAMA_CURL=ON", - "cmake --build build -j --target llama-cli", + "cmake --build build -j --target llama-server", ].join("\n"), - content: command("./build/bin/llama-cli"), + content: command("./build/bin/llama-server"), }, ]; }; From f5fc53139846b08f7c174035d262313e063087dc Mon Sep 17 00:00:00 2001 From: Vaibhavs10 Date: Thu, 22 May 2025 15:43:18 +0200 Subject: [PATCH 2/6] up. --- packages/tasks/src/local-apps.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index a92f20a1e4..c7e92efc18 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -108,11 +108,6 @@ function getQuantTag(filepath?: string): string { const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[] => { const command = (binary: string) => { const snippet = ["# Load and run the model:", `${binary} -hf ${model.id}${getQuantTag(filepath)}`]; - if (!model.tags.includes("conversational")) { - // for non-conversational models, add a prompt - snippet[snippet.length - 1] += " \\"; - snippet.push(' -p "Once upon a time,"'); - } return snippet.join("\n"); }; return [ @@ -140,7 +135,7 @@ const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[] setup: [ "git clone https://github.com/ggerganov/llama.cpp.git", "cd llama.cpp", - "cmake -B build -DLLAMA_CURL=ON", + "cmake -B build", "cmake --build build -j --target llama-server", ].join("\n"), content: command("./build/bin/llama-server"), From 5c37e92c9b5100ed28b97d25f1aef376a2a1fe9e Mon Sep 17 00:00:00 2001 From: Vaibhavs10 Date: Thu, 22 May 2025 16:04:26 +0200 Subject: [PATCH 3/6] issue. --- packages/tasks/src/local-apps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index c7e92efc18..3928033e53 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -108,7 +108,7 @@ function getQuantTag(filepath?: string): string { const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[] => { const command = (binary: string) => { const snippet = ["# Load and run the model:", `${binary} -hf ${model.id}${getQuantTag(filepath)}`]; - return snippet.join("\n"); + return snippet; }; return [ { From 9f9e8bfc8e56fa47624188e7de138e8177036a87 Mon Sep 17 00:00:00 2001 From: Vaibhavs10 Date: Thu, 22 May 2025 17:49:22 +0200 Subject: [PATCH 4/6] fix test. --- packages/tasks/src/local-apps.spec.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/tasks/src/local-apps.spec.ts b/packages/tasks/src/local-apps.spec.ts index f4ec4bca47..6ba6360a9a 100644 --- a/packages/tasks/src/local-apps.spec.ts +++ b/packages/tasks/src/local-apps.spec.ts @@ -13,7 +13,7 @@ describe("local-apps", () => { const snippet = snippetFunc(model); expect(snippet[0].content).toEqual(`# Load and run the model: -llama-cli -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); +llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); }); it("llama.cpp non-conversational", async () => { @@ -26,8 +26,7 @@ llama-cli -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); const snippet = snippetFunc(model); expect(snippet[0].content).toEqual(`# Load and run the model: -llama-cli -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}} \\ - -p "Once upon a time,"`); +llama-server -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}"`); }); it("vLLM conversational llm", async () => { From 6aa4ee65bef714de2e2814d4889a2a83d741aaad Mon Sep 17 00:00:00 2001 From: Vaibhavs10 Date: Thu, 22 May 2025 17:56:43 +0200 Subject: [PATCH 5/6] fix test. --- packages/tasks/src/local-apps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index 3928033e53..c7e92efc18 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -108,7 +108,7 @@ function getQuantTag(filepath?: string): string { const snippetLlamacpp = (model: ModelData, filepath?: string): LocalAppSnippet[] => { const command = (binary: string) => { const snippet = ["# Load and run the model:", `${binary} -hf ${model.id}${getQuantTag(filepath)}`]; - return snippet; + return snippet.join("\n"); }; return [ { From 411b63167e8841334d2281c2dfc1c98be3351433 Mon Sep 17 00:00:00 2001 From: Vaibhavs10 Date: Thu, 22 May 2025 18:25:27 +0200 Subject: [PATCH 6/6] fix test. --- packages/tasks/src/local-apps.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tasks/src/local-apps.spec.ts b/packages/tasks/src/local-apps.spec.ts index 6ba6360a9a..06553c97c8 100644 --- a/packages/tasks/src/local-apps.spec.ts +++ b/packages/tasks/src/local-apps.spec.ts @@ -26,7 +26,7 @@ llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:{{QUANT_TAG}}`); const snippet = snippetFunc(model); expect(snippet[0].content).toEqual(`# Load and run the model: -llama-server -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}"`); +llama-server -hf mlabonne/gemma-2b-GGUF:{{QUANT_TAG}}`); }); it("vLLM conversational llm", async () => {