Also load large LLMs fully on GPU

openradx · Jun 18, 2024 · d989d10 · d989d10
1 parent 417857a
commit d989d10
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/compose/docker-compose.dev.yml b/compose/docker-compose.dev.yml
@@ -89,7 +89,7 @@ services:
   llamacpp_gpu:
     <<: *llamacpp
     image: ghcr.io/ggerganov/llama.cpp:server-cuda
-    entrypoint: "/bin/bash -c '/llama-server -mu $${LLM_MODEL_URL} -ngl 50 -c 4096 --host 0.0.0.0 --port 8080'"
+    entrypoint: "/bin/bash -c '/llama-server -mu $${LLM_MODEL_URL} -ngl 99 -c 4096 --host 0.0.0.0 --port 8080'"
     deploy:
       resources:
         reservations:

diff --git a/compose/docker-compose.prod.yml b/compose/docker-compose.prod.yml
@@ -94,7 +94,7 @@ services:
       - 9610:8080
     volumes:
       - models_data:/models
-    entrypoint: "/bin/bash -c '/llama-server -mu $${LLM_MODEL_URL} -ngl 50 -cb -c 4096 --host 0.0.0.0 --port 8080'"
+    entrypoint: "/bin/bash -c '/llama-server -mu $${LLM_MODEL_URL} -ngl 99 -cb -c 4096 --host 0.0.0.0 --port 8080'"
     deploy:
       # <<: *deploy
       resources: