Add support for starting & changing models through UI

README.md

@@ -87,14 +87,11 @@ cd llama-gpt
 Run LlamaGPT with the following command:
 
 ```
-./run-mac.sh --model 7b
+./run-mac.sh
 ```
 
 You can access LlamaGPT at http://localhost:3000.
 
-> To run 13B or 70B chat models, replace `7b` with `13b` or `70b` respectively.
-> To run 7B, 13B or 34B Code Llama models, replace `7b` with `code-7b`, `code-13b` or `code-34b` respectively.
-
 To stop LlamaGPT, do `Ctrl + C` in Terminal.
 
 ### Install LlamaGPT anywhere else with Docker
@@ -111,20 +108,17 @@ cd llama-gpt
 Run LlamaGPT with the following command:
 
 ```
-./run.sh --model 7b
+./run.sh
 ```
 
 Or if you have an Nvidia GPU, you can run LlamaGPT with CUDA support using the `--with-cuda` flag, like:
 
 ```
-./run.sh --model 7b --with-cuda
+./run.sh --with-cuda
 ```
 
 You can access LlamaGPT at `http://localhost:3000`.
 
-> To run 13B or 70B chat models, replace `7b` with `13b` or `70b` respectively.
-> To run Code Llama 7B, 13B or 34B models, replace `7b` with `code-7b`, `code-13b` or `code-34b` respectively.
-
 To stop LlamaGPT, do `Ctrl + C` in Terminal.
 
 > Note: On the first run, it may take a while for the model to be downloaded to the `/models` directory. You may also see lots of output like this for a few minutes, which is normal:

docker-compose-cuda-ggml.yml

@@ -28,19 +28,19 @@ services:
               count: 1
               capabilities: [gpu]
 
-  llama-gpt-ui:
-    # TODO: Use this image instead of building from source after the next release
-    # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
-    build:
-      context: ./ui
-      dockerfile: Dockerfile
-    ports:
-      - 3000:3000
-    restart: on-failure
-    environment:
-      - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
-      - 'OPENAI_API_HOST=http://llama-gpt-api-cuda-ggml:8000'
-      - 'DEFAULT_MODEL=/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
-      - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
-      - 'WAIT_HOSTS=llama-gpt-api-cuda-ggml:8000'
-      - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'
+  # llama-gpt-ui:
+  #   # TODO: Use this image instead of building from source after the next release
+  #   # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
+  #   build:
+  #     context: ./ui
+  #     dockerfile: Dockerfile
+  #   ports:
+  #     - 3000:3000
+  #   restart: on-failure
+  #   environment:
+  #     - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
+  #     - 'OPENAI_API_HOST=http://llama-gpt-api-cuda-ggml:8000'
+  #     - 'DEFAULT_MODEL=/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
+  #     - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
+  #     - 'WAIT_HOSTS=llama-gpt-api-cuda-ggml:8000'
+  #     - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'

docker-compose-cuda-gguf.yml

@@ -28,19 +28,19 @@ services:
               count: 1
               capabilities: [gpu]
 
-  llama-gpt-ui:
-    # TODO: Use this image instead of building from source after the next release
-    # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
-    build:
-      context: ./ui
-      dockerfile: Dockerfile
-    ports:
-      - 3000:3000
-    restart: on-failure
-    environment:
-      - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
-      - 'OPENAI_API_HOST=http://llama-gpt-api-cuda-gguf:8000'
-      - 'DEFAULT_MODEL=/models/${MODEL_NAME:-code-llama-2-7b-chat.gguf}'
-      - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
-      - 'WAIT_HOSTS=llama-gpt-api-cuda-gguf:8000'
-      - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'
+  # llama-gpt-ui:
+  #   # TODO: Use this image instead of building from source after the next release
+  #   # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
+  #   build:
+  #     context: ./ui
+  #     dockerfile: Dockerfile
+  #   ports:
+  #     - 3000:3000
+  #   restart: on-failure
+  #   environment:
+  #     - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
+  #     - 'OPENAI_API_HOST=http://llama-gpt-api-cuda-gguf:8000'
+  #     - 'DEFAULT_MODEL=/models/${MODEL_NAME:-code-llama-2-7b-chat.gguf}'
+  #     - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
+  #     - 'WAIT_HOSTS=llama-gpt-api-cuda-gguf:8000'
+  #     - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'

docker-compose-gguf.yml

@@ -19,19 +19,19 @@ services:
       - IPC_LOCK
     command: '/bin/sh /api/run.sh'
 
-  llama-gpt-ui:
-    # TODO: Use this image instead of building from source after the next release
-    # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
-    build:
-      context: ./ui
-      dockerfile: Dockerfile
-    ports:
-      - 3000:3000
-    restart: on-failure
-    environment:
-      - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
-      - 'OPENAI_API_HOST=http://llama-gpt-api:8000'
-      - 'DEFAULT_MODEL=/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
-      - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
-      - 'WAIT_HOSTS=llama-gpt-api:8000'
-      - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'
+  # llama-gpt-ui:
+  #   # TODO: Use this image instead of building from source after the next release
+  #   # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
+  #   build:
+  #     context: ./ui
+  #     dockerfile: Dockerfile
+  #   ports:
+  #     - 3000:3000
+  #   restart: on-failure
+  #   environment:
+  #     - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
+  #     - 'OPENAI_API_HOST=http://llama-gpt-api:8000'
+  #     - 'DEFAULT_MODEL=/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
+  #     - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
+  #     - 'WAIT_HOSTS=llama-gpt-api:8000'
+  #     - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'

docker-compose-mac.yml → docker-compose-mac-ui.yml

@@ -11,5 +11,6 @@ services:
     environment:
       - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
       - 'OPENAI_API_HOST=http://host.docker.internal:3001'
+      - 'MODEL_MANAGER_ENDPOINT=http://host.docker.internal:3002'
       - 'DEFAULT_MODEL=$MODEL'
       - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely and use markdown if responding with code."}'

docker-compose-rocm-ggml.yml

@@ -0,0 +1,46 @@
+version: '3.6'
+
+services:
+  llama-gpt-api-rocm-ggml:
+    build:
+      context: ./rocm
+      dockerfile: ggml.Dockerfile
+    restart: on-failure
+    volumes:
+      - './models:/models'
+      - './rocm:/rocm'
+    ports:
+      - 3001:8000
+    environment:
+      MODEL: '/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
+      MODEL_DOWNLOAD_URL: '${MODEL_DOWNLOAD_URL:-https://huggingface.co/TheBloke/Nous-Hermes-Llama-2-7B-GGML/resolve/main/nous-hermes-llama-2-7b.ggmlv3.q4_0.bin}'
+      N_GQA: '${N_GQA:-1}'
+      USE_MLOCK: 1
+    cap_add:
+      - IPC_LOCK
+      - SYS_RESOURCE
+    command: '/bin/sh /rocm/run.sh'
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: amdgpu
+              count: 1
+              capabilities: [gpu]
+
+  llama-gpt-ui:
+    # TODO: Use this image instead of building from source after the next release
+    # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
+    build:
+      context: ./ui
+      dockerfile: Dockerfile
+    ports:
+      - 3000:3000
+    restart: on-failure
+    environment:
+      - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
+      - 'OPENAI_API_HOST=http://llama-gpt-api-rocm-ggml:8000'
+      - 'DEFAULT_MODEL=/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
+      - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
+      - 'WAIT_HOSTS=llama-gpt-api-rocm-ggml:8000'
+      - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'

docker-compose-rocm-gguf.yml

@@ -0,0 +1,46 @@
+version: '3.6'
+
+services:
+  llama-gpt-api-rocm-gguf:
+    build:
+      context: ./rocm
+      dockerfile: gguf.Dockerfile
+    restart: on-failure
+    volumes:
+      - './models:/models'
+      - './rocm:/rocm'
+    ports:
+      - 3001:8000
+    environment:
+      MODEL: '/models/${MODEL_NAME:-code-llama-2-7b-chat.gguf}'
+      MODEL_DOWNLOAD_URL: '${MODEL_DOWNLOAD_URL:-https://huggingface.co/TheBloke/CodeLlama-7B-Instruct-GGUF/resolve/main/codellama-7b-instruct.Q4_K_M.gguf}'
+      N_GQA: '${N_GQA:-1}'
+      USE_MLOCK: 1
+    cap_add:
+      - IPC_LOCK
+      - SYS_RESOURCE
+    command: '/bin/sh /rocm/run.sh'
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: amdgpu
+              count: 1
+              capabilities: [gpu]
+
+  llama-gpt-ui:
+    # TODO: Use this image instead of building from source after the next release
+    # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
+    build:
+      context: ./ui
+      dockerfile: Dockerfile
+    ports:
+      - 3000:3000
+    restart: on-failure
+    environment:
+      - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
+      - 'OPENAI_API_HOST=http://llama-gpt-api-rocm-gguf:8000'
+      - 'DEFAULT_MODEL=/models/${MODEL_NAME:-code-llama-2-7b-chat.gguf}'
+      - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
+      - 'WAIT_HOSTS=llama-gpt-api-rocm-gguf:8000'
+      - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'

docker-compose-ui.yml

@@ -0,0 +1,18 @@
+version: '3.6'
+
+services:
+  llama-gpt-ui:
+    # TODO: Use this image instead of building from source after the next release
+    # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
+    build:
+      context: ./ui
+      dockerfile: Dockerfile
+    ports:
+      - 3000:3000
+    restart: on-failure
+    environment:
+      - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
+      - 'OPENAI_API_HOST=http://llama-gpt-api:8000'
+      - 'MODEL_MANAGER_ENDPOINT=http://host.docker.internal:3002'
+      - 'DEFAULT_MODEL=/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
+      - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'

docker-compose.yml

@@ -19,19 +19,19 @@ services:
       - IPC_LOCK
     command: '/bin/sh /api/run.sh'
 
-  llama-gpt-ui:
-    # TODO: Use this image instead of building from source after the next release
-    # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
-    build:
-      context: ./ui
-      dockerfile: Dockerfile
-    ports:
-      - 3000:3000
-    restart: on-failure
-    environment:
-      - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
-      - 'OPENAI_API_HOST=http://llama-gpt-api:8000'
-      - 'DEFAULT_MODEL=/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
-      - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
-      - 'WAIT_HOSTS=llama-gpt-api:8000'
-      - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'
+  # llama-gpt-ui:
+  #   # TODO: Use this image instead of building from source after the next release
+  #   # image: 'ghcr.io/getumbrel/llama-gpt-ui:latest'
+  #   build:
+  #     context: ./ui
+  #     dockerfile: Dockerfile
+  #   ports:
+  #     - 3000:3000
+  #   restart: on-failure
+  #   environment:
+  #     - 'OPENAI_API_KEY=sk-XXXXXXXXXXXXXXXXXXXX'
+  #     - 'OPENAI_API_HOST=http://llama-gpt-api:8000'
+  #     - 'DEFAULT_MODEL=/models/${MODEL_NAME:-llama-2-7b-chat.bin}'
+  #     - 'NEXT_PUBLIC_DEFAULT_SYSTEM_PROMPT=${DEFAULT_SYSTEM_PROMPT:-"You are a helpful and friendly AI assistant. Respond very concisely."}'
+  #     - 'WAIT_HOSTS=llama-gpt-api:8000'
+  #     - 'WAIT_TIMEOUT=${WAIT_TIMEOUT:-3600}'