feat: Add exllama

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
mudler · Aug 10, 2023 · afa71c8 · afa71c8
1 parent 8c781a6
commit afa71c8
Show file tree

Hide file tree

Showing 4 changed files with 490 additions and 2 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -11,7 +11,7 @@ ARG TARGETARCH
 ARG TARGETVARIANT
 
 ENV BUILD_TYPE=${BUILD_TYPE}
-ENV EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/build/extra/grpc/huggingface/huggingface.py,autogptq:/build/extra/grpc/autogptq/autogptq.py,bark:/build/extra/grpc/bark/ttsbark.py,diffusers:/build/extra/grpc/diffusers/backend_diffusers.py"
+ENV EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/build/extra/grpc/huggingface/huggingface.py,autogptq:/build/extra/grpc/autogptq/autogptq.py,bark:/build/extra/grpc/bark/ttsbark.py,diffusers:/build/extra/grpc/diffusers/backend_diffusers.py,exllama:/build/extra/grpc/exllama/exllama.py"
 ARG GO_TAGS="stablediffusion tts"
 
 RUN apt-get update && \
@@ -42,7 +42,7 @@ RUN if [ "${TARGETARCH}" = "amd64" ]; then \
         pip install git+https://github.com/suno-ai/bark.git diffusers invisible_watermark transformers accelerate safetensors;\
     fi
 RUN if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "amd64" ]; then \
-        pip install torch && pip install auto-gptq;\
+        pip install torch && pip install auto-gptq git+https://github.com/jllllll/exllama;\
     fi
 RUN pip install -r /build/extra/requirements.txt && rm -rf /build/extra/requirements.txt
 

diff --git a/extra/grpc/exllama/backend_pb2.py b/extra/grpc/exllama/backend_pb2.py