From b27dff67cfb4abd4180b2904912f631154af117e Mon Sep 17 00:00:00 2001 From: letonghan Date: Tue, 14 May 2024 21:13:46 +0800 Subject: [PATCH 1/4] add docker for chatqna mega service Signed-off-by: letonghan --- ChatQnA/microservice/xeon/README.md | 29 ++++++++++-- ChatQnA/microservice/xeon/chatqna.py | 10 ++-- ChatQnA/microservice/xeon/docker/Dockerfile | 51 +++++++++++++++++++++ 3 files changed, 81 insertions(+), 9 deletions(-) create mode 100644 ChatQnA/microservice/xeon/docker/Dockerfile diff --git a/ChatQnA/microservice/xeon/README.md b/ChatQnA/microservice/xeon/README.md index 0463d07cda..af5221139f 100644 --- a/ChatQnA/microservice/xeon/README.md +++ b/ChatQnA/microservice/xeon/README.md @@ -17,8 +17,6 @@ First of all, you need to build Docker Images locally and install the python pac ```bash git clone https://github.com/opea-project/GenAIComps.git cd GenAIComps -pip install -r requirements.txt -pip install . ``` ### 1. Build Embedding Image @@ -36,7 +34,7 @@ docker build -t opea/gen-ai-comps:retriever-redis-server --build-arg https_proxy ### 3. Build Rerank Image ```bash -docker build -t opea/gen-ai-comps:reranking-tei-xeon-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/docker/Dockerfile . +docker build -t opea/gen-ai-comps:reranking-tei-xeon-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/reranks/langchain/docker/Dockerfile . ``` ### 4. Build LLM Image @@ -146,9 +144,9 @@ curl http://${host_ip}:9009/generate \ 7. LLM Microservice ```bash -curl http://${host_ip}:9000/v1/chat/completions\ +curl http://${your_ip}:9000/v1/chat/completions\ -X POST \ - -d '{"text":"What is Deep Learning?"}' \ + -d '{"query":"What is Deep Learning?","max_new_tokens":17,"top_k":10,"top_p":0.95,"typical_p":0.95,"temperature":0.01,"repetition_penalty":1.03,"streaming":true}' \ -H 'Content-Type: application/json' ``` @@ -160,6 +158,27 @@ Modify the `initial_inputs` of line 34 in `chatqna.py`, then you will get the Ch All of the intermediate results will be printed for each microservices. Users can check the accuracy of the results to make targeted modifications. +### Run Mega Service with Python + ```bash +# install packages +cd /GenAIComps +pip install -r requirements.txt +pip install . +# run chatqna service +cd /GenAIExamples/ChatQnA/microservice/xeon python chatqna.py ``` + +### Run Mega Service with Docker +To run ChatQnA service with Docker, remember to pass the `${micro_service_host_ip}` variable into docker container, which is the real host ip of your microservices. +```bash +docker build -t opea/gen-ai-comps:chatqna-xeon-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . +docker run -d --name="chatqna-xeon-server" -p 8888:8888 --ipc=host -e https_proxy=$https_proxy -e http_proxy=$http_proxy -e SERVICE_SERVICE_HOST_IP=${micro_service_host_ip} opea/gen-ai-comps:chatqna-xeon-server +``` + +Then you can check the result of your chatqna service with the command below. + +```bash +docker logs chatqna-xeon-server +``` \ No newline at end of file diff --git a/ChatQnA/microservice/xeon/chatqna.py b/ChatQnA/microservice/xeon/chatqna.py index 61598a0c1a..ee78e06d21 100644 --- a/ChatQnA/microservice/xeon/chatqna.py +++ b/ChatQnA/microservice/xeon/chatqna.py @@ -13,8 +13,10 @@ # limitations under the License. +import os from comps import MicroService, ServiceOrchestrator +SERVICE_HOST_IP = os.getenv("SERVICE_SERVICE_HOST_IP", "0.0.0.0") class ChatQnAService: def __init__(self, port=8000): @@ -22,16 +24,16 @@ def __init__(self, port=8000): def add_remote_service(self): embedding = MicroService( - name="embedding", host="0.0.0.0", port=6000, expose_endpoint="/v1/embeddings", use_remote_service=True + name="embedding", host=SERVICE_HOST_IP, port=6000, expose_endpoint="/v1/embeddings", use_remote_service=True ) retriever = MicroService( - name="retriever", host="0.0.0.0", port=7000, expose_endpoint="/v1/retrieval", use_remote_service=True + name="retriever", host=SERVICE_HOST_IP, port=7000, expose_endpoint="/v1/retrieval", use_remote_service=True ) rerank = MicroService( - name="rerank", host="0.0.0.0", port=8000, expose_endpoint="/v1/reranking", use_remote_service=True + name="rerank", host=SERVICE_HOST_IP, port=8000, expose_endpoint="/v1/reranking", use_remote_service=True ) llm = MicroService( - name="llm", host="0.0.0.0", port=9000, expose_endpoint="/v1/chat/completions", use_remote_service=True + name="llm", host=SERVICE_HOST_IP, port=9000, expose_endpoint="/v1/chat/completions", use_remote_service=True ) self.service_builder.add(embedding).add(retriever).add(rerank).add(llm) self.service_builder.flow_to(embedding, retriever) diff --git a/ChatQnA/microservice/xeon/docker/Dockerfile b/ChatQnA/microservice/xeon/docker/Dockerfile new file mode 100644 index 0000000000..728820160b --- /dev/null +++ b/ChatQnA/microservice/xeon/docker/Dockerfile @@ -0,0 +1,51 @@ +# Copyright (c) 2024 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SCRIPT USAGE NOTICE: By downloading and using any script file included +# with the associated software package (such as files with .bat, .cmd, or +# .JS extensions, Docker files, or any other type of file that, when executed, +# automatically downloads and/or installs files onto your system) (the “Script File”), +# it is your obligation to review the Script File to understand what files (e.g., +# other software, AI models, AI Datasets) the Script File will download to your system +# (“Downloaded Files”). Furthermore, by downloading and using the Downloaded Files, +# even if they are installed through a silent install, you agree to any and all +# terms and conditions associated with such files, including but not limited to, +# license terms, notices, or disclaimers. + +FROM langchain/langchain:latest + +RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \ + libgl1-mesa-glx \ + libjemalloc-dev \ + vim + +RUN useradd -m -s /bin/bash user && \ + mkdir -p /home/user && \ + chown -R user /home/user/ + +RUN cd /home/user/ && \ + git clone https://github.com/opea-project/GenAIComps.git + +RUN cd /home/user/GenAIComps && pip install --no-cache-dir --upgrade pip && \ + pip install -r requirements.txt && pip install . + +COPY ../chatqna.py /home/user/chatqna.py + +ENV PYTHONPATH=$PYTHONPATH:/home/user + +USER user + +WORKDIR /home/user + +ENTRYPOINT ["python", "chatqna.py"] From 61a0a850dc6067adc79e694444d3125332e17bc0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 May 2024 13:16:28 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- ChatQnA/microservice/xeon/README.md | 4 +++- ChatQnA/microservice/xeon/chatqna.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChatQnA/microservice/xeon/README.md b/ChatQnA/microservice/xeon/README.md index af5221139f..7a52f3d7e2 100644 --- a/ChatQnA/microservice/xeon/README.md +++ b/ChatQnA/microservice/xeon/README.md @@ -171,7 +171,9 @@ python chatqna.py ``` ### Run Mega Service with Docker + To run ChatQnA service with Docker, remember to pass the `${micro_service_host_ip}` variable into docker container, which is the real host ip of your microservices. + ```bash docker build -t opea/gen-ai-comps:chatqna-xeon-server --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f docker/Dockerfile . docker run -d --name="chatqna-xeon-server" -p 8888:8888 --ipc=host -e https_proxy=$https_proxy -e http_proxy=$http_proxy -e SERVICE_SERVICE_HOST_IP=${micro_service_host_ip} opea/gen-ai-comps:chatqna-xeon-server @@ -181,4 +183,4 @@ Then you can check the result of your chatqna service with the command below. ```bash docker logs chatqna-xeon-server -``` \ No newline at end of file +``` diff --git a/ChatQnA/microservice/xeon/chatqna.py b/ChatQnA/microservice/xeon/chatqna.py index ee78e06d21..52851ce64b 100644 --- a/ChatQnA/microservice/xeon/chatqna.py +++ b/ChatQnA/microservice/xeon/chatqna.py @@ -14,10 +14,12 @@ import os + from comps import MicroService, ServiceOrchestrator SERVICE_HOST_IP = os.getenv("SERVICE_SERVICE_HOST_IP", "0.0.0.0") + class ChatQnAService: def __init__(self, port=8000): self.service_builder = ServiceOrchestrator(host="0.0.0.0", port=port, endpoint="/v1/chatqna") From 2b8a988d3406e41d167a5c29184cc556cc1e4e24 Mon Sep 17 00:00:00 2001 From: letonghan Date: Tue, 14 May 2024 21:17:08 +0800 Subject: [PATCH 3/4] update readme Signed-off-by: letonghan --- ChatQnA/microservice/xeon/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChatQnA/microservice/xeon/README.md b/ChatQnA/microservice/xeon/README.md index af5221139f..8f5abbb73a 100644 --- a/ChatQnA/microservice/xeon/README.md +++ b/ChatQnA/microservice/xeon/README.md @@ -144,7 +144,7 @@ curl http://${host_ip}:9009/generate \ 7. LLM Microservice ```bash -curl http://${your_ip}:9000/v1/chat/completions\ +curl http://${host_ip}:9000/v1/chat/completions\ -X POST \ -d '{"query":"What is Deep Learning?","max_new_tokens":17,"top_k":10,"top_p":0.95,"typical_p":0.95,"temperature":0.01,"repetition_penalty":1.03,"streaming":true}' \ -H 'Content-Type: application/json' From ad407e6ad6847a7581ee312a9d6e142a19d2290d Mon Sep 17 00:00:00 2001 From: letonghan Date: Tue, 14 May 2024 21:18:30 +0800 Subject: [PATCH 4/4] update dockerfile Signed-off-by: letonghan --- ChatQnA/microservice/xeon/docker/Dockerfile | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ChatQnA/microservice/xeon/docker/Dockerfile b/ChatQnA/microservice/xeon/docker/Dockerfile index 728820160b..6ed5693526 100644 --- a/ChatQnA/microservice/xeon/docker/Dockerfile +++ b/ChatQnA/microservice/xeon/docker/Dockerfile @@ -12,16 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# SCRIPT USAGE NOTICE: By downloading and using any script file included -# with the associated software package (such as files with .bat, .cmd, or -# .JS extensions, Docker files, or any other type of file that, when executed, -# automatically downloads and/or installs files onto your system) (the “Script File”), -# it is your obligation to review the Script File to understand what files (e.g., -# other software, AI models, AI Datasets) the Script File will download to your system -# (“Downloaded Files”). Furthermore, by downloading and using the Downloaded Files, -# even if they are installed through a silent install, you agree to any and all -# terms and conditions associated with such files, including but not limited to, -# license terms, notices, or disclaimers. FROM langchain/langchain:latest