Skip to content

Commit a54ffd2

Browse files
lvliang-intellvliang-intel
authored
Support ChatQnA pipeline without rerank microservice (#643)
Signed-off-by: lvliang-intel <liang1.lv@intel.com>
1 parent f3ffcd5 commit a54ffd2

7 files changed

+910
-0
lines changed

‎ChatQnA/docker/Dockerfile_without_rerank

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
2+
3+
# Copyright (C) 2024 Intel Corporation
4+
# SPDX-License-Identifier: Apache-2.0
5+
6+
FROM python:3.11-slim
7+
8+
RUN apt-get update -y && apt-get install -y --no-install-recommends --fix-missing \
9+
libgl1-mesa-glx \
10+
libjemalloc-dev \
11+
vim \
12+
git
13+
14+
RUN useradd -m -s /bin/bash user && \
15+
mkdir -p /home/user && \
16+
chown -R user /home/user/
17+
18+
WORKDIR /home/user/
19+
RUN git clone https://github.com/opea-project/GenAIComps.git
20+
21+
WORKDIR /home/user/GenAIComps
22+
RUN pip install --no-cache-dir --upgrade pip && \
23+
pip install --no-cache-dir -r /home/user/GenAIComps/requirements.txt
24+
25+
COPY ./chatqna_without_rerank.py /home/user/chatqna_without_rerank.py
26+
27+
ENV PYTHONPATH=$PYTHONPATH:/home/user/GenAIComps
28+
29+
USER user
30+
31+
WORKDIR /home/user
32+
33+
ENTRYPOINT ["python", "chatqna_without_rerank.py"]

‎ChatQnA/docker/chatqna_without_rerank.py

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
import os
5+
6+
from comps import ChatQnAGateway, MicroService, ServiceOrchestrator, ServiceType
7+
8+
MEGA_SERVICE_HOST_IP = os.getenv("MEGA_SERVICE_HOST_IP", "0.0.0.0")
9+
MEGA_SERVICE_PORT = int(os.getenv("MEGA_SERVICE_PORT", 8888))
10+
EMBEDDING_SERVICE_HOST_IP = os.getenv("EMBEDDING_SERVICE_HOST_IP", "0.0.0.0")
11+
EMBEDDING_SERVICE_PORT = int(os.getenv("EMBEDDING_SERVICE_PORT", 6000))
12+
RETRIEVER_SERVICE_HOST_IP = os.getenv("RETRIEVER_SERVICE_HOST_IP", "0.0.0.0")
13+
RETRIEVER_SERVICE_PORT = int(os.getenv("RETRIEVER_SERVICE_PORT", 7000))
14+
LLM_SERVICE_HOST_IP = os.getenv("LLM_SERVICE_HOST_IP", "0.0.0.0")
15+
LLM_SERVICE_PORT = int(os.getenv("LLM_SERVICE_PORT", 9000))
16+
17+
18+
class ChatQnAService:
19+
def __init__(self, host="0.0.0.0", port=8000):
20+
self.host = host
21+
self.port = port
22+
self.megaservice = ServiceOrchestrator()
23+
24+
def add_remote_service(self):
25+
embedding = MicroService(
26+
name="embedding",
27+
host=EMBEDDING_SERVICE_HOST_IP,
28+
port=EMBEDDING_SERVICE_PORT,
29+
endpoint="/v1/embeddings",
30+
use_remote_service=True,
31+
service_type=ServiceType.EMBEDDING,
32+
)
33+
retriever = MicroService(
34+
name="retriever",
35+
host=RETRIEVER_SERVICE_HOST_IP,
36+
port=RETRIEVER_SERVICE_PORT,
37+
endpoint="/v1/retrieval",
38+
use_remote_service=True,
39+
service_type=ServiceType.RETRIEVER,
40+
)
41+
llm = MicroService(
42+
name="llm",
43+
host=LLM_SERVICE_HOST_IP,
44+
port=LLM_SERVICE_PORT,
45+
endpoint="/v1/chat/completions",
46+
use_remote_service=True,
47+
service_type=ServiceType.LLM,
48+
)
49+
self.megaservice.add(embedding).add(retriever).add(llm)
50+
self.megaservice.flow_to(embedding, retriever)
51+
self.megaservice.flow_to(retriever, llm)
52+
self.gateway = ChatQnAGateway(megaservice=self.megaservice, host="0.0.0.0", port=self.port)
53+
54+
55+
if __name__ == "__main__":
56+
chatqna = ChatQnAService(host=MEGA_SERVICE_HOST_IP, port=MEGA_SERVICE_PORT)
57+
chatqna.add_remote_service()

‎ChatQnA/docker/docker_build_compose.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ services:
1515
dockerfile: ./Dockerfile_guardrails
1616
extends: chatqna
1717
image: ${REGISTRY:-opea}/chatqna-guardrails:${TAG:-latest}
18+
chatqna-without-rerank:
19+
build:
20+
dockerfile: ./Dockerfile_without_rerank
21+
extends: chatqna
22+
image: ${REGISTRY:-opea}/chatqna-without-rerank:${TAG:-latest}
1823
chatqna-ui:
1924
build:
2025
context: ui

‎ChatQnA/docker/gaudi/compose_without_rerank.yaml

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
services:
5+
redis-vector-db:
6+
image: redis/redis-stack:7.2.0-v9
7+
container_name: redis-vector-db
8+
ports:
9+
- "6379:6379"
10+
- "8001:8001"
11+
dataprep-redis-service:
12+
image: ${REGISTRY:-opea}/dataprep-redis:${TAG:-latest}
13+
container_name: dataprep-redis-server
14+
depends_on:
15+
- redis-vector-db
16+
- tei-embedding-service
17+
ports:
18+
- "6007:6007"
19+
environment:
20+
no_proxy: ${no_proxy}
21+
http_proxy: ${http_proxy}
22+
https_proxy: ${https_proxy}
23+
REDIS_URL: ${REDIS_URL}
24+
INDEX_NAME: ${INDEX_NAME}
25+
TEI_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
26+
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
27+
tei-embedding-service:
28+
image: ${REGISTRY:-opea}/tei-gaudi:${TAG:-latest}
29+
container_name: tei-embedding-gaudi-server
30+
ports:
31+
- "8090:80"
32+
volumes:
33+
- "./data:/data"
34+
runtime: habana
35+
cap_add:
36+
- SYS_NICE
37+
ipc: host
38+
environment:
39+
no_proxy: ${no_proxy}
40+
http_proxy: ${http_proxy}
41+
https_proxy: ${https_proxy}
42+
HABANA_VISIBLE_DEVICES: all
43+
OMPI_MCA_btl_vader_single_copy_mechanism: none
44+
MAX_WARMUP_SEQUENCE_LENGTH: 512
45+
INIT_HCCL_ON_ACQUIRE: 0
46+
ENABLE_EXPERIMENTAL_FLAGS: true
47+
command: --model-id ${EMBEDDING_MODEL_ID} --auto-truncate
48+
embedding:
49+
image: ${REGISTRY:-opea}/embedding-tei:${TAG:-latest}
50+
container_name: embedding-tei-server
51+
depends_on:
52+
- tei-embedding-service
53+
ports:
54+
- "6000:6000"
55+
ipc: host
56+
environment:
57+
no_proxy: ${no_proxy}
58+
http_proxy: ${http_proxy}
59+
https_proxy: ${https_proxy}
60+
TEI_EMBEDDING_ENDPOINT: ${TEI_EMBEDDING_ENDPOINT}
61+
restart: unless-stopped
62+
retriever:
63+
image: ${REGISTRY:-opea}/retriever-redis:${TAG:-latest}
64+
container_name: retriever-redis-server
65+
depends_on:
66+
- redis-vector-db
67+
ports:
68+
- "7000:7000"
69+
ipc: host
70+
environment:
71+
no_proxy: ${no_proxy}
72+
http_proxy: ${http_proxy}
73+
https_proxy: ${https_proxy}
74+
REDIS_URL: ${REDIS_URL}
75+
INDEX_NAME: ${INDEX_NAME}
76+
restart: unless-stopped
77+
tgi-service:
78+
image: ghcr.io/huggingface/tgi-gaudi:2.0.1
79+
container_name: tgi-gaudi-server
80+
ports:
81+
- "8005:80"
82+
volumes:
83+
- "./data:/data"
84+
environment:
85+
no_proxy: ${no_proxy}
86+
http_proxy: ${http_proxy}
87+
https_proxy: ${https_proxy}
88+
HF_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
89+
HF_HUB_DISABLE_PROGRESS_BARS: 1
90+
HF_HUB_ENABLE_HF_TRANSFER: 0
91+
HABANA_VISIBLE_DEVICES: all
92+
OMPI_MCA_btl_vader_single_copy_mechanism: none
93+
runtime: habana
94+
cap_add:
95+
- SYS_NICE
96+
ipc: host
97+
command: --model-id ${LLM_MODEL_ID} --max-input-length 1024 --max-total-tokens 2048
98+
llm:
99+
image: ${REGISTRY:-opea}/llm-tgi:${TAG:-latest}
100+
container_name: llm-tgi-gaudi-server
101+
depends_on:
102+
- tgi-service
103+
ports:
104+
- "9000:9000"
105+
ipc: host
106+
environment:
107+
no_proxy: ${no_proxy}
108+
http_proxy: ${http_proxy}
109+
https_proxy: ${https_proxy}
110+
TGI_LLM_ENDPOINT: ${TGI_LLM_ENDPOINT}
111+
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
112+
HF_HUB_DISABLE_PROGRESS_BARS: 1
113+
HF_HUB_ENABLE_HF_TRANSFER: 0
114+
restart: unless-stopped
115+
chaqna-gaudi-backend-server:
116+
image: ${REGISTRY:-opea}/chatqna-without-rerank:${TAG:-latest}
117+
container_name: chatqna-gaudi-backend-server
118+
depends_on:
119+
- redis-vector-db
120+
- tei-embedding-service
121+
- embedding
122+
- retriever
123+
- tgi-service
124+
- llm
125+
ports:
126+
- "8888:8888"
127+
environment:
128+
- no_proxy=${no_proxy}
129+
- https_proxy=${https_proxy}
130+
- http_proxy=${http_proxy}
131+
- MEGA_SERVICE_HOST_IP=${MEGA_SERVICE_HOST_IP}
132+
- EMBEDDING_SERVICE_HOST_IP=${EMBEDDING_SERVICE_HOST_IP}
133+
- RETRIEVER_SERVICE_HOST_IP=${RETRIEVER_SERVICE_HOST_IP}
134+
- LLM_SERVICE_HOST_IP=${LLM_SERVICE_HOST_IP}
135+
ipc: host
136+
restart: always
137+
chaqna-gaudi-ui-server:
138+
image: ${REGISTRY:-opea}/chatqna-ui:${TAG:-latest}
139+
container_name: chatqna-gaudi-ui-server
140+
depends_on:
141+
- chaqna-gaudi-backend-server
142+
ports:
143+
- "5173:5173"
144+
environment:
145+
- no_proxy=${no_proxy}
146+
- https_proxy=${https_proxy}
147+
- http_proxy=${http_proxy}
148+
- CHAT_BASE_URL=${BACKEND_SERVICE_ENDPOINT}
149+
- UPLOAD_FILE_BASE_URL=${DATAPREP_SERVICE_ENDPOINT}
150+
- GET_FILE=${DATAPREP_GET_FILE_ENDPOINT}
151+
- DELETE_FILE=${DATAPREP_DELETE_FILE_ENDPOINT}
152+
ipc: host
153+
restart: always
154+
155+
networks:
156+
default:
157+
driver: bridge

0 commit comments

Comments
 (0)