Skip to content

Commit df7c192

Browse files
artem-astafevartem-astafev
authored
Added docker compose example for AgentQnA deployment on AMD ROCm (#1166)
Signed-off-by: artem-astafev <a.astafev@datamonsters.com>
1 parent f930638 commit df7c192

File tree

5 files changed

+341
-0
lines changed

5 files changed

+341
-0
lines changed

‎AgentQnA/docker_compose/amd/gpu/rocm/compose.yaml

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
services:
5+
agent-tgi-server:
6+
image: ${AGENTQNA_TGI_IMAGE}
7+
container_name: agent-tgi-server
8+
ports:
9+
- "${AGENTQNA_TGI_SERVICE_PORT-8085}:80"
10+
volumes:
11+
- /var/opea/agent-service/:/data
12+
environment:
13+
no_proxy: ${no_proxy}
14+
http_proxy: ${http_proxy}
15+
https_proxy: ${https_proxy}
16+
TGI_LLM_ENDPOINT: "http://${HOST_IP}:${AGENTQNA_TGI_SERVICE_PORT}"
17+
HUGGING_FACE_HUB_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
18+
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
19+
shm_size: 1g
20+
devices:
21+
- /dev/kfd:/dev/kfd
22+
- /dev/dri/${AGENTQNA_CARD_ID}:/dev/dri/${AGENTQNA_CARD_ID}
23+
- /dev/dri/${AGENTQNA_RENDER_ID}:/dev/dri/${AGENTQNA_RENDER_ID}
24+
cap_add:
25+
- SYS_PTRACE
26+
group_add:
27+
- video
28+
security_opt:
29+
- seccomp:unconfined
30+
ipc: host
31+
command: --model-id ${LLM_MODEL_ID} --max-input-length 4096 --max-total-tokens 8192
32+
33+
worker-rag-agent:
34+
image: opea/agent-langchain:latest
35+
container_name: rag-agent-endpoint
36+
volumes:
37+
# - ${WORKDIR}/GenAIExamples/AgentQnA/docker_image_build/GenAIComps/comps/agent/langchain/:/home/user/comps/agent/langchain/
38+
- ${TOOLSET_PATH}:/home/user/tools/
39+
ports:
40+
- "9095:9095"
41+
ipc: host
42+
environment:
43+
ip_address: ${ip_address}
44+
strategy: rag_agent_llama
45+
recursion_limit: ${recursion_limit_worker}
46+
llm_engine: tgi
47+
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
48+
llm_endpoint_url: ${LLM_ENDPOINT_URL}
49+
model: ${LLM_MODEL_ID}
50+
temperature: ${temperature}
51+
max_new_tokens: ${max_new_tokens}
52+
streaming: false
53+
tools: /home/user/tools/worker_agent_tools.yaml
54+
require_human_feedback: false
55+
RETRIEVAL_TOOL_URL: ${RETRIEVAL_TOOL_URL}
56+
no_proxy: ${no_proxy}
57+
http_proxy: ${http_proxy}
58+
https_proxy: ${https_proxy}
59+
LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
60+
LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
61+
LANGCHAIN_PROJECT: "opea-worker-agent-service"
62+
port: 9095
63+
64+
supervisor-react-agent:
65+
image: opea/agent-langchain:latest
66+
container_name: react-agent-endpoint
67+
depends_on:
68+
- agent-tgi-server
69+
- worker-rag-agent
70+
volumes:
71+
# - ${WORKDIR}/GenAIExamples/AgentQnA/docker_image_build/GenAIComps/comps/agent/langchain/:/home/user/comps/agent/langchain/
72+
- ${TOOLSET_PATH}:/home/user/tools/
73+
ports:
74+
- "${AGENTQNA_FRONTEND_PORT}:9090"
75+
ipc: host
76+
environment:
77+
ip_address: ${ip_address}
78+
strategy: react_langgraph
79+
recursion_limit: ${recursion_limit_supervisor}
80+
llm_engine: tgi
81+
HUGGINGFACEHUB_API_TOKEN: ${HUGGINGFACEHUB_API_TOKEN}
82+
llm_endpoint_url: ${LLM_ENDPOINT_URL}
83+
model: ${LLM_MODEL_ID}
84+
temperature: ${temperature}
85+
max_new_tokens: ${max_new_tokens}
86+
streaming: false
87+
tools: /home/user/tools/supervisor_agent_tools.yaml
88+
require_human_feedback: false
89+
no_proxy: ${no_proxy}
90+
http_proxy: ${http_proxy}
91+
https_proxy: ${https_proxy}
92+
LANGCHAIN_API_KEY: ${LANGCHAIN_API_KEY}
93+
LANGCHAIN_TRACING_V2: ${LANGCHAIN_TRACING_V2}
94+
LANGCHAIN_PROJECT: "opea-supervisor-agent-service"
95+
CRAG_SERVER: $CRAG_SERVER
96+
WORKER_AGENT_URL: $WORKER_AGENT_URL
97+
port: 9090

‎AgentQnA/docker_compose/amd/gpu/rocm/launch_agent_service_tgi_rocm.sh

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Copyright (C) 2024 Advanced Micro Devices, Inc.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
WORKPATH=$(dirname "$PWD")/..
5+
export ip_address=${host_ip}
6+
export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
7+
export AGENTQNA_TGI_IMAGE=ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
8+
export AGENTQNA_TGI_SERVICE_PORT="8085"
9+
10+
# LLM related environment variables
11+
export AGENTQNA_CARD_ID="card1"
12+
export AGENTQNA_RENDER_ID="renderD136"
13+
export HF_CACHE_DIR=${HF_CACHE_DIR}
14+
ls $HF_CACHE_DIR
15+
export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
16+
#export NUM_SHARDS=4
17+
export LLM_ENDPOINT_URL="http://${ip_address}:${AGENTQNA_TGI_SERVICE_PORT}"
18+
export temperature=0.01
19+
export max_new_tokens=512
20+
21+
# agent related environment variables
22+
export AGENTQNA_WORKER_AGENT_SERVICE_PORT="9095"
23+
export TOOLSET_PATH=/home/huggingface/datamonsters/amd-opea/GenAIExamples/AgentQnA/tools/
24+
echo "TOOLSET_PATH=${TOOLSET_PATH}"
25+
export recursion_limit_worker=12
26+
export recursion_limit_supervisor=10
27+
export WORKER_AGENT_URL="http://${ip_address}:${AGENTQNA_WORKER_AGENT_SERVICE_PORT}/v1/chat/completions"
28+
export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
29+
export CRAG_SERVER=http://${ip_address}:18881
30+
31+
export AGENTQNA_FRONTEND_PORT="9090"
32+
33+
#retrieval_tool
34+
export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
35+
export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
36+
export REDIS_URL="redis://${host_ip}:26379"
37+
export INDEX_NAME="rag-redis"
38+
export MEGA_SERVICE_HOST_IP=${host_ip}
39+
export EMBEDDING_SERVICE_HOST_IP=${host_ip}
40+
export RETRIEVER_SERVICE_HOST_IP=${host_ip}
41+
export RERANK_SERVICE_HOST_IP=${host_ip}
42+
export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
43+
export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep"
44+
export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get_file"
45+
export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/delete_file"
46+
47+
docker compose -f compose.yaml up -d

‎AgentQnA/docker_compose/amd/gpu/rocm/set_env.sh

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright (C) 2024 Advanced Micro Devices, Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
6+
WORKPATH=$(dirname "$PWD")/..
7+
export ip_address=${host_ip}
8+
export HUGGINGFACEHUB_API_TOKEN=${your_hf_api_token}
9+
export AGENTQNA_TGI_IMAGE=ghcr.io/huggingface/text-generation-inference:2.3.1-rocm
10+
export AGENTQNA_TGI_SERVICE_PORT="19001"
11+
12+
# LLM related environment variables
13+
export AGENTQNA_CARD_ID="card1"
14+
export AGENTQNA_RENDER_ID="renderD136"
15+
export HF_CACHE_DIR=${HF_CACHE_DIR}
16+
ls $HF_CACHE_DIR
17+
export LLM_MODEL_ID="meta-llama/Meta-Llama-3-8B-Instruct"
18+
export NUM_SHARDS=4
19+
export LLM_ENDPOINT_URL="http://${ip_address}:${AGENTQNA_TGI_SERVICE_PORT}"
20+
export temperature=0.01
21+
export max_new_tokens=512
22+
23+
# agent related environment variables
24+
export AGENTQNA_WORKER_AGENT_SERVICE_PORT="9095"
25+
export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
26+
export recursion_limit_worker=12
27+
export recursion_limit_supervisor=10
28+
export WORKER_AGENT_URL="http://${ip_address}:${AGENTQNA_WORKER_AGENT_SERVICE_PORT}/v1/chat/completions"
29+
export RETRIEVAL_TOOL_URL="http://${ip_address}:8889/v1/retrievaltool"
30+
export CRAG_SERVER=http://${ip_address}:18881
31+
32+
export AGENTQNA_FRONTEND_PORT="15557"
33+
34+
#retrieval_tool
35+
export TEI_EMBEDDING_ENDPOINT="http://${host_ip}:6006"
36+
export TEI_RERANKING_ENDPOINT="http://${host_ip}:8808"
37+
export REDIS_URL="redis://${host_ip}:26379"
38+
export INDEX_NAME="rag-redis"
39+
export MEGA_SERVICE_HOST_IP=${host_ip}
40+
export EMBEDDING_SERVICE_HOST_IP=${host_ip}
41+
export RETRIEVER_SERVICE_HOST_IP=${host_ip}
42+
export RERANK_SERVICE_HOST_IP=${host_ip}
43+
export BACKEND_SERVICE_ENDPOINT="http://${host_ip}:8889/v1/retrievaltool"
44+
export DATAPREP_SERVICE_ENDPOINT="http://${host_ip}:6007/v1/dataprep"
45+
export DATAPREP_GET_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/get_file"
46+
export DATAPREP_DELETE_FILE_ENDPOINT="http://${host_ip}:6007/v1/dataprep/delete_file"

‎AgentQnA/tests/step4a_launch_and_validate_agent_tgi_on_rocm.sh

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/bin/bash
2+
# Copyright (C) 2024 Intel Corporation
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -ex
6+
7+
WORKPATH=$(dirname "$PWD")
8+
export WORKDIR=$WORKPATH/../../
9+
echo "WORKDIR=${WORKDIR}"
10+
export ip_address=$(hostname -I | awk '{print $1}')
11+
export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
12+
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
13+
14+
export HF_CACHE_DIR=$WORKDIR/hf_cache
15+
if [ ! -d "$HF_CACHE_DIR" ]; then
16+
mkdir -p "$HF_CACHE_DIR"
17+
fi
18+
ls $HF_CACHE_DIR
19+
20+
21+
function start_agent_and_api_server() {
22+
echo "Starting CRAG server"
23+
docker run -d --runtime=runc --name=kdd-cup-24-crag-service -p=8080:8000 docker.io/aicrowd/kdd-cup-24-crag-mock-api:v0
24+
25+
echo "Starting Agent services"
26+
cd $WORKDIR/GenAIExamples/AgentQnA/docker_compose/amd/gpu/rocm
27+
bash launch_agent_service_tgi_rocm.sh
28+
}
29+
30+
function validate() {
31+
local CONTENT="$1"
32+
local EXPECTED_RESULT="$2"
33+
local SERVICE_NAME="$3"
34+
35+
if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then
36+
echo "[ $SERVICE_NAME ] Content is as expected: $CONTENT"
37+
echo 0
38+
else
39+
echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT"
40+
echo 1
41+
fi
42+
}
43+
44+
function validate_agent_service() {
45+
echo "----------------Test agent ----------------"
46+
local CONTENT=$(http_proxy="" curl http://${ip_address}:9095/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
47+
"query": "Tell me about Michael Jackson song thriller"
48+
}')
49+
local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint")
50+
docker logs rag-agent-endpoint
51+
if [ "$EXIT_CODE" == "1" ]; then
52+
exit 1
53+
fi
54+
55+
local CONTENT=$(http_proxy="" curl http://${ip_address}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
56+
"query": "Tell me about Michael Jackson song thriller"
57+
}')
58+
local EXIT_CODE=$(validate "$CONTENT" "Thriller" "react-agent-endpoint")
59+
docker logs react-agent-endpoint
60+
if [ "$EXIT_CODE" == "1" ]; then
61+
exit 1
62+
fi
63+
64+
}
65+
66+
function main() {
67+
echo "==================== Start agent ===================="
68+
start_agent_and_api_server
69+
echo "==================== Agent started ===================="
70+
71+
echo "==================== Validate agent service ===================="
72+
validate_agent_service
73+
echo "==================== Agent service validated ===================="
74+
}
75+
76+
main

‎AgentQnA/tests/test_compose_on_rocm.sh

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/bin/bash
2+
# Copyright (C) 2024 Advanced Micro Devices, Inc.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
set -e
6+
7+
WORKPATH=$(dirname "$PWD")
8+
export WORKDIR=$WORKPATH/../../
9+
echo "WORKDIR=${WORKDIR}"
10+
export ip_address=$(hostname -I | awk '{print $1}')
11+
export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN}
12+
export TOOLSET_PATH=$WORKDIR/GenAIExamples/AgentQnA/tools/
13+
14+
function stop_crag() {
15+
cid=$(docker ps -aq --filter "name=kdd-cup-24-crag-service")
16+
echo "Stopping container kdd-cup-24-crag-service with cid $cid"
17+
if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
18+
}
19+
20+
function stop_agent_docker() {
21+
cd $WORKPATH/docker_compose/amd/gpu/rocm
22+
# docker compose -f compose.yaml down
23+
container_list=$(cat compose.yaml | grep container_name | cut -d':' -f2)
24+
for container_name in $container_list; do
25+
cid=$(docker ps -aq --filter "name=$container_name")
26+
echo "Stopping container $container_name"
27+
if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
28+
done
29+
}
30+
31+
function stop_retrieval_tool() {
32+
echo "Stopping Retrieval tool"
33+
local RETRIEVAL_TOOL_PATH=$WORKPATH/../DocIndexRetriever
34+
cd $RETRIEVAL_TOOL_PATH/docker_compose/intel/cpu/xeon/
35+
# docker compose -f compose.yaml down
36+
container_list=$(cat compose.yaml | grep container_name | cut -d':' -f2)
37+
for container_name in $container_list; do
38+
cid=$(docker ps -aq --filter "name=$container_name")
39+
echo "Stopping container $container_name"
40+
if [[ ! -z "$cid" ]]; then docker rm $cid -f && sleep 1s; fi
41+
done
42+
}
43+
echo "workpath: $WORKPATH"
44+
echo "=================== Stop containers ===================="
45+
stop_crag
46+
stop_agent_docker
47+
stop_retrieval_tool
48+
49+
cd $WORKPATH/tests
50+
51+
echo "=================== #1 Building docker images===================="
52+
bash step1_build_images.sh
53+
echo "=================== #1 Building docker images completed===================="
54+
55+
echo "=================== #2 Start retrieval tool===================="
56+
bash step2_start_retrieval_tool.sh
57+
echo "=================== #2 Retrieval tool started===================="
58+
59+
echo "=================== #3 Ingest data and validate retrieval===================="
60+
bash step3_ingest_data_and_validate_retrieval.sh
61+
echo "=================== #3 Data ingestion and validation completed===================="
62+
63+
echo "=================== #4 Start agent and API server===================="
64+
bash step4a_launch_and_validate_agent_tgi_on_rocm.sh
65+
echo "=================== #4 Agent test passed ===================="
66+
67+
echo "=================== #5 Stop agent and API server===================="
68+
stop_crag
69+
stop_agent_docker
70+
stop_retrieval_tool
71+
echo "=================== #5 Agent and API server stopped===================="
72+
73+
echo y | docker system prune
74+
75+
echo "ALL DONE!"

0 commit comments

Comments
 (0)