diff --git a/.github/workflows/chatqna_benchmark.yml b/.github/workflows/chatqna_benchmark.yml new file mode 100644 index 0000000000..a76c43a9af --- /dev/null +++ b/.github/workflows/chatqna_benchmark.yml @@ -0,0 +1,84 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: E2E test with chatqna_benchmark + +on: + pull_request: + branches: [chatqna_benchmark] + + # schedule: + # - cron: "35 1 * * 5" + workflow_dispatch: + +jobs: + # job1: + # uses: ./.github/workflows/reuse-get-test-matrix.yml + # with: + # diff_excluded_files: '.github|README.md|*.txt|deprecate|kubernetes|manifest|gmc|assets' + + Example-test: + # needs: [job1] + # strategy: + # matrix: ${{ fromJSON(needs.job1.outputs.run_matrix) }} + # runs-on: ${{ matrix.hardware }} + runs-on: ubuntu-latest + # continue-on-error: true + steps: + # - name: Test example + # run: | + # echo "Matrix - example ${{ matrix.example }}, hardware ${{ matrix.hardware }}" + + - name: Clean Up Working Directory + run: sudo rm -rf ${{github.workspace}}/* + + - name: Checkout out Repo + uses: actions/checkout@v4 + with: + ref: "refs/pull/${{ github.event.number }}/merge" + + - name: Clone repo GenAIEval + run: | + cd .. + git clone https://github.com/opea-project/GenAIEval.git + + - name: Run test + env: + HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }} + GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + # example: ${{ matrix.example }} + # hardware: ${{ matrix.hardware }} + IMAGE_TAG: ${{ needs.mega-image-build.outputs.image_tag }} + IMAGE_REPO_GAUDI: ${{ vars.IMAGE_REPO_GAUDI }} + IMAGE_REPO_XEON: ${{ vars.IMAGE_REPO_XEON }} + run: | + # cd ${{ github.workspace }}/$example/tests + cd ${{ github.workspace }}/.github/workflows + cp ../../../GenAIEval/evals/benchmark/chatqna_benchmark.py . + # if [ "$hardware" == "gaudi" ]; then IMAGE_REPO=$IMAGE_REPO_GAUDI; else IMAGE_REPO=$IMAGE_REPO_XEON; fi + # export IMAGE_REPO=${IMAGE_REPO} + # example_l=$(echo $example | tr '[:upper:]' '[:lower:]') + # if [ -f test_${example_l}_on_${hardware}.sh ]; then timeout 30m bash test_${example_l}_on_${hardware}.sh; else echo "Test script not found, skip test!"; fi + if [ -f test_chatqna.sh ]; then timeout 30m bash test_chatqna.sh; else echo "Test script not found, skip test!"; fi + + # - name: Clean up container + # env: + # example: ${{ matrix.example }} + # hardware: ${{ matrix.hardware }} + # if: cancelled() || failure() + # run: | + # cd ${{ github.workspace }}/$example/docker/$hardware + # container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2) + # for container_name in $container_list; do + # cid=$(docker ps -aq --filter "name=$container_name") + # if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi + # done + # echo y | docker system prune + + # - name: Publish pipeline artifact + # if: ${{ !cancelled() }} + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ matrix.example }}-${{ matrix.hardware }} + # path: ${{ github.workspace }}/${{ matrix.example }}/tests/*.log diff --git a/.github/workflows/test_chatqna_on_gaudi.sh b/.github/workflows/test_chatqna_on_gaudi.sh new file mode 100644 index 0000000000..23f7efb99e --- /dev/null +++ b/.github/workflows/test_chatqna_on_gaudi.sh @@ -0,0 +1,240 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +echo "IMAGE_REPO=${IMAGE_REPO}" + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + git clone https://github.com/opea-project/GenAIComps.git + cd GenAIComps + + docker build -t opea/embedding-tei:latest -f comps/embeddings/langchain/docker/Dockerfile . + docker build -t opea/retriever-redis:latest -f comps/retrievers/langchain/redis/docker/Dockerfile . + docker build -t opea/reranking-tei:latest -f comps/reranks/langchain/docker/Dockerfile . + docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile . + docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/docker/Dockerfile . + + cd .. + git clone https://github.com/huggingface/tei-gaudi + cd tei-gaudi/ + docker build --no-cache -f Dockerfile-hpu -t opea/tei-gaudi:latest . + + docker pull ghcr.io/huggingface/tgi-gaudi:2.0.0 + docker pull ghcr.io/huggingface/text-embeddings-inference:cpu-1.2 + + cd $WORKPATH../ChatQnA/docker + docker build --no-cache -t opea/chatqna:latest -f Dockerfile . + + cd $WORKPATH../ChatQnA/docker/ui + docker build --no-cache -t opea/chatqna-ui:latest -f docker/Dockerfile . + + docker images +} + +function start_services() { + cd $WORKPATH../ChatQnA/docker/gaudi + + export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" + export RERANK_MODEL_ID="BAAI/bge-reranker-base" + export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" + export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:8090" + export TEI_RERANKING_ENDPOINT="http://${ip_address}:8808" + export TGI_LLM_ENDPOINT="http://${ip_address}:8008" + export REDIS_URL="redis://${ip_address}:6379" + export INDEX_NAME="rag-redis" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MEGA_SERVICE_HOST_IP=${ip_address} + export EMBEDDING_SERVICE_HOST_IP=${ip_address} + export RETRIEVER_SERVICE_HOST_IP=${ip_address} + export RERANK_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_HOST_IP=${ip_address} + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/chatqna" + export DATAPREP_SERVICE_ENDPOINT="http://${ip_address}:6007/v1/dataprep" + + sed -i "s/backend_address/$ip_address/g" $WORKPATH../ChatQnA/docker/ui/svelte/.env + + if [[ "$IMAGE_REPO" != "" ]]; then + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + fi + + # Start Docker Containers + docker compose -f docker_compose.yaml up -d + n=0 + until [[ "$n" -ge 400 ]]; do + docker logs tgi-gaudi-server > tgi_service_start.log + if grep -q Connected tgi_service_start.log; then + break + fi + sleep 1s + n=$((n+1)) + done +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # tei for embedding service + validate_services \ + "${ip_address}:8090/embed" \ + "\[\[" \ + "tei-embedding" \ + "tei-embedding-gaudi-server" \ + '{"inputs":"What is Deep Learning?"}' + + # embedding microservice + validate_services \ + "${ip_address}:6000/v1/embeddings" \ + '"text":"What is Deep Learning?","embedding":\[' \ + "embedding" \ + "embedding-tei-server" \ + '{"text":"What is Deep Learning?"}' + + sleep 1m # retrieval can't curl as expected, try to wait for more time + + # retrieval microservice + test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + validate_services \ + "${ip_address}:7000/v1/retrieval" \ + " " \ + "retrieval" \ + "retriever-redis-server" \ + "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" + + # tei for rerank microservice + validate_services \ + "${ip_address}:8808/rerank" \ + '{"index":1,"score":' \ + "tei-rerank" \ + "tei-reranking-gaudi-server" \ + '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' + + # rerank microservice + validate_services \ + "${ip_address}:8000/v1/reranking" \ + "Deep learning is..." \ + "rerank" \ + "reranking-tei-gaudi-server" \ + '{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}' + + # tgi for llm service + validate_services \ + "${ip_address}:8008/generate" \ + "generated_text" \ + "tgi-llm" \ + "tgi-gaudi-server" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + + # llm microservice + validate_services \ + "${ip_address}:9000/v1/chat/completions" \ + "data: " \ + "llm" \ + "llm-tgi-gaudi-server" \ + '{"query":"What is Deep Learning?"}' + +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${ip_address}:8888/v1/chatqna" \ + "billion" \ + "mega-chatqna" \ + "chatqna-gaudi-backend-server" \ + '{"messages": "What is the revenue of Nike in 2023?"}' + +} + +function validate_frontend() { + cd $WORKPATH../ChatQnA/docker/ui/svelte + local conda_env_name="OPEA_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH +# conda remove -n ${conda_env_name} --all -y +# conda create -n ${conda_env_name} python=3.12 -y + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + +# conda install -c conda-forge nodejs -y + npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi +} + +function stop_docker() { + cd $WORKPATH/../ChatQnA/docker/gaudi + container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2) + for container_name in $container_list; do + cid=$(docker ps -aq --filter "name=$container_name") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi + done +} + +function main() { + + stop_docker + if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi + start_time=$(date +%s) + start_services + end_time=$(date +%s) + duration=$((end_time-start_time)) + echo "Mega service start duration is $duration s" + + python3 chatqna_benchmark.py + # validate_microservices + # validate_megaservice + # validate_frontend + + stop_docker + echo y | docker system prune + +} + +main diff --git a/.github/workflows/test_chatqna_on_xeon.sh b/.github/workflows/test_chatqna_on_xeon.sh new file mode 100755 index 0000000000..b98f4ff9e1 --- /dev/null +++ b/.github/workflows/test_chatqna_on_xeon.sh @@ -0,0 +1,232 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +set -e +echo "IMAGE_REPO=${IMAGE_REPO}" + +WORKPATH=$(dirname "$PWD") +LOG_PATH="$WORKPATH/tests" +ip_address=$(hostname -I | awk '{print $1}') + +function build_docker_images() { + cd $WORKPATH + git clone https://github.com/opea-project/GenAIComps.git + cd GenAIComps + + docker build -t opea/embedding-tei:latest -f comps/embeddings/langchain/docker/Dockerfile . + docker build -t opea/retriever-redis:latest -f comps/retrievers/langchain/redis/docker/Dockerfile . + docker build -t opea/reranking-tei:latest -f comps/reranks/tei/docker/Dockerfile . + docker build -t opea/llm-tgi:latest -f comps/llms/text-generation/tgi/Dockerfile . + docker build -t opea/dataprep-redis:latest -f comps/dataprep/redis/langchain/docker/Dockerfile . + + cd $WORKPATH../ChatQnA/docker + docker build --no-cache -t opea/chatqna:latest -f Dockerfile . + + cd $WORKPATH../ChatQnA/docker/ui + docker build --no-cache -t opea/chatqna-ui:latest -f docker/Dockerfile . + + docker images +} + +function start_services() { + cd $WORKPATH../ChatQnA/docker/xeon + + export EMBEDDING_MODEL_ID="BAAI/bge-base-en-v1.5" + export RERANK_MODEL_ID="BAAI/bge-reranker-base" + export LLM_MODEL_ID="Intel/neural-chat-7b-v3-3" + export TEI_EMBEDDING_ENDPOINT="http://${ip_address}:6006" + export TEI_RERANKING_ENDPOINT="http://${ip_address}:8808" + export TGI_LLM_ENDPOINT="http://${ip_address}:9009" + export REDIS_URL="redis://${ip_address}:6379" + export INDEX_NAME="rag-redis" + export HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} + export MEGA_SERVICE_HOST_IP=${ip_address} + export EMBEDDING_SERVICE_HOST_IP=${ip_address} + export RETRIEVER_SERVICE_HOST_IP=${ip_address} + export RERANK_SERVICE_HOST_IP=${ip_address} + export LLM_SERVICE_HOST_IP=${ip_address} + export BACKEND_SERVICE_ENDPOINT="http://${ip_address}:8888/v1/chatqna" + export DATAPREP_SERVICE_ENDPOINT="http://${ip_address}:6007/v1/dataprep" + + sed -i "s/backend_address/$ip_address/g" $WORKPATH../ChatQnA/docker/ui/svelte/.env + + if [[ "$IMAGE_REPO" != "" ]]; then + # Replace the container name with a test-specific name + echo "using image repository $IMAGE_REPO and image tag $IMAGE_TAG" + sed -i "s#image: opea/chatqna:latest#image: opea/chatqna:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/chatqna-ui:latest#image: opea/chatqna-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/chatqna-conversation-ui:latest#image: opea/chatqna-conversation-ui:${IMAGE_TAG}#g" docker_compose.yaml + sed -i "s#image: opea/*#image: ${IMAGE_REPO}opea/#g" docker_compose.yaml + fi + + # Start Docker Containers + docker compose -f docker_compose.yaml up -d + n=0 + until [[ "$n" -ge 200 ]]; do + docker logs tgi-service > tgi_service_start.log + if grep -q Connected tgi_service_start.log; then + break + fi + sleep 1s + n=$((n+1)) + done +} + +function validate_services() { + local URL="$1" + local EXPECTED_RESULT="$2" + local SERVICE_NAME="$3" + local DOCKER_NAME="$4" + local INPUT_DATA="$5" + + local HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL") + if [ "$HTTP_STATUS" -eq 200 ]; then + echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..." + + local CONTENT=$(curl -s -X POST -d "$INPUT_DATA" -H 'Content-Type: application/json' "$URL" | tee ${LOG_PATH}/${SERVICE_NAME}.log) + + if echo "$CONTENT" | grep -q "$EXPECTED_RESULT"; then + echo "[ $SERVICE_NAME ] Content is as expected." + else + echo "[ $SERVICE_NAME ] Content does not match the expected result: $CONTENT" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + else + echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS" + docker logs ${DOCKER_NAME} >> ${LOG_PATH}/${SERVICE_NAME}.log + exit 1 + fi + sleep 1s +} + +function validate_microservices() { + # Check if the microservices are running correctly. + + # tei for embedding service + validate_services \ + "${ip_address}:6006/embed" \ + "\[\[" \ + "tei-embedding" \ + "tei-embedding-server" \ + '{"inputs":"What is Deep Learning?"}' + + # embedding microservice + validate_services \ + "${ip_address}:6000/v1/embeddings" \ + '"text":"What is Deep Learning?","embedding":\[' \ + "embedding" \ + "embedding-tei-server" \ + '{"text":"What is Deep Learning?"}' + + sleep 1m # retrieval can't curl as expected, try to wait for more time + + # retrieval microservice + test_embedding=$(python3 -c "import random; embedding = [random.uniform(-1, 1) for _ in range(768)]; print(embedding)") + validate_services \ + "${ip_address}:7000/v1/retrieval" \ + " " \ + "retrieval" \ + "retriever-redis-server" \ + "{\"text\":\"What is the revenue of Nike in 2023?\",\"embedding\":${test_embedding}}" + + # tei for rerank microservice + validate_services \ + "${ip_address}:8808/rerank" \ + '{"index":1,"score":' \ + "tei-rerank" \ + "tei-reranking-server" \ + '{"query":"What is Deep Learning?", "texts": ["Deep Learning is not...", "Deep learning is..."]}' + + # rerank microservice + validate_services \ + "${ip_address}:8000/v1/reranking" \ + "Deep learning is..." \ + "rerank" \ + "reranking-tei-xeon-server" \ + '{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}' + + # tgi for llm service + validate_services \ + "${ip_address}:9009/generate" \ + "generated_text" \ + "tgi-llm" \ + "tgi-service" \ + '{"inputs":"What is Deep Learning?","parameters":{"max_new_tokens":17, "do_sample": true}}' + + # llm microservice + validate_services \ + "${ip_address}:9000/v1/chat/completions" \ + "data: " \ + "llm" \ + "llm-tgi-server" \ + '{"query":"What is Deep Learning?"}' + +} + +function validate_megaservice() { + # Curl the Mega Service + validate_services \ + "${ip_address}:8888/v1/chatqna" \ + "billion" \ + "mega-chatqna" \ + "chatqna-xeon-backend-server" \ + '{"messages": "What is the revenue of Nike in 2023?"}' + +} + +function validate_frontend() { + cd $WORKPATH../ChatQnA/docker/ui/svelte + local conda_env_name="OPEA_e2e" + export PATH=${HOME}/miniforge3/bin/:$PATH +# conda remove -n ${conda_env_name} --all -y +# conda create -n ${conda_env_name} python=3.12 -y + source activate ${conda_env_name} + + sed -i "s/localhost/$ip_address/g" playwright.config.ts + +# conda install -c conda-forge nodejs -y + npm install && npm ci && npx playwright install --with-deps + node -v && npm -v && pip list + + exit_status=0 + npx playwright test || exit_status=$? + + if [ $exit_status -ne 0 ]; then + echo "[TEST INFO]: ---------frontend test failed---------" + exit $exit_status + else + echo "[TEST INFO]: ---------frontend test passed---------" + fi +} + +function stop_docker() { + cd $WORKPATH/../ChatQnA/docker/xeon + container_list=$(cat docker_compose.yaml | grep container_name | cut -d':' -f2) + for container_name in $container_list; do + cid=$(docker ps -aq --filter "name=$container_name") + if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi + done +} + +function main() { + + stop_docker + if [[ "$IMAGE_REPO" == "" ]]; then build_docker_images; fi + start_time=$(date +%s) + start_services + end_time=$(date +%s) + duration=$((end_time-start_time)) + echo "Mega service start duration is $duration s" && sleep 1s + python3 chatqna_benchmark.py + # validate_microservices + # validate_megaservice + # validate_frontend + + stop_docker + echo y | docker system prune + +} + +main