Add validate result detection. (#492)

Signed-off-by: zepan <ze.pan@intel.com>
opea-project · Aug 15, 2024 · cf15b91 · cf15b91
1 parent a56a847
commit cf15b91
Show file tree

Hide file tree

Showing 16 changed files with 149 additions and 32 deletions.
diff --git a/comps/dataprep/redis/llama_index/requirements.txt b/comps/dataprep/redis/llama_index/requirements.txt
@@ -12,6 +12,7 @@ opentelemetry-exporter-otlp
 opentelemetry-sdk
 prometheus-fastapi-instrumentator
 python-bidi==0.4.2
+python-multipart
 redis
 sentence_transformers
 shortuuid

diff --git a/comps/guardrails/pii_detection/requirements.txt b/comps/guardrails/pii_detection/requirements.txt
@@ -20,6 +20,7 @@ prometheus-fastapi-instrumentator
 pyarrow
 pymupdf
 python-docx
+python-multipart
 ray
 redis
 scikit-learn

diff --git a/tests/test_agent_langchain.sh b/tests/test_agent_langchain.sh
@@ -34,7 +34,7 @@ function start_service() {
     docker logs test-comps-tgi-gaudi-service
 
     echo "Starting agent microservice"
-    docker run -d --runtime=runc --name="test-comps-langchain-agent-endpoint" -v $WORKPATH/comps/agent/langchain/tools:/home/user/comps/agent/langchain/tools -p 9090:9090 --ipc=host -e HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -e model=${model} -e strategy=react -e llm_endpoint_url=http://${ip_address}:8080 -e llm_engine=tgi -e recursion_limit=5 -e require_human_feedback=false -e tools=/home/user/comps/agent/langchain/tools/custom_tools.yaml opea/comps-agent-langchain:comps
+    docker run -d --runtime=runc --name="test-comps-langchain-agent-endpoint" -v $WORKPATH/comps/agent/langchain/tools:/home/user/comps/agent/langchain/tools -p 5042:9090 --ipc=host -e HUGGINGFACEHUB_API_TOKEN=${HUGGINGFACEHUB_API_TOKEN} -e model=${model} -e strategy=react -e llm_endpoint_url=http://${ip_address}:8080 -e llm_engine=tgi -e recursion_limit=5 -e require_human_feedback=false -e tools=/home/user/comps/agent/langchain/tools/custom_tools.yaml opea/comps-agent-langchain:comps
     sleep 5s
     docker logs test-comps-langchain-agent-endpoint
 
@@ -69,7 +69,7 @@ function validate() {
 
 function validate_microservice() {
     echo "Testing agent service"
-    local CONTENT=$(curl http://${ip_address}:9090/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
+    local CONTENT=$(curl http://${ip_address}:5042/v1/chat/completions -X POST -H "Content-Type: application/json" -d '{
      "query": "What is Intel OPEA project?"
     }' | tee ${LOG_PATH}/test-agent-langchain.log)
     local EXIT_CODE=$(validate "$CONTENT" "OPEA" "test-agent-langchain")

diff --git a/tests/test_dataprep_pinecone.sh b/tests/test_dataprep_pinecone.sh
@@ -24,20 +24,33 @@ function start_service() {
     export PINECONE_INDEX_NAME="test-index"
     export HUGGINGFACEHUB_API_TOKEN=$HF_TOKEN
 
-    docker run -d --name="test-comps-dataprep-pinecone" -p 6007:6007 -p 6008:6008 -p 6009:6009 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e PINECONE_API_KEY=$PINECONE_API_KEY -e PINECONE_INDEX_NAME=$PINECONE_INDEX_NAME opea/dataprep-pinecone:comps
+    docker run -d --name="test-comps-dataprep-pinecone" -p 5039:6007 -p 5040:6008 -p 5041:6009 --ipc=host -e http_proxy=$http_proxy -e https_proxy=$https_proxy -e no_proxy=$no_proxy -e PINECONE_API_KEY=$PINECONE_API_KEY -e PINECONE_INDEX_NAME=$PINECONE_INDEX_NAME opea/dataprep-pinecone:comps
 
     sleep 1m
 }
 
 function validate_microservice() {
-    URL="http://$ip_address:6007/v1/dataprep"
+    URL="http://$ip_address:5039/v1/dataprep"
     echo 'The OPEA platform includes: Detailed framework of composable building blocks for state-of-the-art generative AI systems including LLMs, data stores, and prompt engines' > ./dataprep_file.txt
     result=$(curl --noproxy $ip_address --location --request POST \
       --form 'files=@./dataprep_file.txt' $URL)
-
-    DELETE_URL="http://$ip_address:6009/v1/dataprep/delete_file"
-    result_2=$(curl --noproxy $ip_address --location --request POST \
+    if [[ $result == *"200"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-dataprep-pinecone
+        exit 1
+    fi
+    DELETE_URL="http://$ip_address:5041/v1/dataprep/delete_file"
+    result=$(curl --noproxy $ip_address --location --request POST \
       -d '{"file_path": "all"}' -H 'Content-Type: application/json' $DELETE_URL)
+    if [[ $result == *"true"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-dataprep-pinecone
+        exit 1
+    fi
 }
 
 function stop_docker() {

diff --git a/tests/test_dataprep_qdrant_langchain.sh b/tests/test_dataprep_qdrant_langchain.sh
@@ -57,13 +57,19 @@ function validate_services() {
     # check response status
     if [ "$HTTP_STATUS" -ne "200" ]; then
         echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs test-comps-dataprep-qdrant-langchain
+        docker logs test-comps-dataprep-qdrant-langchain-tei
+        docker logs test-comps-dataprep-qdrant-langchain-server
         exit 1
     else
         echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
     fi
     # check response body
     if [[ "$RESPONSE_BODY" != *"$EXPECTED_RESULT"* ]]; then
         echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY"
+        docker logs test-comps-dataprep-qdrant-langchain
+        docker logs test-comps-dataprep-qdrant-langchain-tei
+        docker logs test-comps-dataprep-qdrant-langchain-server
         exit 1
     else
         echo "[ $SERVICE_NAME ] Content is as expected."

diff --git a/tests/test_dataprep_redis_langchain.sh b/tests/test_dataprep_redis_langchain.sh
@@ -39,16 +39,17 @@ function validate_microservice() {
     HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
     RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
     SERVICE_NAME="dataprep - upload - file"
-    docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_upload_file.log
 
     if [ "$HTTP_STATUS" -ne "200" ]; then
         echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_upload_file.log
         exit 1
     else
         echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
     fi
     if [[ "$RESPONSE_BODY" != *"Data preparation succeeded"* ]]; then
         echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY"
+        docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_upload_file.log
         exit 1
     else
         echo "[ $SERVICE_NAME ] Content is as expected."
@@ -60,16 +61,18 @@ function validate_microservice() {
     HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
     RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
     SERVICE_NAME="dataprep - upload - link"
-    docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_upload_link.log
+
 
     if [ "$HTTP_STATUS" -ne "200" ]; then
         echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_upload_link.log
         exit 1
     else
         echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
     fi
     if [[ "$RESPONSE_BODY" != *"Data preparation succeeded"* ]]; then
         echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY"
+        docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_upload_link.log
         exit 1
     else
         echo "[ $SERVICE_NAME ] Content is as expected."
@@ -81,16 +84,17 @@ function validate_microservice() {
     HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
     RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
     SERVICE_NAME="dataprep - get"
-    docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_file.log
 
     if [ "$HTTP_STATUS" -ne "200" ]; then
         echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_file.log
         exit 1
     else
         echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
     fi
     if [[ "$RESPONSE_BODY" != *'{"name":'* ]]; then
         echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY"
+        docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_file.log
         exit 1
     else
         echo "[ $SERVICE_NAME ] Content is as expected."
@@ -102,18 +106,19 @@ function validate_microservice() {
     HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
     RESPONSE_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
     SERVICE_NAME="dataprep - del"
-    docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_del.log
 
     # check response status
     if [ "$HTTP_STATUS" -ne "200" ]; then
         echo "[ $SERVICE_NAME ] HTTP status is not 200. Received status was $HTTP_STATUS"
+        docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_del.log
         exit 1
     else
         echo "[ $SERVICE_NAME ] HTTP status is 200. Checking content..."
     fi
     # check response body
     if [[ "$RESPONSE_BODY" != *'{"status":true}'* ]]; then
         echo "[ $SERVICE_NAME ] Content does not match the expected result: $RESPONSE_BODY"
+        docker logs test-comps-dataprep-redis-langchain-server >> ${LOG_PATH}/dataprep_del.log
         exit 1
     else
         echo "[ $SERVICE_NAME ] Content is as expected."

diff --git a/tests/test_embeddings_langchain.sh b/tests/test_embeddings_langchain.sh
@@ -37,6 +37,14 @@ function validate_microservice() {
         -X POST \
         -d '{"text":"What is Deep Learning?"}' \
         -H 'Content-Type: application/json')
+    if [[ $result == *"embedding"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-embedding-tei-endpoint
+        docker logs test-comps-embedding-tei-server
+        exit 1
+    fi
 }
 
 function stop_docker() {

diff --git a/tests/test_guardrails_pii_detection.sh b/tests/test_guardrails_pii_detection.sh
@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -xe
+set -x
 
 WORKPATH=$(dirname "$PWD")
 ip_address=$(hostname -I | awk '{print $1}')
@@ -31,15 +31,42 @@ function validate_microservice() {
     export PATH="${HOME}/miniforge3/bin:$PATH"
     source activate
     echo "test 1 - single task - ner"
-    python comps/guardrails/pii_detection/test.py --test_text --batch_size 1 --ip_addr $ip_address --strategy ner
+    result=$(python comps/guardrails/pii_detection/test.py --test_text --batch_size 1 --ip_addr $ip_address --strategy ner)
+    if [[ $result == *"An error occurred"* ]]; then
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-guardrails-pii-detection-endpoint
+        exit 1
+    else
+        echo "Result correct."
+    fi
     echo "test 2 - 20 tasks in parallel - ner"
-    python comps/guardrails/pii_detection/test.py --test_text --batch_size 20 --ip_addr $ip_address --strategy ner
+    result=$(python comps/guardrails/pii_detection/test.py --test_text --batch_size 20 --ip_addr $ip_address --strategy ner)
+    if [[ $result == *"An error occurred"* ]]; then
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-guardrails-pii-detection-endpoint
+        exit 1
+    else
+        echo "Result correct."
+    fi
     echo "test 3 - single task - ml"
-    python comps/guardrails/pii_detection/test.py --test_text --batch_size 1 --ip_addr $ip_address --strategy ml
+    result=$(python comps/guardrails/pii_detection/test.py --test_text --batch_size 1 --ip_addr $ip_address --strategy ml)
+    if [[ $result == *"An error occurred"* ]]; then
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-guardrails-pii-detection-endpoint
+        exit 1
+    else
+        echo "Result correct."
+    fi
     echo "test 4 - 20 tasks in parallel - ml"
-    python comps/guardrails/pii_detection/test.py --test_text --batch_size 20 --ip_addr $ip_address --strategy ml
+    result=$(python comps/guardrails/pii_detection/test.py --test_text --batch_size 20 --ip_addr $ip_address --strategy ml)
+    if [[ $result == *"An error occurred"* ]]; then
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-guardrails-pii-detection-endpoint
+        exit 1
+    else
+        echo "Result correct."
+    fi
     echo "Validate microservice completed"
-    docker logs test-comps-guardrails-pii-detection-endpoint
 }
 
 function stop_docker() {

diff --git a/tests/test_llms_text-generation_vllm-openvino.sh b/tests/test_llms_text-generation_vllm-openvino.sh
@@ -102,6 +102,7 @@ function test_api_endpoint {
     else
         echo "FAIL: $endpoint returned unexpected status code: $response (expected: $expected_status)"
         docker logs $CONTAINER_NAME
+        exit 1
     fi
 }
 # Main function

diff --git a/tests/test_llms_text-generation_vllm-ray.sh b/tests/test_llms_text-generation_vllm-ray.sh
@@ -76,12 +76,26 @@ function validate_microservice() {
     result=$(http_proxy="" curl http://${ip_address}:5031/v1/chat/completions \
         -H "Content-Type: application/json" \
         -d '{"model": "facebook/opt-125m", "messages": [{"role": "user", "content": "How are you?"}]}')
-    result_2=$(http_proxy="" curl http://${ip_address}:5032/v1/chat/completions \
+    if [[ $result == *"message"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-vllm-ray-service
+        docker logs test-comps-vllm-ray-microservice
+        exit 1
+    fi
+    result=$(http_proxy="" curl http://${ip_address}:5032/v1/chat/completions \
         -X POST \
         -d '{"query":"What is Deep Learning?","max_new_tokens":17,"top_k":10,"top_p":0.95,"typical_p":0.95,"temperature":0.01,"repetition_penalty":1.03,"streaming":false}' \
         -H 'Content-Type: application/json')
-    docker logs test-comps-vllm-ray-service
-    docker logs test-comps-vllm-ray-microservice
+    if [[ $result == *"text"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-vllm-ray-service
+        docker logs test-comps-vllm-ray-microservice
+        exit 1
+    fi
 }
 
 function stop_docker() {

diff --git a/tests/test_llms_text-generation_vllm.sh b/tests/test_llms_text-generation_vllm.sh
@@ -74,20 +74,34 @@ function start_service() {
 }
 
 function validate_microservice() {
-    result=$(http_proxy="" curl http://${ip_address}:8008/v1/completions \
+    result=$(http_proxy="" curl http://${ip_address}:5025/v1/completions \
         -H "Content-Type: application/json" \
         -d '{
         "model": "facebook/opt-125m",
         "prompt": "What is Deep Learning?",
         "max_tokens": 32,
         "temperature": 0
         }')
-    result_2=$(http_proxy="" curl http://${ip_address}:5030/v1/chat/completions \
+    if [[ $result == *"text"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-vllm-service
+        docker logs test-comps-vllm-microservice
+        exit 1
+    fi
+    result=$(http_proxy="" curl http://${ip_address}:5030/v1/chat/completions \
         -X POST \
         -d '{"query":"What is Deep Learning?","max_new_tokens":17,"top_p":0.95,"temperature":0.01,"streaming":false}' \
         -H 'Content-Type: application/json')
-            docker logs test-comps-vllm-service
-            docker logs test-comps-vllm-microservice
+    if [[ $result == *"text"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-vllm-service
+        docker logs test-comps-vllm-microservice
+        exit 1
+    fi
 }
 
 function stop_docker() {

diff --git a/tests/test_reranks_fastrag.sh b/tests/test_reranks_fastrag.sh
@@ -31,7 +31,13 @@ function validate_microservice() {
         -X POST \
         -d '{"initial_query":"What is Deep Learning?", "retrieved_docs": [{"text":"Deep Learning is not..."}, {"text":"Deep learning is..."}]}' \
         -H 'Content-Type: application/json')
-    docker logs test-comps-reranking-fastrag-server
+    if [[ $result == *"reranked_docs"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-reranking-fastrag-server
+        exit 1
+    fi
 }
 
 function stop_docker() {

diff --git a/tests/test_retrievers_haystack_qdrant.sh b/tests/test_retrievers_haystack_qdrant.sh
@@ -49,8 +49,14 @@ function validate_microservice() {
         -X POST \
         -d "{\"text\":\"test\",\"embedding\":${test_embedding}}" \
         -H 'Content-Type: application/json')
-    docker logs test-comps-retriever-qdrant-server
-    docker logs test-comps-retriever-tei-endpoint
+    if [[ $result == *"retrieved_docs"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-retriever-qdrant-server
+        docker logs test-comps-retriever-tei-endpoint
+        exit 1
+    fi
 }
 
 function stop_docker() {

diff --git a/tests/test_retrievers_langchain_pgvector.sh b/tests/test_retrievers_langchain_pgvector.sh
@@ -47,8 +47,14 @@ function validate_microservice() {
         -X POST \
         -d "{\"text\":\"test\",\"embedding\":${test_embedding}}" \
         -H 'Content-Type: application/json')
-    docker logs test-comps-vectorstore-postgres
-    docker logs test-comps-retriever-tei-endpoint
+    if [[ $result == *"retrieved_docs"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-vectorstore-postgres
+        docker logs test-comps-retriever-tei-endpoint
+        exit 1
+    fi
 }
 
 function stop_docker() {

diff --git a/tests/test_retrievers_langchain_pinecone.sh b/tests/test_retrievers_langchain_pinecone.sh
@@ -44,8 +44,14 @@ function validate_microservice() {
         -X POST \
         -d "{\"text\":\"test\",\"embedding\":${test_embedding}}" \
         -H 'Content-Type: application/json')
-    docker logs test-comps-retriever-pinecone-server
-    docker logs test-comps-retriever-tei-endpoint
+    if [[ $result == *"retrieved_docs"* ]]; then
+        echo "Result correct."
+    else
+        echo "Result wrong. Received was $result"
+        docker logs test-comps-retriever-pinecone-server
+        docker logs test-comps-retriever-tei-endpoint
+        exit 1
+    fi
 }
 
 function stop_docker() {
-Original file line number
+Diff line change
@@ Expand Up / @@ -20,6 +20,7 @@ prometheus-fastapi-instrumentator @@
     pyarrow
     pymupdf
     python-docx
+    python-multipart
     ray
     redis
     scikit-learn
@@ Expand Down @@