Skip to content

Commit

Permalink
add manual helm e2e test flow (#331)
Browse files Browse the repository at this point in the history 
Signed-off-by: Yingchun Guo <yingchun.guo@intel.com>
  • Loading branch information
@daisy-ycguo
daisy-ycguo authored Aug 21, 2024
1 parent 8a85364 commit 3b5f62e
Show file tree
Hide file tree
Showing 4 changed files with 376 additions and 125 deletions.
171 changes: 171 additions & 0 deletions .github/workflows/_helm-e2e.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

name: To Test a Helm Job for call
permissions: read-all
on:
workflow_call:
inputs:
workload:
default: "chatqna"
required: true
type: string
description: "workload to test, chatqna or common/asr"
tag:
default: "latest"
required: false
type: string
opea_branch:
default: "main"
required: false
type: string
hardware:
default: "xeon"
required: true
type: string

env:
CHARTS_DIR: "helm-charts"

jobs:
####################################################################################################
# Helm Test
####################################################################################################
helm-test:
runs-on: ${{ inputs.hardware }}
continue-on-error: true
outputs:
should_cleanup: ${{ steps.set_boolean.outputs.should_cleanup }}
steps:
- name: Clean Up Working Directory
run: sudo rm -rf ${{github.workspace}}/*

- name: Get checkout ref
id: get-checkout-ref
run: |
if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
else
CHECKOUT_REF=${{ github.ref }}
fi
echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
echo "checkout ref ${CHECKOUT_REF}"
- name: Checkout out Repo
uses: actions/checkout@v4
with:
ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
fetch-depth: 0

- name: Set variables
env:
workload_path: ${{ inputs.workload }}
run: |
CHART_NAME="${workload_path##*/}"
echo "CHART_NAME=$CHART_NAME" >> $GITHUB_ENV
echo "RELEASE_NAME=${CHART_NAME}$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
echo "NAMESPACE=${CHART_NAME}-$(date +%Y%m%d%H%M%S)" >> $GITHUB_ENV
echo "ROLLOUT_TIMEOUT_SECONDS=600s" >> $GITHUB_ENV
echo "KUBECTL_TIMEOUT_SECONDS=60s" >> $GITHUB_ENV
echo "should_cleanup=false" >> $GITHUB_ENV
echo "skip_validate=false" >> $GITHUB_ENV
echo "RELEASENAME=$RELEASE_NAME"
echo "NAMESPACE=$NAMESPACE"
echo "CHART_FOLDER=$CHARTS_DIR/${workload_path}" >> $GITHUB_ENV
- name: Initialize chart testing
run: |
# Replace values for CI test environment
USER_ID=$(whoami)
CHART_MOUNT=/home/$USER_ID/.cache/huggingface/hub
HFTOKEN=$(cat /home/$USER_ID/.cache/huggingface/token)
pushd helm-charts
# insert a prefix before opea/.*, the prefix is OPEA_IMAGE_REPO
find . -name '*values.yaml' -type f -exec sed -i "s#repository: opea/*#repository: ${OPEA_IMAGE_REPO}opea/#g" {} \;
# set OPEA image tag to ${{ inputs.tag }}
find . -name '*values.yaml' -type f -exec sed -i 's#tag: ""#tag: ${{ inputs.tag }}#g' {} \;
# set huggingface token
find . -name '*values.yaml' -type f -exec sed -i "s#insert-your-huggingface-token-here#${HFTOKEN}#g" {} \;
# replace the mount dir "Volume: *" with "Volume: $CHART_MOUNT"
find . -name '*values.yaml' -type f -exec sed -i "s#modelUseHostPath: .*#modelUseHostPath: $CHART_MOUNT#g" {} \;
# replace the pull policy "IfNotPresent" with "Always"
find . -name '*values.yaml' -type f -exec sed -i "s#pullPolicy: IfNotPresent#pullPolicy: Always#g" {} \;
popd
- name: Helm install
id: install
env:
GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
run: |
set -xe
echo "should_cleanup=true" >> $GITHUB_ENV
helm-charts/update_dependency.sh && helm dependency update ${{ env.CHART_FOLDER }}
value_file="values.yaml"
if [ "${{ inputs.hardware }}" == "gaudi" ]; then
value_file="gaudi-values.yaml"
fi
if [[ ! -f ${{ env.CHART_FOLDER }}/${value_file} ]]; then
echo "No value file found, exist test!"
echo "skip_validate=true" >> $GITHUB_ENV
echo "should_cleanup=false" >> $GITHUB_ENV
exit 0
fi
if ! helm install --create-namespace --namespace $NAMESPACE --wait \
--timeout "$ROLLOUT_TIMEOUT_SECONDS" \
--set autodependency.enabled=true \
--set GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
--set GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
--values ${{ env.CHART_FOLDER }}/${value_file} \
$RELEASE_NAME ${{ env.CHART_FOLDER }} ; then
echo "Failed to install chart ${{ inputs.workload }}"
echo "skip_validate=true" >> $GITHUB_ENV
.github/workflows/scripts/e2e/chart_test.sh dump_pods_status $NAMESPACE
exit 1
fi
- name: Validate e2e test
if: always()
run: |
set -xe
if $skip_validate; then
echo "Skip validate"
else
LOG_PATH=/home/$(whoami)/logs
chart=${{ env.CHART_NAME }}
helm test -n $NAMESPACE $RELEASE_NAME --logs |tee ${LOG_PATH}/charts-${chart}.log
exit_code=$?
if [ $exit_code -ne 0 ]; then
echo "Chart ${chart} test failed, please check the logs in ${LOG_PATH}!"
exit 1
fi
echo "Checking response results, make sure the output is reasonable. "
teststatus=false
if [[ -f $LOG_PATH/charts-${chart}.log ]] && \
[[ $(grep -c "^Phase:.*Failed" $LOG_PATH/charts-${chart}.log) != 0 ]]; then
teststatus=false
.github/workflows/scripts/e2e/chart_test.sh dump_all_pod_logs $NAMESPACE
else
teststatus=true
fi
if [ $teststatus == false ]; then
echo "Response check failed, please check the logs in artifacts!"
exit 1
else
echo "Response check succeed!"
exit 0
fi
fi
- name: Helm uninstall
if: always()
run: |
if $should_cleanup; then
helm uninstall $RELEASE_NAME --namespace $NAMESPACE
if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then
kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all
kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
fi
fi
2 changes: 1 addition & 1 deletion .github/workflows/manual-freeze-tag.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
run: |
git config --global user.name "NeuralChatBot"
git config --global user.email "grp_neural_chat_bot@intel.com"
git remote set-url origin https://NeuralChatBot:"${{ secrets.ACTION_TOKEN }}"@github.com/opea-project/GenAIExamples.git
git remote set-url origin https://NeuralChatBot:"${{ secrets.ACTION_TOKEN }}"@github.com/opea-project/GenAIInfra.git
- name: Run script
env:
Expand Down
68 changes: 68 additions & 0 deletions .github/workflows/manual-helm-workflow
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

name: GenAIInfra Helm CD workflow on manual event
on:
workflow_dispatch:
inputs:
workloads:
default: ""
required: true
type: string
description: "workloads to test, empty for testing all helm charts"
tag:
default: "latest"
description: "Tag to apply to images"
required: true
type: string
infra_branch:
default: true
description: 'Build test required images for Examples'
required: false
type: string
nodes:
default: "xeon,gaudi"
required: true
type: string

env:
CHARTS_DIR: "helm-charts"

jobs:
get-build-matrix:
runs-on: ubuntu-latest
outputs:
services: ${{ steps.get-services.outputs.services }}
nodes: ${{ steps.get-services.outputs.nodes }}
steps:
- name: Get test Services
id: get-services
run: |
set -x
if [[ -z "${{ inputs.workloads }}" ]]; then
subfolders=$(find "$CHARTS_DIR" -mindepth 1 -maxdepth 1 -type d ! -name "common" -exec basename {} \; | sed 's/^/"/; s/$/"/' | paste -sd, -)
common_subfolders=$(find "$CHARTS_DIR/common" -mindepth 1 -maxdepth 1 -type d -exec basename {} \; | sed 's|^|"common/|; s/$/"/' | paste -sd, -)
services="[$subfolders,$common_subfolders]"
echo "services=$services" >> $GITHUB_OUTPUT
else
service_list=($(echo ${{ github.event.inputs.workloads }} | tr ',' ' '))
services=$(printf '%s\n' "${service_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
echo "services=$services" >> $GITHUB_OUTPUT
fi
node_list=($(echo ${{ github.event.inputs.nodes }} | tr ',' ' '))
nodes=$(printf '%s\n' "${node_list[@]}" | sort -u | jq -R '.' | jq -sc '.')
echo "nodes=$nodes" >> $GITHUB_OUTPUT

gmc-release:
needs: get-build-matrix
strategy:
matrix:
workload: ${{ fromJSON(needs.get-build-matrix.outputs.services) }}
node: ${{ fromJSON(needs.get-build-matrix.outputs.nodes) }}
uses: ./.github/workflows/_helm-e2e.yaml
with:
tag: ${{ inputs.tag }}
workload: ${{ matrix.workload }}
hardware: ${{ matrix.node }}
opea_branch: ${{ inputs.infra_branch }}
secrets: inherit
Loading

0 comments on commit 3b5f62e

Please sign in to comment.