merge-trigger-tke.yaml

name:  MO Checkin Regression On TKE

on:
  pull_request_target:
    branches: [ main,'[0-9]+.[0-9]+*' ]
    types:
      - closed
    paths:
      - '**.go'
      - '**.c'
      - '**.h'
      - '.github/workflows/merge-trigger-tke.yaml'

jobs:
  docker_image_build:
    runs-on: amd64-mo-guangzhou-dind
    if: ${{ github.event.pull_request.merged == true }}
    timeout-minutes: 30
    name: BUILD MO DOCKER IMAGE
    outputs:
       version: ${{ steps.prep.outputs.version }}
       commit_id: ${{ steps.prep.outputs.commit_id }}
    steps:
      - name: Clone Repo Matrixorigin/Matrixone
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          path: ./matrixone
      - name: Prepare And Build
        id: prep
        run: |
          cd $GITHUB_WORKSPACE/matrixone

          LAST_COMMIT_ID=$(git rev-parse --short HEAD)
          DOCKER_IMAGE=ccr.ccs.tencentyun.com/matrixone-dev/matrixone
          VERSION=commit-$LAST_COMMIT_ID
          TAGS="${DOCKER_IMAGE}:${VERSION}"
          
          # Set output parameters.
          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
          echo "docker_image=${DOCKER_IMAGE}" >> $GITHUB_OUTPUT
          echo "version=${VERSION}" >> $GITHUB_OUTPUT
          echo "commit_id=${LAST_COMMIT_ID}" >> $GITHUB_OUTPUT
          echo ${TAGS}
          echo ${{ secrets.TCR_TOKEN }} | docker login ccr.ccs.tencentyun.com -u ${{ secrets.TCR_USERNAME }} --password-stdin

          # add tencentyun docker image mirror and tencentyun apt mirror
          sed -i 's#^RUN apt-get update#RUN sed -i "s?http.*ubuntu\.com?http://mirrors\.tencentyun\.com?g" /etc/apt/sources.list  \&\& apt-get clean all \&\& apt-get update#g' optools/images/Dockerfile
          sed -i 's#FROM golang:1.20.7-bookworm as builder#FROM ccr.ccs.tencentyun.com/cicd-runner/golang:1.20.7-bookworm as builder#g' optools/images/Dockerfile
          sed -i 's#FROM ubuntu:latest#FROM ccr.ccs.tencentyun.com/cicd-runner/ubuntu:latest#g' optools/images/Dockerfile

          docker build -t $TAGS -f optools/images/Dockerfile . --build-arg GOPROXY='https://goproxy.cn,direct'
          docker push $TAGS

  setup_mo_test_env:
    if: ${{ always() && !cancelled() && needs.docker_image_build.result == 'success' }}
    needs: docker_image_build
    runs-on: amd64-mo-guangzhou-medium8
    timeout-minutes: 40
    name: SETUP MO TEST ENV
    outputs:
      mo_svc_ip: ${{ steps.get_mo_addr.outputs.mo_svc_ip }}
      cn_0: ${{ steps.get_mo_addr.outputs.cn_0 }}
      cn_1: ${{ steps.get_mo_addr.outputs.cn_1 }}
      cn_ips: ${{ steps.get_mo_addr.outputs.cn_ips }}
    steps:
      - name: Clone Repo Matrixorigin/Matrixone
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          path: ./matrixone
      - name: Prepare Registry Token
        run: |
          # set docker login info
          DOCKER_REGISTRY_SERVER=ccr.ccs.tencentyun.com
          DOCKER_USER=${{ secrets.TCR_USERNAME }}
          DOCKER_PASSWORD=${{ secrets.TCR_TOKEN }}
          if [ "$(kubectl get namespaces| grep -c mo-checkin-regression-${{ github.event.pull_request.number }})" -eq 0 ];then
            kubectl create namespace mo-checkin-regression-${{ github.event.pull_request.number }};
            kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} create secret generic tencent-token --from-literal=AWS_ACCESS_KEY_ID=${{ secrets.TENCENT_AK }} --from-literal=AWS_SECRET_ACCESS_KEY=${{ secrets.TENCENT_SK }};
            kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} create secret docker-registry tke-registry --docker-server=$DOCKER_REGISTRY_SERVER --docker-username=$DOCKER_USER --docker-password=$DOCKER_PASSWORD;
          fi       
      - name: ECHO mo-checkin-regression.yaml
        run: |
          echo '${{ needs.docker_image_build.outputs.version }}'
          cd $GITHUB_WORKSPACE/matrixone
          sed -i 's/nsformocheckin/mo-checkin-regression-${{ github.event.pull_request.number }}/g' $GITHUB_WORKSPACE/matrixone/optools/mo_checkin_regression/mo_checkin_regression_tke.yaml
          sed -i 's/s3pathformocheckin/mo-checkin-gz-1308875761\/${{ github.event.pull_request.number }}/g' $GITHUB_WORKSPACE/matrixone/optools/mo_checkin_regression/mo_checkin_regression_tke.yaml
          sed -i 's/regionformocheckin/ap-guangzhou/g' $GITHUB_WORKSPACE/matrixone/optools/mo_checkin_regression/mo_checkin_regression_tke.yaml
          sed -i 's@endpointformocheckin@https://cos.ap-guangzhou.myqcloud.com@g' $GITHUB_WORKSPACE/matrixone/optools/mo_checkin_regression/mo_checkin_regression_tke.yaml
          sed -i "s/imagetag/${{ needs.docker_image_build.outputs.version }}/g" $GITHUB_WORKSPACE/matrixone/optools/mo_checkin_regression/mo_checkin_regression_tke.yaml
         
          cat $GITHUB_WORKSPACE/matrixone/optools/mo_checkin_regression/mo_checkin_regression_tke.yaml
      - name: Clean TKE ENV
        run: |
          if [ "$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pod|grep -c "mo-checkin-regression-*")" -ne 0 ];then 
            kubectl get matrixonecluster -n mo-checkin-regression-${{ github.event.pull_request.number }} | awk 'NR>1{system("kubectl delete matrixonecluster/"$1" -n mo-checkin-regression-${{ github.event.pull_request.number }}")}'
            sleep 60s;
          fi
          kubectl get bucket -n mo-checkin-regression-${{ github.event.pull_request.number }} | grep bucket | awk '{system("kubectl delete bucket/"$1" -n mo-checkin-regression-${{ github.event.pull_request.number }}")}'                
      - name: Create MO Cluster
        run: |
          kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} apply -f $GITHUB_WORKSPACE/matrixone/optools/mo_checkin_regression/mo_checkin_regression_tke.yaml
          sleep 100s
          while true; do
           if [ "$(kubectl get mo -n mo-checkin-regression-${{ github.event.pull_request.number }} | grep ${{ needs.docker_image_build.outputs.version }} | grep -iv NotReady | grep -ic Ready )" -eq 0 ]; then
              kubectl get mo -n mo-checkin-regression-${{ github.event.pull_request.number }};
              sleep 15s;
              continue;
            else
              break;
            fi
          done
          kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pod -owide
          if [ ! -d $GITHUB_WORKSPACE/mo-server-log ]; then
            mkdir $GITHUB_WORKSPACE/mo-server-log;
          fi
      - name: Get MO Cluster Addr
        id: get_mo_addr
        timeout-minutes: 2
        run: |
          mo_svc_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get svc | grep "6001/TCP"|awk '{print $3}')
          echo mo_svc_ip=$mo_svc_ip
          echo "mo_svc_ip=${mo_svc_ip}" >> $GITHUB_OUTPUT
          kubectl get pod -n mo-checkin-regression-${{ github.event.pull_request.number }} -l matrixorigin.io/component=CNSet | awk 'NR>1{arr[a++]=$1} END {for(i in arr){system("echo cn_"i"="arr[i]" >> $GITHUB_OUTPUT")}}'
          cn_ips=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=CNSet -o=jsonpath='{.items[*].status.podIP}' | sed 's/ /,/g')
          echo "cn_ips is $cn_ips"
          echo "cn_ips=$cn_ips"  >>  $GITHUB_OUTPUT
      - name: Prepare Data in cn-0
        timeout-minutes: 5
        run: |
          kubectl exec -it ${{ steps.get_mo_addr.outputs.cn_0 }} -n mo-checkin-regression-${{ github.event.pull_request.number }} -- bash -c '
          set -x;
          export https_proxy=http://proxy-service.proxy.svc.cluster.local:8001
          git config --global init.defaultBranch main
          success="no"
          path_matrixone=$(pwd);
          for i in {1..5}; do
            rm -rf $path_matrixone/matrixone;
            mkdir $path_matrixone/matrixone;
            git config --global --add safe.directory $path_matrixone/matrixone;
            cd $path_matrixone/matrixone && git init;
            git remote add origin https://github.com/matrixorigin/matrixone.git;
            git config --local gc.auto 0;
            timeout 120 git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +${{ github.sha }}:refs/remotes/origin/${{ github.ref_name }}
            git checkout --progress --force -B ${{ github.ref_name }} refs/remotes/origin/${{ github.ref_name }}
            if [ "$(git log -1 --format='%H')" == "${{ github.sha }}" ]; then
              success="yes";
              break;
            fi
            sleep 15;
          done

          if [ $success == "no" ]; then
            exit 1;
          fi
          exit 0;
          '
      - name: Prepare Data in cn-1
        timeout-minutes: 5
        run: |
          kubectl exec -it ${{ steps.get_mo_addr.outputs.cn_1 }} -n mo-checkin-regression-${{ github.event.pull_request.number }} -- bash -c '
          set -x;
          export https_proxy=http://proxy-service.proxy.svc.cluster.local:8001
          git config --global init.defaultBranch main
          success="no"
          path_matrixone=$(pwd);
          for i in {1..5}; do
            rm -rf $path_matrixone/matrixone;
            mkdir $path_matrixone/matrixone;
            git config --global --add safe.directory $path_matrixone/matrixone;
            cd $path_matrixone/matrixone && git init;
            git remote add origin https://github.com/matrixorigin/matrixone.git;
            git config --local gc.auto 0;
            timeout 120 git -c protocol.version=2 fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin +${{ github.sha }}:refs/remotes/origin/${{ github.ref_name }}
            git checkout --progress --force -B ${{ github.ref_name }} refs/remotes/origin/${{ github.ref_name }}
            if [ "$(git log -1 --format='%H')" == "${{ github.sha }}" ]; then
              success="yes";
              break;
            fi
            sleep 15;
          done

          if [ $success == "no" ]; then
            exit 1;
          fi
          exit 0;
          '
  bvt_test:
    if: ${{ always() && !cancelled() && needs.setup_mo_test_env.result == 'success' }}
    name: MO CHECKIN BVT TEST
    runs-on: amd64-mo-guangzhou-medium8
    needs: [setup_mo_test_env,docker_image_build]
    steps:
      - name: Clone MO-Tester
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/mo-tester
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-tester
          ref: main
      - name: Clone matrixone Repo
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          path: ./matrixone
      - name: Set JAVA PATH
        run: |
          echo "$JAVA_HOME/bin" >> $GITHUB_PATH
      - name: Generate MO-Tester Config and Start BVT Test
        if: ${{ always() && !cancelled() }}
        timeout-minutes: 20
        run: |
          export LC_ALL="C.UTF-8"
          locale
          sed -i "s/127.0.0.1:6001/${{ needs.setup_mo_test_env.outputs.mo_svc_ip }}:6001/" ./mo-tester/mo.yml
          sed -i 's/socketTimeout:.*/socketTimeout: 300000/g' ./mo-tester/mo.yml
          sed -i 's/  serverIP: "127.0.0.1"/  serverIP: "${{ needs.setup_mo_test_env.outputs.cn_ips }}"/g' ./mo-tester/mo.yml
          sed -i 's/waittime:.*/waittime: 2000/g' ./mo-tester/run.yml
          cat ./mo-tester/mo.yml
          echo "=========================="
          cat ./mo-tester/run.yml
          echo "=========================="

          cd $GITHUB_WORKSPACE/mo-tester
          ./run.sh -n -g -o -p $GITHUB_WORKSPACE/matrixone/test/distributed/cases -s /matrixone/test/distributed/resources -e optimistic  2>&1
          rm -rf .git lib
      - name: Collect Upload files
        if: ${{ always() || cancelled() }}
        run: |
          mkdir -p $GITHUB_WORKSPACE/reports/mo-tester
          mkdir -p $GITHUB_WORKSPACE/mo-tester && cp -r $GITHUB_WORKSPACE/mo-tester $GITHUB_WORKSPACE/reports/mo-tester
      - uses: actions/upload-artifact@v3
        if: ${{ failure() || cancelled()}}
        continue-on-error: true
        with:
          name: mo-checkin-regression-${{ github.event.pull_request.number }}-bvt
          path: |
             ${{ github.workspace }}/reports
  ssb_and_tpch_test:
    if: ${{ always() && !cancelled() && needs.setup_mo_test_env.result == 'success' }}
    name: MO CHECKIN SSB AND TPCH TEST
    runs-on: amd64-mo-guangzhou-2xlarge16
    needs: [setup_mo_test_env,bvt_test]
    steps:
      - name: Clone mo-load-data Repo
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/mo-load-data
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-load-data
          ref: main

      - name: Clone mo-ssb Repo
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/mo-ssb
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-ssb
          ref: main

      - name: Clone mo-tpch Repo
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/mo-tpch
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-tpch
          ref: main
      - name: Clone mo-night-regression Repo
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/mo-nightly-regression
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-nightly-regression
          ref: main
      
      - name: Create Account for Test
        timeout-minutes: 2
        id: account_for_ssb_and_tpch
        run: |
          mysql -h ${{ needs.setup_mo_test_env.outputs.mo_svc_ip }} -P 6001 -udump -p111 -e "create account ssb_tpch_test admin_name = 'admin' identified by '111' comment 'for ssb and tpch test';"
          echo "ssb_tpch_account=ssb_tpch_test:admin" >> $GITHUB_OUTPUT
          echo "ssb_tpch_password=111" >> $GITHUB_OUTPUT

      - name: Modify Tool's Setting And Set PATH
        run: |
          cd $GITHUB_WORKSPACE/mo-load-data
          sed -i '/.*ENDPOINT: ""*/c\ENDPOINT: "cos.ap-guangzhou.myqcloud.com"' access.yml
          sed -i '/.*ACCESS_KEY_ID: ""*/c\ACCESS_KEY_ID: "${{secrets.TENCENT_AK}}"' access.yml
          sed -i '/.*SECRET_ACCESS_KEY: ""*/c\SECRET_ACCESS_KEY: "${{secrets.TENCENT_SK}}"' access.yml
          echo "$JAVA_HOME/bin" >> $GITHUB_PATH

      - name: Start Load SSB 10G Data And Test
        if: ${{ always() && !cancelled() }}
        id: step_ssb_test
        timeout-minutes: 20
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale
          sleep 60s
          cd $GITHUB_WORKSPACE/mo-load-data
          mkdir -p report
          ./load.sh -u ${{ steps.account_for_ssb_and_tpch.outputs.ssb_tpch_account }} -p ${{ steps.account_for_ssb_and_tpch.outputs.ssb_tpch_password }} -h ${{ needs.setup_mo_test_env.outputs.mo_svc_ip }} -c cases/02_from_cos/ssb_10 -r -m -g | tee report/mo-load-data-ssb.log
          # start test
          cd $GITHUB_WORKSPACE/mo-ssb
          mkdir -p report
          ./run.sh -u ${{ steps.account_for_ssb_and_tpch.outputs.ssb_tpch_account }} -p ${{ steps.account_for_ssb_and_tpch.outputs.ssb_tpch_password }} -h ${{ needs.setup_mo_test_env.outputs.mo_svc_ip }} -q all -s 10 -t 5 | tee report/mo-ssb.log

      - name: Check MO Status and Collect Trace
        if: ${{  always() && steps.step_ssb_test.conclusion != 'success' }}
        run: |
          set -x
          unset https_proxy
          unset http_proxy
          kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pod -owide
          rm -rf $GITHUB_WORKSPACE/upload && mkdir -p $GITHUB_WORKSPACE/upload && cd $GITHUB_WORKSPACE/upload
          dn_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=DNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}')
          curl "http://$dn_ip:6060/debug/pprof/trace?seconds=30" -o dn_trace.out
          curl "http://$dn_ip:6060/debug/pprof/goroutine?debug=2" -o dn_groutine.out

          cn_0_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=CNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' | head -n 1)
          curl "http://$cn_0_ip:6060/debug/pprof/trace?seconds=30" -o cn_0_trace.out
          curl "http://$cn_0_ip:6060/debug/pprof/goroutine?debug=2" -o cn_0_groutine.out

          cn_1_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=CNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' | grep -v $cn_0_ip)
          curl "http://$cn_1_ip:6060/debug/pprof/trace?seconds=30" -o cn_1_trace.out
          curl "http://$cn_1_ip:6060/debug/pprof/goroutine?debug=2" -o cn_1_groutine.out

      - uses: actions/upload-artifact@v3
        if: ${{ always() && steps.step_ssb_test.conclusion != 'success' }}
        with:
          name: mo-cluster-trace-${{ github.event.pull_request.number }}-ssb
          path: |
            ${{ github.workspace }}/upload
          retention-days: 7


      # start tpch test
      - name: Load TPCH 10G Test Data
        if: ${{ always() && !cancelled() }}
        timeout-minutes: 20
        id: load_tpch_data
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale
          sleep 60s
          cd $GITHUB_WORKSPACE/mo-load-data
          mkdir -p report
          ./load.sh -u ${{ steps.account_for_ssb_and_tpch.outputs.ssb_tpch_account }} -p ${{ steps.account_for_ssb_and_tpch.outputs.ssb_tpch_password }} -h ${{ needs.setup_mo_test_env.outputs.mo_svc_ip }} -c cases/02_from_cos/tpch_10 -r -m -g | tee report/mo-load-data-tpch.log

      - name: Check MO Status and Collect Trace
        if: ${{ always() && steps.load_tpch_data.conclusion != 'success' }}
        run: |
          set -x
          unset https_proxy
          unset http_proxy
          kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pod -owide
          rm -rf $GITHUB_WORKSPACE/upload && mkdir -p $GITHUB_WORKSPACE/upload && cd $GITHUB_WORKSPACE/upload
          dn_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=DNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}')
          curl "http://$dn_ip:6060/debug/pprof/trace?seconds=30" -o dn_trace.out
          curl "http://$dn_ip:6060/debug/pprof/goroutine?debug=2" -o dn_groutine.out

          cn_0_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=CNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' | head -n 1)
          curl "http://$cn_0_ip:6060/debug/pprof/trace?seconds=30" -o cn_0_trace.out
          curl "http://$cn_0_ip:6060/debug/pprof/goroutine?debug=2" -o cn_0_groutine.out

          cn_1_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=CNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' | grep -v $cn_0_ip)
          curl "http://$cn_1_ip:6060/debug/pprof/trace?seconds=30" -o cn_1_trace.out
          curl "http://$cn_1_ip:6060/debug/pprof/goroutine?debug=2" -o cn_1_groutine.out

      - uses: actions/upload-artifact@v3
        if: ${{ always() && steps.load_tpch_data.conclusion != 'success' }}
        with:
          name: mo-cluster-trace-${{ github.event.pull_request.number }}-load-tpch
          path: |
            ${{ github.workspace }}/upload
          retention-days: 7


      - name: Start TPCH 10G Test
        if: ${{ always() && !cancelled() && steps.load_tpch_data.conclusion == 'success' }}
        id: step_tpch_test
        timeout-minutes: 20
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale
          sleep 60s
          cd $GITHUB_WORKSPACE/mo-tpch
          mkdir -p report
          ./run.sh -u ${{ steps.account_for_ssb_and_tpch.outputs.ssb_tpch_account }} -p ${{ steps.account_for_ssb_and_tpch.outputs.ssb_tpch_password }} -h ${{ needs.setup_mo_test_env.outputs.mo_svc_ip }} -q all -s 10 -t 5 | tee report/tpch.log

      - name: Check MO Status and Collect Trace
        if: ${{ always() && steps.step_tpch_test.conclusion != 'success' }}
        run: |
          set -x
          unset https_proxy
          unset http_proxy
          kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pod -owide
          rm -rf $GITHUB_WORKSPACE/upload && mkdir -p $GITHUB_WORKSPACE/upload && cd $GITHUB_WORKSPACE/upload
          dn_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=DNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}')
          curl "http://$dn_ip:6060/debug/pprof/trace?seconds=30" -o dn_trace.out
          curl "http://$dn_ip:6060/debug/pprof/goroutine?debug=2" -o dn_groutine.out

          cn_0_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=CNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' | head -n 1)
          curl "http://$cn_0_ip:6060/debug/pprof/trace?seconds=30" -o cn_0_trace.out
          curl "http://$cn_0_ip:6060/debug/pprof/goroutine?debug=2" -o cn_0_groutine.out

          cn_1_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=CNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' | grep -v $cn_0_ip)
          curl "http://$cn_1_ip:6060/debug/pprof/trace?seconds=30" -o cn_1_trace.out
          curl "http://$cn_1_ip:6060/debug/pprof/goroutine?debug=2" -o cn_1_groutine.out

      - uses: actions/upload-artifact@v3
        if: ${{ always() && steps.step_tpch_test.conclusion != 'success' }}
        with:
          name: mo-cluster-trace-${{ github.event.pull_request.number }}-tpch
          path: |
            ${{ github.workspace }}/upload
          retention-days: 7

      - name: Collect Upload files
        if: ${{ failure() || cancelled() }}
        run: |
          mkdir -p $GITHUB_WORKSPACE/reports
          mkdir -p $GITHUB_WORKSPACE/reports/mo-ssb
          mkdir -p $GITHUB_WORKSPACE/reports/mo-load-data
          mkdir -p $GITHUB_WORKSPACE/reports/mo-tpch
          mkdir -p $GITHUB_WORKSPACE/mo-ssb/report && cp -r $GITHUB_WORKSPACE/mo-ssb/report $GITHUB_WORKSPACE/reports/mo-ssb
          mkdir -p $GITHUB_WORKSPACE/mo-load-data/report && cp -r $GITHUB_WORKSPACE/mo-load-data/report $GITHUB_WORKSPACE/reports/mo-load-data
          mkdir -p $GITHUB_WORKSPACE/mo-tpch/report && cp -r $GITHUB_WORKSPACE/mo-tpch/report $GITHUB_WORKSPACE/reports/mo-tpch

      - uses: actions/upload-artifact@v3
        if: ${{ failure() || cancelled()}}
        continue-on-error: true
        with:
          name: mo-checkin-regression-${{ github.event.pull_request.number }}-ssb-tpch
          path: |
             ${{ github.workspace }}/reports
          retention-days: 7

  sysbench_test:
    if: ${{ always() && !cancelled() && needs.setup_mo_test_env.result == 'success' }}
    name: SYSBENCH TEST
    runs-on: amd64-mo-guangzhou-2xlarge16
    needs: [setup_mo_test_env,bvt_test]
    steps:
      - name: Clone mo-load Repo
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/mo-load
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-load
          ref: main
      - name: Clone mo-night-regression Repo
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/mo-nightly-regression
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-nightly-regression
          ref: main
          
      - name: Create Account for Test And Set PATH
        timeout-minutes: 2
        id: account_for_sysbench
        run: |
          echo "sysbench_account=dump" >> $GITHUB_OUTPUT
          echo "sysbench_password=111" >> $GITHUB_OUTPUT
          echo "$JAVA_HOME/bin" >> $GITHUB_PATH

 
      # start sysbench test
      - name: Start OLTP Mixed Test With 100 Threads
        if: ${{ always() && !cancelled() }}
        timeout-minutes: 30
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale
          sleep 60s

          cd $GITHUB_WORKSPACE/mo-load
          mkdir -p report
          sed -i 's/127.0.0.1/${{ needs.setup_mo_test_env.outputs.mo_svc_ip }}/g' mo.yml
          sed -i 's/name:.*$/name: "${{ steps.account_for_sysbench.outputs.sysbench_account }}"/g' mo.yml
          sed -i 's/password:.*$/password: "${{ steps.account_for_sysbench.outputs.sysbench_password }}"/g' mo.yml
          
          echo 'Start Prepare Data of 10-tables-100000-per-table'
          ./start.sh -m SYSBENCH -n 10 -s 100000 -t 1 | tee report/load-oltp-mix.log
          sleep 10
          
          echo ''
          echo 'Start OLTP Insert Test With 100 threads'
          ./start.sh -c cases/sysbench/mixed_10_100000 -t 100 -d 20 -g -b sbtest | tee report/oltp-mix.log
     
      - name: Result Summary
        if: ${{ always() && !cancelled()  }}
        run: |
          cd $GITHUB_WORKSPACE/mo-load

          files=("report/oltp-mix.log")
          for filename in "${files[@]}"
          do
              if [ ! -f $filename ]; then
                  continue
              fi
              echo -e "\n[$filename]"
              grep "tps=" $filename > tmp
              N=$(wc -l < tmp)
              head -n 3 tmp
              echo "..."
              middle_start=$((N / 2 - 1))
              middle_end=$((N / 2 + 1))
              head -n $middle_end tmp | tail -n +$middle_start
              echo "..."
              tail -n 3 tmp 
          done

      - name: Collect Upload files
        if: ${{ always() || cancelled() }}
        run: |
          mkdir -p $GITHUB_WORKSPACE/reports/mo-load
          mkdir -p $GITHUB_WORKSPACE/mo-load/report && cp -r $GITHUB_WORKSPACE/mo-load/report $GITHUB_WORKSPACE/reports/mo-load
          
      - uses: actions/upload-artifact@v3
        if: ${{ failure() || cancelled()}}
        continue-on-error: true
        with:
          name: mo-checkin-regression-${{ github.event.pull_request.number }}-sysbench
          path: |
             ${{ github.workspace }}/reports
          retention-days: 7

  tpcc_test:
    if: ${{ always() && !cancelled() && needs.setup_mo_test_env.result == 'success'}}
    name: TPCC TEST
    runs-on: amd64-mo-guangzhou-2xlarge16
    needs: [setup_mo_test_env,bvt_test]
    steps:
      - name: Clone mo-load-data Repo
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/mo-load-data
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-load-data
          ref: main

      - name: Clone mo-tpcc Repo
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/mo-tpcc
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-tpcc
          ref: main
      - name: Clone mo-night-regression Repo
        if: ${{ always() && !cancelled() }}
        uses: actions/checkout@v3
        with:
          repository: matrixorigin/mo-nightly-regression
          token: ${{secrets.TOKEN_ACTION}}
          path: ./mo-nightly-regression
          ref: main
          
      - name: Create Account for Test
        timeout-minutes: 2
        id: account_for_tpcc
        run: |
          mysql -h ${{ needs.setup_mo_test_env.outputs.mo_svc_ip }} -P 6001 -udump -p111 -e "create account tpcc_test admin_name = 'admin' identified by '111' comment 'for tpcc test';"
          echo "tpcc_account=tpcc_test:admin" >> $GITHUB_OUTPUT
          echo "tpcc_password=111" >> $GITHUB_OUTPUT
          

      - name: Modify Tool's Setting And Set PATH
        run: |
          cd $GITHUB_WORKSPACE/mo-load-data
          sed -i '/.*ENDPOINT: ""*/c\ENDPOINT: "cos.ap-guangzhou.myqcloud.com"' access.yml
          sed -i '/.*ACCESS_KEY_ID: ""*/c\ACCESS_KEY_ID: "${{secrets.TENCENT_AK}}"' access.yml
          sed -i '/.*SECRET_ACCESS_KEY: ""*/c\SECRET_ACCESS_KEY: "${{secrets.TENCENT_SK}}"' access.yml

          echo "$JAVA_HOME/bin" >> $GITHUB_PATH
    
     # start tpcc test
      - name: Load TPCC 10 Warehouse Data
        if: ${{ always() && !cancelled() }}
        timeout-minutes: 20
        id: load_tpcc_data
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale
          sleep 60s
          cd $GITHUB_WORKSPACE/mo-load-data
          mkdir -p report
          ./load.sh -u ${{ steps.account_for_tpcc.outputs.tpcc_account }} -p ${{ steps.account_for_tpcc.outputs.tpcc_password }} -h ${{ needs.setup_mo_test_env.outputs.mo_svc_ip }} -c cases/02_from_cos/tpcc_10 -r -m -g | tee report/mo-load-data-tpcc.log

      - name: Start TPCC 10 Warehouses 100 terminals Test
        if: ${{ always() && !cancelled() && steps.load_tpcc_data.conclusion == 'success' }}
        timeout-minutes: 25
        run: |
          set -uo pipefail
          export LC_ALL="C.UTF-8"
          locale
          
          cd $GITHUB_WORKSPACE/mo-tpcc
          mkdir -p report
          cp props.mo props_10.mo
          sed -i '/.*terminals=*/c\terminals=100' props_10.mo
          sed -i '/.*warehouses=*/c\warehouses=10' props_10.mo
          sed -i 's/tpcc/tpcc_10/g' props_10.mo
          sed -i 's/127.0.0.1/${{ needs.setup_mo_test_env.outputs.mo_svc_ip }}/g' props_10.mo
          sed -i '/runMins=*/c\runMins=15' props_10.mo
          sed -i '/.*expectedErrorCodes=*/c\expectedErrorCodes=20619,1062' props_10.mo
          sed -i 's/user=.*/user=${{ steps.account_for_tpcc.outputs.tpcc_account }}/g' props_10.mo
          sed -i 's/password=.*/password=${{ steps.account_for_tpcc.outputs.tpcc_password }}/g' props_10.mo
          ./runBenchmark.sh props_10.mo | tee report/tpcc-benchmark-10-100.log
          ./runVerify.sh props_10.mo | tee report/tpcc-verify-10-100.log
          
      - name: Result Summary
        if: ${{ always() && !cancelled()  }}
        run: |
          cd $GITHUB_WORKSPACE/mo-tpcc
          files=("report/tpcc-benchmark-10-100.log")
          for filename in "${files[@]}"
          do
              if [ ! -f $filename ]; then
                  continue
              fi
              echo -e "\n[$filename]"
              grep "Running" $filename > tmp
              N=$(wc -l < tmp)
              head -n 3 tmp
              echo "..."
              middle_start=$((N / 2 - 1))
              middle_end=$((N / 2 + 1))
              head -n $middle_end tmp | tail -n +$middle_start
              echo "..."
              tail -n 3 tmp 
          done
      
      - name: Collect Upload files
        if: ${{ always() || cancelled() }}
        run: |
          mkdir -p $GITHUB_WORKSPACE/reports/mo-load-data
          mkdir -p $GITHUB_WORKSPACE/reports/mo-tpcc
          mkdir -p $GITHUB_WORKSPACE/mo-load-data/report && cp -r $GITHUB_WORKSPACE/mo-load-data/report $GITHUB_WORKSPACE/reports/mo-load-data
          mkdir -p $GITHUB_WORKSPACE/mo-tpcc/report && cp -r $GITHUB_WORKSPACE/mo-tpcc/report $GITHUB_WORKSPACE/reports/mo-tpcc     
      
      - uses: actions/upload-artifact@v3
        if: ${{ failure() || cancelled()}}
        continue-on-error: true
        with:
          name: mo-checkin-regression-${{ github.event.pull_request.number }}-tpcc
          path: |
             ${{ github.workspace }}/reports
          retention-days: 7

  clean_up_env:
    if: ${{ always() && github.event.pull_request.merged == true }}
    name: CLEAN ENV AND NOTICE
    runs-on: amd64-mo-guangzhou-medium8
    needs: [docker_image_build,setup_mo_test_env,bvt_test,ssb_and_tpch_test,sysbench_test,tpcc_test]
    steps:
      - name: Check and Recreate Namespace
        if: ${{ needs.docker_image_build.result == 'success' }}
        run: |
          if [ "$(kubectl get namespaces| grep -c mo-checkin-regression-${{ github.event.pull_request.number }})" -eq 0 ];then
            kubectl create namespace mo-checkin-regression-${{ github.event.pull_request.number }};
          fi
      - name: Check MO Status and Collect Trace
        if: ${{ needs.docker_image_build.result == 'success' }}
        continue-on-error: true
        run: |
          set -x
          unset https_proxy
          unset http_proxy
          kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pod -owide
          mkdir -p $GITHUB_WORKSPACE/upload && cd $GITHUB_WORKSPACE/upload
          dn_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=DNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}')
          curl "http://$dn_ip:6060/debug/pprof/trace?seconds=30" -o dn_trace.out
          curl "http://$dn_ip:6060/debug/pprof/goroutine?debug=2" -o dn_groutine.out

          cn_0_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=CNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' | head -n 1)
          curl "http://$cn_0_ip:6060/debug/pprof/trace?seconds=30" -o cn_0_trace.out
          curl "http://$cn_0_ip:6060/debug/pprof/goroutine?debug=2" -o cn_0_groutine.out

          cn_1_ip=$(kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pods -l matrixorigin.io/component=CNSet -o=jsonpath='{range .items[*]}{.status.podIP}{"\n"}{end}' | grep -v $cn_0_ip)
          curl "http://$cn_1_ip:6060/debug/pprof/trace?seconds=30" -o cn_1_trace.out
          curl "http://$cn_1_ip:6060/debug/pprof/goroutine?debug=2" -o cn_1_groutine.out

      - uses: actions/upload-artifact@v3
        if: ${{ needs.docker_image_build.result == 'success' }}
        continue-on-error: true
        with:
          name: mo-cluster-trace-${{ github.event.pull_request.number }}
          path: |
            ${{ github.workspace }}/upload
          retention-days: 7
        
      - name: Check MO Status and Clean ENV
        if: ${{ needs.docker_image_build.result == 'success' && needs.setup_mo_test_env.result == 'success' && needs.bvt_test.result == 'success' && needs.ssb_and_tpch_test.result == 'success' && needs.sysbench_test.result == 'success' && needs.tpcc_test.result == 'success'  }}
        run: |
          cd $GITHUB_WORKSPACE
          kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pod -owide
          kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get matrixonecluster        
          #delete matrixone cluster
          kubectl get matrixonecluster -n mo-checkin-regression-${{ github.event.pull_request.number }} | awk 'NR>1{system("kubectl delete matrixonecluster/"$1" -n mo-checkin-regression-${{ github.event.pull_request.number }}")}'
          sleep 60      
          kubectl -n mo-checkin-regression-${{ github.event.pull_request.number }} get pod -owide
          
          #delete bucket
          kubectl get bucket -n mo-checkin-regression-${{ github.event.pull_request.number }} | grep bucket | awk '{system("kubectl delete bucket/"$1" -n mo-checkin-regression-${{ github.event.pull_request.number }}")}'
          #delete namespace
          kubectl delete namespace mo-checkin-regression-${{ github.event.pull_request.number }}
      - name: NOTICE FAIL RESULT
        if: ${{ failure() || cancelled() || needs.docker_image_build.result != 'success'|| needs.setup_mo_test_env.result != 'success' || needs.bvt_test.result != 'success' || needs.ssb_and_tpch_test.result != 'success' || needs.sysbench_test.result != 'success' || needs.tpcc_test.result != 'success' }}
        uses: guguducken/action-wechat-work@master
        env:
          WECHAT_WORK_BOT_WEBHOOK: ${{ secrets.WECHAT_MO_CHECKIN_BOT_WEBHOOK }}
        with:
          msgtype: markdown
          content: "MO Checkin Regression On TKE failed: [${{ github.event.pull_request.title }}](${{ github.event.pull_request.html_url }})\nCreator:<font color=\"warning\">${{ github.event.pull_request.user.login }}</font>\nAction Link: [Link](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})<@shuqi.qin><@JiejieJia><@SuDong>"