test: Add test pipeline

.dockerignore

@@ -0,0 +1,6 @@
+.git
+.vscode
+scripts
+logs
+tests
+deployment

.github/workflows/ci.yaml

@@ -0,0 +1,145 @@
+name: Test
+
+on:
+    push:
+        branches:
+        - main
+    pull_request:           
+        branches:
+        - main
+
+jobs:
+    milvus-cdc-test:
+        runs-on: ubuntu-latest
+        steps:
+        - uses: actions/checkout@v2
+
+        - uses: actions/setup-python@v4
+          with:
+            python-version: 3.8
+            cache: pip
+            cache-dependency-path: 'tests/requirements.txt'      
+        - name: Deploy Source Milvus
+          timeout-minutes: 15
+          shell: bash
+          working-directory: tests/deployment/upstream/
+          run: |
+            docker-compose up -d
+            bash ../../../scripts/check_healthy.sh
+            docker-compose ps -a
+
+        - name: Creating kind cluster
+          uses: helm/kind-action@v1.2.0
+
+        - name: Print cluster information
+          run: |
+            kubectl config view
+            kubectl cluster-info
+            kubectl get nodes
+            kubectl get pods -n kube-system
+            helm version
+            kubectl version 
+
+        - name: Deploy Target Milvus
+          timeout-minutes: 15
+          shell: bash
+          working-directory: tests/deployment/downstream
+          run: |
+            helm repo add milvus https://milvus-io.github.io/milvus-helm
+            helm repo update
+            helm install --wait --timeout 720s cdc-target milvus/milvus -f standalone-values.yaml;
+            kubectl get pods
+            sleep 20s
+            kubectl get pods
+            kubectl port-forward service/cdc-target-milvus 19500:19530 >/dev/null 2>&1 &
+            sleep 20s
+            # check whether port-forward success
+            nc -vz 127.0.0.1 19500
+
+        - uses: actions/setup-go@v4
+          with:
+            go-version: '1.18'
+            cache-dependency-path: server/go.sum
+            cache: true
+        - name: Build CDC
+          timeout-minutes: 15
+          working-directory: server
+          shell: bash
+          run: |
+            go build -o milvus-cdc main/main.go
+            ls -l
+
+        - name: Deploy Milvus CDC
+          timeout-minutes: 15
+          working-directory: server
+          shell: bash
+          run: |
+            ./milvus-cdc > server.log 2>&1 &
+            sleep 20s
+            nc -vz 127.0.0.1 8444
+            sleep 20s
+            curl --location '127.0.0.1:8444/cdc' \
+            --header 'Content-Type: application/json' \
+            --data '{
+                "request_type": "create",
+                "request_data": {
+                    "milvus_connect_param": {
+                        "host": "127.0.0.1",
+                        "port": 19500,
+                        "username": "",
+                        "password": "",
+                        "enable_tls": false,
+                        "ignore_partition": true,
+                        "connect_timeout": 10
+                    },
+                    "collection_infos": [
+                        {
+                            "name": "sift_128_euclidean_HNSW"
+                        }
+                    ]
+                }
+            }'
+
+        - name: Install dependencies
+          timeout-minutes: 5
+          working-directory: tests
+          run: |
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
+
+        - name: Download data
+          timeout-minutes: 5
+          working-directory: tests/assets/ann_hdf5
+          run: |
+            bash download.sh
+
+        - name: Run tests for source milvus
+          timeout-minutes: 15
+          working-directory: tests/scripts
+          run: |
+            python source_test.py --host 127.0.0.1 --port 19530
+
+        - name: Wait for 1 minutes
+          timeout-minutes: 5
+          run: |
+            sleep 60s
+
+        - name: Run tests for target milvus
+          timeout-minutes: 15
+          working-directory: tests/scripts
+          run: |
+            python target_test.py --host 127.0.0.1 --port 19500
+
+        - name: Verify deletion
+          timeout-minutes: 15
+          working-directory: tests/scripts
+          run: |
+            python verify_delete.py --source_host 127.0.0.1 --source_port 19530 --target_host 127.0.0.1 --target_port 19500
+
+        - name: Upload logs
+          if: ${{ ! success() }}
+          uses: actions/upload-artifact@v2
+          with:
+            name: logs
+            path: |
+              server/server.log

.gitignore

@@ -1,2 +1,5 @@
 .DS_Store
+.vscode
 **/.idea/*
+**/dist/*
+**/*.log/*

.goreleaser.yaml

@@ -0,0 +1,48 @@
+# This is an example .goreleaser.yml file with some sensible defaults.
+# Make sure to check the documentation at https://goreleaser.com
+before:
+  hooks:
+    # You may remove this if you don't use go modules.
+    - go mod tidy
+    # you may remove this if you don't need go generate
+    # - go generate ./...
+builds:
+  - env:
+      - CGO_ENABLED=1
+    main: ./main
+    goos:
+      # - linux
+      # - windows
+      - darwin
+    goarch:
+      - amd64
+      - arm64
+archives:
+  - format: tar.gz
+    # this name template makes the OS and Arch compatible with the results of uname.
+    name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end }}
+    # use zip for windows archives
+    format_overrides:
+    - goos: windows
+      format: zip
+checksum:
+  name_template: 'checksums.txt'
+snapshot:
+  name_template: "{{ incpatch .Version }}-next"
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - '^docs:'
+      - '^test:'
+
+# The lines beneath this are called `modelines`. See `:help modeline`
+# Feel free to remove those if you don't want/use them.
+# yaml-language-server: $schema=https://goreleaser.com/static/schema.json
+# vim: set ts=2 sw=2 tw=0 fo=cnqoj

Dockerfile

@@ -0,0 +1,14 @@
+FROM  golang:1.18 AS builder
+ENV CGO_ENABLED=1
+WORKDIR /app
+COPY . .
+RUN go mod tidy
+RUN cd server && go build -o /app/milvus-cdc main/main.go
+
+FROM debian:bullseye
+WORKDIR /app
+COPY --from=builder /app/milvus-cdc ./
+COPY --from=builder /app/server/configs ./configs
+EXPOSE 8444
+
+CMD ["/bin/bash", "-c", "cat /app/configs/cdc.yaml;/app/milvus-cdc"]

README.md

@@ -0,0 +1,3 @@
+# Milvus-CDC
+
+Milvus-CDC is a change data capture tool for Milvus. It can capture the changes of upstream Milvus collections and sink them to downstream Milvus.

build_image.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# get current git branch
+BRANCH=$(git symbolic-ref --short HEAD)
+
+# get current git commit id
+COMMIT_ID=$(git rev-parse --short HEAD)
+
+# build docker image
+docker build -t milvus-cdc:${BRANCH}-${COMMIT_ID} .
+docker tag milvus-cdc:${BRANCH}-${COMMIT_ID} milvus-cdc:latest

deployment/k8s/configmap.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: milvus-cdc-config
+  namespace: chaos-testing
+data:
+  cdc.yaml: |
+    address: 0.0.0.0
+    port: 8444
+    etcd.endpoints: [10.101.143.170:2379] # etcd endpoints
+    etcd.rootpath: cdc_demo
+    source:
+      etcd.endpoints: [10.101.143.170:2379]  # etcd endpoints
+      etcd.rootpath: cdc-test-source  # etcd rootpath
+      etcd.meta.path: meta  # etcd meta sub path
+      defaultPartitionName: _default
+      reader.buffer.len: 10
+      mqtype: pulsar
+      pulsar:
+        address: 10.101.152.110
+        port: 6650
+        web.address:
+        web.port: 80
+        max.message.size: 5242880
+        tenant: public
+        namespace: default
+      kafka:
+        broker_list: localhost:9092    

deployment/k8s/deployment.yaml

@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: milvus-cdc
+spec:
+  selector:
+    matchLabels:
+      app: milvus-cdc
+      instance: my-release
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: milvus-cdc
+        instance: my-release
+    spec:
+      containers:
+        - name: milvus-cdc
+          image: harbor.milvus.io/qa/milvus-cdc:0.1
+          imagePullPolicy: Always
+          resources:
+            limits:
+              cpu: 500m
+              memory: 1Gi
+            requests:
+              cpu: 250m
+              memory: 256Mi
+          securityContext:
+            privileged: false
+          volumeMounts:
+          - name: cdc-config
+            mountPath: /app/configs/cdc.yaml
+            subPath: cdc.yaml
+      volumes:
+        - name: cdc-config
+          configMap:
+            defaultMode: 420
+            name: milvus-cdc-config
+      restartPolicy: Always
+

deployment/k8s/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: milvus-cdc-service
+  labels:
+    app: milvus-cdc
+    instance: my-release
+spec:
+  selector:
+    app: milvus-cdc
+    instance: my-release
+  ports:
+  - name: milvus-cdc
+    protocol: TCP
+    port: 8444
+    targetPort: 8444

scripts/check_healthy.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+#to check containers all running and minio is healthy
+function check_healthy {
+    Expect=$(yq '.services | length' 'docker-compose.yml')
+    Expect_health=$(yq '.services' 'docker-compose.yml' |grep 'healthcheck'|wc -l)
+    cnt=$(docker-compose ps | grep -E "running|Running|Up|up" | wc -l)
+    healthy=$(docker-compose ps | grep "healthy" | wc -l)
+    time_cnt=0
+    echo "running num $cnt expect num $Expect"
+    echo "healthy num $healthy expect num $Expect_health"
+    while [[ $cnt -ne $Expect || $healthy -ne 1 ]];
+    do
+    printf "waiting all containers getting running\n"
+    sleep 5
+    let time_cnt+=5
+    # if time is greater than 300s, the condition still not satisfied, we regard it as a failure
+    if [ $time_cnt -gt 300 ];
+    then
+        printf "timeout,there are some issues with deployment!"
+        exit 1
+    fi
+    cnt=$(docker-compose ps | grep -E "running|Running|Up|up" | wc -l)
+    healthy=$(docker-compose ps | grep "healthy" | wc -l)
+    echo "running num $cnt expect num $Expect"
+    echo "healthy num $healthy expect num $Expect_health"
+    done
+}
+
+check_healthy

scripts/export_log_docker.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# Exit immediately for non zero status
+set -e
+
+log_dir=${1:-"logs"}
+array=($(docker-compose ps -a|awk 'NR == 1 {next} {print $1}'))
+echo ${array[@]}
+if [ ! -d $log_dir ];
+then
+    mkdir -p $log_dir
+fi
+echo "export logs start"
+for container in ${array[*]}
+do
+if [[ $container == milvus-* ]];
+then
+    echo "export logs for container $container "
+    docker logs $container > ./$log_dir/$container.log 2>&1 || echo "export logs for container $container failed"
+fi
+done
+echo "export logs done"

scripts/uninstall_milvus.sh

@@ -0,0 +1,7 @@
+# Exit immediately for non zero status
+set -e
+release=${1:-"milvus-chaos"}
+ns=${2:-"chaos-testing"}
+helm uninstall ${release} -n=${ns}
+kubectl delete pvc -l release=${release} -n=${ns}
+kubectl delete pvc -l app.kubernetes.io/instance=${release} -n=${ns}