-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
- Loading branch information
1 parent
365f83d
commit 81d96b2
Showing
39 changed files
with
1,144 additions
and
760 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
.git | ||
.vscode | ||
scripts | ||
logs | ||
tests | ||
deployment |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,145 @@ | ||
name: Test | ||
|
||
on: | ||
push: | ||
branches: | ||
- main | ||
pull_request: | ||
branches: | ||
- main | ||
|
||
jobs: | ||
milvus-cdc-test: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- uses: actions/checkout@v2 | ||
|
||
- uses: actions/setup-python@v4 | ||
with: | ||
python-version: 3.8 | ||
cache: pip | ||
cache-dependency-path: 'tests/requirements.txt' | ||
- name: Deploy Source Milvus | ||
timeout-minutes: 15 | ||
shell: bash | ||
working-directory: tests/deployment/upstream/ | ||
run: | | ||
docker-compose up -d | ||
bash ../../../scripts/check_healthy.sh | ||
docker-compose ps -a | ||
- name: Creating kind cluster | ||
uses: helm/kind-action@v1.2.0 | ||
|
||
- name: Print cluster information | ||
run: | | ||
kubectl config view | ||
kubectl cluster-info | ||
kubectl get nodes | ||
kubectl get pods -n kube-system | ||
helm version | ||
kubectl version | ||
- name: Deploy Target Milvus | ||
timeout-minutes: 15 | ||
shell: bash | ||
working-directory: tests/deployment/downstream | ||
run: | | ||
helm repo add milvus https://milvus-io.github.io/milvus-helm | ||
helm repo update | ||
helm install --wait --timeout 720s cdc-target milvus/milvus -f standalone-values.yaml; | ||
kubectl get pods | ||
sleep 20s | ||
kubectl get pods | ||
kubectl port-forward service/cdc-target-milvus 19500:19530 >/dev/null 2>&1 & | ||
sleep 20s | ||
# check whether port-forward success | ||
nc -vz 127.0.0.1 19500 | ||
- uses: actions/setup-go@v4 | ||
with: | ||
go-version: '1.18' | ||
cache-dependency-path: server/go.sum | ||
cache: true | ||
- name: Build CDC | ||
timeout-minutes: 15 | ||
working-directory: server | ||
shell: bash | ||
run: | | ||
go build -o milvus-cdc main/main.go | ||
ls -l | ||
- name: Deploy Milvus CDC | ||
timeout-minutes: 15 | ||
working-directory: server | ||
shell: bash | ||
run: | | ||
./milvus-cdc > server.log 2>&1 & | ||
sleep 20s | ||
nc -vz 127.0.0.1 8444 | ||
sleep 20s | ||
curl --location '127.0.0.1:8444/cdc' \ | ||
--header 'Content-Type: application/json' \ | ||
--data '{ | ||
"request_type": "create", | ||
"request_data": { | ||
"milvus_connect_param": { | ||
"host": "127.0.0.1", | ||
"port": 19500, | ||
"username": "", | ||
"password": "", | ||
"enable_tls": false, | ||
"ignore_partition": true, | ||
"connect_timeout": 10 | ||
}, | ||
"collection_infos": [ | ||
{ | ||
"name": "sift_128_euclidean_HNSW" | ||
} | ||
] | ||
} | ||
}' | ||
- name: Install dependencies | ||
timeout-minutes: 5 | ||
working-directory: tests | ||
run: | | ||
python -m pip install --upgrade pip | ||
pip install -r requirements.txt | ||
- name: Download data | ||
timeout-minutes: 5 | ||
working-directory: tests/assets/ann_hdf5 | ||
run: | | ||
bash download.sh | ||
- name: Run tests for source milvus | ||
timeout-minutes: 15 | ||
working-directory: tests/scripts | ||
run: | | ||
python source_test.py --host 127.0.0.1 --port 19530 | ||
- name: Wait for 1 minutes | ||
timeout-minutes: 5 | ||
run: | | ||
sleep 60s | ||
- name: Run tests for target milvus | ||
timeout-minutes: 15 | ||
working-directory: tests/scripts | ||
run: | | ||
python target_test.py --host 127.0.0.1 --port 19500 | ||
- name: Verify deletion | ||
timeout-minutes: 15 | ||
working-directory: tests/scripts | ||
run: | | ||
python verify_delete.py --source_host 127.0.0.1 --source_port 19530 --target_host 127.0.0.1 --target_port 19500 | ||
- name: Upload logs | ||
if: ${{ ! success() }} | ||
uses: actions/upload-artifact@v2 | ||
with: | ||
name: logs | ||
path: | | ||
server/server.log |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,2 +1,5 @@ | ||
.DS_Store | ||
.vscode | ||
**/.idea/* | ||
**/dist/* | ||
**/*.log/* |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
# This is an example .goreleaser.yml file with some sensible defaults. | ||
# Make sure to check the documentation at https://goreleaser.com | ||
before: | ||
hooks: | ||
# You may remove this if you don't use go modules. | ||
- go mod tidy | ||
# you may remove this if you don't need go generate | ||
# - go generate ./... | ||
builds: | ||
- env: | ||
- CGO_ENABLED=1 | ||
main: ./main | ||
goos: | ||
# - linux | ||
# - windows | ||
- darwin | ||
goarch: | ||
- amd64 | ||
- arm64 | ||
archives: | ||
- format: tar.gz | ||
# this name template makes the OS and Arch compatible with the results of uname. | ||
name_template: >- | ||
{{ .ProjectName }}_ | ||
{{- title .Os }}_ | ||
{{- if eq .Arch "amd64" }}x86_64 | ||
{{- else if eq .Arch "386" }}i386 | ||
{{- else }}{{ .Arch }}{{ end }} | ||
{{- if .Arm }}v{{ .Arm }}{{ end }} | ||
# use zip for windows archives | ||
format_overrides: | ||
- goos: windows | ||
format: zip | ||
checksum: | ||
name_template: 'checksums.txt' | ||
snapshot: | ||
name_template: "{{ incpatch .Version }}-next" | ||
changelog: | ||
sort: asc | ||
filters: | ||
exclude: | ||
- '^docs:' | ||
- '^test:' | ||
|
||
# The lines beneath this are called `modelines`. See `:help modeline` | ||
# Feel free to remove those if you don't want/use them. | ||
# yaml-language-server: $schema=https://goreleaser.com/static/schema.json | ||
# vim: set ts=2 sw=2 tw=0 fo=cnqoj |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
FROM golang:1.18 AS builder | ||
ENV CGO_ENABLED=1 | ||
WORKDIR /app | ||
COPY . . | ||
RUN go mod tidy | ||
RUN cd server && go build -o /app/milvus-cdc main/main.go | ||
|
||
FROM debian:bullseye | ||
WORKDIR /app | ||
COPY --from=builder /app/milvus-cdc ./ | ||
COPY --from=builder /app/server/configs ./configs | ||
EXPOSE 8444 | ||
|
||
CMD ["/bin/bash", "-c", "cat /app/configs/cdc.yaml;/app/milvus-cdc"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Milvus-CDC | ||
|
||
Milvus-CDC is a change data capture tool for Milvus. It can capture the changes of upstream Milvus collections and sink them to downstream Milvus. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
#!/bin/bash | ||
|
||
# get current git branch | ||
BRANCH=$(git symbolic-ref --short HEAD) | ||
|
||
# get current git commit id | ||
COMMIT_ID=$(git rev-parse --short HEAD) | ||
|
||
# build docker image | ||
docker build -t milvus-cdc:${BRANCH}-${COMMIT_ID} . | ||
docker tag milvus-cdc:${BRANCH}-${COMMIT_ID} milvus-cdc:latest |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
apiVersion: v1 | ||
kind: ConfigMap | ||
metadata: | ||
name: milvus-cdc-config | ||
namespace: chaos-testing | ||
data: | ||
cdc.yaml: | | ||
address: 0.0.0.0 | ||
port: 8444 | ||
etcd.endpoints: [10.101.143.170:2379] # etcd endpoints | ||
etcd.rootpath: cdc_demo | ||
source: | ||
etcd.endpoints: [10.101.143.170:2379] # etcd endpoints | ||
etcd.rootpath: cdc-test-source # etcd rootpath | ||
etcd.meta.path: meta # etcd meta sub path | ||
defaultPartitionName: _default | ||
reader.buffer.len: 10 | ||
mqtype: pulsar | ||
pulsar: | ||
address: 10.101.152.110 | ||
port: 6650 | ||
web.address: | ||
web.port: 80 | ||
max.message.size: 5242880 | ||
tenant: public | ||
namespace: default | ||
kafka: | ||
broker_list: localhost:9092 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
apiVersion: apps/v1 | ||
kind: Deployment | ||
metadata: | ||
name: milvus-cdc | ||
spec: | ||
selector: | ||
matchLabels: | ||
app: milvus-cdc | ||
instance: my-release | ||
replicas: 2 | ||
template: | ||
metadata: | ||
labels: | ||
app: milvus-cdc | ||
instance: my-release | ||
spec: | ||
containers: | ||
- name: milvus-cdc | ||
image: harbor.milvus.io/qa/milvus-cdc:0.1 | ||
imagePullPolicy: Always | ||
resources: | ||
limits: | ||
cpu: 500m | ||
memory: 1Gi | ||
requests: | ||
cpu: 250m | ||
memory: 256Mi | ||
securityContext: | ||
privileged: false | ||
volumeMounts: | ||
- name: cdc-config | ||
mountPath: /app/configs/cdc.yaml | ||
subPath: cdc.yaml | ||
volumes: | ||
- name: cdc-config | ||
configMap: | ||
defaultMode: 420 | ||
name: milvus-cdc-config | ||
restartPolicy: Always | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
apiVersion: v1 | ||
kind: Service | ||
metadata: | ||
name: milvus-cdc-service | ||
labels: | ||
app: milvus-cdc | ||
instance: my-release | ||
spec: | ||
selector: | ||
app: milvus-cdc | ||
instance: my-release | ||
ports: | ||
- name: milvus-cdc | ||
protocol: TCP | ||
port: 8444 | ||
targetPort: 8444 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
#!/bin/bash | ||
|
||
#to check containers all running and minio is healthy | ||
function check_healthy { | ||
Expect=$(yq '.services | length' 'docker-compose.yml') | ||
Expect_health=$(yq '.services' 'docker-compose.yml' |grep 'healthcheck'|wc -l) | ||
cnt=$(docker-compose ps | grep -E "running|Running|Up|up" | wc -l) | ||
healthy=$(docker-compose ps | grep "healthy" | wc -l) | ||
time_cnt=0 | ||
echo "running num $cnt expect num $Expect" | ||
echo "healthy num $healthy expect num $Expect_health" | ||
while [[ $cnt -ne $Expect || $healthy -ne 1 ]]; | ||
do | ||
printf "waiting all containers getting running\n" | ||
sleep 5 | ||
let time_cnt+=5 | ||
# if time is greater than 300s, the condition still not satisfied, we regard it as a failure | ||
if [ $time_cnt -gt 300 ]; | ||
then | ||
printf "timeout,there are some issues with deployment!" | ||
exit 1 | ||
fi | ||
cnt=$(docker-compose ps | grep -E "running|Running|Up|up" | wc -l) | ||
healthy=$(docker-compose ps | grep "healthy" | wc -l) | ||
echo "running num $cnt expect num $Expect" | ||
echo "healthy num $healthy expect num $Expect_health" | ||
done | ||
} | ||
|
||
check_healthy |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
#!/bin/bash | ||
|
||
# Exit immediately for non zero status | ||
set -e | ||
|
||
log_dir=${1:-"logs"} | ||
array=($(docker-compose ps -a|awk 'NR == 1 {next} {print $1}')) | ||
echo ${array[@]} | ||
if [ ! -d $log_dir ]; | ||
then | ||
mkdir -p $log_dir | ||
fi | ||
echo "export logs start" | ||
for container in ${array[*]} | ||
do | ||
if [[ $container == milvus-* ]]; | ||
then | ||
echo "export logs for container $container " | ||
docker logs $container > ./$log_dir/$container.log 2>&1 || echo "export logs for container $container failed" | ||
fi | ||
done | ||
echo "export logs done" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
# Exit immediately for non zero status | ||
set -e | ||
release=${1:-"milvus-chaos"} | ||
ns=${2:-"chaos-testing"} | ||
helm uninstall ${release} -n=${ns} | ||
kubectl delete pvc -l release=${release} -n=${ns} | ||
kubectl delete pvc -l app.kubernetes.io/instance=${release} -n=${ns} |
Binary file not shown.
Oops, something went wrong.