Milvus cdc server and core lib, initial commit

Signed-off-by: SimFG <bang.fu@zilliz.com>

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+**/.idea/*

Makefile

@@ -0,0 +1,5 @@
+PWD 	  := $(shell pwd)
+
+test:
+	@echo "Running go unittests..."
+	@(env bash $(PWD)/scripts/run_go_unittest.sh)

core/config/common.go

@@ -0,0 +1,27 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+type Option[T any] interface {
+	Apply(object T)
+}
+
+type OptionFunc[T any] func(object T)
+
+func (o OptionFunc[T]) Apply(object T) {
+	o(object)
+}

core/config/mq.go

@@ -0,0 +1,110 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"strconv"
+
+	"github.com/milvus-io/milvus/pkg/config"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
+)
+
+var configManager = config.NewManager()
+
+func NewParamItem(value string) paramtable.ParamItem {
+	item := paramtable.ParamItem{
+		Formatter: func(_ string) string {
+			return value
+		},
+	}
+	item.Init(configManager)
+	return item
+}
+
+func NewParamGroup() paramtable.ParamGroup {
+	group := paramtable.ParamGroup{
+		GetFunc: func() map[string]string {
+			return map[string]string{}
+		},
+	}
+	return group
+}
+
+type KafkaConfig struct {
+	Address string
+}
+
+func NewKafkaConfig(options ...Option[*KafkaConfig]) KafkaConfig {
+	var k KafkaConfig
+	for _, option := range options {
+		option.Apply(&k)
+	}
+	return k
+}
+
+func KafkaAddressOption(address string) Option[*KafkaConfig] {
+	return OptionFunc[*KafkaConfig](func(object *KafkaConfig) {
+		object.Address = address
+	})
+}
+
+type PulsarConfig struct {
+	Address        string
+	Port           string
+	WebAddress     string
+	WebPort        int
+	MaxMessageSize string
+
+	// support tenant
+	Tenant    string
+	Namespace string
+}
+
+func NewPulsarConfig(options ...Option[*PulsarConfig]) PulsarConfig {
+	p := PulsarConfig{}
+	for _, option := range options {
+		option.Apply(&p)
+	}
+	return p
+}
+
+func PulsarAddressOption(address string) Option[*PulsarConfig] {
+	return OptionFunc[*PulsarConfig](func(object *PulsarConfig) {
+		object.Address = address
+	})
+}
+
+func PulsarWebAddressOption(address string, port int) Option[*PulsarConfig] {
+	return OptionFunc[*PulsarConfig](func(object *PulsarConfig) {
+		object.WebAddress = address
+		object.WebPort = port
+	})
+}
+
+// PulsarMaxMessageSizeOption size unit: Bytes
+func PulsarMaxMessageSizeOption(size int64) Option[*PulsarConfig] {
+	return OptionFunc[*PulsarConfig](func(object *PulsarConfig) {
+		object.MaxMessageSize = strconv.FormatInt(size, 10)
+	})
+}
+
+func PulsarTenantOption(tenant string, namespace string) Option[*PulsarConfig] {
+	return OptionFunc[*PulsarConfig](func(object *PulsarConfig) {
+		object.Tenant = tenant
+		object.Namespace = namespace
+	})
+}

core/config/reader.go

@@ -0,0 +1,75 @@
+// Licensed to the LF AI & Data foundation under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+type MilvusMQConfig struct {
+	Pulsar PulsarConfig
+	Kafka  KafkaConfig
+}
+
+type MilvusEtcdConfig struct {
+	Endpoints   []string
+	RootPath    string
+	MetaSubPath string
+
+	// the key of the collection meta, default: root-coord/collection
+	CollectionKey string
+	// default: root-coord/partitions
+	PartitionKey string
+	// default: root-coord/fields
+	FiledKey string
+	// default: _default
+	DefaultPartitionName string
+}
+
+func defaultMilvusEtcdConfig() MilvusEtcdConfig {
+	return MilvusEtcdConfig{
+		Endpoints:            []string{"localhost:2379"},
+		RootPath:             "by-dev",
+		MetaSubPath:          "meta",
+		CollectionKey:        "root-coord/collection",
+		PartitionKey:         "root-coord/partitions",
+		FiledKey:             "root-coord/fields",
+		DefaultPartitionName: "_default",
+	}
+}
+
+func NewMilvusEtcdConfig(options ...Option[*MilvusEtcdConfig]) MilvusEtcdConfig {
+	c := defaultMilvusEtcdConfig()
+	for _, option := range options {
+		option.Apply(&c)
+	}
+	return c
+}
+
+func MilvusEtcdEndpointsOption(endpoints []string) Option[*MilvusEtcdConfig] {
+	return OptionFunc[*MilvusEtcdConfig](func(object *MilvusEtcdConfig) {
+		object.Endpoints = endpoints
+	})
+}
+
+func MilvusEtcdRootPathOption(rootPath string) Option[*MilvusEtcdConfig] {
+	return OptionFunc[*MilvusEtcdConfig](func(object *MilvusEtcdConfig) {
+		object.RootPath = rootPath
+	})
+}
+
+func MilvusEtcdMetaSubPathOption(metaSubPath string) Option[*MilvusEtcdConfig] {
+	return OptionFunc[*MilvusEtcdConfig](func(object *MilvusEtcdConfig) {
+		object.MetaSubPath = metaSubPath
+	})
+}

core/go.mod

@@ -0,0 +1,150 @@
+module github.com/zilliztech/milvus-cdc/core
+
+go 1.18
+
+require (
+	github.com/cockroachdb/errors v1.9.1
+	github.com/goccy/go-json v0.10.0
+	github.com/golang/protobuf v1.5.2
+	github.com/milvus-io/milvus-proto/go-api v0.0.0-20230303054144-16f081962572
+	github.com/milvus-io/milvus-sdk-go/v2 v2.2.1-0.20230228025743-8fff5cdd8006
+	github.com/milvus-io/milvus/pkg v0.0.1
+	github.com/samber/lo v1.27.0
+	github.com/stretchr/testify v1.8.1
+	go.etcd.io/etcd/api/v3 v3.5.5
+	go.etcd.io/etcd/client/v3 v3.5.5
+	go.uber.org/zap v1.17.0
+)
+
+require (
+	github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
+	github.com/99designs/keyring v1.2.1 // indirect
+	github.com/AthenZ/athenz v1.10.39 // indirect
+	github.com/DataDog/zstd v1.5.0 // indirect
+	github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 // indirect
+	github.com/ardielle/ardielle-go v1.5.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blang/semver/v4 v4.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
+	github.com/cockroachdb/redact v1.1.3 // indirect
+	github.com/confluentinc/confluent-kafka-go v1.9.1 // indirect
+	github.com/containerd/cgroups v1.0.4 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/danieljoos/wincred v1.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/dvsekhvalnov/jose2go v1.5.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/getsentry/sentry-go v0.12.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect
+	github.com/godbus/dbus/v5 v5.0.4 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/btree v1.0.1 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+	github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/jonboulle/clockwork v0.2.2 // indirect
+	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/klauspost/compress v1.14.4 // indirect
+	github.com/kr/pretty v0.3.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/linkedin/goavro/v2 v2.11.1 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/mtibben/percent v0.2.1 // indirect
+	github.com/opencontainers/runtime-spec v1.0.2 // indirect
+	github.com/pelletier/go-toml v1.9.3 // indirect
+	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/prometheus/client_golang v1.11.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.26.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/rogpeppe/go-internal v1.8.1 // indirect
+	github.com/shirou/gopsutil/v3 v3.22.9 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/soheilhy/cmux v0.1.5 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/viper v1.8.1 // indirect
+	github.com/streamnative/pulsarctl v0.5.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
+	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
+	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	github.com/yusufpapurcu/wmi v1.2.2 // indirect
+	go.etcd.io/bbolt v1.3.6 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
+	go.etcd.io/etcd/client/v2 v2.305.5 // indirect
+	go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect
+	go.etcd.io/etcd/raft/v3 v3.5.5 // indirect
+	go.etcd.io/etcd/server/v3 v3.5.5 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 // indirect
+	go.opentelemetry.io/otel v1.11.2 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2 // indirect
+	go.opentelemetry.io/otel/metric v0.34.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.11.2 // indirect
+	go.opentelemetry.io/otel/trace v1.11.2 // indirect
+	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/automaxprocs v1.4.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
+	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
+	golang.org/x/net v0.6.0 // indirect
+	golang.org/x/oauth2 v0.5.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/term v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29 // indirect
+	google.golang.org/grpc v1.51.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
+)
+
+replace (
+	github.com/apache/pulsar-client-go => github.com/milvus-io/pulsar-client-go v0.6.10
+	github.com/milvus-io/milvus/pkg => github.com/jaime0815/milvus/pkg v0.0.0-20230306083504-7b3f06609d62
+	github.com/streamnative/pulsarctl => github.com/xiaofan-luan/pulsarctl v0.5.1
+	github.com/tecbot/gorocksdb => ./../rocksdb
+)
+
+exclude (
+	cloud.google.com/go/compute/metadata v0.2.0
+	github.com/apache/pulsar-client-go/oauth2 v0.0.0-20211108044248-fe3b7c4e445b
+)