From 6423b6c71858e214999935ccb1a603106e62e9ea Mon Sep 17 00:00:00 2001 From: jaime Date: Wed, 26 Jun 2024 13:36:05 +0800 Subject: [PATCH] enhance: move rocksmq from internal to pkg (#34165) pr: https://github.com/milvus-io/milvus/pull/33881 issue: https://github.com/milvus-io/milvus/issues/33956 Signed-off-by: jaime --- .gitignore | 2 - cmd/milvus/mck.go | 2 +- cmd/roles/roles.go | 2 +- cmd/tools/migration/backend/etcd.go | 2 +- cmd/tools/migration/mmap/tool/main.go | 2 +- go.mod | 9 +- go.sum | 23 +- internal/allocator/global_id_allocator.go | 2 +- internal/datacoord/channel_checker.go | 2 +- internal/datacoord/channel_manager.go | 2 +- internal/datacoord/channel_manager_test.go | 2 +- internal/datacoord/channel_manager_v2.go | 2 +- internal/datacoord/channel_manager_v2_test.go | 2 +- internal/datacoord/channel_store.go | 2 +- internal/datacoord/channel_store_test.go | 4 +- internal/datacoord/channel_store_v2.go | 2 +- internal/datacoord/channel_store_v2_test.go | 2 +- internal/datacoord/cluster_test.go | 2 +- internal/datacoord/meta_test.go | 2 +- internal/datacoord/server.go | 6 +- internal/datanode/data_node.go | 2 +- internal/datanode/event_manager.go | 2 +- .../flow_graph_dmstream_input_node.go | 6 +- .../flow_graph_dmstream_input_node_test.go | 4 +- internal/datanode/mock_test.go | 2 +- internal/kv/etcd/embed_etcd_kv.go | 4 +- internal/kv/etcd/embed_etcd_kv_test.go | 4 +- internal/kv/etcd/etcd_kv.go | 2 +- internal/kv/etcd/etcd_kv_test.go | 2 +- internal/kv/etcd/metakv_factory.go | 2 +- internal/kv/etcd/util.go | 2 +- internal/kv/etcd/util_test.go | 2 +- internal/kv/mem/mem_kv.go | 2 +- internal/kv/mem/mem_kv_test.go | 2 +- internal/kv/mock_snapshot_kv.go | 3 +- internal/kv/mocks/meta_kv.go | 2 +- internal/kv/mocks/txn_kv.go | 2 +- internal/kv/mocks/watch_kv.go | 2 +- internal/kv/tikv/txn_tikv.go | 4 +- internal/kv/tikv/txn_tikv_test.go | 2 +- internal/metastore/kv/datacoord/kv_catalog.go | 2 +- .../metastore/kv/datacoord/kv_catalog_test.go | 2 +- .../metastore/kv/querycoord/kv_catalog.go | 2 +- .../kv/querycoord/kv_catalog_test.go | 2 +- internal/metastore/kv/rootcoord/kv_catalog.go | 2 +- .../metastore/kv/rootcoord/suffix_snapshot.go | 2 +- internal/mq/msgstream/mq_factory.go | 26 -- internal/mq/msgstream/mq_factory_test.go | 43 --- internal/proxy/impl_test.go | 12 +- internal/proxy/mock_test.go | 4 +- .../channel_level_score_balancer_test.go | 2 +- .../balance/rowcount_based_balancer_test.go | 4 +- .../balance/score_based_balancer_test.go | 2 +- .../checkers/balance_checker_test.go | 2 +- .../checkers/channel_checker_test.go | 2 +- .../checkers/controller_base_test.go | 2 +- .../querycoordv2/checkers/controller_test.go | 2 +- .../checkers/index_checker_test.go | 2 +- .../checkers/leader_checker_test.go | 2 +- .../checkers/segment_checker_test.go | 2 +- .../querycoordv2/dist/dist_controller_test.go | 2 +- internal/querycoordv2/job/job_test.go | 2 +- .../meta/collection_manager_test.go | 2 +- .../querycoordv2/meta/replica_manager_test.go | 2 +- .../meta/resource_manager_test.go | 2 +- .../querycoordv2/meta/target_manager_test.go | 2 +- .../observers/collection_observer_test.go | 2 +- .../observers/replica_observer_test.go | 2 +- .../observers/resource_observer_test.go | 2 +- .../observers/target_observer_test.go | 2 +- internal/querycoordv2/ops_service_test.go | 2 +- internal/querycoordv2/server.go | 2 +- internal/querycoordv2/services_test.go | 2 +- internal/querycoordv2/task/task_test.go | 2 +- .../querynodev2/delegator/delegator_data.go | 4 +- internal/querynodev2/pipeline/manager_test.go | 4 +- .../querynodev2/pipeline/pipeline_test.go | 4 +- internal/rootcoord/dml_channels.go | 4 +- internal/rootcoord/dml_channels_test.go | 4 +- internal/rootcoord/root_coord.go | 2 +- internal/tso/global_allocator.go | 2 +- internal/tso/tso.go | 2 +- internal/util/dependency/factory.go | 7 +- internal/util/flowgraph/input_node_test.go | 6 +- internal/util/flowgraph/node_test.go | 4 +- internal/util/pipeline/stream_pipeline.go | 4 +- .../util/pipeline/stream_pipeline_test.go | 4 +- internal/util/tsoutil/tso.go | 2 +- pkg/go.mod | 4 + pkg/go.sum | 24 ++ {internal => pkg}/kv/kv.go | 2 +- .../kv/predicates/mock_predicate.go | 0 {internal => pkg}/kv/predicates/predicate.go | 0 .../kv/predicates/predicate_test.go | 0 .../kv/rocksdb/rocks_iterator.go | 0 {internal => pkg}/kv/rocksdb/rocksdb_kv.go | 4 +- .../kv/rocksdb/rocksdb_kv_test.go | 4 +- pkg/mq/{msgstream/mqwrapper => common}/id.go | 2 +- .../mqwrapper => common}/message.go | 35 +- .../mqwrapper => common}/mock_id.go | 4 +- .../mq/mqimpl/rocksmq/client/client.go | 2 +- .../mq/mqimpl/rocksmq/client/client_impl.go | 8 +- .../mqimpl/rocksmq/client/client_impl_test.go | 36 +- .../mq/mqimpl/rocksmq/client/consumer.go | 10 +- .../mq/mqimpl/rocksmq/client/consumer_impl.go | 10 +- .../rocksmq/client/consumer_impl_test.go | 14 +- .../mq/mqimpl/rocksmq/client/error.go | 0 .../mq/mqimpl/rocksmq/client/producer.go | 6 +- .../mq/mqimpl/rocksmq/client/producer_impl.go | 6 +- .../rocksmq/client/producer_impl_test.go | 0 .../mq/mqimpl/rocksmq/client/rmq_message.go | 8 +- .../mq/mqimpl/rocksmq/client/test_helper.go | 10 +- .../mq/mqimpl/rocksmq/client/util.go | 0 .../mq/mqimpl/rocksmq/server/global_rmq.go | 2 +- .../mqimpl/rocksmq/server/global_rmq_test.go | 0 .../mq/mqimpl/rocksmq/server/mock_rocksmq.go | 2 +- .../mq/mqimpl/rocksmq/server/rmq_id.go | 4 +- .../mq/mqimpl/rocksmq/server/rmq_id_test.go | 0 .../mq/mqimpl/rocksmq/server/rocksmq.go | 0 .../mq/mqimpl/rocksmq/server/rocksmq_impl.go | 92 +++-- .../rocksmq/server/rocksmq_impl_test.go | 306 ++-------------- .../rocksmq/server/rocksmq_retention.go | 2 +- .../rocksmq/server/rocksmq_retention_test.go | 29 +- pkg/mq/msgdispatcher/client.go | 4 +- pkg/mq/msgdispatcher/client_test.go | 10 +- pkg/mq/msgdispatcher/dispatcher.go | 4 +- pkg/mq/msgdispatcher/dispatcher_test.go | 10 +- pkg/mq/msgdispatcher/manager.go | 4 +- pkg/mq/msgdispatcher/manager_test.go | 28 +- pkg/mq/msgdispatcher/mock_client.go | 24 +- pkg/mq/msgdispatcher/mock_test.go | 4 +- pkg/mq/msgstream/common_mq_factory.go | 3 +- pkg/mq/msgstream/factory_stream_test.go | 10 +- pkg/mq/msgstream/mock_msgstream.go | 46 +-- pkg/mq/msgstream/mq_factory.go | 21 +- pkg/mq/msgstream/mq_factory_test.go | 19 + pkg/mq/msgstream/mq_kafka_msgstream_test.go | 14 +- pkg/mq/msgstream/mq_msgstream.go | 13 +- pkg/mq/msgstream/mq_msgstream_test.go | 31 +- .../mq/msgstream/mq_rocksmq_msgstream_test.go | 333 +++++------------- pkg/mq/msgstream/mqwrapper/client.go | 12 +- pkg/mq/msgstream/mqwrapper/consumer.go | 24 +- .../msgstream/mqwrapper/kafka/kafka_client.go | 11 +- .../mqwrapper/kafka/kafka_client_test.go | 38 +- .../mqwrapper/kafka/kafka_consumer.go | 19 +- .../mqwrapper/kafka/kafka_consumer_test.go | 28 +- pkg/mq/msgstream/mqwrapper/kafka/kafka_id.go | 4 +- .../mqwrapper/kafka/kafka_message.go | 4 +- .../mqwrapper/kafka/kafka_producer.go | 3 +- .../mqwrapper/kafka/kafka_producer_test.go | 12 +- pkg/mq/msgstream/mqwrapper/nmq/nmq_client.go | 13 +- .../mqwrapper/nmq/nmq_client_test.go | 21 +- .../msgstream/mqwrapper/nmq/nmq_consumer.go | 13 +- .../mqwrapper/nmq/nmq_consumer_test.go | 53 +-- pkg/mq/msgstream/mqwrapper/nmq/nmq_id.go | 6 +- pkg/mq/msgstream/mqwrapper/nmq/nmq_message.go | 6 +- .../msgstream/mqwrapper/nmq/nmq_producer.go | 3 +- .../mqwrapper/nmq/nmq_producer_test.go | 6 +- pkg/mq/msgstream/mqwrapper/producer.go | 25 +- .../mqwrapper/pulsar/pulsar_client.go | 11 +- .../mqwrapper/pulsar/pulsar_client_test.go | 51 +-- .../mqwrapper/pulsar/pulsar_consumer.go | 13 +- .../mqwrapper/pulsar/pulsar_consumer_test.go | 18 +- .../msgstream/mqwrapper/pulsar/pulsar_id.go | 4 +- .../mqwrapper/pulsar/pulsar_message.go | 6 +- .../mqwrapper/pulsar/pulsar_producer.go | 3 +- .../mqwrapper/pulsar/pulsar_producer_test.go | 6 +- .../mq/msgstream/mqwrapper/rmq/rmq_client.go | 17 +- .../mqwrapper/rmq/rmq_client_test.go | 47 ++- .../msgstream/mqwrapper/rmq/rmq_consumer.go | 18 +- .../msgstream/mqwrapper/rmq/rmq_producer.go | 7 +- pkg/mq/msgstream/msgstream.go | 6 +- pkg/mq/msgstream/msgstream_util.go | 4 +- pkg/mq/msgstream/msgstream_util_test.go | 4 +- pkg/mq/msgstream/stream_bench_test.go | 7 +- pkg/mq/msgstream/stream_test.go | 37 +- 176 files changed, 872 insertions(+), 1219 deletions(-) delete mode 100644 internal/mq/msgstream/mq_factory.go delete mode 100644 internal/mq/msgstream/mq_factory_test.go rename {internal => pkg}/kv/kv.go (98%) rename {internal => pkg}/kv/predicates/mock_predicate.go (100%) rename {internal => pkg}/kv/predicates/predicate.go (100%) rename {internal => pkg}/kv/predicates/predicate_test.go (100%) rename {internal => pkg}/kv/rocksdb/rocks_iterator.go (100%) rename {internal => pkg}/kv/rocksdb/rocksdb_kv.go (99%) rename {internal => pkg}/kv/rocksdb/rocksdb_kv_test.go (98%) rename pkg/mq/{msgstream/mqwrapper => common}/id.go (98%) rename pkg/mq/{msgstream/mqwrapper => common}/message.go (52%) rename pkg/mq/{msgstream/mqwrapper => common}/mock_id.go (98%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/client.go (94%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/client_impl.go (94%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/client_impl_test.go (89%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/consumer.go (88%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/consumer_impl.go (94%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/consumer_impl_test.go (89%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/error.go (100%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/producer.go (88%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/producer_impl.go (91%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/producer_impl_test.go (100%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/rmq_message.go (86%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/test_helper.go (86%) rename {internal => pkg}/mq/mqimpl/rocksmq/client/util.go (100%) rename {internal => pkg}/mq/mqimpl/rocksmq/server/global_rmq.go (97%) rename {internal => pkg}/mq/mqimpl/rocksmq/server/global_rmq_test.go (100%) rename {internal => pkg}/mq/mqimpl/rocksmq/server/mock_rocksmq.go (99%) rename {internal => pkg}/mq/mqimpl/rocksmq/server/rmq_id.go (95%) rename {internal => pkg}/mq/mqimpl/rocksmq/server/rmq_id_test.go (100%) rename {internal => pkg}/mq/mqimpl/rocksmq/server/rocksmq.go (100%) rename {internal => pkg}/mq/mqimpl/rocksmq/server/rocksmq_impl.go (93%) rename {internal => pkg}/mq/mqimpl/rocksmq/server/rocksmq_impl_test.go (75%) rename {internal => pkg}/mq/mqimpl/rocksmq/server/rocksmq_retention.go (99%) rename {internal => pkg}/mq/mqimpl/rocksmq/server/rocksmq_retention_test.go (96%) rename internal/mq/msgstream/mqwrapper/rmq/rocksmq_msgstream_test.go => pkg/mq/msgstream/mq_rocksmq_msgstream_test.go (62%) rename {internal => pkg}/mq/msgstream/mqwrapper/rmq/rmq_client.go (88%) rename {internal => pkg}/mq/msgstream/mqwrapper/rmq/rmq_client_test.go (82%) rename {internal => pkg}/mq/msgstream/mqwrapper/rmq/rmq_consumer.go (82%) rename {internal => pkg}/mq/msgstream/mqwrapper/rmq/rmq_producer.go (86%) diff --git a/.gitignore b/.gitignore index 17c251a7be6a..b6adfcbdb4b4 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,6 @@ internal/core/build/* **/.idea/* internal/msgstream/pulsarms/client-cpp/build/ internal/msgstream/pulsarms/client-cpp/build/* -internal/kv/rocksdb/cwrapper/output/ tests/python_client/default.etcd/ # vscode generated files @@ -97,7 +96,6 @@ deployments/docker/gpu/*/volumes # rocksdb cwrapper_rocksdb_build/ -internal/kv/rocksdb/cwrapper/ # local file data **/data/* diff --git a/cmd/milvus/mck.go b/cmd/milvus/mck.go index 2347398bf0f5..5129a67d935b 100644 --- a/cmd/milvus/mck.go +++ b/cmd/milvus/mck.go @@ -17,12 +17,12 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/proto/datapb" pb "github.com/milvus-io/milvus/internal/proto/etcdpb" "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/logutil" diff --git a/cmd/roles/roles.go b/cmd/roles/roles.go index fa850518a20d..c0877ffc98ec 100644 --- a/cmd/roles/roles.go +++ b/cmd/roles/roles.go @@ -35,7 +35,6 @@ import ( "github.com/milvus-io/milvus/cmd/components" "github.com/milvus-io/milvus/internal/http" "github.com/milvus-io/milvus/internal/http/healthz" - rocksmqimpl "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/internal/util/dependency" kvfactory "github.com/milvus-io/milvus/internal/util/dependency/kv" "github.com/milvus-io/milvus/internal/util/initcore" @@ -43,6 +42,7 @@ import ( "github.com/milvus-io/milvus/pkg/config" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + rocksmqimpl "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/nmq" "github.com/milvus-io/milvus/pkg/tracer" "github.com/milvus-io/milvus/pkg/util/etcd" diff --git a/cmd/tools/migration/backend/etcd.go b/cmd/tools/migration/backend/etcd.go index cc1562d9b9a1..8cc29dddef3f 100644 --- a/cmd/tools/migration/backend/etcd.go +++ b/cmd/tools/migration/backend/etcd.go @@ -4,8 +4,8 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" "github.com/milvus-io/milvus/cmd/tools/migration/configs" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" ) diff --git a/cmd/tools/migration/mmap/tool/main.go b/cmd/tools/migration/mmap/tool/main.go index a22c652a64ae..8975ffe59ef5 100644 --- a/cmd/tools/migration/mmap/tool/main.go +++ b/cmd/tools/migration/mmap/tool/main.go @@ -8,7 +8,6 @@ import ( "time" "github.com/milvus-io/milvus/cmd/tools/migration/mmap" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" kv_tikv "github.com/milvus-io/milvus/internal/kv/tikv" "github.com/milvus-io/milvus/internal/metastore" @@ -17,6 +16,7 @@ import ( "github.com/milvus-io/milvus/internal/rootcoord" "github.com/milvus-io/milvus/internal/tso" "github.com/milvus-io/milvus/internal/util/tsoutil" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util" "github.com/milvus-io/milvus/pkg/util/etcd" diff --git a/go.mod b/go.mod index 63b4f1036e41..24ffebf7bd3a 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/aliyun/credentials-go v1.2.7 github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e github.com/apache/arrow/go/v12 v12.0.1 - github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 + github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 // indirect github.com/bits-and-blooms/bloom/v3 v3.0.1 github.com/blang/semver/v4 v4.0.0 github.com/casbin/casbin/v2 v2.44.2 @@ -39,7 +39,7 @@ require ( github.com/spf13/cast v1.3.1 github.com/spf13/viper v1.8.1 github.com/stretchr/testify v1.8.4 - github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.865 github.com/tidwall/gjson v1.14.4 github.com/tikv/client-go/v2 v2.0.4 @@ -69,6 +69,7 @@ require ( github.com/milvus-io/milvus/pkg v0.0.0-00010101000000-000000000000 github.com/pkg/errors v0.9.1 github.com/valyala/fastjson v1.6.4 + google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -110,9 +111,6 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/expr-lang/expr v1.15.7 // indirect - github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect - github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect - github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -235,7 +233,6 @@ require ( google.golang.org/genproto v0.0.0-20230706204954-ccb25ca9f130 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230629202037-9506855d4529 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda // indirect - google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index a7e4cbac8d28..0c36e4a79155 100644 --- a/go.sum +++ b/go.sum @@ -56,12 +56,14 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9Orh github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= @@ -169,6 +171,7 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= @@ -215,6 +218,7 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -235,6 +239,7 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8= +github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= @@ -245,6 +250,7 @@ github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+ne github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -257,6 +263,7 @@ github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03D github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -290,7 +297,6 @@ github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2C github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -300,6 +306,7 @@ github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AE github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -338,6 +345,7 @@ github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGw github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -401,6 +409,7 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -517,7 +526,6 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= -github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -548,7 +556,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kris-nova/logger v0.0.0-20181127235838-fd0d87064b06 h1:vN4d3jSss3ExzUn2cE0WctxztfOgiKvMKnDrydBsg00= +github.com/kris-nova/logger v0.0.0-20181127235838-fd0d87064b06/go.mod h1:++9BgZujZd4v0ZTZCb5iPsaomXdZWyxotIAh1IiDm44= github.com/kris-nova/lolgopher v0.0.0-20180921204813-313b3abb0d9b h1:xYEM2oBUhBEhQjrV+KJ9lEWDWYZoNVZUaBF++Wyljq4= +github.com/kris-nova/lolgopher v0.0.0-20180921204813-313b3abb0d9b/go.mod h1:V0HF/ZBlN86HqewcDC/cVxMmYDiRukWjSrgKLUAn9Js= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= @@ -556,6 +566,7 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76 h1:IVlcvV0CjvfBYYod5ePe89l+3LBAl//6n9kJ9Vr2i0k= +github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76/go.mod h1:Iu9BHUvTh8/KpbuSoKx/CaJEdJvFxSverxIy7I+nq7s= github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM= github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA= @@ -582,6 +593,7 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0= +github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -653,6 +665,7 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -713,6 +726,7 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -838,6 +852,7 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.865 h1:LcUqBlKC4j15LhT303yQDX/XxyHG4haEQqbHgZZA4SY= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.865/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= @@ -899,6 +914,7 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -969,6 +985,7 @@ go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnw go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= @@ -1456,6 +1473,7 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1512,3 +1530,4 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= +stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= diff --git a/internal/allocator/global_id_allocator.go b/internal/allocator/global_id_allocator.go index 1a0dd877edc2..12d72cc43719 100644 --- a/internal/allocator/global_id_allocator.go +++ b/internal/allocator/global_id_allocator.go @@ -17,8 +17,8 @@ package allocator import ( - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/tso" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/typeutil" ) diff --git a/internal/datacoord/channel_checker.go b/internal/datacoord/channel_checker.go index 9ab1555b72cf..764ca2800f63 100644 --- a/internal/datacoord/channel_checker.go +++ b/internal/datacoord/channel_checker.go @@ -27,8 +27,8 @@ import ( "go.uber.org/atomic" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/typeutil" ) diff --git a/internal/datacoord/channel_manager.go b/internal/datacoord/channel_manager.go index 396fcd6eafb8..1ba48aae4589 100644 --- a/internal/datacoord/channel_manager.go +++ b/internal/datacoord/channel_manager.go @@ -27,9 +27,9 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/mq/msgstream" "github.com/milvus-io/milvus/pkg/util/funcutil" diff --git a/internal/datacoord/channel_manager_test.go b/internal/datacoord/channel_manager_test.go index 5866d638bc83..1dd12e52012b 100644 --- a/internal/datacoord/channel_manager_test.go +++ b/internal/datacoord/channel_manager_test.go @@ -30,10 +30,10 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/atomic" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/util/dependency" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/datacoord/channel_manager_v2.go b/internal/datacoord/channel_manager_v2.go index cee393b1b7dd..e79896a3deb7 100644 --- a/internal/datacoord/channel_manager_v2.go +++ b/internal/datacoord/channel_manager_v2.go @@ -27,8 +27,8 @@ import ( "github.com/samber/lo" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/conc" "github.com/milvus-io/milvus/pkg/util/merr" diff --git a/internal/datacoord/channel_manager_v2_test.go b/internal/datacoord/channel_manager_v2_test.go index b2093b9881b0..4bce9cf93f74 100644 --- a/internal/datacoord/channel_manager_v2_test.go +++ b/internal/datacoord/channel_manager_v2_test.go @@ -29,8 +29,8 @@ import ( "go.uber.org/zap" kvmock "github.com/milvus-io/milvus/internal/kv/mocks" - "github.com/milvus-io/milvus/internal/kv/predicates" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/datacoord/channel_store.go b/internal/datacoord/channel_store.go index 90590f2244d7..2da7137c34f9 100644 --- a/internal/datacoord/channel_store.go +++ b/internal/datacoord/channel_store.go @@ -28,8 +28,8 @@ import ( "go.uber.org/zap" "go.uber.org/zap/zapcore" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/datacoord/channel_store_test.go b/internal/datacoord/channel_store_test.go index 0790fc6a2ea1..4201259fb81a 100644 --- a/internal/datacoord/channel_store_test.go +++ b/internal/datacoord/channel_store_test.go @@ -26,10 +26,10 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/kv/mocks" - "github.com/milvus-io/milvus/internal/kv/predicates" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/pkg/kv" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/testutils" diff --git a/internal/datacoord/channel_store_v2.go b/internal/datacoord/channel_store_v2.go index e159756d76fa..5ad2d71b9270 100644 --- a/internal/datacoord/channel_store_v2.go +++ b/internal/datacoord/channel_store_v2.go @@ -8,8 +8,8 @@ import ( "github.com/samber/lo" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util/timerecord" diff --git a/internal/datacoord/channel_store_v2_test.go b/internal/datacoord/channel_store_v2_test.go index 2228ff971499..501d4a9d74ba 100644 --- a/internal/datacoord/channel_store_v2_test.go +++ b/internal/datacoord/channel_store_v2_test.go @@ -12,8 +12,8 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus/internal/kv/mocks" - "github.com/milvus-io/milvus/internal/kv/predicates" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util/testutils" diff --git a/internal/datacoord/cluster_test.go b/internal/datacoord/cluster_test.go index fee6d1938673..0d788b5d547f 100644 --- a/internal/datacoord/cluster_test.go +++ b/internal/datacoord/cluster_test.go @@ -26,10 +26,10 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/kv/mocks" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/testutils" ) diff --git a/internal/datacoord/meta_test.go b/internal/datacoord/meta_test.go index eb26acfb7272..b37bda537a90 100644 --- a/internal/datacoord/meta_test.go +++ b/internal/datacoord/meta_test.go @@ -30,7 +30,6 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus/internal/datacoord/broker" - "github.com/milvus-io/milvus/internal/kv" mockkv "github.com/milvus-io/milvus/internal/kv/mocks" "github.com/milvus-io/milvus/internal/metastore/kv/datacoord" mocks2 "github.com/milvus-io/milvus/internal/metastore/mocks" @@ -38,6 +37,7 @@ import ( "github.com/milvus-io/milvus/internal/mocks" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util" "github.com/milvus-io/milvus/pkg/util/merr" diff --git a/internal/datacoord/server.go b/internal/datacoord/server.go index 6d8bf51531df..86367a93d6be 100644 --- a/internal/datacoord/server.go +++ b/internal/datacoord/server.go @@ -38,7 +38,6 @@ import ( datanodeclient "github.com/milvus-io/milvus/internal/distributed/datanode/client" indexnodeclient "github.com/milvus-io/milvus/internal/distributed/indexnode/client" rootcoordclient "github.com/milvus-io/milvus/internal/distributed/rootcoord/client" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/kv/tikv" "github.com/milvus-io/milvus/internal/metastore/kv/datacoord" @@ -47,10 +46,11 @@ import ( "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/util/dependency" "github.com/milvus-io/milvus/internal/util/sessionutil" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util" "github.com/milvus-io/milvus/pkg/util/expr" "github.com/milvus-io/milvus/pkg/util/logutil" @@ -729,7 +729,7 @@ func (s *Server) startDataNodeTtLoop(ctx context.Context) { } subName := fmt.Sprintf("%s-%d-datanodeTl", Params.CommonCfg.DataCoordSubName.GetValue(), paramtable.GetNodeID()) - ttMsgStream.AsConsumer(context.TODO(), []string{timeTickChannel}, subName, mqwrapper.SubscriptionPositionLatest) + ttMsgStream.AsConsumer(context.TODO(), []string{timeTickChannel}, subName, common.SubscriptionPositionLatest) log.Info("DataCoord creates the timetick channel consumer", zap.String("timeTickChannel", timeTickChannel), zap.String("subscription", subName)) diff --git a/internal/datanode/data_node.go b/internal/datanode/data_node.go index 89734ec8ea9d..a6ab491af68d 100644 --- a/internal/datanode/data_node.go +++ b/internal/datanode/data_node.go @@ -39,12 +39,12 @@ import ( "github.com/milvus-io/milvus/internal/datanode/importv2" "github.com/milvus-io/milvus/internal/datanode/syncmgr" "github.com/milvus-io/milvus/internal/datanode/writebuffer" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/util/dependency" "github.com/milvus-io/milvus/internal/util/sessionutil" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/mq/msgdispatcher" diff --git a/internal/datanode/event_manager.go b/internal/datanode/event_manager.go index 4d0ceea6c2fc..2e78195b51cb 100644 --- a/internal/datanode/event_manager.go +++ b/internal/datanode/event_manager.go @@ -29,8 +29,8 @@ import ( "go.uber.org/atomic" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/logutil" ) diff --git a/internal/datanode/flow_graph_dmstream_input_node.go b/internal/datanode/flow_graph_dmstream_input_node.go index 21242e2a5cfb..2d0ed625134e 100644 --- a/internal/datanode/flow_graph_dmstream_input_node.go +++ b/internal/datanode/flow_graph_dmstream_input_node.go @@ -27,9 +27,9 @@ import ( "github.com/milvus-io/milvus/internal/util/flowgraph" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgdispatcher" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/tsoutil" "github.com/milvus-io/milvus/pkg/util/typeutil" @@ -46,7 +46,7 @@ func newDmInputNode(initCtx context.Context, dispatcherClient msgdispatcher.Clie var err error var input <-chan *msgstream.MsgPack if seekPos != nil && len(seekPos.MsgID) != 0 { - input, err = dispatcherClient.Register(initCtx, dmNodeConfig.vChannelName, seekPos, mqwrapper.SubscriptionPositionUnknown) + input, err = dispatcherClient.Register(initCtx, dmNodeConfig.vChannelName, seekPos, common.SubscriptionPositionUnknown) if err != nil { return nil, err } @@ -55,7 +55,7 @@ func newDmInputNode(initCtx context.Context, dispatcherClient msgdispatcher.Clie zap.Time("tsTime", tsoutil.PhysicalTime(seekPos.GetTimestamp())), zap.Duration("tsLag", time.Since(tsoutil.PhysicalTime(seekPos.GetTimestamp())))) } else { - input, err = dispatcherClient.Register(initCtx, dmNodeConfig.vChannelName, nil, mqwrapper.SubscriptionPositionEarliest) + input, err = dispatcherClient.Register(initCtx, dmNodeConfig.vChannelName, nil, common.SubscriptionPositionEarliest) if err != nil { return nil, err } diff --git a/internal/datanode/flow_graph_dmstream_input_node_test.go b/internal/datanode/flow_graph_dmstream_input_node_test.go index ae804fe19e5d..14315aae988b 100644 --- a/internal/datanode/flow_graph_dmstream_input_node_test.go +++ b/internal/datanode/flow_graph_dmstream_input_node_test.go @@ -26,9 +26,9 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/util/dependency" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgdispatcher" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -73,7 +73,7 @@ func (mtm *mockTtMsgStream) Chan() <-chan *msgstream.MsgPack { func (mtm *mockTtMsgStream) AsProducer(channels []string) {} -func (mtm *mockTtMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position mqwrapper.SubscriptionInitialPosition) error { +func (mtm *mockTtMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position common.SubscriptionInitialPosition) error { return nil } diff --git a/internal/datanode/mock_test.go b/internal/datanode/mock_test.go index 92286c4bbfcb..e945973971a7 100644 --- a/internal/datanode/mock_test.go +++ b/internal/datanode/mock_test.go @@ -39,7 +39,6 @@ import ( "github.com/milvus-io/milvus/internal/datanode/metacache" "github.com/milvus-io/milvus/internal/datanode/syncmgr" "github.com/milvus-io/milvus/internal/datanode/writebuffer" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/etcdpb" @@ -49,6 +48,7 @@ import ( "github.com/milvus-io/milvus/internal/util/dependency" "github.com/milvus-io/milvus/internal/util/sessionutil" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/mq/msgdispatcher" "github.com/milvus-io/milvus/pkg/mq/msgstream" diff --git a/internal/kv/etcd/embed_etcd_kv.go b/internal/kv/etcd/embed_etcd_kv.go index ba393e45ca00..ccb6682b3193 100644 --- a/internal/kv/etcd/embed_etcd_kv.go +++ b/internal/kv/etcd/embed_etcd_kv.go @@ -29,8 +29,8 @@ import ( "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/merr" ) diff --git a/internal/kv/etcd/embed_etcd_kv_test.go b/internal/kv/etcd/embed_etcd_kv_test.go index fa4272d4c9a1..7e01b5b5516b 100644 --- a/internal/kv/etcd/embed_etcd_kv_test.go +++ b/internal/kv/etcd/embed_etcd_kv_test.go @@ -28,9 +28,9 @@ import ( "github.com/stretchr/testify/suite" "golang.org/x/exp/maps" - "github.com/milvus-io/milvus/internal/kv" embed_etcd_kv "github.com/milvus-io/milvus/internal/kv/etcd" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/util/funcutil" "github.com/milvus-io/milvus/pkg/util/metricsinfo" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/kv/etcd/etcd_kv.go b/internal/kv/etcd/etcd_kv.go index 79cb91d212e2..5003d0fae4b1 100644 --- a/internal/kv/etcd/etcd_kv.go +++ b/internal/kv/etcd/etcd_kv.go @@ -26,7 +26,7 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util/merr" diff --git a/internal/kv/etcd/etcd_kv_test.go b/internal/kv/etcd/etcd_kv_test.go index 76908530fba9..d4ec49d062fd 100644 --- a/internal/kv/etcd/etcd_kv_test.go +++ b/internal/kv/etcd/etcd_kv_test.go @@ -31,7 +31,7 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" "golang.org/x/exp/maps" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/funcutil" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/kv/etcd/metakv_factory.go b/internal/kv/etcd/metakv_factory.go index 0f2dd64f41f2..8c575c81a77c 100644 --- a/internal/kv/etcd/metakv_factory.go +++ b/internal/kv/etcd/metakv_factory.go @@ -22,7 +22,7 @@ import ( "go.etcd.io/etcd/server/v3/embed" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/kv/etcd/util.go b/internal/kv/etcd/util.go index 6363ddb5f9ad..bc0a34ef3701 100644 --- a/internal/kv/etcd/util.go +++ b/internal/kv/etcd/util.go @@ -6,7 +6,7 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/util/merr" ) diff --git a/internal/kv/etcd/util_test.go b/internal/kv/etcd/util_test.go index 331f4845ae48..f39b7a162aa5 100644 --- a/internal/kv/etcd/util_test.go +++ b/internal/kv/etcd/util_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv/predicates" ) type EtcdKVUtilSuite struct { diff --git a/internal/kv/mem/mem_kv.go b/internal/kv/mem/mem_kv.go index d4309e879aec..0b95569ca74d 100644 --- a/internal/kv/mem/mem_kv.go +++ b/internal/kv/mem/mem_kv.go @@ -22,7 +22,7 @@ import ( "github.com/google/btree" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/util/merr" ) diff --git a/internal/kv/mem/mem_kv_test.go b/internal/kv/mem/mem_kv_test.go index 76e8896827f7..7fbb66bc13e2 100644 --- a/internal/kv/mem/mem_kv_test.go +++ b/internal/kv/mem/mem_kv_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/util/merr" ) diff --git a/internal/kv/mock_snapshot_kv.go b/internal/kv/mock_snapshot_kv.go index 35cc851853dc..6637d7ac9644 100644 --- a/internal/kv/mock_snapshot_kv.go +++ b/internal/kv/mock_snapshot_kv.go @@ -1,11 +1,12 @@ package kv import ( + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/typeutil" ) type mockSnapshotKV struct { - SnapShotKV + kv.SnapShotKV SaveFunc func(key string, value string, ts typeutil.Timestamp) error LoadFunc func(key string, ts typeutil.Timestamp) (string, error) MultiSaveFunc func(kvs map[string]string, ts typeutil.Timestamp) error diff --git a/internal/kv/mocks/meta_kv.go b/internal/kv/mocks/meta_kv.go index 5a615ff5250d..1119c6a45bea 100644 --- a/internal/kv/mocks/meta_kv.go +++ b/internal/kv/mocks/meta_kv.go @@ -3,7 +3,7 @@ package mocks import ( - predicates "github.com/milvus-io/milvus/internal/kv/predicates" + predicates "github.com/milvus-io/milvus/pkg/kv/predicates" mock "github.com/stretchr/testify/mock" ) diff --git a/internal/kv/mocks/txn_kv.go b/internal/kv/mocks/txn_kv.go index 25bbb438ff95..2168180ce379 100644 --- a/internal/kv/mocks/txn_kv.go +++ b/internal/kv/mocks/txn_kv.go @@ -3,7 +3,7 @@ package mocks import ( - predicates "github.com/milvus-io/milvus/internal/kv/predicates" + predicates "github.com/milvus-io/milvus/pkg/kv/predicates" mock "github.com/stretchr/testify/mock" ) diff --git a/internal/kv/mocks/watch_kv.go b/internal/kv/mocks/watch_kv.go index c49ff4a924c7..4fe5aebe5bec 100644 --- a/internal/kv/mocks/watch_kv.go +++ b/internal/kv/mocks/watch_kv.go @@ -7,7 +7,7 @@ import ( mock "github.com/stretchr/testify/mock" - predicates "github.com/milvus-io/milvus/internal/kv/predicates" + predicates "github.com/milvus-io/milvus/pkg/kv/predicates" ) // WatchKV is an autogenerated mock type for the WatchKV type diff --git a/internal/kv/tikv/txn_tikv.go b/internal/kv/tikv/txn_tikv.go index 8fca5d0d4567..f7fcd7d444a7 100644 --- a/internal/kv/tikv/txn_tikv.go +++ b/internal/kv/tikv/txn_tikv.go @@ -32,8 +32,8 @@ import ( "github.com/tikv/client-go/v2/txnkv/txnsnapshot" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util/merr" diff --git a/internal/kv/tikv/txn_tikv_test.go b/internal/kv/tikv/txn_tikv_test.go index 6a7dcad0f19e..eeff2338ffb9 100644 --- a/internal/kv/tikv/txn_tikv_test.go +++ b/internal/kv/tikv/txn_tikv_test.go @@ -30,7 +30,7 @@ import ( "github.com/tikv/client-go/v2/txnkv/transaction" "golang.org/x/exp/maps" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv/predicates" ) func TestTiKVLoad(te *testing.T) { diff --git a/internal/metastore/kv/datacoord/kv_catalog.go b/internal/metastore/kv/datacoord/kv_catalog.go index 4272458cc3c4..69dced301feb 100644 --- a/internal/metastore/kv/datacoord/kv_catalog.go +++ b/internal/metastore/kv/datacoord/kv_catalog.go @@ -30,7 +30,6 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/kv/binlog" "github.com/milvus-io/milvus/internal/metastore/model" @@ -38,6 +37,7 @@ import ( "github.com/milvus-io/milvus/internal/proto/indexpb" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/util/segmentutil" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util" diff --git a/internal/metastore/kv/datacoord/kv_catalog_test.go b/internal/metastore/kv/datacoord/kv_catalog_test.go index 8ff58679be1e..fab1a9f9418b 100644 --- a/internal/metastore/kv/datacoord/kv_catalog_test.go +++ b/internal/metastore/kv/datacoord/kv_catalog_test.go @@ -36,11 +36,11 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/kv/mocks" - "github.com/milvus-io/milvus/internal/kv/predicates" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/model" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/indexpb" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/metautil" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/metastore/kv/querycoord/kv_catalog.go b/internal/metastore/kv/querycoord/kv_catalog.go index 5c590cf96eb6..97b141ed154d 100644 --- a/internal/metastore/kv/querycoord/kv_catalog.go +++ b/internal/metastore/kv/querycoord/kv_catalog.go @@ -13,8 +13,8 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/compressor" ) diff --git a/internal/metastore/kv/querycoord/kv_catalog_test.go b/internal/metastore/kv/querycoord/kv_catalog_test.go index 001ac310fe52..6dbdadfb1f00 100644 --- a/internal/metastore/kv/querycoord/kv_catalog_test.go +++ b/internal/metastore/kv/querycoord/kv_catalog_test.go @@ -8,11 +8,11 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/kv/mocks" "github.com/milvus-io/milvus/internal/proto/querypb" . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/metastore/kv/rootcoord/kv_catalog.go b/internal/metastore/kv/rootcoord/kv_catalog.go index 3ef8f7d770f4..384a9c575b35 100644 --- a/internal/metastore/kv/rootcoord/kv_catalog.go +++ b/internal/metastore/kv/rootcoord/kv_catalog.go @@ -12,12 +12,12 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/model" pb "github.com/milvus-io/milvus/internal/proto/etcdpb" "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util" "github.com/milvus-io/milvus/pkg/util/crypto" diff --git a/internal/metastore/kv/rootcoord/suffix_snapshot.go b/internal/metastore/kv/rootcoord/suffix_snapshot.go index 832bfd45d9aa..017085ac21e3 100644 --- a/internal/metastore/kv/rootcoord/suffix_snapshot.go +++ b/internal/metastore/kv/rootcoord/suffix_snapshot.go @@ -30,8 +30,8 @@ import ( "github.com/cockroachdb/errors" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/retry" diff --git a/internal/mq/msgstream/mq_factory.go b/internal/mq/msgstream/mq_factory.go deleted file mode 100644 index 6d19d2abf0a5..000000000000 --- a/internal/mq/msgstream/mq_factory.go +++ /dev/null @@ -1,26 +0,0 @@ -package msgstream - -import ( - "go.uber.org/zap" - - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" - "github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/rmq" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/util/paramtable" -) - -// NewRocksmqFactory creates a new message stream factory based on rocksmq. -func NewRocksmqFactory(path string, cfg *paramtable.ServiceParam) msgstream.Factory { - if err := server.InitRocksMQ(path); err != nil { - log.Fatal("fail to init rocksmq", zap.Error(err)) - } - log.Info("init rocksmq msgstream success", zap.String("path", path)) - - return &msgstream.CommonFactory{ - Newer: rmq.NewClientWithDefaultOptions, - DispatcherFactory: msgstream.ProtoUDFactory{}, - ReceiveBufSize: cfg.MQCfg.ReceiveBufSize.GetAsInt64(), - MQBufSize: cfg.MQCfg.MQBufSize.GetAsInt64(), - } -} diff --git a/internal/mq/msgstream/mq_factory_test.go b/internal/mq/msgstream/mq_factory_test.go deleted file mode 100644 index dc0e9213c1c0..000000000000 --- a/internal/mq/msgstream/mq_factory_test.go +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package msgstream - -import ( - "context" - "os" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/milvus-io/milvus/pkg/util/paramtable" -) - -func TestRmsFactory(t *testing.T) { - defer os.Unsetenv("ROCKSMQ_PATH") - paramtable.Init() - - dir := t.TempDir() - - rmsFactory := NewRocksmqFactory(dir, ¶mtable.Get().ServiceParam) - - ctx := context.Background() - _, err := rmsFactory.NewMsgStream(ctx) - assert.NoError(t, err) - - _, err = rmsFactory.NewTtMsgStream(ctx) - assert.NoError(t, err) -} diff --git a/internal/proxy/impl_test.go b/internal/proxy/impl_test.go index 9577ee9b6d80..88c40e88f2b8 100644 --- a/internal/proxy/impl_test.go +++ b/internal/proxy/impl_test.go @@ -45,8 +45,8 @@ import ( "github.com/milvus-io/milvus/internal/util/sessionutil" "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/log" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/commonpbutil" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" @@ -1378,7 +1378,7 @@ func TestProxy_ReplicateMessage(t *testing.T) { factory := dependency.NewMockFactory(t) stream := msgstream.NewMockMsgStream(t) - mockMsgID := mqwrapper.NewMockMessageID(t) + mockMsgID := mqcommon.NewMockMessageID(t) factory.EXPECT().NewMsgStream(mock.Anything).Return(stream, nil).Once() mockMsgID.EXPECT().Serialize().Return([]byte("mock")).Once() @@ -1474,10 +1474,10 @@ func TestProxy_ReplicateMessage(t *testing.T) { msgStreamObj.EXPECT().AsProducer(mock.Anything).Return() msgStreamObj.EXPECT().EnableProduce(mock.Anything).Return() msgStreamObj.EXPECT().Close().Return() - mockMsgID1 := mqwrapper.NewMockMessageID(t) - mockMsgID2 := mqwrapper.NewMockMessageID(t) + mockMsgID1 := mqcommon.NewMockMessageID(t) + mockMsgID2 := mqcommon.NewMockMessageID(t) mockMsgID2.EXPECT().Serialize().Return([]byte("mock message id 2")) - broadcastMock := msgStreamObj.EXPECT().Broadcast(mock.Anything).Return(map[string][]mqwrapper.MessageID{ + broadcastMock := msgStreamObj.EXPECT().Broadcast(mock.Anything).Return(map[string][]mqcommon.MessageID{ "unit_test_replicate_message": {mockMsgID1, mockMsgID2}, }, nil) @@ -1565,7 +1565,7 @@ func TestProxy_ReplicateMessage(t *testing.T) { } { broadcastMock.Unset() - broadcastMock = msgStreamObj.EXPECT().Broadcast(mock.Anything).Return(map[string][]mqwrapper.MessageID{ + broadcastMock = msgStreamObj.EXPECT().Broadcast(mock.Anything).Return(map[string][]mqcommon.MessageID{ "unit_test_replicate_message": {}, }, nil) resp, err := node.ReplicateMessage(context.TODO(), replicateRequest) diff --git a/internal/proxy/mock_test.go b/internal/proxy/mock_test.go index 96ee30669a0f..d7d69a90c137 100644 --- a/internal/proxy/mock_test.go +++ b/internal/proxy/mock_test.go @@ -27,8 +27,8 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/allocator" "github.com/milvus-io/milvus/internal/proto/rootcoordpb" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/funcutil" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" @@ -257,7 +257,7 @@ func (ms *simpleMockMsgStream) Chan() <-chan *msgstream.MsgPack { func (ms *simpleMockMsgStream) AsProducer(channels []string) { } -func (ms *simpleMockMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position mqwrapper.SubscriptionInitialPosition) error { +func (ms *simpleMockMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position common.SubscriptionInitialPosition) error { return nil } diff --git a/internal/querycoordv2/balance/channel_level_score_balancer_test.go b/internal/querycoordv2/balance/channel_level_score_balancer_test.go index 219ee694d349..6e1fbddd19de 100644 --- a/internal/querycoordv2/balance/channel_level_score_balancer_test.go +++ b/internal/querycoordv2/balance/channel_level_score_balancer_test.go @@ -22,7 +22,6 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -33,6 +32,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/balance/rowcount_based_balancer_test.go b/internal/querycoordv2/balance/rowcount_based_balancer_test.go index 41662d00d3f6..f3ad8d9fae3d 100644 --- a/internal/querycoordv2/balance/rowcount_based_balancer_test.go +++ b/internal/querycoordv2/balance/rowcount_based_balancer_test.go @@ -21,10 +21,9 @@ import ( "testing" "github.com/samber/lo" - mock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -35,6 +34,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" diff --git a/internal/querycoordv2/balance/score_based_balancer_test.go b/internal/querycoordv2/balance/score_based_balancer_test.go index 90401cec640a..b209e71fa39a 100644 --- a/internal/querycoordv2/balance/score_based_balancer_test.go +++ b/internal/querycoordv2/balance/score_based_balancer_test.go @@ -22,7 +22,6 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -33,6 +32,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/checkers/balance_checker_test.go b/internal/querycoordv2/checkers/balance_checker_test.go index e389ab64f937..8f9333d3471f 100644 --- a/internal/querycoordv2/checkers/balance_checker_test.go +++ b/internal/querycoordv2/checkers/balance_checker_test.go @@ -23,7 +23,6 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -34,6 +33,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/session" "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/checkers/channel_checker_test.go b/internal/querycoordv2/checkers/channel_checker_test.go index 149123194d99..58b5a7a75fc3 100644 --- a/internal/querycoordv2/checkers/channel_checker_test.go +++ b/internal/querycoordv2/checkers/channel_checker_test.go @@ -24,7 +24,6 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -34,6 +33,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/session" "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/checkers/controller_base_test.go b/internal/querycoordv2/checkers/controller_base_test.go index 762a8a2bde59..0d8e301492b5 100644 --- a/internal/querycoordv2/checkers/controller_base_test.go +++ b/internal/querycoordv2/checkers/controller_base_test.go @@ -21,7 +21,6 @@ import ( "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/querycoordv2/balance" @@ -30,6 +29,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/session" "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/checkers/controller_test.go b/internal/querycoordv2/checkers/controller_test.go index c04f4ecaea17..95087bf25689 100644 --- a/internal/querycoordv2/checkers/controller_test.go +++ b/internal/querycoordv2/checkers/controller_test.go @@ -24,7 +24,6 @@ import ( "github.com/stretchr/testify/suite" "go.uber.org/atomic" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -34,6 +33,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/session" "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/checkers/index_checker_test.go b/internal/querycoordv2/checkers/index_checker_test.go index ef9b80e50de3..c62685690e9e 100644 --- a/internal/querycoordv2/checkers/index_checker_test.go +++ b/internal/querycoordv2/checkers/index_checker_test.go @@ -24,7 +24,6 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/indexpb" @@ -34,6 +33,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/session" "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/checkers/leader_checker_test.go b/internal/querycoordv2/checkers/leader_checker_test.go index f01b8bb34d7e..b2bb3b10dea8 100644 --- a/internal/querycoordv2/checkers/leader_checker_test.go +++ b/internal/querycoordv2/checkers/leader_checker_test.go @@ -24,7 +24,6 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -34,6 +33,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/session" "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/checkers/segment_checker_test.go b/internal/querycoordv2/checkers/segment_checker_test.go index c6fdd0344039..3d43a4037656 100644 --- a/internal/querycoordv2/checkers/segment_checker_test.go +++ b/internal/querycoordv2/checkers/segment_checker_test.go @@ -25,7 +25,6 @@ import ( "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -36,6 +35,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/dist/dist_controller_test.go b/internal/querycoordv2/dist/dist_controller_test.go index 8ecaa0e410ba..9929962039ef 100644 --- a/internal/querycoordv2/dist/dist_controller_test.go +++ b/internal/querycoordv2/dist/dist_controller_test.go @@ -25,7 +25,6 @@ import ( "github.com/stretchr/testify/suite" "go.uber.org/atomic" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/querypb" @@ -33,6 +32,7 @@ import ( . "github.com/milvus-io/milvus/internal/querycoordv2/params" "github.com/milvus-io/milvus/internal/querycoordv2/session" "github.com/milvus-io/milvus/internal/querycoordv2/task" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/querycoordv2/job/job_test.go b/internal/querycoordv2/job/job_test.go index 40b2628c045f..e31e1e062f00 100644 --- a/internal/querycoordv2/job/job_test.go +++ b/internal/querycoordv2/job/job_test.go @@ -27,7 +27,6 @@ import ( "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v2/rgpb" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" @@ -39,6 +38,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/observers" . "github.com/milvus-io/milvus/internal/querycoordv2/params" "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/querycoordv2/meta/collection_manager_test.go b/internal/querycoordv2/meta/collection_manager_test.go index 6cfeb58c815f..320075bb977c 100644 --- a/internal/querycoordv2/meta/collection_manager_test.go +++ b/internal/querycoordv2/meta/collection_manager_test.go @@ -26,12 +26,12 @@ import ( "github.com/stretchr/testify/suite" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/querypb" . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/querycoordv2/meta/replica_manager_test.go b/internal/querycoordv2/meta/replica_manager_test.go index 1520c89241f4..36db70a73845 100644 --- a/internal/querycoordv2/meta/replica_manager_test.go +++ b/internal/querycoordv2/meta/replica_manager_test.go @@ -24,12 +24,12 @@ import ( "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/querypb" . "github.com/milvus-io/milvus/internal/querycoordv2/params" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" diff --git a/internal/querycoordv2/meta/resource_manager_test.go b/internal/querycoordv2/meta/resource_manager_test.go index 2a93b6e4e47c..ca58a8e899e2 100644 --- a/internal/querycoordv2/meta/resource_manager_test.go +++ b/internal/querycoordv2/meta/resource_manager_test.go @@ -22,12 +22,12 @@ import ( "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v2/rgpb" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/kv/mocks" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/querycoordv2/params" "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/querycoordv2/meta/target_manager_test.go b/internal/querycoordv2/meta/target_manager_test.go index daba49b4398b..5aad9d8a7c49 100644 --- a/internal/querycoordv2/meta/target_manager_test.go +++ b/internal/querycoordv2/meta/target_manager_test.go @@ -27,7 +27,6 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" @@ -35,6 +34,7 @@ import ( "github.com/milvus-io/milvus/internal/proto/querypb" . "github.com/milvus-io/milvus/internal/querycoordv2/params" "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/querycoordv2/observers/collection_observer_test.go b/internal/querycoordv2/observers/collection_observer_test.go index b23a57342662..6e8d4f541d77 100644 --- a/internal/querycoordv2/observers/collection_observer_test.go +++ b/internal/querycoordv2/observers/collection_observer_test.go @@ -26,7 +26,6 @@ import ( "github.com/stretchr/testify/suite" clientv3 "go.etcd.io/etcd/client/v3" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" @@ -36,6 +35,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/meta" . "github.com/milvus-io/milvus/internal/querycoordv2/params" "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/merr" diff --git a/internal/querycoordv2/observers/replica_observer_test.go b/internal/querycoordv2/observers/replica_observer_test.go index 9ddfb7a0190e..9f9062488cb8 100644 --- a/internal/querycoordv2/observers/replica_observer_test.go +++ b/internal/querycoordv2/observers/replica_observer_test.go @@ -22,13 +22,13 @@ import ( "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v2/rgpb" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/querycoordv2/meta" . "github.com/milvus-io/milvus/internal/querycoordv2/params" "github.com/milvus-io/milvus/internal/querycoordv2/session" "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" diff --git a/internal/querycoordv2/observers/resource_observer_test.go b/internal/querycoordv2/observers/resource_observer_test.go index f75dae11e2a9..07a5c4151124 100644 --- a/internal/querycoordv2/observers/resource_observer_test.go +++ b/internal/querycoordv2/observers/resource_observer_test.go @@ -25,12 +25,12 @@ import ( "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v2/rgpb" - "github.com/milvus-io/milvus/internal/kv" etcdKV "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/mocks" "github.com/milvus-io/milvus/internal/querycoordv2/meta" . "github.com/milvus-io/milvus/internal/querycoordv2/params" "github.com/milvus-io/milvus/internal/querycoordv2/session" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/observers/target_observer_test.go b/internal/querycoordv2/observers/target_observer_test.go index e3553c5c1606..4116b06b8a4a 100644 --- a/internal/querycoordv2/observers/target_observer_test.go +++ b/internal/querycoordv2/observers/target_observer_test.go @@ -24,7 +24,6 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -33,6 +32,7 @@ import ( . "github.com/milvus-io/milvus/internal/querycoordv2/params" "github.com/milvus-io/milvus/internal/querycoordv2/session" "github.com/milvus-io/milvus/internal/querycoordv2/utils" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/paramtable" ) diff --git a/internal/querycoordv2/ops_service_test.go b/internal/querycoordv2/ops_service_test.go index c9d062d631a3..c25eb6f01718 100644 --- a/internal/querycoordv2/ops_service_test.go +++ b/internal/querycoordv2/ops_service_test.go @@ -26,7 +26,6 @@ import ( "go.uber.org/atomic" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" @@ -43,6 +42,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" "github.com/milvus-io/milvus/internal/util/sessionutil" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/metricsinfo" diff --git a/internal/querycoordv2/server.go b/internal/querycoordv2/server.go index 41dbdcafca3b..95590a9a7a6a 100644 --- a/internal/querycoordv2/server.go +++ b/internal/querycoordv2/server.go @@ -35,7 +35,6 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus/internal/allocator" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/kv/tikv" "github.com/milvus-io/milvus/internal/metastore" @@ -55,6 +54,7 @@ import ( "github.com/milvus-io/milvus/internal/util/sessionutil" "github.com/milvus-io/milvus/internal/util/tsoutil" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util" diff --git a/internal/querycoordv2/services_test.go b/internal/querycoordv2/services_test.go index e4fb877d0101..f7d12cce6d27 100644 --- a/internal/querycoordv2/services_test.go +++ b/internal/querycoordv2/services_test.go @@ -31,7 +31,6 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus-proto/go-api/v2/rgpb" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" @@ -49,6 +48,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/task" "github.com/milvus-io/milvus/internal/querycoordv2/utils" "github.com/milvus-io/milvus/internal/util/sessionutil" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/metricsinfo" diff --git a/internal/querycoordv2/task/task_test.go b/internal/querycoordv2/task/task_test.go index 2272d1cc3408..f0f3651dad1f 100644 --- a/internal/querycoordv2/task/task_test.go +++ b/internal/querycoordv2/task/task_test.go @@ -31,7 +31,6 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore" "github.com/milvus-io/milvus/internal/metastore/kv/querycoord" @@ -43,6 +42,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/session" "github.com/milvus-io/milvus/internal/querycoordv2/utils" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" diff --git a/internal/querynodev2/delegator/delegator_data.go b/internal/querynodev2/delegator/delegator_data.go index 02a9509fe621..1ca7111b2b23 100644 --- a/internal/querynodev2/delegator/delegator_data.go +++ b/internal/querynodev2/delegator/delegator_data.go @@ -42,8 +42,8 @@ import ( "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/commonpbutil" "github.com/milvus-io/milvus/pkg/util/conc" "github.com/milvus-io/milvus/pkg/util/funcutil" @@ -763,7 +763,7 @@ func (sd *shardDelegator) readDeleteFromMsgstream(ctx context.Context, position // Random the subname in case we trying to load same delta at the same time subName := fmt.Sprintf("querynode-delta-loader-%d-%d-%d", paramtable.GetNodeID(), sd.collectionID, rand.Int()) log.Info("from dml check point load delete", zap.Any("position", position), zap.String("vChannel", vchannelName), zap.String("subName", subName), zap.Time("positionTs", ts)) - err = stream.AsConsumer(context.TODO(), []string{pChannelName}, subName, mqwrapper.SubscriptionPositionUnknown) + err = stream.AsConsumer(context.TODO(), []string{pChannelName}, subName, mqcommon.SubscriptionPositionUnknown) if err != nil { return nil, err } diff --git a/internal/querynodev2/pipeline/manager_test.go b/internal/querynodev2/pipeline/manager_test.go index e9869ac9b3d3..e1654cd462df 100644 --- a/internal/querynodev2/pipeline/manager_test.go +++ b/internal/querynodev2/pipeline/manager_test.go @@ -27,9 +27,9 @@ import ( "github.com/milvus-io/milvus/internal/querynodev2/delegator" "github.com/milvus-io/milvus/internal/querynodev2/segments" "github.com/milvus-io/milvus/internal/querynodev2/tsafe" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgdispatcher" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -80,7 +80,7 @@ func (suite *PipelineManagerTestSuite) TestBasic() { // mock collection manager suite.collectionManager.EXPECT().Get(suite.collectionID).Return(&segments.Collection{}) // mock mq factory - suite.msgDispatcher.EXPECT().Register(mock.Anything, suite.channel, mock.Anything, mqwrapper.SubscriptionPositionUnknown).Return(suite.msgChan, nil) + suite.msgDispatcher.EXPECT().Register(mock.Anything, suite.channel, mock.Anything, common.SubscriptionPositionUnknown).Return(suite.msgChan, nil) suite.msgDispatcher.EXPECT().Deregister(suite.channel) // build manager diff --git a/internal/querynodev2/pipeline/pipeline_test.go b/internal/querynodev2/pipeline/pipeline_test.go index ecad79b9ee13..3dca8e674cc5 100644 --- a/internal/querynodev2/pipeline/pipeline_test.go +++ b/internal/querynodev2/pipeline/pipeline_test.go @@ -30,9 +30,9 @@ import ( "github.com/milvus-io/milvus/internal/querynodev2/delegator" "github.com/milvus-io/milvus/internal/querynodev2/segments" "github.com/milvus-io/milvus/internal/querynodev2/tsafe" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgdispatcher" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" ) @@ -115,7 +115,7 @@ func (suite *PipelineTestSuite) TestBasic() { suite.collectionManager.EXPECT().Get(suite.collectionID).Return(collection) // mock mq factory - suite.msgDispatcher.EXPECT().Register(mock.Anything, suite.channel, mock.Anything, mqwrapper.SubscriptionPositionUnknown).Return(suite.msgChan, nil) + suite.msgDispatcher.EXPECT().Register(mock.Anything, suite.channel, mock.Anything, common.SubscriptionPositionUnknown).Return(suite.msgChan, nil) suite.msgDispatcher.EXPECT().Deregister(suite.channel) // mock delegator diff --git a/internal/rootcoord/dml_channels.go b/internal/rootcoord/dml_channels.go index b3055c3c7b1e..058fc24c8ba9 100644 --- a/internal/rootcoord/dml_channels.go +++ b/internal/rootcoord/dml_channels.go @@ -29,8 +29,8 @@ import ( "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -186,7 +186,7 @@ func newDmlChannels(ctx context.Context, factory msgstream.Factory, chanNamePref if params.PreCreatedTopicEnabled.GetAsBool() { subName := fmt.Sprintf("pre-created-topic-check-%s", name) - ms.AsConsumer(ctx, []string{name}, subName, mqwrapper.SubscriptionPositionUnknown) + ms.AsConsumer(ctx, []string{name}, subName, common.SubscriptionPositionUnknown) // check if topic is existed // kafka and rmq will err if the topic does not yet exist, pulsar will not // allow topics is not empty, for the reason that when restart or upgrade, the topic is not empty diff --git a/internal/rootcoord/dml_channels_test.go b/internal/rootcoord/dml_channels_test.go index e27117b0268a..c7ce78b6ce0c 100644 --- a/internal/rootcoord/dml_channels_test.go +++ b/internal/rootcoord/dml_channels_test.go @@ -28,8 +28,8 @@ import ( "github.com/stretchr/testify/require" "github.com/milvus-io/milvus/internal/util/dependency" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/funcutil" "github.com/milvus-io/milvus/pkg/util/paramtable" ) @@ -281,7 +281,7 @@ func (ms *FailMsgStream) Close() {} func (ms *FailMsgStream) Chan() <-chan *msgstream.MsgPack { return nil } func (ms *FailMsgStream) AsProducer(channels []string) {} func (ms *FailMsgStream) AsReader(channels []string, subName string) {} -func (ms *FailMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position mqwrapper.SubscriptionInitialPosition) error { +func (ms *FailMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position common.SubscriptionInitialPosition) error { return nil } func (ms *FailMsgStream) SetRepackFunc(repackFunc msgstream.RepackFunc) {} diff --git a/internal/rootcoord/root_coord.go b/internal/rootcoord/root_coord.go index 62f6cef9fd17..b83c96add7dc 100644 --- a/internal/rootcoord/root_coord.go +++ b/internal/rootcoord/root_coord.go @@ -36,7 +36,6 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/allocator" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/kv/tikv" "github.com/milvus-io/milvus/internal/metastore" @@ -53,6 +52,7 @@ import ( "github.com/milvus-io/milvus/internal/util/sessionutil" tsoutil2 "github.com/milvus-io/milvus/internal/util/tsoutil" "github.com/milvus-io/milvus/pkg/common" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/util" diff --git a/internal/tso/global_allocator.go b/internal/tso/global_allocator.go index 7d737387a590..cf63b97e74c6 100644 --- a/internal/tso/global_allocator.go +++ b/internal/tso/global_allocator.go @@ -36,7 +36,7 @@ import ( "github.com/cockroachdb/errors" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/tsoutil" "github.com/milvus-io/milvus/pkg/util/typeutil" diff --git a/internal/tso/tso.go b/internal/tso/tso.go index 495ec510d157..b41ef20605de 100644 --- a/internal/tso/tso.go +++ b/internal/tso/tso.go @@ -38,7 +38,7 @@ import ( "github.com/cockroachdb/errors" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/kv" + "github.com/milvus-io/milvus/pkg/kv" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/tsoutil" "github.com/milvus-io/milvus/pkg/util/typeutil" diff --git a/internal/util/dependency/factory.go b/internal/util/dependency/factory.go index 106cb3263561..e0aa141ec271 100644 --- a/internal/util/dependency/factory.go +++ b/internal/util/dependency/factory.go @@ -6,7 +6,6 @@ import ( "github.com/cockroachdb/errors" "go.uber.org/zap" - smsgstream "github.com/milvus-io/milvus/internal/mq/msgstream" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" @@ -40,7 +39,7 @@ type DefaultFactory struct { func NewDefaultFactory(standAlone bool) *DefaultFactory { return &DefaultFactory{ standAlone: standAlone, - msgStreamFactory: smsgstream.NewRocksmqFactory("/tmp/milvus/rocksmq/", ¶mtable.Get().ServiceParam), + msgStreamFactory: msgstream.NewRocksmqFactory("/tmp/milvus/rocksmq/", ¶mtable.Get().ServiceParam), chunkManagerFactory: storage.NewChunkManagerFactory("local", storage.RootPath("/tmp/milvus")), } @@ -50,7 +49,7 @@ func NewDefaultFactory(standAlone bool) *DefaultFactory { func MockDefaultFactory(standAlone bool, params *paramtable.ComponentParam) *DefaultFactory { return &DefaultFactory{ standAlone: standAlone, - msgStreamFactory: smsgstream.NewRocksmqFactory("/tmp/milvus/rocksmq/", ¶mtable.Get().ServiceParam), + msgStreamFactory: msgstream.NewRocksmqFactory("/tmp/milvus/rocksmq/", ¶mtable.Get().ServiceParam), chunkManagerFactory: storage.NewChunkManagerFactoryWithParam(params), } } @@ -89,7 +88,7 @@ func (f *DefaultFactory) initMQ(standalone bool, params *paramtable.ComponentPar case mqTypeNatsmq: f.msgStreamFactory = msgstream.NewNatsmqFactory() case mqTypeRocksmq: - f.msgStreamFactory = smsgstream.NewRocksmqFactory(params.RocksmqCfg.Path.GetValue(), ¶ms.ServiceParam) + f.msgStreamFactory = msgstream.NewRocksmqFactory(params.RocksmqCfg.Path.GetValue(), ¶ms.ServiceParam) case mqTypePulsar: f.msgStreamFactory = msgstream.NewPmsFactory(¶ms.ServiceParam) case mqTypeKafka: diff --git a/internal/util/flowgraph/input_node_test.go b/internal/util/flowgraph/input_node_test.go index ff205a0ae4a4..03c8d38d909a 100644 --- a/internal/util/flowgraph/input_node_test.go +++ b/internal/util/flowgraph/input_node_test.go @@ -27,7 +27,7 @@ import ( "github.com/milvus-io/milvus/internal/util/dependency" "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -38,7 +38,7 @@ func TestInputNode(t *testing.T) { msgStream, _ := factory.NewMsgStream(context.TODO()) channels := []string{"cc"} - msgStream.AsConsumer(context.Background(), channels, "sub", mqwrapper.SubscriptionPositionEarliest) + msgStream.AsConsumer(context.Background(), channels, "sub", common.SubscriptionPositionEarliest) msgPack := generateMsgPack() produceStream, _ := factory.NewMsgStream(context.TODO()) @@ -81,7 +81,7 @@ func Test_InputNodeSkipMode(t *testing.T) { msgStream, _ := factory.NewMsgStream(context.TODO()) channels := []string{"cc" + fmt.Sprint(rand.Int())} - msgStream.AsConsumer(context.Background(), channels, "sub", mqwrapper.SubscriptionPositionEarliest) + msgStream.AsConsumer(context.Background(), channels, "sub", common.SubscriptionPositionEarliest) produceStream, _ := factory.NewMsgStream(context.TODO()) produceStream.AsProducer(channels) diff --git a/internal/util/flowgraph/node_test.go b/internal/util/flowgraph/node_test.go index 551bfb612f6f..850fd183a267 100644 --- a/internal/util/flowgraph/node_test.go +++ b/internal/util/flowgraph/node_test.go @@ -28,8 +28,8 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus/internal/util/dependency" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" ) func generateMsgPack() msgstream.MsgPack { @@ -78,7 +78,7 @@ func TestNodeManager_Start(t *testing.T) { msgStream, _ := factory.NewMsgStream(context.TODO()) channels := []string{"cc"} - msgStream.AsConsumer(context.TODO(), channels, "sub", mqwrapper.SubscriptionPositionEarliest) + msgStream.AsConsumer(context.TODO(), channels, "sub", common.SubscriptionPositionEarliest) produceStream, _ := factory.NewMsgStream(context.TODO()) produceStream.AsProducer(channels) diff --git a/internal/util/pipeline/stream_pipeline.go b/internal/util/pipeline/stream_pipeline.go index 3c22c6e99fec..9485129f891e 100644 --- a/internal/util/pipeline/stream_pipeline.go +++ b/internal/util/pipeline/stream_pipeline.go @@ -25,9 +25,9 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgdispatcher" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/tsoutil" ) @@ -71,7 +71,7 @@ func (p *streamPipeline) ConsumeMsgStream(position *msgpb.MsgPosition) error { } start := time.Now() - p.input, err = p.dispatcher.Register(context.TODO(), p.vChannel, position, mqwrapper.SubscriptionPositionUnknown) + p.input, err = p.dispatcher.Register(context.TODO(), p.vChannel, position, common.SubscriptionPositionUnknown) if err != nil { log.Error("dispatcher register failed", zap.String("channel", position.ChannelName)) return WrapErrRegDispather(err) diff --git a/internal/util/pipeline/stream_pipeline_test.go b/internal/util/pipeline/stream_pipeline_test.go index a731a18ff34b..0f94bd18b52d 100644 --- a/internal/util/pipeline/stream_pipeline_test.go +++ b/internal/util/pipeline/stream_pipeline_test.go @@ -24,9 +24,9 @@ import ( "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgdispatcher" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" ) type StreamPipelineSuite struct { @@ -46,7 +46,7 @@ func (suite *StreamPipelineSuite) SetupTest() { suite.inChannel = make(chan *msgstream.MsgPack, 1) suite.outChannel = make(chan msgstream.Timestamp) suite.msgDispatcher = msgdispatcher.NewMockClient(suite.T()) - suite.msgDispatcher.EXPECT().Register(mock.Anything, suite.channel, mock.Anything, mqwrapper.SubscriptionPositionUnknown).Return(suite.inChannel, nil) + suite.msgDispatcher.EXPECT().Register(mock.Anything, suite.channel, mock.Anything, common.SubscriptionPositionUnknown).Return(suite.inChannel, nil) suite.msgDispatcher.EXPECT().Deregister(suite.channel) suite.pipeline = NewPipelineWithStream(suite.msgDispatcher, 0, false, suite.channel) suite.length = 4 diff --git a/internal/util/tsoutil/tso.go b/internal/util/tsoutil/tso.go index 9d2a83d32fac..20254cc708ba 100644 --- a/internal/util/tsoutil/tso.go +++ b/internal/util/tsoutil/tso.go @@ -22,9 +22,9 @@ import ( "github.com/tikv/client-go/v2/txnkv" clientv3 "go.etcd.io/etcd/client/v3" - "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/kv/tikv" + "github.com/milvus-io/milvus/pkg/kv" ) // NewTSOKVBase returns a kv.TxnKV object diff --git a/pkg/go.mod b/pkg/go.mod index 6f967a0c2260..49e9b30a90df 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -28,6 +28,7 @@ require ( github.com/spf13/viper v1.8.1 github.com/streamnative/pulsarctl v0.5.0 github.com/stretchr/testify v1.8.4 + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c github.com/tikv/client-go/v2 v2.0.4 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/x448/float16 v0.8.4 @@ -77,6 +78,9 @@ require ( github.com/docker/go-units v0.4.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect + github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect + github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect + github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/getsentry/sentry-go v0.12.0 // indirect diff --git a/pkg/go.sum b/pkg/go.sum index be71e1e131f4..beff409b3c29 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -26,7 +26,9 @@ cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUM cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= @@ -119,6 +121,7 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= @@ -182,10 +185,18 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8= +github.com/envoyproxy/protoc-gen-validate v0.10.1/go.mod h1:DRjgyB0I43LtJapqN6NiRwroiAU2PaFuvk/vjgh61ss= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -194,6 +205,7 @@ github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03D github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -249,6 +261,7 @@ github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzq github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -309,6 +322,7 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -447,7 +461,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kris-nova/logger v0.0.0-20181127235838-fd0d87064b06 h1:vN4d3jSss3ExzUn2cE0WctxztfOgiKvMKnDrydBsg00= +github.com/kris-nova/logger v0.0.0-20181127235838-fd0d87064b06/go.mod h1:++9BgZujZd4v0ZTZCb5iPsaomXdZWyxotIAh1IiDm44= github.com/kris-nova/lolgopher v0.0.0-20180921204813-313b3abb0d9b h1:xYEM2oBUhBEhQjrV+KJ9lEWDWYZoNVZUaBF++Wyljq4= +github.com/kris-nova/lolgopher v0.0.0-20180921204813-313b3abb0d9b/go.mod h1:V0HF/ZBlN86HqewcDC/cVxMmYDiRukWjSrgKLUAn9Js= github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76 h1:IVlcvV0CjvfBYYod5ePe89l+3LBAl//6n9kJ9Vr2i0k= @@ -486,6 +502,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8= +github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4= github.com/milvus-io/milvus-proto/go-api/v2 v2.4.5 h1:83SZKox70jyABAPY7HdbvrwI3cmgwzyuc7dO4dYmoC0= github.com/milvus-io/milvus-proto/go-api/v2 v2.4.5/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A= @@ -534,6 +552,7 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.1 h1:b3iUnf1v+ppJiOfNX4yxxqfWKMQPZR5yoh8urCTFX88= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -585,6 +604,7 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -703,6 +723,7 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= github.com/tikv/client-go/v2 v2.0.4 h1:cPtMXTExqjzk8L40qhrgB/mXiBXKP5LRU0vwjtI2Xxo= @@ -819,6 +840,7 @@ go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnw go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= @@ -1265,6 +1287,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v1 v1.0.0/go.mod h1:CxwszS/Xz1C49Ucd2i6Zil5UToP1EmyrFhKaMVbg1mk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1318,3 +1341,4 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= +stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= diff --git a/internal/kv/kv.go b/pkg/kv/kv.go similarity index 98% rename from internal/kv/kv.go rename to pkg/kv/kv.go index 14091cdc1e84..9f051870a10a 100644 --- a/internal/kv/kv.go +++ b/pkg/kv/kv.go @@ -19,7 +19,7 @@ package kv import ( clientv3 "go.etcd.io/etcd/client/v3" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/util/typeutil" ) diff --git a/internal/kv/predicates/mock_predicate.go b/pkg/kv/predicates/mock_predicate.go similarity index 100% rename from internal/kv/predicates/mock_predicate.go rename to pkg/kv/predicates/mock_predicate.go diff --git a/internal/kv/predicates/predicate.go b/pkg/kv/predicates/predicate.go similarity index 100% rename from internal/kv/predicates/predicate.go rename to pkg/kv/predicates/predicate.go diff --git a/internal/kv/predicates/predicate_test.go b/pkg/kv/predicates/predicate_test.go similarity index 100% rename from internal/kv/predicates/predicate_test.go rename to pkg/kv/predicates/predicate_test.go diff --git a/internal/kv/rocksdb/rocks_iterator.go b/pkg/kv/rocksdb/rocks_iterator.go similarity index 100% rename from internal/kv/rocksdb/rocks_iterator.go rename to pkg/kv/rocksdb/rocks_iterator.go diff --git a/internal/kv/rocksdb/rocksdb_kv.go b/pkg/kv/rocksdb/rocksdb_kv.go similarity index 99% rename from internal/kv/rocksdb/rocksdb_kv.go rename to pkg/kv/rocksdb/rocksdb_kv.go index f8854138910a..b81e7ead385f 100644 --- a/internal/kv/rocksdb/rocksdb_kv.go +++ b/pkg/kv/rocksdb/rocksdb_kv.go @@ -22,8 +22,8 @@ import ( "github.com/cockroachdb/errors" "github.com/tecbot/gorocksdb" - "github.com/milvus-io/milvus/internal/kv" - "github.com/milvus-io/milvus/internal/kv/predicates" + "github.com/milvus-io/milvus/pkg/kv" + "github.com/milvus-io/milvus/pkg/kv/predicates" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/typeutil" ) diff --git a/internal/kv/rocksdb/rocksdb_kv_test.go b/pkg/kv/rocksdb/rocksdb_kv_test.go similarity index 98% rename from internal/kv/rocksdb/rocksdb_kv_test.go rename to pkg/kv/rocksdb/rocksdb_kv_test.go index b1d07010b8ec..14c05a030933 100644 --- a/internal/kv/rocksdb/rocksdb_kv_test.go +++ b/pkg/kv/rocksdb/rocksdb_kv_test.go @@ -25,8 +25,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/milvus-io/milvus/internal/kv/predicates" - rocksdbkv "github.com/milvus-io/milvus/internal/kv/rocksdb" + "github.com/milvus-io/milvus/pkg/kv/predicates" + rocksdbkv "github.com/milvus-io/milvus/pkg/kv/rocksdb" "github.com/milvus-io/milvus/pkg/util/merr" ) diff --git a/pkg/mq/msgstream/mqwrapper/id.go b/pkg/mq/common/id.go similarity index 98% rename from pkg/mq/msgstream/mqwrapper/id.go rename to pkg/mq/common/id.go index cd32d843fc6b..2fc5e6212aa7 100644 --- a/pkg/mq/msgstream/mqwrapper/id.go +++ b/pkg/mq/common/id.go @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package mqwrapper +package common // MessageID is the interface that provides operations of message is type MessageID interface { diff --git a/pkg/mq/msgstream/mqwrapper/message.go b/pkg/mq/common/message.go similarity index 52% rename from pkg/mq/msgstream/mqwrapper/message.go rename to pkg/mq/common/message.go index dbb13484e549..bd8d231c4874 100644 --- a/pkg/mq/msgstream/mqwrapper/message.go +++ b/pkg/mq/common/message.go @@ -14,7 +14,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package mqwrapper +package common + +// ProducerOptions contains the options of a producer +type ProducerOptions struct { + // The topic that this Producer will publish + Topic string + + // Enable compression + // For Pulsar, this enables ZSTD compression with default compression level + EnableCompression bool +} + +// ProducerMessage contains the messages of a producer +type ProducerMessage struct { + // Payload get the payload of the message + Payload []byte + // Properties are application defined key/value pairs that will be attached to the message. + // Return the properties attached to the message. + Properties map[string]string +} // Message is the interface that provides operations of a consumer type Message interface { @@ -32,3 +51,17 @@ type Message interface { // The message id can be used to univocally refer to a message without having the keep the entire payload in memory. ID() MessageID } + +// SubscriptionInitialPosition is the type of a subscription initial position +type SubscriptionInitialPosition int + +const ( + // SubscriptionPositionLatest is latest position which means the start consuming position will be the last message + SubscriptionPositionLatest SubscriptionInitialPosition = iota + + // SubscriptionPositionEarliest is earliest position which means the start consuming position will be the first message + SubscriptionPositionEarliest + + // SubscriptionPositionUnkown indicates we don't care about the consumer location, since we are doing another seek or only some meta api over that + SubscriptionPositionUnknown +) diff --git a/pkg/mq/msgstream/mqwrapper/mock_id.go b/pkg/mq/common/mock_id.go similarity index 98% rename from pkg/mq/msgstream/mqwrapper/mock_id.go rename to pkg/mq/common/mock_id.go index a8dad1aa9a2e..83e95716bfb4 100644 --- a/pkg/mq/msgstream/mqwrapper/mock_id.go +++ b/pkg/mq/common/mock_id.go @@ -1,8 +1,8 @@ // Code generated by mockery v2.32.4. DO NOT EDIT. -package mqwrapper +package common -import mock "github.com/stretchr/testify/mock" +import "github.com/stretchr/testify/mock" // MockMessageID is an autogenerated mock type for the MessageID type type MockMessageID struct { diff --git a/internal/mq/mqimpl/rocksmq/client/client.go b/pkg/mq/mqimpl/rocksmq/client/client.go similarity index 94% rename from internal/mq/mqimpl/rocksmq/client/client.go rename to pkg/mq/mqimpl/rocksmq/client/client.go index 8bc6aab90d4c..fbd705dc5281 100644 --- a/internal/mq/mqimpl/rocksmq/client/client.go +++ b/pkg/mq/mqimpl/rocksmq/client/client.go @@ -11,7 +11,7 @@ package client -import "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" +import "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" // RocksMQ is the type server.RocksMQ type RocksMQ = server.RocksMQ diff --git a/internal/mq/mqimpl/rocksmq/client/client_impl.go b/pkg/mq/mqimpl/rocksmq/client/client_impl.go similarity index 94% rename from internal/mq/mqimpl/rocksmq/client/client_impl.go rename to pkg/mq/mqimpl/rocksmq/client/client_impl.go index f0cce8653092..06ab4717b055 100644 --- a/internal/mq/mqimpl/rocksmq/client/client_impl.go +++ b/pkg/mq/mqimpl/rocksmq/client/client_impl.go @@ -17,9 +17,9 @@ import ( "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" ) type client struct { @@ -83,7 +83,7 @@ func (c *client) Subscribe(options ConsumerOptions) (Consumer, error) { if err != nil { return nil, err } - if options.SubscriptionInitialPosition == mqwrapper.SubscriptionPositionLatest { + if options.SubscriptionInitialPosition == common.SubscriptionPositionLatest { err = c.server.SeekToLatest(options.Topic, options.SubscriptionName) if err != nil { return nil, err @@ -110,7 +110,7 @@ func (c *client) Subscribe(options ConsumerOptions) (Consumer, error) { } c.server.RegisterConsumer(cons) - if options.SubscriptionInitialPosition == mqwrapper.SubscriptionPositionLatest { + if options.SubscriptionInitialPosition == common.SubscriptionPositionLatest { err = c.server.SeekToLatest(options.Topic, options.SubscriptionName) if err != nil { return nil, err diff --git a/internal/mq/mqimpl/rocksmq/client/client_impl_test.go b/pkg/mq/mqimpl/rocksmq/client/client_impl_test.go similarity index 89% rename from internal/mq/mqimpl/rocksmq/client/client_impl_test.go rename to pkg/mq/mqimpl/rocksmq/client/client_impl_test.go index 7f2822bf0aa8..52a8d650891d 100644 --- a/internal/mq/mqimpl/rocksmq/client/client_impl_test.go +++ b/pkg/mq/mqimpl/rocksmq/client/client_impl_test.go @@ -23,9 +23,9 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" + server2 "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/util/paramtable" ) @@ -97,7 +97,7 @@ func TestClient_Subscribe(t *testing.T) { consumer, err := client.Subscribe(ConsumerOptions{ Topic: newTopicName(), SubscriptionName: newConsumerName(), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) assert.Error(t, err) assert.Nil(t, consumer) @@ -116,7 +116,7 @@ func TestClient_Subscribe(t *testing.T) { opt := ConsumerOptions{ Topic: newTopicName(), SubscriptionName: newConsumerName(), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, } consumer1, err := client1.Subscribe(opt) assert.NoError(t, err) @@ -128,7 +128,7 @@ func TestClient_Subscribe(t *testing.T) { opt1 := ConsumerOptions{ Topic: newTopicName(), SubscriptionName: newConsumerName(), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionLatest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionLatest, } consumer3, err := client1.Subscribe(opt1) assert.NoError(t, err) @@ -145,7 +145,7 @@ func TestClient_Subscribe(t *testing.T) { } func TestClient_SubscribeError(t *testing.T) { - mockMQ := server.NewMockRocksMQ(t) + mockMQ := server2.NewMockRocksMQ(t) client, err := NewClient(Options{ Server: mockMQ, }) @@ -161,7 +161,7 @@ func TestClient_SubscribeError(t *testing.T) { consumer, err := client.Subscribe(ConsumerOptions{ Topic: testTopic, SubscriptionName: testGroupName, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionLatest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionLatest, }) assert.Error(t, err) assert.Nil(t, consumer) @@ -183,7 +183,7 @@ func TestClient_SeekLatest(t *testing.T) { opt := ConsumerOptions{ Topic: topicName, SubscriptionName: newConsumerName(), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, } consumer1, err := client.Subscribe(opt) assert.NoError(t, err) @@ -194,7 +194,7 @@ func TestClient_SeekLatest(t *testing.T) { }) assert.NotNil(t, producer) assert.NoError(t, err) - msg := &mqwrapper.ProducerMessage{ + msg := &mqcommon.ProducerMessage{ Payload: make([]byte, 10), Properties: map[string]string{}, } @@ -204,14 +204,14 @@ func TestClient_SeekLatest(t *testing.T) { msgChan := consumer1.Chan() msgRead, ok := <-msgChan assert.Equal(t, ok, true) - assert.Equal(t, msgRead.ID(), &server.RmqID{MessageID: id}) + assert.Equal(t, msgRead.ID(), &server2.RmqID{MessageID: id}) consumer1.Close() opt1 := ConsumerOptions{ Topic: topicName, SubscriptionName: newConsumerName(), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionLatest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionLatest, } consumer2, err := client.Subscribe(opt1) assert.NoError(t, err) @@ -227,7 +227,7 @@ func TestClient_SeekLatest(t *testing.T) { assert.Equal(t, len(msg.Payload()), 8) loop = false case <-ticker.C: - msg := &mqwrapper.ProducerMessage{ + msg := &mqcommon.ProducerMessage{ Payload: make([]byte, 8), } _, err = producer.Send(msg) @@ -263,13 +263,13 @@ func TestClient_consume(t *testing.T) { opt := ConsumerOptions{ Topic: topicName, SubscriptionName: newConsumerName(), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, } consumer, err := client.Subscribe(opt) assert.NoError(t, err) assert.NotNil(t, consumer) - msg := &mqwrapper.ProducerMessage{ + msg := &mqcommon.ProducerMessage{ Payload: make([]byte, 10), } id, err := producer.Send(msg) @@ -278,7 +278,7 @@ func TestClient_consume(t *testing.T) { msgChan := consumer.Chan() msgConsume, ok := <-msgChan assert.Equal(t, ok, true) - assert.Equal(t, &server.RmqID{MessageID: id}, msgConsume.ID()) + assert.Equal(t, &server2.RmqID{MessageID: id}, msgConsume.ID()) } func TestRocksmq_Properties(t *testing.T) { @@ -302,7 +302,7 @@ func TestRocksmq_Properties(t *testing.T) { opt := ConsumerOptions{ Topic: topicName, SubscriptionName: newConsumerName(), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, } consumer, err := client.Subscribe(opt) assert.NoError(t, err) @@ -322,7 +322,7 @@ func TestRocksmq_Properties(t *testing.T) { header, err := UnmarshalHeader(msgb) assert.NoError(t, err) assert.NotNil(t, header) - msg := &mqwrapper.ProducerMessage{ + msg := &mqcommon.ProducerMessage{ Payload: msgb, Properties: map[string]string{common.TraceIDKey: "a"}, } @@ -340,7 +340,7 @@ func TestRocksmq_Properties(t *testing.T) { // rocksmq consumer needs produce to notify to receive msg // if produce all in the begin, it will stuck if consume not that fast // related with https://github.com/milvus-io/milvus/issues/27801 - msg = &mqwrapper.ProducerMessage{ + msg = &mqcommon.ProducerMessage{ Payload: msgb, Properties: map[string]string{common.TraceIDKey: "b"}, } diff --git a/internal/mq/mqimpl/rocksmq/client/consumer.go b/pkg/mq/mqimpl/rocksmq/client/consumer.go similarity index 88% rename from internal/mq/mqimpl/rocksmq/client/consumer.go rename to pkg/mq/mqimpl/rocksmq/client/consumer.go index 6790f5a520de..48efa18ef1e4 100644 --- a/internal/mq/mqimpl/rocksmq/client/consumer.go +++ b/pkg/mq/mqimpl/rocksmq/client/consumer.go @@ -12,8 +12,8 @@ package client import ( - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" ) // UniqueID is the type of message ID @@ -34,11 +34,11 @@ type ConsumerOptions struct { // InitialPosition at which the cursor will be set when subscribe // Default is `Latest` - mqwrapper.SubscriptionInitialPosition + common.SubscriptionInitialPosition // Message for this consumer // When a message is received, it will be pushed to this channel for consumption - MessageChannel chan mqwrapper.Message + MessageChannel chan common.Message } // Consumer interface provide operations for a consumer @@ -53,7 +53,7 @@ type Consumer interface { MsgMutex() chan struct{} // Message channel - Chan() <-chan mqwrapper.Message + Chan() <-chan common.Message // Seek to the uniqueID position Seek(UniqueID) error //nolint:govet diff --git a/internal/mq/mqimpl/rocksmq/client/consumer_impl.go b/pkg/mq/mqimpl/rocksmq/client/consumer_impl.go similarity index 94% rename from internal/mq/mqimpl/rocksmq/client/consumer_impl.go rename to pkg/mq/mqimpl/rocksmq/client/consumer_impl.go index 2957300b657b..fb907b4defc7 100644 --- a/internal/mq/mqimpl/rocksmq/client/consumer_impl.go +++ b/pkg/mq/mqimpl/rocksmq/client/consumer_impl.go @@ -17,7 +17,7 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) type consumer struct { @@ -30,7 +30,7 @@ type consumer struct { msgMutex chan struct{} initCh chan struct{} - messageCh chan mqwrapper.Message + messageCh chan common.Message } func newConsumer(c *client, options ConsumerOptions) (*consumer, error) { @@ -48,7 +48,7 @@ func newConsumer(c *client, options ConsumerOptions) (*consumer, error) { messageCh := options.MessageChannel if options.MessageChannel == nil { - messageCh = make(chan mqwrapper.Message, 1) + messageCh = make(chan common.Message, 1) } // only used for initCh := make(chan struct{}, 1) @@ -80,7 +80,7 @@ func getExistedConsumer(c *client, options ConsumerOptions, msgMutex chan struct messageCh := options.MessageChannel if options.MessageChannel == nil { - messageCh = make(chan mqwrapper.Message, 1) + messageCh = make(chan common.Message, 1) } return &consumer{ @@ -109,7 +109,7 @@ func (c *consumer) MsgMutex() chan struct{} { } // Chan start consume goroutine and return message channel -func (c *consumer) Chan() <-chan mqwrapper.Message { +func (c *consumer) Chan() <-chan common.Message { c.startOnce.Do(func() { c.client.wg.Add(1) go c.client.consume(c) diff --git a/internal/mq/mqimpl/rocksmq/client/consumer_impl_test.go b/pkg/mq/mqimpl/rocksmq/client/consumer_impl_test.go similarity index 89% rename from internal/mq/mqimpl/rocksmq/client/consumer_impl_test.go rename to pkg/mq/mqimpl/rocksmq/client/consumer_impl_test.go index 646d8b54836a..e0d8f355d0ba 100644 --- a/internal/mq/mqimpl/rocksmq/client/consumer_impl_test.go +++ b/pkg/mq/mqimpl/rocksmq/client/consumer_impl_test.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) func TestConsumer_newConsumer(t *testing.T) { @@ -26,7 +26,7 @@ func TestConsumer_newConsumer(t *testing.T) { consumer, err := newConsumer(nil, ConsumerOptions{ Topic: newTopicName(), SubscriptionName: newConsumerName(), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, }) assert.Nil(t, consumer) assert.Error(t, err) @@ -65,7 +65,7 @@ func TestConsumer_newConsumer(t *testing.T) { consumer1, err := newConsumer(client, ConsumerOptions{ Topic: newTopicName(), SubscriptionName: consumerName, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, }) assert.NoError(t, err) assert.NotNil(t, consumer1) @@ -88,7 +88,7 @@ func TestConsumer_newConsumer(t *testing.T) { consumer4, err := getExistedConsumer(client, ConsumerOptions{ Topic: newTopicName(), SubscriptionName: newConsumerName(), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, }, nil) assert.NoError(t, err) assert.NotNil(t, consumer4) @@ -102,7 +102,7 @@ func TestConsumer_newConsumer(t *testing.T) { consumer6, err := getExistedConsumer(client, ConsumerOptions{ Topic: newTopicName(), SubscriptionName: "", - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, }, nil) assert.Error(t, err) assert.Nil(t, consumer6) @@ -114,7 +114,7 @@ func TestConsumer_Subscription(t *testing.T) { consumer, err := newConsumer(newMockClient(), ConsumerOptions{ Topic: topicName, SubscriptionName: consumerName, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, }) assert.Nil(t, consumer) assert.Error(t, err) @@ -139,7 +139,7 @@ func TestConsumer_Seek(t *testing.T) { consumer, err := newConsumer(client, ConsumerOptions{ Topic: topicName, SubscriptionName: consumerName, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, }) assert.NoError(t, err) assert.NotNil(t, consumer) diff --git a/internal/mq/mqimpl/rocksmq/client/error.go b/pkg/mq/mqimpl/rocksmq/client/error.go similarity index 100% rename from internal/mq/mqimpl/rocksmq/client/error.go rename to pkg/mq/mqimpl/rocksmq/client/error.go diff --git a/internal/mq/mqimpl/rocksmq/client/producer.go b/pkg/mq/mqimpl/rocksmq/client/producer.go similarity index 88% rename from internal/mq/mqimpl/rocksmq/client/producer.go rename to pkg/mq/mqimpl/rocksmq/client/producer.go index 6a9f74f8e6cf..65fc19a8dc4b 100644 --- a/internal/mq/mqimpl/rocksmq/client/producer.go +++ b/pkg/mq/mqimpl/rocksmq/client/producer.go @@ -11,7 +11,9 @@ package client -import "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" +import ( + "github.com/milvus-io/milvus/pkg/mq/common" +) // ProducerOptions is the options of a producer type ProducerOptions struct { @@ -24,7 +26,7 @@ type Producer interface { Topic() string // publish a message - Send(message *mqwrapper.ProducerMessage) (UniqueID, error) + Send(message *common.ProducerMessage) (UniqueID, error) // Close a producer Close() diff --git a/internal/mq/mqimpl/rocksmq/client/producer_impl.go b/pkg/mq/mqimpl/rocksmq/client/producer_impl.go similarity index 91% rename from internal/mq/mqimpl/rocksmq/client/producer_impl.go rename to pkg/mq/mqimpl/rocksmq/client/producer_impl.go index ad27f449b04a..f858bce63cc3 100644 --- a/internal/mq/mqimpl/rocksmq/client/producer_impl.go +++ b/pkg/mq/mqimpl/rocksmq/client/producer_impl.go @@ -14,9 +14,9 @@ package client import ( "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" ) // assertion make sure implementation @@ -51,7 +51,7 @@ func (p *producer) Topic() string { } // Send produce message in rocksmq -func (p *producer) Send(message *mqwrapper.ProducerMessage) (UniqueID, error) { +func (p *producer) Send(message *common.ProducerMessage) (UniqueID, error) { // NOTICE: this is the hack. // we should not unmarshal the payload here but we can not extend the payload byte payload := message.Payload diff --git a/internal/mq/mqimpl/rocksmq/client/producer_impl_test.go b/pkg/mq/mqimpl/rocksmq/client/producer_impl_test.go similarity index 100% rename from internal/mq/mqimpl/rocksmq/client/producer_impl_test.go rename to pkg/mq/mqimpl/rocksmq/client/producer_impl_test.go diff --git a/internal/mq/mqimpl/rocksmq/client/rmq_message.go b/pkg/mq/mqimpl/rocksmq/client/rmq_message.go similarity index 86% rename from internal/mq/mqimpl/rocksmq/client/rmq_message.go rename to pkg/mq/mqimpl/rocksmq/client/rmq_message.go index 7133f392344e..e2fb4376aa57 100644 --- a/internal/mq/mqimpl/rocksmq/client/rmq_message.go +++ b/pkg/mq/mqimpl/rocksmq/client/rmq_message.go @@ -12,13 +12,13 @@ package client import ( - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/util/typeutil" ) // Check rmqMessage implements ConsumerMessage -var _ mqwrapper.Message = (*RmqMessage)(nil) +var _ common.Message = (*RmqMessage)(nil) // rmqMessage wraps the message for rocksmq type RmqMessage struct { @@ -44,6 +44,6 @@ func (rm *RmqMessage) Payload() []byte { } // ID returns the id of rocksmq message -func (rm *RmqMessage) ID() mqwrapper.MessageID { +func (rm *RmqMessage) ID() common.MessageID { return &server.RmqID{MessageID: rm.msgID} } diff --git a/internal/mq/mqimpl/rocksmq/client/test_helper.go b/pkg/mq/mqimpl/rocksmq/client/test_helper.go similarity index 86% rename from internal/mq/mqimpl/rocksmq/client/test_helper.go rename to pkg/mq/mqimpl/rocksmq/client/test_helper.go index cdff81465c42..c75c68a919b7 100644 --- a/internal/mq/mqimpl/rocksmq/client/test_helper.go +++ b/pkg/mq/mqimpl/rocksmq/client/test_helper.go @@ -20,8 +20,8 @@ import ( "github.com/stretchr/testify/assert" "go.uber.org/zap" - server2 "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/util/paramtable" ) @@ -33,8 +33,8 @@ func newConsumerName() string { return fmt.Sprintf("my-consumer-%v", time.Now().Nanosecond()) } -func newMockRocksMQ() server2.RocksMQ { - var rocksmq server2.RocksMQ +func newMockRocksMQ() server.RocksMQ { + var rocksmq server.RocksMQ return rocksmq } @@ -45,10 +45,10 @@ func newMockClient() *client { return client } -func newRocksMQ(t *testing.T, rmqPath string) server2.RocksMQ { +func newRocksMQ(t *testing.T, rmqPath string) server.RocksMQ { rocksdbPath := rmqPath paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := server2.NewRocksMQ(rocksdbPath, nil) + rmq, err := server.NewRocksMQ(rocksdbPath) assert.NoError(t, err) return rmq } diff --git a/internal/mq/mqimpl/rocksmq/client/util.go b/pkg/mq/mqimpl/rocksmq/client/util.go similarity index 100% rename from internal/mq/mqimpl/rocksmq/client/util.go rename to pkg/mq/mqimpl/rocksmq/client/util.go diff --git a/internal/mq/mqimpl/rocksmq/server/global_rmq.go b/pkg/mq/mqimpl/rocksmq/server/global_rmq.go similarity index 97% rename from internal/mq/mqimpl/rocksmq/server/global_rmq.go rename to pkg/mq/mqimpl/rocksmq/server/global_rmq.go index e668c63d8de8..bccebb56bc87 100644 --- a/internal/mq/mqimpl/rocksmq/server/global_rmq.go +++ b/pkg/mq/mqimpl/rocksmq/server/global_rmq.go @@ -51,7 +51,7 @@ func InitRocksMQ(path string) error { return } } - Rmq, finalErr = NewRocksMQ(path, nil) + Rmq, finalErr = NewRocksMQ(path) }) return finalErr } diff --git a/internal/mq/mqimpl/rocksmq/server/global_rmq_test.go b/pkg/mq/mqimpl/rocksmq/server/global_rmq_test.go similarity index 100% rename from internal/mq/mqimpl/rocksmq/server/global_rmq_test.go rename to pkg/mq/mqimpl/rocksmq/server/global_rmq_test.go diff --git a/internal/mq/mqimpl/rocksmq/server/mock_rocksmq.go b/pkg/mq/mqimpl/rocksmq/server/mock_rocksmq.go similarity index 99% rename from internal/mq/mqimpl/rocksmq/server/mock_rocksmq.go rename to pkg/mq/mqimpl/rocksmq/server/mock_rocksmq.go index 1731d01f4feb..2723cdf2dcf0 100644 --- a/internal/mq/mqimpl/rocksmq/server/mock_rocksmq.go +++ b/pkg/mq/mqimpl/rocksmq/server/mock_rocksmq.go @@ -2,7 +2,7 @@ package server -import mock "github.com/stretchr/testify/mock" +import "github.com/stretchr/testify/mock" // MockRocksMQ is an autogenerated mock type for the RocksMQ type type MockRocksMQ struct { diff --git a/internal/mq/mqimpl/rocksmq/server/rmq_id.go b/pkg/mq/mqimpl/rocksmq/server/rmq_id.go similarity index 95% rename from internal/mq/mqimpl/rocksmq/server/rmq_id.go rename to pkg/mq/mqimpl/rocksmq/server/rmq_id.go index 8e252e334619..91817c6e2e26 100644 --- a/internal/mq/mqimpl/rocksmq/server/rmq_id.go +++ b/pkg/mq/mqimpl/rocksmq/server/rmq_id.go @@ -18,7 +18,7 @@ package server import ( "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" ) // rmqID wraps message ID for rocksmq @@ -27,7 +27,7 @@ type RmqID struct { } // Check if rmqID implements MessageID interface -var _ mqwrapper.MessageID = &RmqID{} +var _ mqcommon.MessageID = &RmqID{} // Serialize convert rmq message id to []byte func (rid *RmqID) Serialize() []byte { diff --git a/internal/mq/mqimpl/rocksmq/server/rmq_id_test.go b/pkg/mq/mqimpl/rocksmq/server/rmq_id_test.go similarity index 100% rename from internal/mq/mqimpl/rocksmq/server/rmq_id_test.go rename to pkg/mq/mqimpl/rocksmq/server/rmq_id_test.go diff --git a/internal/mq/mqimpl/rocksmq/server/rocksmq.go b/pkg/mq/mqimpl/rocksmq/server/rocksmq.go similarity index 100% rename from internal/mq/mqimpl/rocksmq/server/rocksmq.go rename to pkg/mq/mqimpl/rocksmq/server/rocksmq.go diff --git a/internal/mq/mqimpl/rocksmq/server/rocksmq_impl.go b/pkg/mq/mqimpl/rocksmq/server/rocksmq_impl.go similarity index 93% rename from internal/mq/mqimpl/rocksmq/server/rocksmq_impl.go rename to pkg/mq/mqimpl/rocksmq/server/rocksmq_impl.go index cb45b1b70c65..195604b5eeb7 100644 --- a/internal/mq/mqimpl/rocksmq/server/rocksmq_impl.go +++ b/pkg/mq/mqimpl/rocksmq/server/rocksmq_impl.go @@ -25,14 +25,14 @@ import ( "github.com/tecbot/gorocksdb" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/allocator" - "github.com/milvus-io/milvus/internal/kv" - rocksdbkv "github.com/milvus-io/milvus/internal/kv/rocksdb" + "github.com/milvus-io/milvus/pkg/kv" + rocksdb "github.com/milvus-io/milvus/pkg/kv/rocksdb" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/hardware" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/retry" + "github.com/milvus-io/milvus/pkg/util/tsoutil" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -116,15 +116,14 @@ type rocksmq struct { store *gorocksdb.DB cfh []*gorocksdb.ColumnFamilyHandle kv kv.BaseKV - idAllocator allocator.Interface storeMu *sync.Mutex - topicLastID sync.Map consumers sync.Map consumersID sync.Map - retentionInfo *retentionInfo - readers sync.Map - state RmqState + retentionInfo *retentionInfo + readers sync.Map + state RmqState + topicName2LatestMsgID sync.Map } func parseCompressionType(params *paramtable.ComponentParam) ([]gorocksdb.CompressionType, error) { @@ -150,7 +149,7 @@ func parseCompressionType(params *paramtable.ComponentParam) ([]gorocksdb.Compre // 1. New rocksmq instance based on rocksdb with name and rocksdbkv with kvname // 2. Init retention info, load retention info to memory // 3. Start retention goroutine -func NewRocksMQ(name string, idAllocator allocator.Interface) (*rocksmq, error) { +func NewRocksMQ(name string) (*rocksmq, error) { params := paramtable.Get() // TODO we should use same rocksdb instance with different cfs maxProcs := hardware.GetCPUNum() @@ -201,7 +200,7 @@ func NewRocksMQ(name string, idAllocator allocator.Interface) (*rocksmq, error) // finish rocks KV kvName := name + kvSuffix - kv, err := rocksdbkv.NewRocksdbKVWithOpts(kvName, optsKV) + kv, err := rocksdb.NewRocksdbKVWithOpts(kvName, optsKV) if err != nil { return nil, err } @@ -230,28 +229,14 @@ func NewRocksMQ(name string, idAllocator allocator.Interface) (*rocksmq, error) return nil, err } - var mqIDAllocator allocator.Interface - // if user didn't specify id allocator, init one with kv - if idAllocator == nil { - allocator := allocator.NewGlobalIDAllocator("rmq_id", kv) - err = allocator.Initialize() - if err != nil { - return nil, err - } - mqIDAllocator = allocator - } else { - mqIDAllocator = idAllocator - } - rmq := &rocksmq{ - store: db, - cfh: cfHandles, - kv: kv, - idAllocator: mqIDAllocator, - storeMu: &sync.Mutex{}, - consumers: sync.Map{}, - readers: sync.Map{}, - topicLastID: sync.Map{}, + store: db, + cfh: cfHandles, + kv: kv, + storeMu: &sync.Mutex{}, + consumers: sync.Map{}, + readers: sync.Map{}, + topicName2LatestMsgID: sync.Map{}, } ri, err := initRetentionInfo(kv, db) @@ -294,6 +279,36 @@ func (rmq *rocksmq) isClosed() bool { return atomic.LoadInt64(&rmq.state) != RmqStateHealthy } +// The format of old key is: topicName/Message. In order to keep the lexicographical order of keys in kv engine, +// new message id still need to use same format by compose method of tsoutil package, it should greater than the +// previous message id as well if the topic already exists. +// return a range value [start, end) if msgIDs are allocated successfully. +func (rmq *rocksmq) allocMsgID(topicName string, delta int) (UniqueID, UniqueID, error) { + v, ok := rmq.topicName2LatestMsgID.Load(topicName) + var msgID int64 + if !ok { + // try to get the latest message id from the topic + var err error + msgID, err = rmq.getLatestMsg(topicName) + if err != nil { + return 0, 0, err + } + + if msgID == DefaultMessageID { + // initialize a new message id if not found the latest msg in the topic + msgID = UniqueID(tsoutil.ComposeTSByTime(time.Now(), 0)) + log.Warn("init new message id", zap.String("topicName", topicName), zap.Error(err)) + } + log.Info("init the latest message id done", zap.String("topicName", topicName), zap.Int64("msgID", msgID)) + } else { + msgID = v.(int64) + } + + newMsgID := msgID + int64(delta) + rmq.topicName2LatestMsgID.Store(topicName, newMsgID) + return msgID + 1, newMsgID + 1, nil +} + // Close step: // 1. Stop retention // 2. Destroy all consumer groups and topics @@ -440,6 +455,7 @@ func (rmq *rocksmq) DestroyTopic(topicName string) error { defer lock.Unlock() rmq.consumers.Delete(topicName) + rmq.topicName2LatestMsgID.Delete(topicName) // clean the topic data it self fixTopicName := topicName + "/" @@ -582,6 +598,7 @@ func (rmq *rocksmq) destroyConsumerGroupInternal(topicName, groupName string) er defer lock.Unlock() key := constructCurrentID(topicName, groupName) rmq.consumersID.Delete(key) + rmq.topicName2LatestMsgID.Delete(topicName) if vals, ok := rmq.consumers.Load(topicName); ok { consumers := vals.([]*Consumer) for index, v := range consumers { @@ -601,6 +618,9 @@ func (rmq *rocksmq) destroyConsumerGroupInternal(topicName, groupName string) er // Produce produces messages for topic and updates page infos for retention func (rmq *rocksmq) Produce(topicName string, messages []ProducerMessage) ([]UniqueID, error) { + if messages == nil { + return []UniqueID{}, fmt.Errorf("messages are empty") + } if rmq.isClosed() { return nil, errors.New(RmqNotServingErrMsg) } @@ -619,7 +639,7 @@ func (rmq *rocksmq) Produce(topicName string, messages []ProducerMessage) ([]Uni getLockTime := time.Since(start).Milliseconds() msgLen := len(messages) - idStart, idEnd, err := rmq.idAllocator.Alloc(uint32(msgLen)) + idStart, idEnd, err := rmq.allocMsgID(topicName, msgLen) if err != nil { return []UniqueID{}, err } @@ -676,7 +696,6 @@ func (rmq *rocksmq) Produce(topicName string, messages []ProducerMessage) ([]Uni ) } - rmq.topicLastID.Store(topicName, msgIDs[len(msgIDs)-1]) return msgIDs, nil } @@ -768,7 +787,7 @@ func (rmq *rocksmq) Consume(topicName string, groupName string, n int) ([]Consum readOpts := gorocksdb.NewDefaultReadOptions() defer readOpts.Destroy() prefix := topicName + "/" - iter := rocksdbkv.NewRocksIteratorCFWithUpperBound(rmq.store, rmq.cfh[0], typeutil.AddOne(prefix), readOpts) + iter := rocksdb.NewRocksIteratorCFWithUpperBound(rmq.store, rmq.cfh[0], typeutil.AddOne(prefix), readOpts) defer iter.Close() var dataKey string @@ -777,6 +796,7 @@ func (rmq *rocksmq) Consume(topicName string, groupName string, n int) ([]Consum } else { dataKey = path.Join(topicName, strconv.FormatInt(currentID, 10)) } + iter.Seek([]byte(dataKey)) consumerMessage := make([]ConsumerMessage, 0, n) @@ -986,7 +1006,7 @@ func (rmq *rocksmq) SeekToLatest(topicName, groupName string) error { func (rmq *rocksmq) getLatestMsg(topicName string) (int64, error) { readOpts := gorocksdb.NewDefaultReadOptions() defer readOpts.Destroy() - iter := rocksdbkv.NewRocksIteratorCF(rmq.store, rmq.cfh[0], readOpts) + iter := rocksdb.NewRocksIteratorCF(rmq.store, rmq.cfh[0], readOpts) defer iter.Close() prefix := topicName + "/" @@ -1046,7 +1066,7 @@ func (rmq *rocksmq) updateAckedInfo(topicName, groupName string, firstID UniqueI defer readOpts.Destroy() pageMsgFirstKey := pageMsgPrefix + strconv.FormatInt(firstID, 10) - iter := rocksdbkv.NewRocksIteratorWithUpperBound(rmq.kv.(*rocksdbkv.RocksdbKV).DB, typeutil.AddOne(pageMsgPrefix), readOpts) + iter := rocksdb.NewRocksIteratorWithUpperBound(rmq.kv.(*rocksdb.RocksdbKV).DB, typeutil.AddOne(pageMsgPrefix), readOpts) defer iter.Close() var pageIDs []UniqueID diff --git a/internal/mq/mqimpl/rocksmq/server/rocksmq_impl_test.go b/pkg/mq/mqimpl/rocksmq/server/rocksmq_impl_test.go similarity index 75% rename from internal/mq/mqimpl/rocksmq/server/rocksmq_impl_test.go rename to pkg/mq/mqimpl/rocksmq/server/rocksmq_impl_test.go index 7dbfdd077892..77a375e02673 100644 --- a/internal/mq/mqimpl/rocksmq/server/rocksmq_impl_test.go +++ b/pkg/mq/mqimpl/rocksmq/server/rocksmq_impl_test.go @@ -14,7 +14,6 @@ package server import ( "fmt" "os" - "path" "strconv" "strings" "sync" @@ -24,14 +23,10 @@ import ( "github.com/cockroachdb/errors" "github.com/stretchr/testify/assert" - "github.com/tecbot/gorocksdb" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/allocator" - etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" - rocksdbkv "github.com/milvus-io/milvus/internal/kv/rocksdb" + rocksdbkv "github.com/milvus-io/milvus/pkg/kv/rocksdb" "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/etcd" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" ) @@ -52,125 +47,19 @@ type producerMessageBefore2 struct { Payload []byte } -func InitIDAllocator(kvPath string) *allocator.GlobalIDAllocator { - rocksdbKV, err := rocksdbkv.NewRocksdbKV(kvPath) - if err != nil { - panic(err) - } - idAllocator := allocator.NewGlobalIDAllocator("rmq_id", rocksdbKV) - _ = idAllocator.Initialize() - return idAllocator -} - func newChanName() string { return fmt.Sprintf("my-chan-%v", time.Now().Nanosecond()) } -func newGroupName() string { - return fmt.Sprintf("my-group-%v", time.Now().Nanosecond()) -} - -func etcdEndpoints() []string { - endpoints := os.Getenv("ETCD_ENDPOINTS") - if endpoints == "" { - endpoints = "localhost:2379" - } - etcdEndpoints := strings.Split(endpoints, ",") - return etcdEndpoints -} - -// to test compatibility concern -func (rmq *rocksmq) produceBefore2(topicName string, messages []producerMessageBefore2) ([]UniqueID, error) { - if rmq.isClosed() { - return nil, errors.New(RmqNotServingErrMsg) - } - start := time.Now() - ll, ok := topicMu.Load(topicName) - if !ok { - return []UniqueID{}, fmt.Errorf("topic name = %s not exist", topicName) - } - lock, ok := ll.(*sync.Mutex) - if !ok { - return []UniqueID{}, fmt.Errorf("get mutex failed, topic name = %s", topicName) - } - lock.Lock() - defer lock.Unlock() - - getLockTime := time.Since(start).Milliseconds() - - msgLen := len(messages) - idStart, idEnd, err := rmq.idAllocator.Alloc(uint32(msgLen)) - if err != nil { - return []UniqueID{}, err - } - allocTime := time.Since(start).Milliseconds() - if UniqueID(msgLen) != idEnd-idStart { - return []UniqueID{}, errors.New("Obtained id length is not equal that of message") - } - - // Insert data to store system - batch := gorocksdb.NewWriteBatch() - defer batch.Destroy() - msgSizes := make(map[UniqueID]int64) - msgIDs := make([]UniqueID, msgLen) - for i := 0; i < msgLen && idStart+UniqueID(i) < idEnd; i++ { - msgID := idStart + UniqueID(i) - key := path.Join(topicName, strconv.FormatInt(msgID, 10)) - batch.Put([]byte(key), messages[i].Payload) - msgIDs[i] = msgID - msgSizes[msgID] = int64(len(messages[i].Payload)) - } - - opts := gorocksdb.NewDefaultWriteOptions() - defer opts.Destroy() - err = rmq.store.Write(opts, batch) - if err != nil { - return []UniqueID{}, err - } - writeTime := time.Since(start).Milliseconds() - if vals, ok := rmq.consumers.Load(topicName); ok { - for _, v := range vals.([]*Consumer) { - select { - case v.MsgMutex <- struct{}{}: - continue - default: - continue - } - } - } - - // Update message page info - err = rmq.updatePageInfo(topicName, msgIDs, msgSizes) - if err != nil { - return []UniqueID{}, err - } - - getProduceTime := time.Since(start).Milliseconds() - if getProduceTime > 200 { - log.Warn("rocksmq produce too slowly", zap.String("topic", topicName), - zap.Int64("get lock elapse", getLockTime), - zap.Int64("alloc elapse", allocTime-getLockTime), - zap.Int64("write elapse", writeTime-allocTime), - zap.Int64("updatePage elapse", getProduceTime-writeTime), - zap.Int64("produce total elapse", getProduceTime), - ) - } - return msgIDs, nil -} - func TestRocksmq_RegisterConsumer(t *testing.T) { suffix := "_register" - kvPath := rmqPath + kvPathSuffix + suffix - defer os.RemoveAll(kvPath) - idAllocator := InitIDAllocator(kvPath) - rocksdbPath := rmqPath + suffix defer os.RemoveAll(rocksdbPath + kvSuffix) defer os.RemoveAll(rocksdbPath) paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(rocksdbPath, idAllocator) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() @@ -225,17 +114,12 @@ func TestRocksmq_RegisterConsumer(t *testing.T) { func TestRocksmq_Basic(t *testing.T) { suffix := "_rmq" - - kvPath := rmqPath + kvPathSuffix + suffix - defer os.RemoveAll(kvPath) - idAllocator := InitIDAllocator(kvPath) - rocksdbPath := rmqPath + suffix defer os.RemoveAll(rocksdbPath + kvSuffix) defer os.RemoveAll(rocksdbPath) paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(rocksdbPath, idAllocator) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() @@ -281,10 +165,6 @@ func TestRocksmq_Basic(t *testing.T) { func TestRocksmq_MultiConsumer(t *testing.T) { suffix := "rmq_multi_consumer" - kvPath := rmqPath + kvPathSuffix + suffix - defer os.RemoveAll(kvPath) - idAllocator := InitIDAllocator(kvPath) - rocksdbPath := rmqPath + suffix defer os.RemoveAll(rocksdbPath + kvSuffix) defer os.RemoveAll(rocksdbPath) @@ -292,7 +172,7 @@ func TestRocksmq_MultiConsumer(t *testing.T) { params := paramtable.Get() params.Save(params.RocksmqCfg.PageSize.Key, "10") paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(rocksdbPath, idAllocator) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() @@ -336,20 +216,16 @@ func TestRocksmq_MultiConsumer(t *testing.T) { func TestRocksmq_Dummy(t *testing.T) { suffix := "_dummy" - kvPath := rmqPath + kvPathSuffix + suffix - defer os.RemoveAll(kvPath) - idAllocator := InitIDAllocator(kvPath) - rocksdbPath := rmqPath + suffix defer os.RemoveAll(rocksdbPath + kvSuffix) defer os.RemoveAll(rocksdbPath) paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(rocksdbPath, idAllocator) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() - _, err = NewRocksMQ("", idAllocator) + _, err = NewRocksMQ("") assert.Error(t, err) channelName := "channel_a" @@ -406,22 +282,18 @@ func TestRocksmq_Dummy(t *testing.T) { func TestRocksmq_Seek(t *testing.T) { suffix := "_seek" - kvPath := rmqPath + kvPathSuffix + suffix - defer os.RemoveAll(kvPath) - idAllocator := InitIDAllocator(kvPath) - rocksdbPath := rmqPath + suffix defer os.RemoveAll(rocksdbPath + kvSuffix) defer os.RemoveAll(rocksdbPath) paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(rocksdbPath, idAllocator) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - _, err = NewRocksMQ("", idAllocator) + _, err = NewRocksMQ("") assert.Error(t, err) defer os.RemoveAll("_meta_kv") @@ -466,16 +338,6 @@ func TestRocksmq_Seek(t *testing.T) { } func TestRocksmq_Loop(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - defer etcdCli.Close() - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - assert.NoError(t, err) - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_1" _ = os.RemoveAll(name) defer os.RemoveAll(name) @@ -485,7 +347,7 @@ func TestRocksmq_Loop(t *testing.T) { paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -540,16 +402,6 @@ func TestRocksmq_Loop(t *testing.T) { } func TestRocksmq_Goroutines(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - defer etcdCli.Close() - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - assert.NoError(t, err) - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_goroutines" defer os.RemoveAll(name) kvName := name + "_meta_kv" @@ -558,7 +410,7 @@ func TestRocksmq_Goroutines(t *testing.T) { paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -620,16 +472,6 @@ func TestRocksmq_Goroutines(t *testing.T) { Consume: 90000 message / s */ func TestRocksmq_Throughout(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - defer etcdCli.Close() - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - assert.NoError(t, err) - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_3" defer os.RemoveAll(name) kvName := name + "_meta_kv" @@ -638,7 +480,7 @@ func TestRocksmq_Throughout(t *testing.T) { paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -686,16 +528,6 @@ func TestRocksmq_Throughout(t *testing.T) { } func TestRocksmq_MultiChan(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - defer etcdCli.Close() - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - assert.NoError(t, err) - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_multichan" defer os.RemoveAll(name) kvName := name + "_meta_kv" @@ -704,7 +536,7 @@ func TestRocksmq_MultiChan(t *testing.T) { paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -741,16 +573,6 @@ func TestRocksmq_MultiChan(t *testing.T) { } func TestRocksmq_CopyData(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - defer etcdCli.Close() - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - assert.NoError(t, err) - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_copydata" defer os.RemoveAll(name) kvName := name + "_meta_kv" @@ -759,7 +581,7 @@ func TestRocksmq_CopyData(t *testing.T) { paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -810,16 +632,6 @@ func TestRocksmq_CopyData(t *testing.T) { } func TestRocksmq_SeekToLatest(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - defer etcdCli.Close() - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - assert.NoError(t, err) - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_seektolatest" defer os.RemoveAll(name) kvName := name + "_meta_kv" @@ -828,7 +640,7 @@ func TestRocksmq_SeekToLatest(t *testing.T) { paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -904,23 +716,13 @@ func TestRocksmq_SeekToLatest(t *testing.T) { } func TestRocksmq_GetLatestMsg(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - defer etcdCli.Close() - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - assert.NoError(t, err) - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_data" defer os.RemoveAll(name) kvName := name + "_meta_kv" _ = os.RemoveAll(kvName) defer os.RemoveAll(kvName) paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) channelName := newChanName() @@ -980,16 +782,12 @@ func TestRocksmq_GetLatestMsg(t *testing.T) { func TestRocksmq_CheckPreTopicValid(t *testing.T) { suffix := "_topic" - kvPath := rmqPath + kvPathSuffix + suffix - defer os.RemoveAll(kvPath) - idAllocator := InitIDAllocator(kvPath) - rocksdbPath := rmqPath + suffix defer os.RemoveAll(rocksdbPath + kvSuffix) defer os.RemoveAll(rocksdbPath) paramtable.Init() paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(rocksdbPath, idAllocator) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() @@ -1029,23 +827,13 @@ func TestRocksmq_CheckPreTopicValid(t *testing.T) { } func TestRocksmq_Close(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - defer etcdCli.Close() - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - assert.NoError(t, err) - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_close" defer os.RemoveAll(name) kvName := name + "_meta_kv" _ = os.RemoveAll(kvName) defer os.RemoveAll(kvName) paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -1064,21 +852,13 @@ func TestRocksmq_Close(t *testing.T) { } func TestRocksmq_SeekWithNoConsumerError(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_seekerror" defer os.RemoveAll(name) kvName := name + "_meta_kv" _ = os.RemoveAll(kvName) defer os.RemoveAll(kvName) paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -1090,21 +870,13 @@ func TestRocksmq_SeekWithNoConsumerError(t *testing.T) { } func TestRocksmq_SeekTopicNotExistError(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_seekerror2" defer os.RemoveAll(name) kvName := name + "_meta_kv" _ = os.RemoveAll(kvName) defer os.RemoveAll(kvName) paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -1113,21 +885,13 @@ func TestRocksmq_SeekTopicNotExistError(t *testing.T) { } func TestRocksmq_SeekTopicMutexError(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_seekerror2" defer os.RemoveAll(name) kvName := name + "_meta_kv" _ = os.RemoveAll(kvName) defer os.RemoveAll(kvName) paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -1137,21 +901,13 @@ func TestRocksmq_SeekTopicMutexError(t *testing.T) { } func TestRocksmq_moveConsumePosError(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_moveconsumeposerror" defer os.RemoveAll(name) kvName := name + "_meta_kv" _ = os.RemoveAll(kvName) defer os.RemoveAll(kvName) paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -1160,14 +916,6 @@ func TestRocksmq_moveConsumePosError(t *testing.T) { } func TestRocksmq_updateAckedInfoErr(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_updateackedinfoerror" defer os.RemoveAll(name) kvName := name + "_meta_kv" @@ -1176,7 +924,7 @@ func TestRocksmq_updateAckedInfoErr(t *testing.T) { params := paramtable.Get() params.Save(params.RocksmqCfg.PageSize.Key, "10") paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() @@ -1220,14 +968,6 @@ func TestRocksmq_updateAckedInfoErr(t *testing.T) { } func TestRocksmq_Info(t *testing.T) { - ep := etcdEndpoints() - etcdCli, err := etcd.GetRemoteEtcdClient(ep) - assert.NoError(t, err) - etcdKV := etcdkv.NewEtcdKV(etcdCli, "/etcd/test/root") - defer etcdKV.Close() - idAllocator := allocator.NewGlobalIDAllocator("dummy", etcdKV) - _ = idAllocator.Initialize() - name := "/tmp/rocksmq_testinfo" defer os.RemoveAll(name) kvName := name + "_meta_kv" @@ -1236,7 +976,7 @@ func TestRocksmq_Info(t *testing.T) { params := paramtable.Get() params.Save(params.RocksmqCfg.PageSize.Key, "10") paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - rmq, err := NewRocksMQ(name, idAllocator) + rmq, err := NewRocksMQ(name) assert.NoError(t, err) defer rmq.Close() diff --git a/internal/mq/mqimpl/rocksmq/server/rocksmq_retention.go b/pkg/mq/mqimpl/rocksmq/server/rocksmq_retention.go similarity index 99% rename from internal/mq/mqimpl/rocksmq/server/rocksmq_retention.go rename to pkg/mq/mqimpl/rocksmq/server/rocksmq_retention.go index 21e847702d15..ccca8a0dc12e 100644 --- a/internal/mq/mqimpl/rocksmq/server/rocksmq_retention.go +++ b/pkg/mq/mqimpl/rocksmq/server/rocksmq_retention.go @@ -21,7 +21,7 @@ import ( "github.com/tecbot/gorocksdb" "go.uber.org/zap" - rocksdbkv "github.com/milvus-io/milvus/internal/kv/rocksdb" + "github.com/milvus-io/milvus/pkg/kv/rocksdb" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" diff --git a/internal/mq/mqimpl/rocksmq/server/rocksmq_retention_test.go b/pkg/mq/mqimpl/rocksmq/server/rocksmq_retention_test.go similarity index 96% rename from internal/mq/mqimpl/rocksmq/server/rocksmq_retention_test.go rename to pkg/mq/mqimpl/rocksmq/server/rocksmq_retention_test.go index ecaf612cdb12..fcb7b143f758 100644 --- a/internal/mq/mqimpl/rocksmq/server/rocksmq_retention_test.go +++ b/pkg/mq/mqimpl/rocksmq/server/rocksmq_retention_test.go @@ -44,7 +44,7 @@ func TestRmqRetention_Basic(t *testing.T) { params.Save(params.RocksmqCfg.PageSize.Key, "10") params.Save(params.RocksmqCfg.TickerTimeInSeconds.Key, "2") - rmq, err := NewRocksMQ(rocksdbPath, nil) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() params.Save(params.RocksmqCfg.RetentionSizeInMB.Key, "0") @@ -139,7 +139,7 @@ func TestRmqRetention_NotConsumed(t *testing.T) { params.Save(params.RocksmqCfg.PageSize.Key, "10") params.Save(params.RocksmqCfg.TickerTimeInSeconds.Key, "2") - rmq, err := NewRocksMQ(rocksdbPath, nil) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() @@ -235,9 +235,6 @@ func TestRmqRetention_MultipleTopic(t *testing.T) { return } defer os.RemoveAll(retentionPath) - kvPath := retentionPath + "kv_multi_topic" - os.RemoveAll(kvPath) - idAllocator := InitIDAllocator(kvPath) rocksdbPath := retentionPath + "db_multi_topic" os.RemoveAll(rocksdbPath) @@ -250,7 +247,7 @@ func TestRmqRetention_MultipleTopic(t *testing.T) { params.Save(params.RocksmqCfg.PageSize.Key, "10") params.Save(params.RocksmqCfg.TickerTimeInSeconds.Key, "1") - rmq, err := NewRocksMQ(rocksdbPath, idAllocator) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() @@ -401,10 +398,6 @@ func TestRetentionInfo_InitRetentionInfo(t *testing.T) { } defer os.RemoveAll(retentionPath) suffix := "init" - kvPath := retentionPath + kvPathSuffix + suffix - defer os.RemoveAll(kvPath) - idAllocator := InitIDAllocator(kvPath) - rocksdbPath := retentionPath + suffix defer os.RemoveAll(rocksdbPath) metaPath := retentionPath + metaPathSuffix + suffix @@ -412,7 +405,7 @@ func TestRetentionInfo_InitRetentionInfo(t *testing.T) { defer os.RemoveAll(metaPath) paramtable.Init() - rmq, err := NewRocksMQ(rocksdbPath, idAllocator) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) assert.NotNil(t, rmq) @@ -421,7 +414,7 @@ func TestRetentionInfo_InitRetentionInfo(t *testing.T) { assert.NoError(t, err) rmq.Close() - rmq, err = NewRocksMQ(rocksdbPath, idAllocator) + rmq, err = NewRocksMQ(rocksdbPath) assert.NoError(t, err) assert.NotNil(t, rmq) @@ -456,10 +449,6 @@ func TestRmqRetention_PageTimeExpire(t *testing.T) { } defer os.RemoveAll(retentionPath) - kvPath := retentionPath + "kv_com1" - os.RemoveAll(kvPath) - idAllocator := InitIDAllocator(kvPath) - rocksdbPath := retentionPath + "db_com1" os.RemoveAll(rocksdbPath) metaPath := retentionPath + "meta_kv_com1" @@ -471,7 +460,7 @@ func TestRmqRetention_PageTimeExpire(t *testing.T) { params.Save(params.RocksmqCfg.PageSize.Key, "10") params.Save(params.RocksmqCfg.TickerTimeInSeconds.Key, "1") - rmq, err := NewRocksMQ(rocksdbPath, idAllocator) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() @@ -582,10 +571,6 @@ func TestRmqRetention_PageSizeExpire(t *testing.T) { return } defer os.RemoveAll(retentionPath) - kvPath := retentionPath + "kv_com2" - os.RemoveAll(kvPath) - idAllocator := InitIDAllocator(kvPath) - rocksdbPath := retentionPath + "db_com2" os.RemoveAll(rocksdbPath) metaPath := retentionPath + "meta_kv_com2" @@ -597,7 +582,7 @@ func TestRmqRetention_PageSizeExpire(t *testing.T) { params.Save(params.RocksmqCfg.PageSize.Key, "10") params.Save(params.RocksmqCfg.TickerTimeInSeconds.Key, "1") - rmq, err := NewRocksMQ(rocksdbPath, idAllocator) + rmq, err := NewRocksMQ(rocksdbPath) assert.NoError(t, err) defer rmq.Close() diff --git a/pkg/mq/msgdispatcher/client.go b/pkg/mq/msgdispatcher/client.go index 6b33334358eb..0f6effc932da 100644 --- a/pkg/mq/msgdispatcher/client.go +++ b/pkg/mq/msgdispatcher/client.go @@ -24,15 +24,15 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/funcutil" ) type ( Pos = msgpb.MsgPosition MsgPack = msgstream.MsgPack - SubPos = mqwrapper.SubscriptionInitialPosition + SubPos = common.SubscriptionInitialPosition ) type Client interface { diff --git a/pkg/mq/msgdispatcher/client_test.go b/pkg/mq/msgdispatcher/client_test.go index 6d24f64cc017..255de91969a1 100644 --- a/pkg/mq/msgdispatcher/client_test.go +++ b/pkg/mq/msgdispatcher/client_test.go @@ -27,16 +27,16 @@ import ( "github.com/stretchr/testify/assert" "go.uber.org/atomic" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/util/typeutil" ) func TestClient(t *testing.T) { client := NewClient(newMockFactory(), typeutil.ProxyRole, 1) assert.NotNil(t, client) - _, err := client.Register(context.Background(), "mock_vchannel_0", nil, mqwrapper.SubscriptionPositionUnknown) + _, err := client.Register(context.Background(), "mock_vchannel_0", nil, common.SubscriptionPositionUnknown) assert.NoError(t, err) - _, err = client.Register(context.Background(), "mock_vchannel_1", nil, mqwrapper.SubscriptionPositionUnknown) + _, err = client.Register(context.Background(), "mock_vchannel_1", nil, common.SubscriptionPositionUnknown) assert.NoError(t, err) assert.NotPanics(t, func() { client.Deregister("mock_vchannel_0") @@ -51,7 +51,7 @@ func TestClient(t *testing.T) { client := NewClient(newMockFactory(), typeutil.DataNodeRole, 1) defer client.Close() assert.NotNil(t, client) - _, err := client.Register(ctx, "mock_vchannel_1", nil, mqwrapper.SubscriptionPositionUnknown) + _, err := client.Register(ctx, "mock_vchannel_1", nil, common.SubscriptionPositionUnknown) assert.Error(t, err) }) } @@ -66,7 +66,7 @@ func TestClient_Concurrency(t *testing.T) { vchannel := fmt.Sprintf("mock-vchannel-%d-%d", i, rand.Int()) wg.Add(1) go func() { - _, err := client1.Register(context.Background(), vchannel, nil, mqwrapper.SubscriptionPositionUnknown) + _, err := client1.Register(context.Background(), vchannel, nil, common.SubscriptionPositionUnknown) assert.NoError(t, err) for j := 0; j < rand.Intn(2); j++ { client1.Deregister(vchannel) diff --git a/pkg/mq/msgdispatcher/dispatcher.go b/pkg/mq/msgdispatcher/dispatcher.go index 4d0ab3e2c606..972b21e1f593 100644 --- a/pkg/mq/msgdispatcher/dispatcher.go +++ b/pkg/mq/msgdispatcher/dispatcher.go @@ -28,8 +28,8 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/funcutil" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/tsoutil" @@ -97,7 +97,7 @@ func NewDispatcher(ctx context.Context, } if position != nil && len(position.MsgID) != 0 { position.ChannelName = funcutil.ToPhysicalChannel(position.ChannelName) - err = stream.AsConsumer(ctx, []string{pchannel}, subName, mqwrapper.SubscriptionPositionUnknown) + err = stream.AsConsumer(ctx, []string{pchannel}, subName, common.SubscriptionPositionUnknown) if err != nil { log.Error("asConsumer failed", zap.Error(err)) return nil, err diff --git a/pkg/mq/msgdispatcher/dispatcher_test.go b/pkg/mq/msgdispatcher/dispatcher_test.go index e7c79b54fc0f..2ee5469b4b25 100644 --- a/pkg/mq/msgdispatcher/dispatcher_test.go +++ b/pkg/mq/msgdispatcher/dispatcher_test.go @@ -26,15 +26,15 @@ import ( "github.com/stretchr/testify/mock" "golang.org/x/net/context" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" ) func TestDispatcher(t *testing.T) { ctx := context.Background() t.Run("test base", func(t *testing.T) { d, err := NewDispatcher(ctx, newMockFactory(), true, "mock_pchannel_0", nil, - "mock_subName_0", mqwrapper.SubscriptionPositionEarliest, nil, nil) + "mock_subName_0", common.SubscriptionPositionEarliest, nil, nil) assert.NoError(t, err) assert.NotPanics(t, func() { d.Handle(start) @@ -62,7 +62,7 @@ func TestDispatcher(t *testing.T) { }, } d, err := NewDispatcher(ctx, factory, true, "mock_pchannel_0", nil, - "mock_subName_0", mqwrapper.SubscriptionPositionEarliest, nil, nil) + "mock_subName_0", common.SubscriptionPositionEarliest, nil, nil) assert.Error(t, err) assert.Nil(t, d) @@ -70,7 +70,7 @@ func TestDispatcher(t *testing.T) { t.Run("test target", func(t *testing.T) { d, err := NewDispatcher(ctx, newMockFactory(), true, "mock_pchannel_0", nil, - "mock_subName_0", mqwrapper.SubscriptionPositionEarliest, nil, nil) + "mock_subName_0", common.SubscriptionPositionEarliest, nil, nil) assert.NoError(t, err) output := make(chan *msgstream.MsgPack, 1024) d.AddTarget(&target{ @@ -133,7 +133,7 @@ func TestDispatcher(t *testing.T) { func BenchmarkDispatcher_handle(b *testing.B) { d, err := NewDispatcher(context.Background(), newMockFactory(), true, "mock_pchannel_0", nil, - "mock_subName_0", mqwrapper.SubscriptionPositionEarliest, nil, nil) + "mock_subName_0", common.SubscriptionPositionEarliest, nil, nil) assert.NoError(b, err) for i := 0; i < b.N; i++ { diff --git a/pkg/mq/msgdispatcher/manager.go b/pkg/mq/msgdispatcher/manager.go index 4f88fd552127..195b588dce43 100644 --- a/pkg/mq/msgdispatcher/manager.go +++ b/pkg/mq/msgdispatcher/manager.go @@ -27,8 +27,8 @@ import ( "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/retry" "github.com/milvus-io/milvus/pkg/util/tsoutil" @@ -234,7 +234,7 @@ func (c *dispatcherManager) split(t *target) { err := retry.Do(context.Background(), func() error { var err error newSolo, err = NewDispatcher(context.Background(), c.factory, false, c.pchannel, t.pos, - c.constructSubName(t.vchannel, false), mqwrapper.SubscriptionPositionUnknown, c.lagNotifyChan, c.lagTargets) + c.constructSubName(t.vchannel, false), common.SubscriptionPositionUnknown, c.lagNotifyChan, c.lagTargets) return err }, retry.Attempts(10)) if err != nil { diff --git a/pkg/mq/msgdispatcher/manager_test.go b/pkg/mq/msgdispatcher/manager_test.go index 51c7790b4081..7591da5a1d96 100644 --- a/pkg/mq/msgdispatcher/manager_test.go +++ b/pkg/mq/msgdispatcher/manager_test.go @@ -29,8 +29,8 @@ import ( "github.com/stretchr/testify/suite" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -47,7 +47,7 @@ func TestManager(t *testing.T) { for j := 0; j < r; j++ { offset++ t.Logf("dyh add, %s", fmt.Sprintf("mock-pchannel-0_vchannel_%d", offset)) - _, err := c.Add(context.Background(), fmt.Sprintf("mock-pchannel-0_vchannel_%d", offset), nil, mqwrapper.SubscriptionPositionUnknown) + _, err := c.Add(context.Background(), fmt.Sprintf("mock-pchannel-0_vchannel_%d", offset), nil, common.SubscriptionPositionUnknown) assert.NoError(t, err) assert.Equal(t, offset, c.Num()) } @@ -65,11 +65,11 @@ func TestManager(t *testing.T) { ctx := context.Background() c := NewDispatcherManager(prefix+"_pchannel_0", typeutil.ProxyRole, 1, newMockFactory()) assert.NotNil(t, c) - _, err := c.Add(ctx, "mock_vchannel_0", nil, mqwrapper.SubscriptionPositionUnknown) + _, err := c.Add(ctx, "mock_vchannel_0", nil, common.SubscriptionPositionUnknown) assert.NoError(t, err) - _, err = c.Add(ctx, "mock_vchannel_1", nil, mqwrapper.SubscriptionPositionUnknown) + _, err = c.Add(ctx, "mock_vchannel_1", nil, common.SubscriptionPositionUnknown) assert.NoError(t, err) - _, err = c.Add(ctx, "mock_vchannel_2", nil, mqwrapper.SubscriptionPositionUnknown) + _, err = c.Add(ctx, "mock_vchannel_2", nil, common.SubscriptionPositionUnknown) assert.NoError(t, err) assert.Equal(t, 3, c.Num()) @@ -90,11 +90,11 @@ func TestManager(t *testing.T) { ctx := context.Background() c := NewDispatcherManager(prefix+"_pchannel_0", typeutil.ProxyRole, 1, newMockFactory()) assert.NotNil(t, c) - _, err := c.Add(ctx, "mock_vchannel_0", nil, mqwrapper.SubscriptionPositionUnknown) + _, err := c.Add(ctx, "mock_vchannel_0", nil, common.SubscriptionPositionUnknown) assert.NoError(t, err) - _, err = c.Add(ctx, "mock_vchannel_1", nil, mqwrapper.SubscriptionPositionUnknown) + _, err = c.Add(ctx, "mock_vchannel_1", nil, common.SubscriptionPositionUnknown) assert.NoError(t, err) - _, err = c.Add(ctx, "mock_vchannel_2", nil, mqwrapper.SubscriptionPositionUnknown) + _, err = c.Add(ctx, "mock_vchannel_2", nil, common.SubscriptionPositionUnknown) assert.NoError(t, err) assert.Equal(t, 3, c.Num()) @@ -120,11 +120,11 @@ func TestManager(t *testing.T) { c := NewDispatcherManager(prefix+"_pchannel_0", typeutil.ProxyRole, 1, newMockFactory()) go c.Run() assert.NotNil(t, c) - _, err := c.Add(ctx, "mock_vchannel_0", nil, mqwrapper.SubscriptionPositionUnknown) + _, err := c.Add(ctx, "mock_vchannel_0", nil, common.SubscriptionPositionUnknown) assert.Error(t, err) - _, err = c.Add(ctx, "mock_vchannel_1", nil, mqwrapper.SubscriptionPositionUnknown) + _, err = c.Add(ctx, "mock_vchannel_1", nil, common.SubscriptionPositionUnknown) assert.Error(t, err) - _, err = c.Add(ctx, "mock_vchannel_2", nil, mqwrapper.SubscriptionPositionUnknown) + _, err = c.Add(ctx, "mock_vchannel_2", nil, common.SubscriptionPositionUnknown) assert.Error(t, err) assert.Equal(t, 0, c.Num()) @@ -282,7 +282,7 @@ func (suite *SimulationSuite) TestDispatchToVchannels() { suite.vchannels = make(map[string]*vchannelHelper, vchannelNum) for i := 0; i < vchannelNum; i++ { vchannel := fmt.Sprintf("%s_vchannelv%d", suite.pchannel, i) - output, err := suite.manager.Add(context.Background(), vchannel, nil, mqwrapper.SubscriptionPositionEarliest) + output, err := suite.manager.Add(context.Background(), vchannel, nil, common.SubscriptionPositionEarliest) assert.NoError(suite.T(), err) suite.vchannels[vchannel] = &vchannelHelper{output: output} } @@ -317,7 +317,7 @@ func (suite *SimulationSuite) TestMerge() { for i := 0; i < vchannelNum; i++ { vchannel := fmt.Sprintf("%s_vchannelv%d", suite.pchannel, i) output, err := suite.manager.Add(context.Background(), vchannel, positions[rand.Intn(len(positions))], - mqwrapper.SubscriptionPositionUnknown) // seek from random position + common.SubscriptionPositionUnknown) // seek from random position assert.NoError(suite.T(), err) suite.vchannels[vchannel] = &vchannelHelper{output: output} } @@ -358,7 +358,7 @@ func (suite *SimulationSuite) TestSplit() { paramtable.Get().Save(targetBufSizeK, "10") } vchannel := fmt.Sprintf("%s_vchannelv%d", suite.pchannel, i) - _, err := suite.manager.Add(context.Background(), vchannel, nil, mqwrapper.SubscriptionPositionEarliest) + _, err := suite.manager.Add(context.Background(), vchannel, nil, common.SubscriptionPositionEarliest) assert.NoError(suite.T(), err) } diff --git a/pkg/mq/msgdispatcher/mock_client.go b/pkg/mq/msgdispatcher/mock_client.go index f2ecaf43414d..af688327ea29 100644 --- a/pkg/mq/msgdispatcher/mock_client.go +++ b/pkg/mq/msgdispatcher/mock_client.go @@ -3,14 +3,14 @@ package msgdispatcher import ( - context "context" + "context" - mqwrapper "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" - mock "github.com/stretchr/testify/mock" + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/stretchr/testify/mock" - msgpb "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" + "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" - msgstream "github.com/milvus-io/milvus/pkg/mq/msgstream" + "github.com/milvus-io/milvus/pkg/mq/msgstream" ) // MockClient is an autogenerated mock type for the Client type @@ -92,15 +92,15 @@ func (_c *MockClient_Deregister_Call) RunAndReturn(run func(string)) *MockClient } // Register provides a mock function with given fields: ctx, vchannel, pos, subPos -func (_m *MockClient) Register(ctx context.Context, vchannel string, pos *msgpb.MsgPosition, subPos mqwrapper.SubscriptionInitialPosition) (<-chan *msgstream.MsgPack, error) { +func (_m *MockClient) Register(ctx context.Context, vchannel string, pos *msgpb.MsgPosition, subPos common.SubscriptionInitialPosition) (<-chan *msgstream.MsgPack, error) { ret := _m.Called(ctx, vchannel, pos, subPos) var r0 <-chan *msgstream.MsgPack var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string, *msgpb.MsgPosition, mqwrapper.SubscriptionInitialPosition) (<-chan *msgstream.MsgPack, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, *msgpb.MsgPosition, common.SubscriptionInitialPosition) (<-chan *msgstream.MsgPack, error)); ok { return rf(ctx, vchannel, pos, subPos) } - if rf, ok := ret.Get(0).(func(context.Context, string, *msgpb.MsgPosition, mqwrapper.SubscriptionInitialPosition) <-chan *msgstream.MsgPack); ok { + if rf, ok := ret.Get(0).(func(context.Context, string, *msgpb.MsgPosition, common.SubscriptionInitialPosition) <-chan *msgstream.MsgPack); ok { r0 = rf(ctx, vchannel, pos, subPos) } else { if ret.Get(0) != nil { @@ -108,7 +108,7 @@ func (_m *MockClient) Register(ctx context.Context, vchannel string, pos *msgpb. } } - if rf, ok := ret.Get(1).(func(context.Context, string, *msgpb.MsgPosition, mqwrapper.SubscriptionInitialPosition) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, string, *msgpb.MsgPosition, common.SubscriptionInitialPosition) error); ok { r1 = rf(ctx, vchannel, pos, subPos) } else { r1 = ret.Error(1) @@ -131,9 +131,9 @@ func (_e *MockClient_Expecter) Register(ctx interface{}, vchannel interface{}, p return &MockClient_Register_Call{Call: _e.mock.On("Register", ctx, vchannel, pos, subPos)} } -func (_c *MockClient_Register_Call) Run(run func(ctx context.Context, vchannel string, pos *msgpb.MsgPosition, subPos mqwrapper.SubscriptionInitialPosition)) *MockClient_Register_Call { +func (_c *MockClient_Register_Call) Run(run func(ctx context.Context, vchannel string, pos *msgpb.MsgPosition, subPos common.SubscriptionInitialPosition)) *MockClient_Register_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string), args[2].(*msgpb.MsgPosition), args[3].(mqwrapper.SubscriptionInitialPosition)) + run(args[0].(context.Context), args[1].(string), args[2].(*msgpb.MsgPosition), args[3].(common.SubscriptionInitialPosition)) }) return _c } @@ -143,7 +143,7 @@ func (_c *MockClient_Register_Call) Return(_a0 <-chan *msgstream.MsgPack, _a1 er return _c } -func (_c *MockClient_Register_Call) RunAndReturn(run func(context.Context, string, *msgpb.MsgPosition, mqwrapper.SubscriptionInitialPosition) (<-chan *msgstream.MsgPack, error)) *MockClient_Register_Call { +func (_c *MockClient_Register_Call) RunAndReturn(run func(context.Context, string, *msgpb.MsgPosition, common.SubscriptionInitialPosition) (<-chan *msgstream.MsgPack, error)) *MockClient_Register_Call { _c.Call.Return(run) return _c } diff --git a/pkg/mq/msgdispatcher/mock_test.go b/pkg/mq/msgdispatcher/mock_test.go index b1685cf0c3db..38b9cc21cc65 100644 --- a/pkg/mq/msgdispatcher/mock_test.go +++ b/pkg/mq/msgdispatcher/mock_test.go @@ -27,8 +27,8 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -66,7 +66,7 @@ func getSeekPositions(factory msgstream.Factory, pchannel string, maxNum int) ([ return nil, err } defer stream.Close() - stream.AsConsumer(context.TODO(), []string{pchannel}, fmt.Sprintf("%d", rand.Int()), mqwrapper.SubscriptionPositionEarliest) + stream.AsConsumer(context.TODO(), []string{pchannel}, fmt.Sprintf("%d", rand.Int()), common.SubscriptionPositionEarliest) positions := make([]*msgstream.MsgPosition, 0) timeoutCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() diff --git a/pkg/mq/msgstream/common_mq_factory.go b/pkg/mq/msgstream/common_mq_factory.go index 9a8bf58cf291..0f3317e70a17 100644 --- a/pkg/mq/msgstream/common_mq_factory.go +++ b/pkg/mq/msgstream/common_mq_factory.go @@ -5,6 +5,7 @@ import ( "github.com/cockroachdb/errors" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" ) @@ -51,7 +52,7 @@ func (f *CommonFactory) NewMsgStreamDisposer(ctx context.Context) func([]string, if err != nil { return err } - msgs.AsConsumer(ctx, channels, subName, mqwrapper.SubscriptionPositionUnknown) + msgs.AsConsumer(ctx, channels, subName, common.SubscriptionPositionUnknown) msgs.Close() return nil } diff --git a/pkg/mq/msgstream/factory_stream_test.go b/pkg/mq/msgstream/factory_stream_test.go index d07e74cdfc0f..0cf5fcbd7acd 100644 --- a/pkg/mq/msgstream/factory_stream_test.go +++ b/pkg/mq/msgstream/factory_stream_test.go @@ -13,7 +13,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/util/funcutil" ) @@ -765,7 +765,7 @@ func consume(ctx context.Context, mq MsgStream) *MsgPack { func createAndSeekConsumer(ctx context.Context, t *testing.T, newer streamNewer, channels []string, seekPositions []*msgpb.MsgPosition) MsgStream { consumer, err := newer(ctx) assert.NoError(t, err) - consumer.AsConsumer(context.Background(), channels, funcutil.RandomString(8), mqwrapper.SubscriptionPositionUnknown) + consumer.AsConsumer(context.Background(), channels, funcutil.RandomString(8), common.SubscriptionPositionUnknown) err = consumer.Seek(context.Background(), seekPositions, false) assert.NoError(t, err) return consumer @@ -781,14 +781,14 @@ func createProducer(ctx context.Context, t *testing.T, newer streamNewer, channe func createConsumer(ctx context.Context, t *testing.T, newer streamNewer, channels []string) MsgStream { consumer, err := newer(ctx) assert.NoError(t, err) - consumer.AsConsumer(context.Background(), channels, funcutil.RandomString(8), mqwrapper.SubscriptionPositionEarliest) + consumer.AsConsumer(context.Background(), channels, funcutil.RandomString(8), common.SubscriptionPositionEarliest) return consumer } func createLatestConsumer(ctx context.Context, t *testing.T, newer streamNewer, channels []string) MsgStream { consumer, err := newer(ctx) assert.NoError(t, err) - consumer.AsConsumer(context.Background(), channels, funcutil.RandomString(8), mqwrapper.SubscriptionPositionLatest) + consumer.AsConsumer(context.Background(), channels, funcutil.RandomString(8), common.SubscriptionPositionLatest) return consumer } @@ -802,7 +802,7 @@ func createStream(ctx context.Context, t *testing.T, newer []streamNewer, channe consumer, err := newer[1](ctx) assert.NoError(t, err) - consumer.AsConsumer(context.Background(), channels, funcutil.RandomString(8), mqwrapper.SubscriptionPositionEarliest) + consumer.AsConsumer(context.Background(), channels, funcutil.RandomString(8), common.SubscriptionPositionEarliest) return producer, consumer } diff --git a/pkg/mq/msgstream/mock_msgstream.go b/pkg/mq/msgstream/mock_msgstream.go index 2b47a0008e0b..8d1d2dbfad1b 100644 --- a/pkg/mq/msgstream/mock_msgstream.go +++ b/pkg/mq/msgstream/mock_msgstream.go @@ -3,12 +3,12 @@ package msgstream import ( - context "context" + "context" - mqwrapper "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" - mock "github.com/stretchr/testify/mock" + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/stretchr/testify/mock" - msgpb "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" + "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" ) // MockMsgStream is an autogenerated mock type for the MsgStream type @@ -25,11 +25,11 @@ func (_m *MockMsgStream) EXPECT() *MockMsgStream_Expecter { } // AsConsumer provides a mock function with given fields: ctx, channels, subName, position -func (_m *MockMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position mqwrapper.SubscriptionInitialPosition) error { +func (_m *MockMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position common.SubscriptionInitialPosition) error { ret := _m.Called(ctx, channels, subName, position) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, []string, string, mqwrapper.SubscriptionInitialPosition) error); ok { + if rf, ok := ret.Get(0).(func(context.Context, []string, string, common.SubscriptionInitialPosition) error); ok { r0 = rf(ctx, channels, subName, position) } else { r0 = ret.Error(0) @@ -52,9 +52,9 @@ func (_e *MockMsgStream_Expecter) AsConsumer(ctx interface{}, channels interface return &MockMsgStream_AsConsumer_Call{Call: _e.mock.On("AsConsumer", ctx, channels, subName, position)} } -func (_c *MockMsgStream_AsConsumer_Call) Run(run func(ctx context.Context, channels []string, subName string, position mqwrapper.SubscriptionInitialPosition)) *MockMsgStream_AsConsumer_Call { +func (_c *MockMsgStream_AsConsumer_Call) Run(run func(ctx context.Context, channels []string, subName string, position common.SubscriptionInitialPosition)) *MockMsgStream_AsConsumer_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].([]string), args[2].(string), args[3].(mqwrapper.SubscriptionInitialPosition)) + run(args[0].(context.Context), args[1].([]string), args[2].(string), args[3].(common.SubscriptionInitialPosition)) }) return _c } @@ -64,7 +64,7 @@ func (_c *MockMsgStream_AsConsumer_Call) Return(_a0 error) *MockMsgStream_AsCons return _c } -func (_c *MockMsgStream_AsConsumer_Call) RunAndReturn(run func(context.Context, []string, string, mqwrapper.SubscriptionInitialPosition) error) *MockMsgStream_AsConsumer_Call { +func (_c *MockMsgStream_AsConsumer_Call) RunAndReturn(run func(context.Context, []string, string, common.SubscriptionInitialPosition) error) *MockMsgStream_AsConsumer_Call { _c.Call.Return(run) return _c } @@ -103,19 +103,19 @@ func (_c *MockMsgStream_AsProducer_Call) RunAndReturn(run func([]string)) *MockM } // Broadcast provides a mock function with given fields: _a0 -func (_m *MockMsgStream) Broadcast(_a0 *MsgPack) (map[string][]mqwrapper.MessageID, error) { +func (_m *MockMsgStream) Broadcast(_a0 *MsgPack) (map[string][]common.MessageID, error) { ret := _m.Called(_a0) - var r0 map[string][]mqwrapper.MessageID + var r0 map[string][]common.MessageID var r1 error - if rf, ok := ret.Get(0).(func(*MsgPack) (map[string][]mqwrapper.MessageID, error)); ok { + if rf, ok := ret.Get(0).(func(*MsgPack) (map[string][]common.MessageID, error)); ok { return rf(_a0) } - if rf, ok := ret.Get(0).(func(*MsgPack) map[string][]mqwrapper.MessageID); ok { + if rf, ok := ret.Get(0).(func(*MsgPack) map[string][]common.MessageID); ok { r0 = rf(_a0) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string][]mqwrapper.MessageID) + r0 = ret.Get(0).(map[string][]common.MessageID) } } @@ -146,12 +146,12 @@ func (_c *MockMsgStream_Broadcast_Call) Run(run func(_a0 *MsgPack)) *MockMsgStre return _c } -func (_c *MockMsgStream_Broadcast_Call) Return(_a0 map[string][]mqwrapper.MessageID, _a1 error) *MockMsgStream_Broadcast_Call { +func (_c *MockMsgStream_Broadcast_Call) Return(_a0 map[string][]common.MessageID, _a1 error) *MockMsgStream_Broadcast_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *MockMsgStream_Broadcast_Call) RunAndReturn(run func(*MsgPack) (map[string][]mqwrapper.MessageID, error)) *MockMsgStream_Broadcast_Call { +func (_c *MockMsgStream_Broadcast_Call) RunAndReturn(run func(*MsgPack) (map[string][]common.MessageID, error)) *MockMsgStream_Broadcast_Call { _c.Call.Return(run) return _c } @@ -307,19 +307,19 @@ func (_c *MockMsgStream_EnableProduce_Call) RunAndReturn(run func(bool)) *MockMs } // GetLatestMsgID provides a mock function with given fields: channel -func (_m *MockMsgStream) GetLatestMsgID(channel string) (mqwrapper.MessageID, error) { +func (_m *MockMsgStream) GetLatestMsgID(channel string) (common.MessageID, error) { ret := _m.Called(channel) - var r0 mqwrapper.MessageID + var r0 common.MessageID var r1 error - if rf, ok := ret.Get(0).(func(string) (mqwrapper.MessageID, error)); ok { + if rf, ok := ret.Get(0).(func(string) (common.MessageID, error)); ok { return rf(channel) } - if rf, ok := ret.Get(0).(func(string) mqwrapper.MessageID); ok { + if rf, ok := ret.Get(0).(func(string) common.MessageID); ok { r0 = rf(channel) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(mqwrapper.MessageID) + r0 = ret.Get(0).(common.MessageID) } } @@ -350,12 +350,12 @@ func (_c *MockMsgStream_GetLatestMsgID_Call) Run(run func(channel string)) *Mock return _c } -func (_c *MockMsgStream_GetLatestMsgID_Call) Return(_a0 mqwrapper.MessageID, _a1 error) *MockMsgStream_GetLatestMsgID_Call { +func (_c *MockMsgStream_GetLatestMsgID_Call) Return(_a0 common.MessageID, _a1 error) *MockMsgStream_GetLatestMsgID_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *MockMsgStream_GetLatestMsgID_Call) RunAndReturn(run func(string) (mqwrapper.MessageID, error)) *MockMsgStream_GetLatestMsgID_Call { +func (_c *MockMsgStream_GetLatestMsgID_Call) RunAndReturn(run func(string) (common.MessageID, error)) *MockMsgStream_GetLatestMsgID_Call { _c.Call.Return(run) return _c } diff --git a/pkg/mq/msgstream/mq_factory.go b/pkg/mq/msgstream/mq_factory.go index 74e324e0cea3..1bd9f8ca3453 100644 --- a/pkg/mq/msgstream/mq_factory.go +++ b/pkg/mq/msgstream/mq_factory.go @@ -30,10 +30,12 @@ import ( "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" kafkawrapper "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/kafka" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/nmq" pulsarmqwrapper "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/pulsar" + "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/rmq" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/retry" ) @@ -205,7 +207,7 @@ func (f *KmsFactory) NewMsgStreamDisposer(ctx context.Context) func([]string, st if err != nil { return err } - msgstream.AsConsumer(ctx, channels, subname, mqwrapper.SubscriptionPositionUnknown) + msgstream.AsConsumer(ctx, channels, subname, common.SubscriptionPositionUnknown) msgstream.Close() return nil } @@ -233,3 +235,18 @@ func NewNatsmqFactory() Factory { MQBufSize: paramtable.MQCfg.MQBufSize.GetAsInt64(), } } + +// NewRocksmqFactory creates a new message stream factory based on rocksmq. +func NewRocksmqFactory(path string, cfg *paramtable.ServiceParam) Factory { + if err := server.InitRocksMQ(path); err != nil { + log.Fatal("fail to init rocksmq", zap.Error(err)) + } + log.Info("init rocksmq msgstream success", zap.String("path", path)) + + return &CommonFactory{ + Newer: rmq.NewClientWithDefaultOptions, + DispatcherFactory: ProtoUDFactory{}, + ReceiveBufSize: cfg.MQCfg.ReceiveBufSize.GetAsInt64(), + MQBufSize: cfg.MQCfg.MQBufSize.GetAsInt64(), + } +} diff --git a/pkg/mq/msgstream/mq_factory_test.go b/pkg/mq/msgstream/mq_factory_test.go index 33566edca0fe..578637e9ce1c 100644 --- a/pkg/mq/msgstream/mq_factory_test.go +++ b/pkg/mq/msgstream/mq_factory_test.go @@ -18,10 +18,13 @@ package msgstream import ( "context" + "os" "testing" "time" "github.com/stretchr/testify/assert" + + "github.com/milvus-io/milvus/pkg/util/paramtable" ) func TestPmsFactory(t *testing.T) { @@ -148,3 +151,19 @@ func TestKafkaFactory(t *testing.T) { }) } } + +func TestRmsFactory(t *testing.T) { + defer os.Unsetenv("ROCKSMQ_PATH") + paramtable.Init() + + dir := t.TempDir() + + rmsFactory := NewRocksmqFactory(dir, ¶mtable.Get().ServiceParam) + + ctx := context.Background() + _, err := rmsFactory.NewMsgStream(ctx) + assert.NoError(t, err) + + _, err = rmsFactory.NewTtMsgStream(ctx) + assert.NoError(t, err) +} diff --git a/pkg/mq/msgstream/mq_kafka_msgstream_test.go b/pkg/mq/msgstream/mq_kafka_msgstream_test.go index fe39f8f082e2..03ab985f798b 100644 --- a/pkg/mq/msgstream/mq_kafka_msgstream_test.go +++ b/pkg/mq/msgstream/mq_kafka_msgstream_test.go @@ -27,7 +27,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" kafkawrapper "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/kafka" "github.com/milvus-io/milvus/pkg/util/funcutil" ) @@ -128,7 +128,7 @@ func TestStream_KafkaMsgStream_SeekToLast(t *testing.T) { // pick a seekPosition var seekPosition *msgpb.MsgPosition - outputStream := getKafkaOutputStream(ctx, kafkaAddress, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + outputStream := getKafkaOutputStream(ctx, kafkaAddress, consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) for i := 0; i < 10; i++ { result := consumer(ctx, outputStream) assert.Equal(t, result.Msgs[0].ID(), int64(i)) @@ -140,7 +140,7 @@ func TestStream_KafkaMsgStream_SeekToLast(t *testing.T) { outputStream.Close() // create a consumer can consume data from seek position to last msg - outputStream2 := getKafkaOutputStream(ctx, kafkaAddress, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionUnknown) + outputStream2 := getKafkaOutputStream(ctx, kafkaAddress, consumerChannels, consumerSubName, common.SubscriptionPositionUnknown) lastMsgID, err := outputStream2.GetLatestMsgID(c) defer outputStream2.Close() assert.NoError(t, err) @@ -407,7 +407,7 @@ func TestStream_KafkaTtMsgStream_DataNodeTimetickMsgstream(t *testing.T) { factory := ProtoUDFactory{} kafkaClient := kafkawrapper.NewKafkaClientInstance(kafkaAddress) outputStream, _ := NewMqTtMsgStream(ctx, 100, 100, kafkaClient, factory.NewUnmarshalDispatcher()) - outputStream.AsConsumer(context.Background(), consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionLatest) + outputStream.AsConsumer(context.Background(), consumerChannels, consumerSubName, common.SubscriptionPositionLatest) var wg sync.WaitGroup wg.Add(1) @@ -457,7 +457,7 @@ func getKafkaInputStream(ctx context.Context, kafkaAddress string, producerChann return inputStream } -func getKafkaOutputStream(ctx context.Context, kafkaAddress string, consumerChannels []string, consumerSubName string, position mqwrapper.SubscriptionInitialPosition) MsgStream { +func getKafkaOutputStream(ctx context.Context, kafkaAddress string, consumerChannels []string, consumerSubName string, position common.SubscriptionInitialPosition) MsgStream { factory := ProtoUDFactory{} kafkaClient := kafkawrapper.NewKafkaClientInstance(kafkaAddress) outputStream, _ := NewMqMsgStream(ctx, 100, 100, kafkaClient, factory.NewUnmarshalDispatcher()) @@ -469,7 +469,7 @@ func getKafkaTtOutputStream(ctx context.Context, kafkaAddress string, consumerCh factory := ProtoUDFactory{} kafkaClient := kafkawrapper.NewKafkaClientInstance(kafkaAddress) outputStream, _ := NewMqTtMsgStream(ctx, 100, 100, kafkaClient, factory.NewUnmarshalDispatcher()) - outputStream.AsConsumer(context.Background(), consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + outputStream.AsConsumer(context.Background(), consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) return outputStream } @@ -481,7 +481,7 @@ func getKafkaTtOutputStreamAndSeek(ctx context.Context, kafkaAddress string, pos for _, c := range positions { consumerName = append(consumerName, c.ChannelName) } - outputStream.AsConsumer(context.Background(), consumerName, funcutil.RandomString(8), mqwrapper.SubscriptionPositionUnknown) + outputStream.AsConsumer(context.Background(), consumerName, funcutil.RandomString(8), common.SubscriptionPositionUnknown) outputStream.Seek(context.Background(), positions, false) return outputStream } diff --git a/pkg/mq/msgstream/mq_msgstream.go b/pkg/mq/msgstream/mq_msgstream.go index a93c9962f414..40f9b21cf002 100644 --- a/pkg/mq/msgstream/mq_msgstream.go +++ b/pkg/mq/msgstream/mq_msgstream.go @@ -35,6 +35,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus/pkg/config" "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" @@ -127,7 +128,7 @@ func (ms *mqMsgStream) AsProducer(channels []string) { } fn := func() error { - pp, err := ms.client.CreateProducer(mqwrapper.ProducerOptions{Topic: channel, EnableCompression: true}) + pp, err := ms.client.CreateProducer(common.ProducerOptions{Topic: channel, EnableCompression: true}) if err != nil { return err } @@ -168,7 +169,7 @@ func (ms *mqMsgStream) CheckTopicValid(channel string) error { // AsConsumerWithPosition Create consumer to receive message from channels, with initial position // if initial position is set to latest, last message in the channel is exclusive -func (ms *mqMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position mqwrapper.SubscriptionInitialPosition) error { +func (ms *mqMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position common.SubscriptionInitialPosition) error { for _, channel := range channels { if _, ok := ms.consumers[channel]; ok { continue @@ -319,7 +320,7 @@ func (ms *mqMsgStream) Produce(msgPack *MsgPack) error { return err } - msg := &mqwrapper.ProducerMessage{Payload: m, Properties: map[string]string{}} + msg := &common.ProducerMessage{Payload: m, Properties: map[string]string{}} InjectCtx(spanCtx, msg.Properties) ms.producerLock.RLock() @@ -362,7 +363,7 @@ func (ms *mqMsgStream) Broadcast(msgPack *MsgPack) (map[string][]MessageID, erro return ids, err } - msg := &mqwrapper.ProducerMessage{Payload: m, Properties: map[string]string{}} + msg := &common.ProducerMessage{Payload: m, Properties: map[string]string{}} InjectCtx(spanCtx, msg.Properties) ms.producerLock.Lock() @@ -382,7 +383,7 @@ func (ms *mqMsgStream) Broadcast(msgPack *MsgPack) (map[string][]MessageID, erro return ids, nil } -func (ms *mqMsgStream) getTsMsgFromConsumerMsg(msg mqwrapper.Message) (TsMsg, error) { +func (ms *mqMsgStream) getTsMsgFromConsumerMsg(msg common.Message) (TsMsg, error) { header := commonpb.MsgHeader{} if msg.Payload() == nil { return nil, fmt.Errorf("failed to unmarshal message header, payload is empty") @@ -567,7 +568,7 @@ func (ms *MqTtMsgStream) addConsumer(consumer mqwrapper.Consumer, channel string } // AsConsumerWithPosition subscribes channels as consumer for a MsgStream and seeks to a certain position. -func (ms *MqTtMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position mqwrapper.SubscriptionInitialPosition) error { +func (ms *MqTtMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position common.SubscriptionInitialPosition) error { for _, channel := range channels { if _, ok := ms.consumers[channel]; ok { continue diff --git a/pkg/mq/msgstream/mq_msgstream_test.go b/pkg/mq/msgstream/mq_msgstream_test.go index ee4a5d57ffac..3bf6e6a0b354 100644 --- a/pkg/mq/msgstream/mq_msgstream_test.go +++ b/pkg/mq/msgstream/mq_msgstream_test.go @@ -36,6 +36,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus/pkg/config" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" kafkawrapper "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/kafka" pulsarwrapper "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/pulsar" @@ -280,7 +281,7 @@ func TestStream_PulsarMsgStream_InsertRepackFunc(t *testing.T) { pulsarClient2, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress}) outputStream, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient2, factory.NewUnmarshalDispatcher()) - outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) var output MsgStream = outputStream err := (*inputStream).Produce(&msgPack) @@ -331,7 +332,7 @@ func TestStream_PulsarMsgStream_DeleteRepackFunc(t *testing.T) { pulsarClient2, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress}) outputStream, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient2, factory.NewUnmarshalDispatcher()) - outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) var output MsgStream = outputStream err := (*inputStream).Produce(&msgPack) @@ -363,7 +364,7 @@ func TestStream_PulsarMsgStream_DefaultRepackFunc(t *testing.T) { pulsarClient2, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress}) outputStream, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient2, factory.NewUnmarshalDispatcher()) - outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) var output MsgStream = outputStream err := (*inputStream).Produce(&msgPack) @@ -512,7 +513,7 @@ func TestStream_PulsarMsgStream_SeekToLast(t *testing.T) { factory := ProtoUDFactory{} pulsarClient, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress}) outputStream2, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient, factory.NewUnmarshalDispatcher()) - outputStream2.AsConsumer(ctx, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + outputStream2.AsConsumer(ctx, consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) lastMsgID, err := outputStream2.GetLatestMsgID(c) defer outputStream2.Close() assert.NoError(t, err) @@ -567,11 +568,11 @@ func TestStream_MsgStream_AsConsumerCtxDone(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Millisecond) defer cancel() <-time.After(2 * time.Millisecond) - err := outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + err := outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) assert.Error(t, err) omsgstream, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient, factory.NewUnmarshalDispatcher()) - err = omsgstream.AsConsumer(ctx, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + err = omsgstream.AsConsumer(ctx, consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) assert.Error(t, err) }) } @@ -945,7 +946,7 @@ func TestStream_MqMsgStream_Seek(t *testing.T) { factory := ProtoUDFactory{} pulsarClient, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress}) outputStream2, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient, factory.NewUnmarshalDispatcher()) - outputStream2.AsConsumer(ctx, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + outputStream2.AsConsumer(ctx, consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) outputStream2.Seek(ctx, []*msgpb.MsgPosition{seekPosition}, false) for i := 6; i < 10; i++ { @@ -986,7 +987,7 @@ func TestStream_MqMsgStream_SeekInvalidMessage(t *testing.T) { factory := ProtoUDFactory{} pulsarClient, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress}) outputStream2, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient, factory.NewUnmarshalDispatcher()) - outputStream2.AsConsumer(ctx, consumerChannels, funcutil.RandomString(8), mqwrapper.SubscriptionPositionEarliest) + outputStream2.AsConsumer(ctx, consumerChannels, funcutil.RandomString(8), common.SubscriptionPositionEarliest) defer outputStream2.Close() messageID, _ := pulsar.DeserializeMessageID(seekPosition.MsgID) // try to seek to not written position @@ -1051,11 +1052,11 @@ func TestSTream_MqMsgStream_SeekBadMessageID(t *testing.T) { factory := ProtoUDFactory{} pulsarClient, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress}) outputStream2, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient, factory.NewUnmarshalDispatcher()) - outputStream2.AsConsumer(ctx, consumerChannels, funcutil.RandomString(8), mqwrapper.SubscriptionPositionLatest) + outputStream2.AsConsumer(ctx, consumerChannels, funcutil.RandomString(8), common.SubscriptionPositionLatest) defer outputStream2.Close() outputStream3, err := NewMqTtMsgStream(ctx, 100, 100, pulsarClient, factory.NewUnmarshalDispatcher()) - outputStream3.AsConsumer(ctx, consumerChannels, funcutil.RandomString(8), mqwrapper.SubscriptionPositionEarliest) + outputStream3.AsConsumer(ctx, consumerChannels, funcutil.RandomString(8), common.SubscriptionPositionEarliest) require.NoError(t, err) defer paramtable.Get().Reset(paramtable.Get().MQCfg.IgnoreBadPosition.Key) @@ -1103,7 +1104,7 @@ func TestStream_MqMsgStream_SeekLatest(t *testing.T) { factory := ProtoUDFactory{} pulsarClient, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress}) outputStream2, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient, factory.NewUnmarshalDispatcher()) - outputStream2.AsConsumer(ctx, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionLatest) + outputStream2.AsConsumer(ctx, consumerChannels, consumerSubName, common.SubscriptionPositionLatest) msgPack.Msgs = nil // produce another 10 tsMs @@ -1224,7 +1225,7 @@ type mockSendFailProducer struct { mqwrapper.Producer } -func (p *mockSendFailProducer) Send(_ context.Context, _ *mqwrapper.ProducerMessage) (MessageID, error) { +func (p *mockSendFailProducer) Send(_ context.Context, _ *common.ProducerMessage) (MessageID, error) { return nil, errors.New("mocked error") } @@ -1445,7 +1446,7 @@ func getPulsarOutputStream(ctx context.Context, pulsarAddress string, consumerCh factory := ProtoUDFactory{} pulsarClient, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress}) outputStream, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient, factory.NewUnmarshalDispatcher()) - outputStream.AsConsumer(context.Background(), consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + outputStream.AsConsumer(context.Background(), consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) return outputStream } @@ -1453,7 +1454,7 @@ func getPulsarTtOutputStream(ctx context.Context, pulsarAddress string, consumer factory := ProtoUDFactory{} pulsarClient, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress}) outputStream, _ := NewMqTtMsgStream(ctx, 100, 100, pulsarClient, factory.NewUnmarshalDispatcher()) - outputStream.AsConsumer(context.Background(), consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest) + outputStream.AsConsumer(context.Background(), consumerChannels, consumerSubName, common.SubscriptionPositionEarliest) return outputStream } @@ -1465,7 +1466,7 @@ func getPulsarTtOutputStreamAndSeek(ctx context.Context, pulsarAddress string, p for _, c := range positions { consumerName = append(consumerName, c.ChannelName) } - outputStream.AsConsumer(context.Background(), consumerName, funcutil.RandomString(8), mqwrapper.SubscriptionPositionUnknown) + outputStream.AsConsumer(context.Background(), consumerName, funcutil.RandomString(8), common.SubscriptionPositionUnknown) outputStream.Seek(context.Background(), positions, false) return outputStream } diff --git a/internal/mq/msgstream/mqwrapper/rmq/rocksmq_msgstream_test.go b/pkg/mq/msgstream/mq_rocksmq_msgstream_test.go similarity index 62% rename from internal/mq/msgstream/mqwrapper/rmq/rocksmq_msgstream_test.go rename to pkg/mq/msgstream/mq_rocksmq_msgstream_test.go index 96be3628c905..b3de2b1895bc 100644 --- a/internal/mq/msgstream/mqwrapper/rmq/rocksmq_msgstream_test.go +++ b/pkg/mq/msgstream/mq_rocksmq_msgstream_test.go @@ -14,12 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package rmq +package msgstream import ( "context" "fmt" - "log" "sync" "testing" @@ -29,27 +28,27 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/mq/msgstream" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/rmq" "github.com/milvus-io/milvus/pkg/util/funcutil" ) func Test_NewMqMsgStream(t *testing.T) { - client, _ := createRmqClient() + client, _ := rmq.NewClientWithDefaultOptions(context.TODO()) defer client.Close() - factory := &msgstream.ProtoUDFactory{} - _, err := msgstream.NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) + factory := &ProtoUDFactory{} + _, err := NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) assert.NoError(t, err) } // TODO(wxyu): add a mock implement of mqwrapper.Client, then inject errors to improve coverage func TestMqMsgStream_AsProducer(t *testing.T) { - client, _ := createRmqClient() + client, _ := rmq.NewClientWithDefaultOptions(context.TODO()) defer client.Close() - factory := &msgstream.ProtoUDFactory{} - m, err := msgstream.NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) + factory := &ProtoUDFactory{} + m, err := NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) assert.NoError(t, err) // empty channel name @@ -58,32 +57,32 @@ func TestMqMsgStream_AsProducer(t *testing.T) { // TODO(wxyu): add a mock implement of mqwrapper.Client, then inject errors to improve coverage func TestMqMsgStream_AsConsumer(t *testing.T) { - client, _ := createRmqClient() + client, _ := rmq.NewClientWithDefaultOptions(context.TODO()) defer client.Close() - factory := &msgstream.ProtoUDFactory{} - m, err := msgstream.NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) + factory := &ProtoUDFactory{} + m, err := NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) assert.NoError(t, err) // repeat calling AsConsumer - m.AsConsumer(context.Background(), []string{"a"}, "b", mqwrapper.SubscriptionPositionUnknown) - m.AsConsumer(context.Background(), []string{"a"}, "b", mqwrapper.SubscriptionPositionUnknown) + m.AsConsumer(context.Background(), []string{"a"}, "b", mqcommon.SubscriptionPositionUnknown) + m.AsConsumer(context.Background(), []string{"a"}, "b", mqcommon.SubscriptionPositionUnknown) } func TestMqMsgStream_ComputeProduceChannelIndexes(t *testing.T) { - client, _ := createRmqClient() + client, _ := rmq.NewClientWithDefaultOptions(context.TODO()) defer client.Close() - factory := &msgstream.ProtoUDFactory{} - m, err := msgstream.NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) + factory := &ProtoUDFactory{} + m, err := NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) assert.NoError(t, err) // empty parameters - reBucketValues := m.ComputeProduceChannelIndexes([]msgstream.TsMsg{}) + reBucketValues := m.ComputeProduceChannelIndexes([]TsMsg{}) assert.Nil(t, reBucketValues) // not called AsProducer yet - insertMsg := &msgstream.InsertMsg{ + insertMsg := &InsertMsg{ BaseMsg: generateBaseMsg(), InsertRequest: msgpb.InsertRequest{ Base: &commonpb.MsgBase{ @@ -105,16 +104,16 @@ func TestMqMsgStream_ComputeProduceChannelIndexes(t *testing.T) { RowData: []*commonpb.Blob{}, }, } - reBucketValues = m.ComputeProduceChannelIndexes([]msgstream.TsMsg{insertMsg}) + reBucketValues = m.ComputeProduceChannelIndexes([]TsMsg{insertMsg}) assert.Nil(t, reBucketValues) } func TestMqMsgStream_GetProduceChannels(t *testing.T) { - client, _ := createRmqClient() + client, _ := rmq.NewClientWithDefaultOptions(context.TODO()) defer client.Close() - factory := &msgstream.ProtoUDFactory{} - m, err := msgstream.NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) + factory := &ProtoUDFactory{} + m, err := NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) assert.NoError(t, err) // empty if not called AsProducer yet @@ -128,15 +127,15 @@ func TestMqMsgStream_GetProduceChannels(t *testing.T) { } func TestMqMsgStream_Produce(t *testing.T) { - client, _ := createRmqClient() + client, _ := rmq.NewClientWithDefaultOptions(context.TODO()) defer client.Close() - factory := &msgstream.ProtoUDFactory{} - m, err := msgstream.NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) + factory := &ProtoUDFactory{} + m, err := NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) assert.NoError(t, err) // Produce before called AsProducer - insertMsg := &msgstream.InsertMsg{ + insertMsg := &InsertMsg{ BaseMsg: generateBaseMsg(), InsertRequest: msgpb.InsertRequest{ Base: &commonpb.MsgBase{ @@ -158,19 +157,19 @@ func TestMqMsgStream_Produce(t *testing.T) { RowData: []*commonpb.Blob{}, }, } - msgPack := &msgstream.MsgPack{ - Msgs: []msgstream.TsMsg{insertMsg}, + msgPack := &MsgPack{ + Msgs: []TsMsg{insertMsg}, } err = m.Produce(msgPack) assert.Error(t, err) } func TestMqMsgStream_Broadcast(t *testing.T) { - client, _ := createRmqClient() + client, _ := rmq.NewClientWithDefaultOptions(context.TODO()) defer client.Close() - factory := &msgstream.ProtoUDFactory{} - m, err := msgstream.NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) + factory := &ProtoUDFactory{} + m, err := NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) assert.NoError(t, err) // Broadcast nil pointer @@ -179,14 +178,14 @@ func TestMqMsgStream_Broadcast(t *testing.T) { } func TestMqMsgStream_Consume(t *testing.T) { - client, _ := createRmqClient() + client, _ := rmq.NewClientWithDefaultOptions(context.TODO()) defer client.Close() - factory := &msgstream.ProtoUDFactory{} + factory := &ProtoUDFactory{} // Consume return nil when ctx canceled var wg sync.WaitGroup ctx, cancel := context.WithCancel(context.Background()) - m, err := msgstream.NewMqMsgStream(ctx, 100, 100, client, factory.NewUnmarshalDispatcher()) + m, err := NewMqMsgStream(ctx, 100, 100, client, factory.NewUnmarshalDispatcher()) assert.NoError(t, err) wg.Add(1) @@ -200,26 +199,12 @@ func TestMqMsgStream_Consume(t *testing.T) { wg.Wait() } -func consumer(ctx context.Context, mq msgstream.MsgStream) *msgstream.MsgPack { - for { - select { - case msgPack, ok := <-mq.Chan(): - if !ok { - panic("Should not reach here") - } - return msgPack - case <-ctx.Done(): - return nil - } - } -} - func TestMqMsgStream_Chan(t *testing.T) { - client, _ := createRmqClient() + client, _ := rmq.NewClientWithDefaultOptions(context.TODO()) defer client.Close() - factory := &msgstream.ProtoUDFactory{} - m, err := msgstream.NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) + factory := &ProtoUDFactory{} + m, err := NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) assert.NoError(t, err) ch := m.Chan() @@ -227,11 +212,11 @@ func TestMqMsgStream_Chan(t *testing.T) { } func TestMqMsgStream_SeekNotSubscribed(t *testing.T) { - client, _ := createRmqClient() + client, _ := rmq.NewClientWithDefaultOptions(context.TODO()) defer client.Close() - factory := &msgstream.ProtoUDFactory{} - m, err := msgstream.NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) + factory := &ProtoUDFactory{} + m, err := NewMqMsgStream(context.Background(), 100, 100, client, factory.NewUnmarshalDispatcher()) assert.NoError(t, err) // seek in not subscribed channel @@ -244,39 +229,28 @@ func TestMqMsgStream_SeekNotSubscribed(t *testing.T) { assert.Error(t, err) } -func generateBaseMsg() msgstream.BaseMsg { - ctx := context.Background() - return msgstream.BaseMsg{ - Ctx: ctx, - BeginTimestamp: msgstream.Timestamp(0), - EndTimestamp: msgstream.Timestamp(1), - HashValues: []uint32{2}, - MsgPosition: nil, - } -} - /****************************************Rmq test******************************************/ func initRmqStream(ctx context.Context, producerChannels []string, consumerChannels []string, consumerGroupName string, - opts ...msgstream.RepackFunc, -) (msgstream.MsgStream, msgstream.MsgStream) { - factory := msgstream.ProtoUDFactory{} + opts ...RepackFunc, +) (MsgStream, MsgStream) { + factory := ProtoUDFactory{} - rmqClient, _ := NewClientWithDefaultOptions(ctx) - inputStream, _ := msgstream.NewMqMsgStream(ctx, 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) + rmqClient, _ := rmq.NewClientWithDefaultOptions(ctx) + inputStream, _ := NewMqMsgStream(ctx, 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) inputStream.AsProducer(producerChannels) for _, opt := range opts { inputStream.SetRepackFunc(opt) } - var input msgstream.MsgStream = inputStream + var input MsgStream = inputStream - rmqClient2, _ := NewClientWithDefaultOptions(ctx) - outputStream, _ := msgstream.NewMqMsgStream(ctx, 100, 100, rmqClient2, factory.NewUnmarshalDispatcher()) - outputStream.AsConsumer(ctx, consumerChannels, consumerGroupName, mqwrapper.SubscriptionPositionEarliest) - var output msgstream.MsgStream = outputStream + rmqClient2, _ := rmq.NewClientWithDefaultOptions(ctx) + outputStream, _ := NewMqMsgStream(ctx, 100, 100, rmqClient2, factory.NewUnmarshalDispatcher()) + outputStream.AsConsumer(ctx, consumerChannels, consumerGroupName, mqcommon.SubscriptionPositionEarliest) + var output MsgStream = outputStream return input, output } @@ -285,22 +259,22 @@ func initRmqTtStream(ctx context.Context, producerChannels []string, consumerChannels []string, consumerGroupName string, - opts ...msgstream.RepackFunc, -) (msgstream.MsgStream, msgstream.MsgStream) { - factory := msgstream.ProtoUDFactory{} + opts ...RepackFunc, +) (MsgStream, MsgStream) { + factory := ProtoUDFactory{} - rmqClient, _ := NewClientWithDefaultOptions(ctx) - inputStream, _ := msgstream.NewMqMsgStream(ctx, 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) + rmqClient, _ := rmq.NewClientWithDefaultOptions(ctx) + inputStream, _ := NewMqMsgStream(ctx, 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) inputStream.AsProducer(producerChannels) for _, opt := range opts { inputStream.SetRepackFunc(opt) } - var input msgstream.MsgStream = inputStream + var input MsgStream = inputStream - rmqClient2, _ := NewClientWithDefaultOptions(ctx) - outputStream, _ := msgstream.NewMqTtMsgStream(ctx, 100, 100, rmqClient2, factory.NewUnmarshalDispatcher()) - outputStream.AsConsumer(ctx, consumerChannels, consumerGroupName, mqwrapper.SubscriptionPositionEarliest) - var output msgstream.MsgStream = outputStream + rmqClient2, _ := rmq.NewClientWithDefaultOptions(ctx) + outputStream, _ := NewMqTtMsgStream(ctx, 100, 100, rmqClient2, factory.NewUnmarshalDispatcher()) + outputStream.AsConsumer(ctx, consumerChannels, consumerGroupName, mqcommon.SubscriptionPositionEarliest) + var output MsgStream = outputStream return input, output } @@ -310,7 +284,7 @@ func TestStream_RmqMsgStream_Insert(t *testing.T) { consumerChannels := []string{"insert1", "insert2"} consumerGroupName := "InsertGroup" - msgPack := msgstream.MsgPack{} + msgPack := MsgPack{} msgPack.Msgs = append(msgPack.Msgs, getTsMsg(commonpb.MsgType_Insert, 1)) msgPack.Msgs = append(msgPack.Msgs, getTsMsg(commonpb.MsgType_Insert, 3)) @@ -329,14 +303,14 @@ func TestStream_RmqTtMsgStream_Insert(t *testing.T) { consumerChannels := []string{"insert1", "insert2"} consumerSubName := "subInsert" - msgPack0 := msgstream.MsgPack{} + msgPack0 := MsgPack{} msgPack0.Msgs = append(msgPack0.Msgs, getTimeTickMsg(0)) - msgPack1 := msgstream.MsgPack{} + msgPack1 := MsgPack{} msgPack1.Msgs = append(msgPack1.Msgs, getTsMsg(commonpb.MsgType_Insert, 1)) msgPack1.Msgs = append(msgPack1.Msgs, getTsMsg(commonpb.MsgType_Insert, 3)) - msgPack2 := msgstream.MsgPack{} + msgPack2 := MsgPack{} msgPack2.Msgs = append(msgPack2.Msgs, getTimeTickMsg(5)) ctx := context.Background() @@ -362,20 +336,20 @@ func TestStream_RmqTtMsgStream_DuplicatedIDs(t *testing.T) { consumerChannels := []string{c1} consumerSubName := funcutil.RandomString(8) - msgPack0 := msgstream.MsgPack{} + msgPack0 := MsgPack{} msgPack0.Msgs = append(msgPack0.Msgs, getTimeTickMsg(0)) - msgPack1 := msgstream.MsgPack{} + msgPack1 := MsgPack{} msgPack1.Msgs = append(msgPack1.Msgs, getTsMsg(commonpb.MsgType_Insert, 1)) msgPack1.Msgs = append(msgPack1.Msgs, getTsMsg(commonpb.MsgType_Insert, 1)) msgPack1.Msgs = append(msgPack1.Msgs, getTsMsg(commonpb.MsgType_Insert, 1)) // would not dedup for non-dml messages - msgPack2 := msgstream.MsgPack{} + msgPack2 := MsgPack{} msgPack2.Msgs = append(msgPack2.Msgs, getTsMsg(commonpb.MsgType_CreateCollection, 2)) msgPack2.Msgs = append(msgPack2.Msgs, getTsMsg(commonpb.MsgType_CreateCollection, 2)) - msgPack3 := msgstream.MsgPack{} + msgPack3 := MsgPack{} msgPack3.Msgs = append(msgPack3.Msgs, getTimeTickMsg(15)) ctx := context.Background() @@ -397,12 +371,12 @@ func TestStream_RmqTtMsgStream_DuplicatedIDs(t *testing.T) { outputStream.Close() - factory := msgstream.ProtoUDFactory{} + factory := ProtoUDFactory{} - rmqClient, _ := NewClientWithDefaultOptions(ctx) - outputStream, _ = msgstream.NewMqTtMsgStream(context.Background(), 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) + rmqClient, _ := rmq.NewClientWithDefaultOptions(ctx) + outputStream, _ = NewMqTtMsgStream(context.Background(), 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) consumerSubName = funcutil.RandomString(8) - outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionUnknown) + outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, mqcommon.SubscriptionPositionUnknown) outputStream.Seek(ctx, receivedMsg.StartPositions, false) seekMsg := consumer(ctx, outputStream) assert.Equal(t, len(seekMsg.Msgs), 1+2) @@ -420,32 +394,32 @@ func TestStream_RmqTtMsgStream_Seek(t *testing.T) { consumerChannels := []string{c1} consumerSubName := funcutil.RandomString(8) - msgPack0 := msgstream.MsgPack{} + msgPack0 := MsgPack{} msgPack0.Msgs = append(msgPack0.Msgs, getTimeTickMsg(0)) - msgPack1 := msgstream.MsgPack{} + msgPack1 := MsgPack{} msgPack1.Msgs = append(msgPack1.Msgs, getTsMsg(commonpb.MsgType_Insert, 1)) msgPack1.Msgs = append(msgPack1.Msgs, getTsMsg(commonpb.MsgType_Insert, 3)) msgPack1.Msgs = append(msgPack1.Msgs, getTsMsg(commonpb.MsgType_Insert, 19)) - msgPack2 := msgstream.MsgPack{} + msgPack2 := MsgPack{} msgPack2.Msgs = append(msgPack2.Msgs, getTimeTickMsg(5)) - msgPack3 := msgstream.MsgPack{} + msgPack3 := MsgPack{} msgPack3.Msgs = append(msgPack3.Msgs, getTsMsg(commonpb.MsgType_Insert, 14)) msgPack3.Msgs = append(msgPack3.Msgs, getTsMsg(commonpb.MsgType_Insert, 9)) - msgPack4 := msgstream.MsgPack{} + msgPack4 := MsgPack{} msgPack4.Msgs = append(msgPack4.Msgs, getTimeTickMsg(11)) - msgPack5 := msgstream.MsgPack{} + msgPack5 := MsgPack{} msgPack5.Msgs = append(msgPack5.Msgs, getTsMsg(commonpb.MsgType_Insert, 12)) msgPack5.Msgs = append(msgPack5.Msgs, getTsMsg(commonpb.MsgType_Insert, 13)) - msgPack6 := msgstream.MsgPack{} + msgPack6 := MsgPack{} msgPack6.Msgs = append(msgPack6.Msgs, getTimeTickMsg(15)) - msgPack7 := msgstream.MsgPack{} + msgPack7 := MsgPack{} msgPack7.Msgs = append(msgPack7.Msgs, getTimeTickMsg(20)) ctx := context.Background() @@ -499,12 +473,12 @@ func TestStream_RmqTtMsgStream_Seek(t *testing.T) { outputStream.Close() - factory := msgstream.ProtoUDFactory{} + factory := ProtoUDFactory{} - rmqClient, _ := NewClientWithDefaultOptions(ctx) - outputStream, _ = msgstream.NewMqTtMsgStream(context.Background(), 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) + rmqClient, _ := rmq.NewClientWithDefaultOptions(ctx) + outputStream, _ = NewMqTtMsgStream(context.Background(), 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) consumerSubName = funcutil.RandomString(8) - outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionUnknown) + outputStream.AsConsumer(ctx, consumerChannels, consumerSubName, mqcommon.SubscriptionPositionUnknown) outputStream.Seek(ctx, receivedMsg3.StartPositions, false) seekMsg := consumer(ctx, outputStream) @@ -532,7 +506,7 @@ func TestStream_RMqMsgStream_SeekInvalidMessage(t *testing.T) { ctx := context.Background() inputStream, outputStream := initRmqStream(ctx, producerChannels, consumerChannels, consumerSubName) - msgPack := &msgstream.MsgPack{} + msgPack := &MsgPack{} for i := 0; i < 10; i++ { insertMsg := getTsMsg(commonpb.MsgType_Insert, int64(i)) msgPack.Msgs = append(msgPack.Msgs, insertMsg) @@ -548,10 +522,10 @@ func TestStream_RMqMsgStream_SeekInvalidMessage(t *testing.T) { } outputStream.Close() - factory := msgstream.ProtoUDFactory{} - rmqClient2, _ := NewClientWithDefaultOptions(ctx) - outputStream2, _ := msgstream.NewMqMsgStream(ctx, 100, 100, rmqClient2, factory.NewUnmarshalDispatcher()) - outputStream2.AsConsumer(ctx, consumerChannels, funcutil.RandomString(8), mqwrapper.SubscriptionPositionUnknown) + factory := ProtoUDFactory{} + rmqClient2, _ := rmq.NewClientWithDefaultOptions(ctx) + outputStream2, _ := NewMqMsgStream(ctx, 100, 100, rmqClient2, factory.NewUnmarshalDispatcher()) + outputStream2.AsConsumer(ctx, consumerChannels, funcutil.RandomString(8), mqcommon.SubscriptionPositionUnknown) id := common.Endian.Uint64(seekPosition.MsgID) + 10 bs := make([]byte, 8) @@ -587,24 +561,24 @@ func TestStream_RmqTtMsgStream_AsConsumerWithPosition(t *testing.T) { consumerChannels := []string{"insert1"} consumerSubName := "subInsert" - factory := msgstream.ProtoUDFactory{} + factory := ProtoUDFactory{} - rmqClient, _ := NewClientWithDefaultOptions(context.Background()) + rmqClient, _ := rmq.NewClientWithDefaultOptions(context.Background()) - otherInputStream, _ := msgstream.NewMqMsgStream(context.Background(), 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) + otherInputStream, _ := NewMqMsgStream(context.Background(), 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) otherInputStream.AsProducer([]string{"root_timetick"}) otherInputStream.Produce(getTimeTickMsgPack(999)) - inputStream, _ := msgstream.NewMqMsgStream(context.Background(), 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) + inputStream, _ := NewMqMsgStream(context.Background(), 100, 100, rmqClient, factory.NewUnmarshalDispatcher()) inputStream.AsProducer(producerChannels) for i := 0; i < 100; i++ { inputStream.Produce(getTimeTickMsgPack(int64(i))) } - rmqClient2, _ := NewClientWithDefaultOptions(context.Background()) - outputStream, _ := msgstream.NewMqMsgStream(context.Background(), 100, 100, rmqClient2, factory.NewUnmarshalDispatcher()) - outputStream.AsConsumer(context.Background(), consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionLatest) + rmqClient2, _ := rmq.NewClientWithDefaultOptions(context.Background()) + outputStream, _ := NewMqMsgStream(context.Background(), 100, 100, rmqClient2, factory.NewUnmarshalDispatcher()) + outputStream.AsConsumer(context.Background(), consumerChannels, consumerSubName, mqcommon.SubscriptionPositionLatest) inputStream.Produce(getTimeTickMsgPack(1000)) pack := <-outputStream.Chan() @@ -615,116 +589,3 @@ func TestStream_RmqTtMsgStream_AsConsumerWithPosition(t *testing.T) { inputStream.Close() outputStream.Close() } - -func getTimeTickMsgPack(reqID msgstream.UniqueID) *msgstream.MsgPack { - msgPack := msgstream.MsgPack{} - msgPack.Msgs = append(msgPack.Msgs, getTimeTickMsg(reqID)) - return &msgPack -} - -func getTsMsg(msgType msgstream.MsgType, reqID msgstream.UniqueID) msgstream.TsMsg { - hashValue := uint32(reqID) - time := uint64(reqID) - switch msgType { - case commonpb.MsgType_Insert: - insertRequest := msgpb.InsertRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Insert, - MsgID: reqID, - Timestamp: time, - SourceID: reqID, - }, - CollectionName: "Collection", - PartitionName: "Partition", - SegmentID: 1, - ShardName: "0", - Timestamps: []msgstream.Timestamp{time}, - RowIDs: []int64{1}, - RowData: []*commonpb.Blob{{}}, - } - insertMsg := &msgstream.InsertMsg{ - BaseMsg: msgstream.BaseMsg{ - BeginTimestamp: 0, - EndTimestamp: 0, - HashValues: []uint32{hashValue}, - }, - InsertRequest: insertRequest, - } - return insertMsg - case commonpb.MsgType_CreateCollection: - createCollectionRequest := msgpb.CreateCollectionRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_CreateCollection, - MsgID: reqID, - Timestamp: 11, - SourceID: reqID, - }, - DbName: "test_db", - CollectionName: "test_collection", - PartitionName: "test_partition", - DbID: 4, - CollectionID: 5, - PartitionID: 6, - Schema: []byte{}, - VirtualChannelNames: []string{}, - PhysicalChannelNames: []string{}, - } - createCollectionMsg := &msgstream.CreateCollectionMsg{ - BaseMsg: msgstream.BaseMsg{ - BeginTimestamp: 0, - EndTimestamp: 0, - HashValues: []uint32{hashValue}, - }, - CreateCollectionRequest: createCollectionRequest, - } - return createCollectionMsg - } - return nil -} - -func getTimeTickMsg(reqID msgstream.UniqueID) msgstream.TsMsg { - hashValue := uint32(reqID) - time := uint64(reqID) - timeTickResult := msgpb.TimeTickMsg{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_TimeTick, - MsgID: reqID, - Timestamp: time, - SourceID: reqID, - }, - } - timeTickMsg := &msgstream.TimeTickMsg{ - BaseMsg: msgstream.BaseMsg{ - BeginTimestamp: 0, - EndTimestamp: 0, - HashValues: []uint32{hashValue}, - }, - TimeTickMsg: timeTickResult, - } - return timeTickMsg -} - -func receiveMsg(ctx context.Context, outputStream msgstream.MsgStream, msgCount int) { - receiveCount := 0 - for { - select { - case <-ctx.Done(): - return - case result, ok := <-outputStream.Chan(): - if !ok || result == nil || len(result.Msgs) == 0 { - return - } - if len(result.Msgs) > 0 { - msgs := result.Msgs - for _, v := range msgs { - receiveCount++ - log.Println("msg type: ", v.Type(), ", msg value: ", v) - } - log.Println("================") - } - if receiveCount >= msgCount { - return - } - } - } -} diff --git a/pkg/mq/msgstream/mqwrapper/client.go b/pkg/mq/msgstream/mqwrapper/client.go index 47f482103ccf..3ec394a4db8d 100644 --- a/pkg/mq/msgstream/mqwrapper/client.go +++ b/pkg/mq/msgstream/mqwrapper/client.go @@ -16,22 +16,26 @@ package mqwrapper +import ( + "github.com/milvus-io/milvus/pkg/mq/common" +) + // Client is the interface that provides operations of message queues type Client interface { // CreateProducer creates a producer instance - CreateProducer(options ProducerOptions) (Producer, error) + CreateProducer(options common.ProducerOptions) (Producer, error) // Subscribe creates a consumer instance and subscribe a topic Subscribe(options ConsumerOptions) (Consumer, error) // Get the earliest MessageID - EarliestMessageID() MessageID + EarliestMessageID() common.MessageID // String to msg ID - StringToMsgID(string) (MessageID, error) + StringToMsgID(string) (common.MessageID, error) // Deserialize MessageId from a byte array - BytesToMsgID([]byte) (MessageID, error) + BytesToMsgID([]byte) (common.MessageID, error) // Close the client and free associated resources Close() diff --git a/pkg/mq/msgstream/mqwrapper/consumer.go b/pkg/mq/msgstream/mqwrapper/consumer.go index f8b49e40601f..41086aa85f64 100644 --- a/pkg/mq/msgstream/mqwrapper/consumer.go +++ b/pkg/mq/msgstream/mqwrapper/consumer.go @@ -16,19 +16,7 @@ package mqwrapper -// SubscriptionInitialPosition is the type of a subscription initial position -type SubscriptionInitialPosition int - -const ( - // SubscriptionPositionLatest is latest position which means the start consuming position will be the last message - SubscriptionPositionLatest SubscriptionInitialPosition = iota - - // SubscriptionPositionEarliest is earliest position which means the start consuming position will be the first message - SubscriptionPositionEarliest - - // SubscriptionPositionUnkown indicates we don't care about the consumer location, since we are doing another seek or only some meta api over that - SubscriptionPositionUnknown -) +import "github.com/milvus-io/milvus/pkg/mq/common" const DefaultPartitionIdx = 0 @@ -45,7 +33,7 @@ type ConsumerOptions struct { // InitialPosition at which the cursor will be set when subscribe // Default is `Latest` - SubscriptionInitialPosition + common.SubscriptionInitialPosition // Set receive channel size BufSize int64 @@ -57,19 +45,19 @@ type Consumer interface { Subscription() string // Get Message channel, once you chan you can not seek again - Chan() <-chan Message + Chan() <-chan common.Message // Seek to the uniqueID position, the second bool param indicates whether the message is included in the position - Seek(MessageID, bool) error //nolint:govet + Seek(common.MessageID, bool) error //nolint:govet // Ack make sure that msg is received - Ack(Message) + Ack(common.Message) // Close consumer Close() // GetLatestMsgID return the latest message ID - GetLatestMsgID() (MessageID, error) + GetLatestMsgID() (common.MessageID, error) // check created topic whether vaild or not CheckTopicValid(channel string) error diff --git a/pkg/mq/msgstream/mqwrapper/kafka/kafka_client.go b/pkg/mq/msgstream/mqwrapper/kafka/kafka_client.go index 5d0cb917e49c..c1e994c3f1a5 100644 --- a/pkg/mq/msgstream/mqwrapper/kafka/kafka_client.go +++ b/pkg/mq/msgstream/mqwrapper/kafka/kafka_client.go @@ -14,6 +14,7 @@ import ( "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/conc" "github.com/milvus-io/milvus/pkg/util/paramtable" @@ -185,7 +186,7 @@ func (kc *kafkaClient) newProducerConfig() *kafka.ConfigMap { return newConf } -func (kc *kafkaClient) newConsumerConfig(group string, offset mqwrapper.SubscriptionInitialPosition) *kafka.ConfigMap { +func (kc *kafkaClient) newConsumerConfig(group string, offset common.SubscriptionInitialPosition) *kafka.ConfigMap { newConf := cloneKafkaConfig(kc.basicConfig) newConf.SetKey("group.id", group) @@ -199,7 +200,7 @@ func (kc *kafkaClient) newConsumerConfig(group string, offset mqwrapper.Subscrip return newConf } -func (kc *kafkaClient) CreateProducer(options mqwrapper.ProducerOptions) (mqwrapper.Producer, error) { +func (kc *kafkaClient) CreateProducer(options common.ProducerOptions) (mqwrapper.Producer, error) { start := timerecord.NewTimeRecorder("create producer") metrics.MsgStreamOpCounter.WithLabelValues(metrics.CreateProducerLabel, metrics.TotalLabel).Inc() @@ -234,11 +235,11 @@ func (kc *kafkaClient) Subscribe(options mqwrapper.ConsumerOptions) (mqwrapper.C return consumer, nil } -func (kc *kafkaClient) EarliestMessageID() mqwrapper.MessageID { +func (kc *kafkaClient) EarliestMessageID() common.MessageID { return &kafkaID{messageID: int64(kafka.OffsetBeginning)} } -func (kc *kafkaClient) StringToMsgID(id string) (mqwrapper.MessageID, error) { +func (kc *kafkaClient) StringToMsgID(id string) (common.MessageID, error) { offset, err := strconv.ParseInt(id, 10, 64) if err != nil { return nil, err @@ -257,7 +258,7 @@ func (kc *kafkaClient) specialExtraConfig(current *kafka.ConfigMap, special kafk } } -func (kc *kafkaClient) BytesToMsgID(id []byte) (mqwrapper.MessageID, error) { +func (kc *kafkaClient) BytesToMsgID(id []byte) (common.MessageID, error) { offset := DeserializeKafkaID(id) return &kafkaID{messageID: offset}, nil } diff --git a/pkg/mq/msgstream/mqwrapper/kafka/kafka_client_test.go b/pkg/mq/msgstream/mqwrapper/kafka/kafka_client_test.go index db5c04b0eaaa..63559ef71a10 100644 --- a/pkg/mq/msgstream/mqwrapper/kafka/kafka_client_test.go +++ b/pkg/mq/msgstream/mqwrapper/kafka/kafka_client_test.go @@ -17,6 +17,7 @@ import ( "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/config" "github.com/milvus-io/milvus/pkg/log" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" ) @@ -62,12 +63,12 @@ func BytesToInt(b []byte) int { } // Consume1 will consume random messages and record the last MessageID it received -func Consume1(ctx context.Context, t *testing.T, kc *kafkaClient, topic string, subName string, c chan mqwrapper.MessageID, total *int) { +func Consume1(ctx context.Context, t *testing.T, kc *kafkaClient, topic string, subName string, c chan mqcommon.MessageID, total *int) { consumer, err := kc.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) assert.NoError(t, err) assert.NotNil(t, consumer) @@ -78,7 +79,7 @@ func Consume1(ctx context.Context, t *testing.T, kc *kafkaClient, topic string, cnt := 1 + rand.Int()%5 log.Info("Consume1 start") - var msg mqwrapper.Message + var msg mqcommon.Message for i := 0; i < cnt; i++ { select { case <-ctx.Done(): @@ -101,12 +102,12 @@ func Consume1(ctx context.Context, t *testing.T, kc *kafkaClient, topic string, } // Consume2 will consume messages from specified MessageID -func Consume2(ctx context.Context, t *testing.T, kc *kafkaClient, topic string, subName string, msgID mqwrapper.MessageID, total *int) { +func Consume2(ctx context.Context, t *testing.T, kc *kafkaClient, topic string, subName string, msgID mqcommon.MessageID, total *int) { consumer, err := kc.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionUnknown, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionUnknown, }) assert.NoError(t, err) assert.NotNil(t, consumer) @@ -142,7 +143,7 @@ func Consume3(ctx context.Context, t *testing.T, kc *kafkaClient, topic string, Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) assert.NoError(t, err) assert.NotNil(t, consumer) @@ -177,7 +178,7 @@ func TestKafkaClient_ConsumeWithAck(t *testing.T) { arr1 := []int{111, 222, 333, 444, 555, 666, 777} arr2 := []string{"111", "222", "333", "444", "555", "666", "777"} - c := make(chan mqwrapper.MessageID, 1) + c := make(chan mqcommon.MessageID, 1) ctx, cancel := context.WithCancel(context.Background()) @@ -228,7 +229,7 @@ func TestKafkaClient_SeekPosition(t *testing.T) { data2 := []string{"1", "2", "3"} ids := produceData(ctx, t, producer, data1, data2) - consumer := createConsumer(t, kc, topic, subName, mqwrapper.SubscriptionPositionUnknown) + consumer := createConsumer(t, kc, topic, subName, mqcommon.SubscriptionPositionUnknown) defer consumer.Close() err := consumer.Seek(ids[2], true) @@ -260,7 +261,7 @@ func TestKafkaClient_ConsumeFromLatest(t *testing.T) { data2 := []string{"1", "2"} produceData(ctx, t, producer, data1, data2) - consumer := createConsumer(t, kc, topic, subName, mqwrapper.SubscriptionPositionLatest) + consumer := createConsumer(t, kc, topic, subName, mqcommon.SubscriptionPositionLatest) defer consumer.Close() go func() { @@ -354,6 +355,12 @@ func withProtocol(v string) kafkaCfgOption { } } +func withKafkaUseSSL(v string) kafkaCfgOption { + return func(cfg *paramtable.KafkaConfig) { + initParamItem(&cfg.KafkaUseSSL, v) + } +} + func createKafkaConfig(opts ...kafkaCfgOption) *paramtable.KafkaConfig { cfg := ¶mtable.KafkaConfig{} for _, opt := range opts { @@ -375,7 +382,8 @@ func TestKafkaClient_NewKafkaClientInstanceWithConfig(t *testing.T) { consumerConfig := make(map[string]string) consumerConfig["client.id"] = "dc" - config := createKafkaConfig(withAddr("addr"), withUsername("username"), withPasswd("password"), withMechanism("sasl"), withProtocol("plain")) + config := createKafkaConfig(withKafkaUseSSL("false"), withAddr("addr"), withUsername("username"), + withPasswd("password"), withMechanism("sasl"), withProtocol("plain")) config.ConsumerExtraConfig = paramtable.ParamGroup{GetFunc: func() map[string]string { return consumerConfig }} config.ProducerExtraConfig = paramtable.ParamGroup{GetFunc: func() map[string]string { return producerConfig }} @@ -408,7 +416,7 @@ func createConsumer(t *testing.T, kc *kafkaClient, topic string, groupID string, - initPosition mqwrapper.SubscriptionInitialPosition, + initPosition mqcommon.SubscriptionInitialPosition, ) mqwrapper.Consumer { consumer, err := kc.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, @@ -421,16 +429,16 @@ func createConsumer(t *testing.T, } func createProducer(t *testing.T, kc *kafkaClient, topic string) mqwrapper.Producer { - producer, err := kc.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + producer, err := kc.CreateProducer(mqcommon.ProducerOptions{Topic: topic}) assert.NoError(t, err) assert.NotNil(t, producer) return producer } -func produceData(ctx context.Context, t *testing.T, producer mqwrapper.Producer, arr []int, pArr []string) []mqwrapper.MessageID { - var msgIDs []mqwrapper.MessageID +func produceData(ctx context.Context, t *testing.T, producer mqwrapper.Producer, arr []int, pArr []string) []mqcommon.MessageID { + var msgIDs []mqcommon.MessageID for k, v := range arr { - msg := &mqwrapper.ProducerMessage{ + msg := &mqcommon.ProducerMessage{ Payload: IntToBytes(v), Properties: map[string]string{ common.TraceIDKey: pArr[k], diff --git a/pkg/mq/msgstream/mqwrapper/kafka/kafka_consumer.go b/pkg/mq/msgstream/mqwrapper/kafka/kafka_consumer.go index c86aecf6ecf6..bf87b260a7be 100644 --- a/pkg/mq/msgstream/mqwrapper/kafka/kafka_consumer.go +++ b/pkg/mq/msgstream/mqwrapper/kafka/kafka_consumer.go @@ -9,6 +9,7 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" @@ -17,7 +18,7 @@ import ( type Consumer struct { c *kafka.Consumer config *kafka.ConfigMap - msgChannel chan mqwrapper.Message + msgChannel chan common.Message hasAssign bool skipMsg bool topic string @@ -30,8 +31,8 @@ type Consumer struct { const timeout = 3000 -func newKafkaConsumer(config *kafka.ConfigMap, bufSize int64, topic string, groupID string, position mqwrapper.SubscriptionInitialPosition) (*Consumer, error) { - msgChannel := make(chan mqwrapper.Message, bufSize) +func newKafkaConsumer(config *kafka.ConfigMap, bufSize int64, topic string, groupID string, position common.SubscriptionInitialPosition) (*Consumer, error) { + msgChannel := make(chan common.Message, bufSize) kc := &Consumer{ config: config, msgChannel: msgChannel, @@ -46,9 +47,9 @@ func newKafkaConsumer(config *kafka.ConfigMap, bufSize int64, topic string, grou } // if it's unknown, we leave the assign to seek - if position != mqwrapper.SubscriptionPositionUnknown { + if position != common.SubscriptionPositionUnknown { var offset kafka.Offset - if position == mqwrapper.SubscriptionPositionEarliest { + if position == common.SubscriptionPositionEarliest { offset, err = kafka.NewOffset("earliest") if err != nil { return nil, err @@ -114,7 +115,7 @@ func (kc *Consumer) Subscription() string { // confluent-kafka-go recommend us to use function-based consumer, // channel-based consumer API had already deprecated, see more details // https://github.com/confluentinc/confluent-kafka-go. -func (kc *Consumer) Chan() <-chan mqwrapper.Message { +func (kc *Consumer) Chan() <-chan common.Message { if !kc.hasAssign { log.Error("can not chan with not assigned channel", zap.String("topic", kc.topic), zap.String("groupID", kc.groupID)) panic("failed to chan a kafka consumer without assign") @@ -155,7 +156,7 @@ func (kc *Consumer) Chan() <-chan mqwrapper.Message { return kc.msgChannel } -func (kc *Consumer) Seek(id mqwrapper.MessageID, inclusive bool) error { +func (kc *Consumer) Seek(id common.MessageID, inclusive bool) error { if kc.hasAssign { return errors.New("kafka consumer is already assigned, can not seek again") } @@ -199,13 +200,13 @@ func (kc *Consumer) internalSeek(offset kafka.Offset, inclusive bool) error { return nil } -func (kc *Consumer) Ack(message mqwrapper.Message) { +func (kc *Consumer) Ack(message common.Message) { // Do nothing // Kafka retention mechanism only depends on retention configuration, // it does not relate to the commit with consumer's offsets. } -func (kc *Consumer) GetLatestMsgID() (mqwrapper.MessageID, error) { +func (kc *Consumer) GetLatestMsgID() (common.MessageID, error) { low, high, err := kc.c.QueryWatermarkOffsets(kc.topic, mqwrapper.DefaultPartitionIdx, timeout) if err != nil { return nil, err diff --git a/pkg/mq/msgstream/mqwrapper/kafka/kafka_consumer_test.go b/pkg/mq/msgstream/mqwrapper/kafka/kafka_consumer_test.go index 43efe783addd..45bec8dad753 100644 --- a/pkg/mq/msgstream/mqwrapper/kafka/kafka_consumer_test.go +++ b/pkg/mq/msgstream/mqwrapper/kafka/kafka_consumer_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" ) func TestKafkaConsumer_Subscription(t *testing.T) { @@ -20,7 +20,7 @@ func TestKafkaConsumer_Subscription(t *testing.T) { topic := fmt.Sprintf("test-topicName-%d", rand.Int()) config := createConfig(groupID) - kc, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionUnknown) + kc, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionUnknown) assert.NoError(t, err) defer kc.Close() assert.Equal(t, kc.Subscription(), groupID) @@ -32,7 +32,7 @@ func TestKafkaConsumer_SeekExclusive(t *testing.T) { topic := fmt.Sprintf("test-topicName-%d", rand.Int()) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionUnknown) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionUnknown) assert.NoError(t, err) defer consumer.Close() @@ -58,7 +58,7 @@ func TestKafkaConsumer_SeekInclusive(t *testing.T) { topic := fmt.Sprintf("test-topicName-%d", rand.Int()) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionUnknown) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionUnknown) assert.NoError(t, err) defer consumer.Close() @@ -84,7 +84,7 @@ func TestKafkaConsumer_GetSeek(t *testing.T) { topic := fmt.Sprintf("test-topicName-%d", rand.Int()) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionUnknown) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionUnknown) assert.NoError(t, err) defer consumer.Close() @@ -101,7 +101,7 @@ func TestKafkaConsumer_ChanWithNoAssign(t *testing.T) { topic := fmt.Sprintf("test-topicName-%d", rand.Int()) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionUnknown) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionUnknown) assert.NoError(t, err) defer consumer.Close() @@ -137,7 +137,7 @@ func TestKafkaConsumer_SeekAfterChan(t *testing.T) { topic := fmt.Sprintf("test-topicName-%d", rand.Int()) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionEarliest) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionEarliest) assert.NoError(t, err) defer consumer.Close() @@ -158,7 +158,7 @@ func TestKafkaConsumer_GetLatestMsgID(t *testing.T) { topic := fmt.Sprintf("test-topicName-%d", rand.Int()) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionUnknown) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionUnknown) assert.NoError(t, err) defer consumer.Close() @@ -185,7 +185,7 @@ func TestKafkaConsumer_ConsumeFromLatest(t *testing.T) { testKafkaConsumerProduceData(t, topic, data1, data2) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionLatest) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionLatest) assert.NoError(t, err) defer consumer.Close() data1 = []int{444, 555} @@ -210,7 +210,7 @@ func TestKafkaConsumer_ConsumeFromEarliest(t *testing.T) { testKafkaConsumerProduceData(t, topic, data1, data2) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionEarliest) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionEarliest) assert.NoError(t, err) msg := <-consumer.Chan() assert.Equal(t, 111, BytesToInt(msg.Payload())) @@ -219,7 +219,7 @@ func TestKafkaConsumer_ConsumeFromEarliest(t *testing.T) { defer consumer.Close() config = createConfig(groupID) - consumer2, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionEarliest) + consumer2, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionEarliest) assert.NoError(t, err) msg = <-consumer2.Chan() assert.Equal(t, 111, BytesToInt(msg.Payload())) @@ -261,7 +261,7 @@ func TestKafkaConsumer_CheckPreTopicValid(t *testing.T) { topic := fmt.Sprintf("test-topicName-%d", rand.Int()) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionEarliest) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionEarliest) assert.NoError(t, err) defer consumer.Close() @@ -280,7 +280,7 @@ func TestKafkaConsumer_Close(t *testing.T) { t.Run("close after only get latest msgID", func(t *testing.T) { groupID := fmt.Sprintf("test-groupid-%d", rand.Int()) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionEarliest) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionEarliest) assert.NoError(t, err) _, err = consumer.GetLatestMsgID() assert.NoError(t, err) @@ -290,7 +290,7 @@ func TestKafkaConsumer_Close(t *testing.T) { t.Run("close after only Chan method is invoked", func(t *testing.T) { groupID := fmt.Sprintf("test-groupid-%d", rand.Int()) config := createConfig(groupID) - consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqwrapper.SubscriptionPositionEarliest) + consumer, err := newKafkaConsumer(config, 16, topic, groupID, mqcommon.SubscriptionPositionEarliest) assert.NoError(t, err) <-consumer.Chan() consumer.Close() diff --git a/pkg/mq/msgstream/mqwrapper/kafka/kafka_id.go b/pkg/mq/msgstream/mqwrapper/kafka/kafka_id.go index 58af67bec9df..2509065c1d4b 100644 --- a/pkg/mq/msgstream/mqwrapper/kafka/kafka_id.go +++ b/pkg/mq/msgstream/mqwrapper/kafka/kafka_id.go @@ -2,14 +2,14 @@ package kafka import ( "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" ) type kafkaID struct { messageID int64 } -var _ mqwrapper.MessageID = &kafkaID{} +var _ mqcommon.MessageID = &kafkaID{} func (kid *kafkaID) Serialize() []byte { return SerializeKafkaID(kid.messageID) diff --git a/pkg/mq/msgstream/mqwrapper/kafka/kafka_message.go b/pkg/mq/msgstream/mqwrapper/kafka/kafka_message.go index d7f09585392c..cc33c8db4090 100644 --- a/pkg/mq/msgstream/mqwrapper/kafka/kafka_message.go +++ b/pkg/mq/msgstream/mqwrapper/kafka/kafka_message.go @@ -3,7 +3,7 @@ package kafka import ( "github.com/confluentinc/confluent-kafka-go/kafka" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) type kafkaMessage struct { @@ -26,7 +26,7 @@ func (km *kafkaMessage) Payload() []byte { return km.msg.Value } -func (km *kafkaMessage) ID() mqwrapper.MessageID { +func (km *kafkaMessage) ID() common.MessageID { kid := &kafkaID{messageID: int64(km.msg.TopicPartition.Offset)} return kid } diff --git a/pkg/mq/msgstream/mqwrapper/kafka/kafka_producer.go b/pkg/mq/msgstream/mqwrapper/kafka/kafka_producer.go index 71fdcf0bcfe1..ae5d1a409be1 100644 --- a/pkg/mq/msgstream/mqwrapper/kafka/kafka_producer.go +++ b/pkg/mq/msgstream/mqwrapper/kafka/kafka_producer.go @@ -12,6 +12,7 @@ import ( "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/timerecord" ) @@ -28,7 +29,7 @@ func (kp *kafkaProducer) Topic() string { return kp.topic } -func (kp *kafkaProducer) Send(ctx context.Context, message *mqwrapper.ProducerMessage) (mqwrapper.MessageID, error) { +func (kp *kafkaProducer) Send(ctx context.Context, message *mqcommon.ProducerMessage) (mqcommon.MessageID, error) { start := timerecord.NewTimeRecorder("send msg to stream") metrics.MsgStreamOpCounter.WithLabelValues(metrics.SendMsgLabel, metrics.TotalLabel).Inc() diff --git a/pkg/mq/msgstream/mqwrapper/kafka/kafka_producer_test.go b/pkg/mq/msgstream/mqwrapper/kafka/kafka_producer_test.go index 3ddbde026927..c2f2b771f5a2 100644 --- a/pkg/mq/msgstream/mqwrapper/kafka/kafka_producer_test.go +++ b/pkg/mq/msgstream/mqwrapper/kafka/kafka_producer_test.go @@ -11,7 +11,7 @@ import ( "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/stretchr/testify/assert" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) func TestKafkaProducer_SendSuccess(t *testing.T) { @@ -23,14 +23,14 @@ func TestKafkaProducer_SendSuccess(t *testing.T) { rand.Seed(time.Now().UnixNano()) topic := fmt.Sprintf("test-topic-%d", rand.Int()) - producer, err := kc.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + producer, err := kc.CreateProducer(common.ProducerOptions{Topic: topic}) assert.NoError(t, err) assert.NotNil(t, producer) kafkaProd := producer.(*kafkaProducer) assert.Equal(t, kafkaProd.Topic(), topic) - msg2 := &mqwrapper.ProducerMessage{ + msg2 := &common.ProducerMessage{ Payload: []byte{}, Properties: map[string]string{}, } @@ -52,7 +52,7 @@ func TestKafkaProducer_SendFail(t *testing.T) { assert.NoError(t, err) producer := &kafkaProducer{p: pp, deliveryChan: deliveryChan, topic: topic} - msg := &mqwrapper.ProducerMessage{ + msg := &common.ProducerMessage{ Payload: []byte{1}, Properties: map[string]string{}, } @@ -76,7 +76,7 @@ func TestKafkaProducer_SendFailAfterClose(t *testing.T) { rand.Seed(time.Now().UnixNano()) topic := fmt.Sprintf("test-topic-%d", rand.Int()) - producer, err := kc.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + producer, err := kc.CreateProducer(common.ProducerOptions{Topic: topic}) assert.Nil(t, err) assert.NotNil(t, producer) @@ -85,7 +85,7 @@ func TestKafkaProducer_SendFailAfterClose(t *testing.T) { kafkaProd := producer.(*kafkaProducer) assert.Equal(t, kafkaProd.Topic(), topic) - msg2 := &mqwrapper.ProducerMessage{ + msg2 := &common.ProducerMessage{ Payload: []byte{}, Properties: map[string]string{}, } diff --git a/pkg/mq/msgstream/mqwrapper/nmq/nmq_client.go b/pkg/mq/msgstream/mqwrapper/nmq/nmq_client.go index 774adb5e7fb2..1a6fb8493c93 100644 --- a/pkg/mq/msgstream/mqwrapper/nmq/nmq_client.go +++ b/pkg/mq/msgstream/mqwrapper/nmq/nmq_client.go @@ -27,6 +27,7 @@ import ( "github.com/nats-io/nats.go" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/timerecord" @@ -79,7 +80,7 @@ func NewClient(url string, options ...nats.Option) (*nmqClient, error) { } // CreateProducer creates a producer for natsmq client -func (nc *nmqClient) CreateProducer(options mqwrapper.ProducerOptions) (mqwrapper.Producer, error) { +func (nc *nmqClient) CreateProducer(options common.ProducerOptions) (mqwrapper.Producer, error) { start := timerecord.NewTimeRecorder("create producer") metrics.MsgStreamOpCounter.WithLabelValues(metrics.CreateProducerLabel, metrics.TotalLabel).Inc() @@ -151,9 +152,9 @@ func (nc *nmqClient) Subscribe(options mqwrapper.ConsumerOptions) (mqwrapper.Con position := options.SubscriptionInitialPosition // TODO: should we only allow exclusive subscribe? Current logic allows double subscribe. switch position { - case mqwrapper.SubscriptionPositionLatest: + case common.SubscriptionPositionLatest: sub, err = js.ChanSubscribe(options.Topic, natsChan, nats.DeliverNew()) - case mqwrapper.SubscriptionPositionEarliest: + case common.SubscriptionPositionEarliest: sub, err = js.ChanSubscribe(options.Topic, natsChan, nats.DeliverAll()) } if err != nil { @@ -176,12 +177,12 @@ func (nc *nmqClient) Subscribe(options mqwrapper.ConsumerOptions) (mqwrapper.Con } // EarliestMessageID returns the earliest message ID for nmq client -func (nc *nmqClient) EarliestMessageID() mqwrapper.MessageID { +func (nc *nmqClient) EarliestMessageID() common.MessageID { return &nmqID{messageID: 1} } // StringToMsgID converts string id to MessageID -func (nc *nmqClient) StringToMsgID(id string) (mqwrapper.MessageID, error) { +func (nc *nmqClient) StringToMsgID(id string) (common.MessageID, error) { rID, err := strconv.ParseUint(id, 10, 64) if err != nil { return nil, errors.Wrap(err, "failed to parse string to MessageID") @@ -190,7 +191,7 @@ func (nc *nmqClient) StringToMsgID(id string) (mqwrapper.MessageID, error) { } // BytesToMsgID converts a byte array to messageID -func (nc *nmqClient) BytesToMsgID(id []byte) (mqwrapper.MessageID, error) { +func (nc *nmqClient) BytesToMsgID(id []byte) (common.MessageID, error) { rID := DeserializeNmqID(id) return &nmqID{messageID: rID}, nil } diff --git a/pkg/mq/msgstream/mqwrapper/nmq/nmq_client_test.go b/pkg/mq/msgstream/mqwrapper/nmq/nmq_client_test.go index c32b325e9828..f2e35b235047 100644 --- a/pkg/mq/msgstream/mqwrapper/nmq/nmq_client_test.go +++ b/pkg/mq/msgstream/mqwrapper/nmq/nmq_client_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" ) @@ -84,7 +85,7 @@ func TestNmqClient_CreateProducer(t *testing.T) { defer client.Close() topic := "TestNmqClient_CreateProducer" - proOpts := mqwrapper.ProducerOptions{Topic: topic} + proOpts := common.ProducerOptions{Topic: topic} producer, err := client.CreateProducer(proOpts) assert.NoError(t, err) assert.NotNil(t, producer) @@ -93,14 +94,14 @@ func TestNmqClient_CreateProducer(t *testing.T) { nmqProducer := producer.(*nmqProducer) assert.Equal(t, nmqProducer.Topic(), topic) - msg := &mqwrapper.ProducerMessage{ + msg := &common.ProducerMessage{ Payload: []byte{}, Properties: nil, } _, err = nmqProducer.Send(context.TODO(), msg) assert.NoError(t, err) - invalidOpts := mqwrapper.ProducerOptions{Topic: ""} + invalidOpts := common.ProducerOptions{Topic: ""} producer, e := client.CreateProducer(invalidOpts) assert.Nil(t, producer) assert.Error(t, e) @@ -112,13 +113,13 @@ func TestNmqClient_GetLatestMsg(t *testing.T) { defer client.Close() topic := fmt.Sprintf("t2GetLatestMsg-%d", rand.Int()) - proOpts := mqwrapper.ProducerOptions{Topic: topic} + proOpts := common.ProducerOptions{Topic: topic} producer, err := client.CreateProducer(proOpts) assert.NoError(t, err) defer producer.Close() for i := 0; i < 10; i++ { - msg := &mqwrapper.ProducerMessage{ + msg := &common.ProducerMessage{ Payload: []byte{byte(i)}, Properties: nil, } @@ -130,7 +131,7 @@ func TestNmqClient_GetLatestMsg(t *testing.T) { consumerOpts := mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: subName, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, } @@ -140,7 +141,7 @@ func TestNmqClient_GetLatestMsg(t *testing.T) { expectLastMsg, err := consumer.GetLatestMsgID() assert.NoError(t, err) - var actualLastMsg mqwrapper.Message + var actualLastMsg common.Message ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() for i := 0; i < 10; i++ { @@ -186,7 +187,7 @@ func TestNmqClient_Subscribe(t *testing.T) { defer client.Close() topic := "TestNmqClient_Subscribe" - proOpts := mqwrapper.ProducerOptions{Topic: topic} + proOpts := common.ProducerOptions{Topic: topic} producer, err := client.CreateProducer(proOpts) assert.NoError(t, err) assert.NotNil(t, producer) @@ -196,7 +197,7 @@ func TestNmqClient_Subscribe(t *testing.T) { consumerOpts := mqwrapper.ConsumerOptions{ Topic: "", SubscriptionName: subName, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, } @@ -211,7 +212,7 @@ func TestNmqClient_Subscribe(t *testing.T) { defer consumer.Close() assert.Equal(t, consumer.Subscription(), subName) - msg := &mqwrapper.ProducerMessage{ + msg := &common.ProducerMessage{ Payload: []byte{1}, Properties: nil, } diff --git a/pkg/mq/msgstream/mqwrapper/nmq/nmq_consumer.go b/pkg/mq/msgstream/mqwrapper/nmq/nmq_consumer.go index 43c4dcee49df..6830f96e9894 100644 --- a/pkg/mq/msgstream/mqwrapper/nmq/nmq_consumer.go +++ b/pkg/mq/msgstream/mqwrapper/nmq/nmq_consumer.go @@ -25,6 +25,7 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/merr" ) @@ -37,7 +38,7 @@ type Consumer struct { topic string groupName string natsChan chan *nats.Msg - msgChan chan mqwrapper.Message + msgChan chan common.Message closeChan chan struct{} once sync.Once closeOnce sync.Once @@ -51,7 +52,7 @@ func (nc *Consumer) Subscription() string { } // Chan returns a channel to read messages from natsmq -func (nc *Consumer) Chan() <-chan mqwrapper.Message { +func (nc *Consumer) Chan() <-chan common.Message { if err := nc.closed(); err != nil { panic(err) } @@ -62,7 +63,7 @@ func (nc *Consumer) Chan() <-chan mqwrapper.Message { } if nc.msgChan == nil { nc.once.Do(func() { - nc.msgChan = make(chan mqwrapper.Message, 256) + nc.msgChan = make(chan common.Message, 256) nc.wg.Add(1) go func() { defer nc.wg.Done() @@ -89,7 +90,7 @@ func (nc *Consumer) Chan() <-chan mqwrapper.Message { } // Seek is used to seek the position in natsmq topic -func (nc *Consumer) Seek(id mqwrapper.MessageID, inclusive bool) error { +func (nc *Consumer) Seek(id common.MessageID, inclusive bool) error { if err := nc.closed(); err != nil { return err } @@ -112,7 +113,7 @@ func (nc *Consumer) Seek(id mqwrapper.MessageID, inclusive bool) error { } // Ack is used to ask a natsmq message -func (nc *Consumer) Ack(message mqwrapper.Message) { +func (nc *Consumer) Ack(message common.Message) { if err := message.(*nmqMessage).raw.Ack(); err != nil { log.Warn("failed to ack message of nmq", zap.String("topic", message.Topic()), zap.Reflect("msgID", message.ID())) } @@ -133,7 +134,7 @@ func (nc *Consumer) Close() { } // GetLatestMsgID returns the ID of the most recent message processed by the consumer. -func (nc *Consumer) GetLatestMsgID() (mqwrapper.MessageID, error) { +func (nc *Consumer) GetLatestMsgID() (common.MessageID, error) { if err := nc.closed(); err != nil { return nil, err } diff --git a/pkg/mq/msgstream/mqwrapper/nmq/nmq_consumer_test.go b/pkg/mq/msgstream/mqwrapper/nmq/nmq_consumer_test.go index 742549387124..bc3652ff71d8 100644 --- a/pkg/mq/msgstream/mqwrapper/nmq/nmq_consumer_test.go +++ b/pkg/mq/msgstream/mqwrapper/nmq/nmq_consumer_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" ) @@ -34,14 +35,14 @@ func TestNatsConsumer_Subscription(t *testing.T) { defer client.Close() topic := t.Name() - proOpts := mqwrapper.ProducerOptions{Topic: topic} + proOpts := common.ProducerOptions{Topic: topic} _, err = client.CreateProducer(proOpts) assert.NoError(t, err) consumer, err := client.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) assert.NoError(t, err) @@ -71,7 +72,7 @@ func Test_BadLatestMessageID(t *testing.T) { consumer, err := client.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) assert.NoError(t, err) @@ -87,13 +88,13 @@ func TestComsumeMessage(t *testing.T) { defer client.Close() topic := t.Name() - p, err := client.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + p, err := client.CreateProducer(common.ProducerOptions{Topic: topic}) assert.NoError(t, err) c, err := client.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) assert.NoError(t, err) @@ -101,7 +102,7 @@ func TestComsumeMessage(t *testing.T) { msg := []byte("test the first message") prop := map[string]string{"k1": "v1", "k2": "v2"} - _, err = p.Send(context.Background(), &mqwrapper.ProducerMessage{ + _, err = p.Send(context.Background(), &common.ProducerMessage{ Payload: msg, Properties: prop, }) @@ -121,7 +122,7 @@ func TestComsumeMessage(t *testing.T) { msg2 := []byte("test the second message") prop2 := map[string]string{"k1": "v3", "k4": "v4"} - _, err = p.Send(context.Background(), &mqwrapper.ProducerMessage{ + _, err = p.Send(context.Background(), &common.ProducerMessage{ Payload: msg2, Properties: prop2, }) @@ -151,7 +152,7 @@ func TestNatsConsumer_Close(t *testing.T) { c, err := client.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) assert.NoError(t, err) @@ -179,7 +180,7 @@ func TestNatsClientErrorOnUnsubscribeTwice(t *testing.T) { consumer, err := client.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) assert.NoError(t, err) @@ -201,7 +202,7 @@ func TestCheckTopicValid(t *testing.T) { consumer, err := client.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) assert.NoError(t, err) @@ -219,11 +220,11 @@ func TestCheckTopicValid(t *testing.T) { assert.Error(t, err) // not empty topic can pass - pub, err := client.CreateProducer(mqwrapper.ProducerOptions{ + pub, err := client.CreateProducer(common.ProducerOptions{ Topic: topic, }) assert.NoError(t, err) - _, err = pub.Send(context.TODO(), &mqwrapper.ProducerMessage{ + _, err = pub.Send(context.TODO(), &common.ProducerMessage{ Payload: []byte("123123123"), }) assert.NoError(t, err) @@ -236,7 +237,7 @@ func TestCheckTopicValid(t *testing.T) { assert.Error(t, err) } -func newTestConsumer(t *testing.T, topic string, position mqwrapper.SubscriptionInitialPosition) (mqwrapper.Consumer, error) { +func newTestConsumer(t *testing.T, topic string, position common.SubscriptionInitialPosition) (mqwrapper.Consumer, error) { client, err := createNmqClient() assert.NoError(t, err) return client.Subscribe(mqwrapper.ConsumerOptions{ @@ -250,14 +251,14 @@ func newTestConsumer(t *testing.T, topic string, position mqwrapper.Subscription func newProducer(t *testing.T, topic string) (*nmqClient, mqwrapper.Producer) { client, err := createNmqClient() assert.NoError(t, err) - producer, err := client.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + producer, err := client.CreateProducer(common.ProducerOptions{Topic: topic}) assert.NoError(t, err) return client, producer } func process(t *testing.T, msgs []string, p mqwrapper.Producer) { for _, msg := range msgs { - _, err := p.Send(context.Background(), &mqwrapper.ProducerMessage{ + _, err := p.Send(context.Background(), &common.ProducerMessage{ Payload: []byte(msg), Properties: map[string]string{}, }) @@ -271,13 +272,13 @@ func TestNmqConsumer_GetLatestMsgID(t *testing.T) { defer client.Close() topic := t.Name() - p, err := client.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + p, err := client.CreateProducer(common.ProducerOptions{Topic: topic}) assert.NoError(t, err) c, err := client.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) assert.NoError(t, err) @@ -300,7 +301,7 @@ func TestNmqConsumer_ConsumeFromLatest(t *testing.T) { defer client.Close() topic := t.Name() - p, err := client.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + p, err := client.CreateProducer(common.ProducerOptions{Topic: topic}) assert.NoError(t, err) msgs := []string{"111", "222", "333"} @@ -309,7 +310,7 @@ func TestNmqConsumer_ConsumeFromLatest(t *testing.T) { c, err := client.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionLatest, + SubscriptionInitialPosition: common.SubscriptionPositionLatest, BufSize: 1024, }) assert.NoError(t, err) @@ -330,7 +331,7 @@ func TestNmqConsumer_ConsumeFromEarliest(t *testing.T) { defer client.Close() topic := t.Name() - p, err := client.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + p, err := client.CreateProducer(common.ProducerOptions{Topic: topic}) assert.NoError(t, err) msgs := []string{"111", "222"} @@ -339,7 +340,7 @@ func TestNmqConsumer_ConsumeFromEarliest(t *testing.T) { c, err := client.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) assert.NoError(t, err) @@ -356,7 +357,7 @@ func TestNmqConsumer_ConsumeFromEarliest(t *testing.T) { c2, err := client.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) assert.NoError(t, err) @@ -381,7 +382,7 @@ func TestNatsConsumer_SeekExclusive(t *testing.T) { process(t, msgs, p) msgID := &nmqID{messageID: 2} - consumer, err := newTestConsumer(t, topic, mqwrapper.SubscriptionPositionUnknown) + consumer, err := newTestConsumer(t, topic, common.SubscriptionPositionUnknown) assert.NoError(t, err) defer consumer.Close() err = consumer.Seek(msgID, false) @@ -404,7 +405,7 @@ func TestNatsConsumer_SeekInclusive(t *testing.T) { process(t, msgs, p) msgID := &nmqID{messageID: 2} - consumer, err := newTestConsumer(t, topic, mqwrapper.SubscriptionPositionUnknown) + consumer, err := newTestConsumer(t, topic, common.SubscriptionPositionUnknown) assert.NoError(t, err) defer consumer.Close() err = consumer.Seek(msgID, true) @@ -423,7 +424,7 @@ func TestNatsConsumer_NoDoubleSeek(t *testing.T) { defer p.Close() msgID := &nmqID{messageID: 2} - consumer, err := newTestConsumer(t, topic, mqwrapper.SubscriptionPositionUnknown) + consumer, err := newTestConsumer(t, topic, common.SubscriptionPositionUnknown) assert.NoError(t, err) defer consumer.Close() err = consumer.Seek(msgID, true) @@ -441,7 +442,7 @@ func TestNatsConsumer_ChanWithNoAssign(t *testing.T) { msgs := []string{"111", "222", "333", "444", "555"} process(t, msgs, p) - consumer, err := newTestConsumer(t, topic, mqwrapper.SubscriptionPositionUnknown) + consumer, err := newTestConsumer(t, topic, common.SubscriptionPositionUnknown) assert.NoError(t, err) defer consumer.Close() diff --git a/pkg/mq/msgstream/mqwrapper/nmq/nmq_id.go b/pkg/mq/msgstream/mqwrapper/nmq/nmq_id.go index a575060a121b..37571d40455d 100644 --- a/pkg/mq/msgstream/mqwrapper/nmq/nmq_id.go +++ b/pkg/mq/msgstream/mqwrapper/nmq/nmq_id.go @@ -18,7 +18,7 @@ package nmq import ( "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" ) // MessageIDType is a type alias for server.UniqueID that represents the ID of a Nmq message. @@ -30,10 +30,10 @@ type nmqID struct { } // Check if nmqID implements MessageID interface -var _ mqwrapper.MessageID = &nmqID{} +var _ mqcommon.MessageID = &nmqID{} // NewNmqID creates and returns a new instance of the nmqID struct with the given MessageID. -func NewNmqID(id MessageIDType) mqwrapper.MessageID { +func NewNmqID(id MessageIDType) mqcommon.MessageID { return &nmqID{ messageID: id, } diff --git a/pkg/mq/msgstream/mqwrapper/nmq/nmq_message.go b/pkg/mq/msgstream/mqwrapper/nmq/nmq_message.go index 833245aa8586..7f6d42981b2b 100644 --- a/pkg/mq/msgstream/mqwrapper/nmq/nmq_message.go +++ b/pkg/mq/msgstream/mqwrapper/nmq/nmq_message.go @@ -22,12 +22,12 @@ import ( "github.com/nats-io/nats.go" "go.uber.org/zap" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) // Check nmqMessage implements ConsumerMessage var ( - _ mqwrapper.Message = (*nmqMessage)(nil) + _ common.Message = (*nmqMessage)(nil) ) // nmqMessage wraps the message for natsmq @@ -63,7 +63,7 @@ func (nm *nmqMessage) Payload() []byte { } // ID returns the id of natsmq message -func (nm *nmqMessage) ID() mqwrapper.MessageID { +func (nm *nmqMessage) ID() common.MessageID { if nm.meta == nil { var err error // raw is always a jetstream message, should never fail. diff --git a/pkg/mq/msgstream/mqwrapper/nmq/nmq_producer.go b/pkg/mq/msgstream/mqwrapper/nmq/nmq_producer.go index 26c627e5aa1f..69a99538242f 100644 --- a/pkg/mq/msgstream/mqwrapper/nmq/nmq_producer.go +++ b/pkg/mq/msgstream/mqwrapper/nmq/nmq_producer.go @@ -24,6 +24,7 @@ import ( "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/timerecord" ) @@ -42,7 +43,7 @@ func (np *nmqProducer) Topic() string { } // Send send the producer messages to natsmq -func (np *nmqProducer) Send(ctx context.Context, message *mqwrapper.ProducerMessage) (mqwrapper.MessageID, error) { +func (np *nmqProducer) Send(ctx context.Context, message *common.ProducerMessage) (common.MessageID, error) { start := timerecord.NewTimeRecorder("send msg to stream") metrics.MsgStreamOpCounter.WithLabelValues(metrics.SendMsgLabel, metrics.TotalLabel).Inc() diff --git a/pkg/mq/msgstream/mqwrapper/nmq/nmq_producer_test.go b/pkg/mq/msgstream/mqwrapper/nmq/nmq_producer_test.go index 44545340a9e7..119e1ef44e62 100644 --- a/pkg/mq/msgstream/mqwrapper/nmq/nmq_producer_test.go +++ b/pkg/mq/msgstream/mqwrapper/nmq/nmq_producer_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) func TestNatsMQProducer(t *testing.T) { @@ -30,7 +30,7 @@ func TestNatsMQProducer(t *testing.T) { assert.NoError(t, err) defer c.Close() topic := t.Name() - pOpts := mqwrapper.ProducerOptions{Topic: topic} + pOpts := common.ProducerOptions{Topic: topic} // Check Topic() p, err := c.CreateProducer(pOpts) @@ -38,7 +38,7 @@ func TestNatsMQProducer(t *testing.T) { assert.Equal(t, p.(*nmqProducer).Topic(), topic) // Check Send() - msg := &mqwrapper.ProducerMessage{ + msg := &common.ProducerMessage{ Payload: []byte{}, Properties: map[string]string{}, } diff --git a/pkg/mq/msgstream/mqwrapper/producer.go b/pkg/mq/msgstream/mqwrapper/producer.go index caf43688d977..12579de297c9 100644 --- a/pkg/mq/msgstream/mqwrapper/producer.go +++ b/pkg/mq/msgstream/mqwrapper/producer.go @@ -16,26 +16,11 @@ package mqwrapper -import "context" +import ( + "context" -// ProducerOptions contains the options of a producer -type ProducerOptions struct { - // The topic that this Producer will publish - Topic string - - // Enable compression - // For Pulsar, this enables ZSTD compression with default compression level - EnableCompression bool -} - -// ProducerMessage contains the messages of a producer -type ProducerMessage struct { - // Payload get the payload of the message - Payload []byte - // Properties are application defined key/value pairs that will be attached to the message. - // Return the properties attached to the message. - Properties map[string]string -} + "github.com/milvus-io/milvus/pkg/mq/common" +) // Producer is the interface that provides operations of producer type Producer interface { @@ -43,7 +28,7 @@ type Producer interface { // Topic() string // publish a message - Send(ctx context.Context, message *ProducerMessage) (MessageID, error) + Send(ctx context.Context, message *common.ProducerMessage) (common.MessageID, error) Close() } diff --git a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_client.go b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_client.go index 8e4e75ea6cdc..f5918870b801 100644 --- a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_client.go +++ b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_client.go @@ -29,6 +29,7 @@ import ( "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/timerecord" ) @@ -65,7 +66,7 @@ func NewClient(tenant string, namespace string, opts pulsar.ClientOptions) (*pul } // CreateProducer create a pulsar producer from options -func (pc *pulsarClient) CreateProducer(options mqwrapper.ProducerOptions) (mqwrapper.Producer, error) { +func (pc *pulsarClient) CreateProducer(options mqcommon.ProducerOptions) (mqwrapper.Producer, error) { start := timerecord.NewTimeRecorder("create producer") metrics.MsgStreamOpCounter.WithLabelValues(metrics.CreateProducerLabel, metrics.TotalLabel).Inc() @@ -125,7 +126,7 @@ func (pc *pulsarClient) Subscribe(options mqwrapper.ConsumerOptions) (mqwrapper. pConsumer := &Consumer{c: consumer, closeCh: make(chan struct{})} // prevent seek to earliest patch applied when using latest position options - if options.SubscriptionInitialPosition == mqwrapper.SubscriptionPositionLatest { + if options.SubscriptionInitialPosition == mqcommon.SubscriptionPositionLatest { pConsumer.AtLatest = true } @@ -162,13 +163,13 @@ func NewAdminClient(address, authPlugin, authParams string) (pulsarctl.Client, e } // EarliestMessageID returns the earliest message id -func (pc *pulsarClient) EarliestMessageID() mqwrapper.MessageID { +func (pc *pulsarClient) EarliestMessageID() mqcommon.MessageID { msgID := pulsar.EarliestMessageID() return &pulsarID{messageID: msgID} } // StringToMsgID converts the string id to MessageID type -func (pc *pulsarClient) StringToMsgID(id string) (mqwrapper.MessageID, error) { +func (pc *pulsarClient) StringToMsgID(id string) (mqcommon.MessageID, error) { pID, err := stringToMsgID(id) if err != nil { return nil, err @@ -177,7 +178,7 @@ func (pc *pulsarClient) StringToMsgID(id string) (mqwrapper.MessageID, error) { } // BytesToMsgID converts []byte id to MessageID type -func (pc *pulsarClient) BytesToMsgID(id []byte) (mqwrapper.MessageID, error) { +func (pc *pulsarClient) BytesToMsgID(id []byte) (mqcommon.MessageID, error) { pID, err := DeserializePulsarMsgID(id) if err != nil { return nil, err diff --git a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_client_test.go b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_client_test.go index 2bf265d832b7..9d8910479bd6 100644 --- a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_client_test.go +++ b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_client_test.go @@ -35,6 +35,7 @@ import ( "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/log" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/retry" @@ -77,14 +78,14 @@ func BytesToInt(b []byte) int { } func Produce(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, arr []int) { - producer, err := pc.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + producer, err := pc.CreateProducer(mqcommon.ProducerOptions{Topic: topic}) assert.NoError(t, err) assert.NotNil(t, producer) log.Info("Produce start") for _, v := range arr { - msg := &mqwrapper.ProducerMessage{ + msg := &mqcommon.ProducerMessage{ Payload: IntToBytes(v), Properties: map[string]string{}, } @@ -96,7 +97,7 @@ func Produce(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, log.Info("Produce done") } -func VerifyMessage(t *testing.T, msg mqwrapper.Message) { +func VerifyMessage(t *testing.T, msg mqcommon.Message) { pload := BytesToInt(msg.Payload()) log.Info("RECV", zap.Any("v", pload)) pm := msg.(*pulsarMessage) @@ -108,12 +109,12 @@ func VerifyMessage(t *testing.T, msg mqwrapper.Message) { } // Consume1 will consume random messages and record the last MessageID it received -func Consume1(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, subName string, c chan mqwrapper.MessageID, total *int) { +func Consume1(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, subName string, c chan mqcommon.MessageID, total *int) { consumer, err := pc.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) assert.NoError(t, err) assert.NotNil(t, consumer) @@ -125,7 +126,7 @@ func Consume1(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, rand.Seed(time.Now().UnixNano()) cnt := 1 + rand.Int()%5 - var msg mqwrapper.Message + var msg mqcommon.Message for i := 0; i < cnt; i++ { select { case <-ctx.Done(): @@ -145,12 +146,12 @@ func Consume1(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, } // Consume2 will consume messages from specified MessageID -func Consume2(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, subName string, msgID mqwrapper.MessageID, total *int) { +func Consume2(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, subName string, msgID mqcommon.MessageID, total *int) { consumer, err := pc.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) assert.NoError(t, err) assert.NotNil(t, consumer) @@ -184,7 +185,7 @@ func Consume3(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) assert.NoError(t, err) assert.NotNil(t, consumer) @@ -217,7 +218,7 @@ func TestPulsarClient_Consume1(t *testing.T) { topic := fmt.Sprintf("test-topic-%d", rand.Int()) subName := fmt.Sprintf("test-subname-%d", rand.Int()) arr := []int{111, 222, 333, 444, 555, 666, 777} - c := make(chan mqwrapper.MessageID, 1) + c := make(chan mqcommon.MessageID, 1) ctx, cancel := context.WithCancel(context.Background()) @@ -256,7 +257,7 @@ func TestPulsarClient_Consume1(t *testing.T) { log.Info("main done") } -func Consume21(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, subName string, c chan mqwrapper.MessageID, total *int) { +func Consume21(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, subName string, c chan mqcommon.MessageID, total *int) { consumer, err := pc.client.Subscribe(pulsar.ConsumerOptions{ Topic: topic, SubscriptionName: subName, @@ -294,7 +295,7 @@ func Consume21(ctx context.Context, t *testing.T, pc *pulsarClient, topic string } // Consume2 will consume messages from specified MessageID -func Consume22(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, subName string, msgID mqwrapper.MessageID, total *int) { +func Consume22(ctx context.Context, t *testing.T, pc *pulsarClient, topic string, subName string, msgID mqcommon.MessageID, total *int) { consumer, err := pc.client.Subscribe(pulsar.ConsumerOptions{ Topic: topic, SubscriptionName: subName, @@ -368,7 +369,7 @@ func TestPulsarClient_Consume2(t *testing.T) { topic := fmt.Sprintf("test-topic-%d", rand.Int()) subName := fmt.Sprintf("test-subname-%d", rand.Int()) arr := []int{111, 222, 333, 444, 555, 666, 777} - c := make(chan mqwrapper.MessageID, 1) + c := make(chan mqcommon.MessageID, 1) ctx, cancel := context.WithCancel(context.Background()) @@ -419,16 +420,16 @@ func TestPulsarClient_SeekPosition(t *testing.T) { topic := fmt.Sprintf("test-topic-%d", rand.Int()) subName := fmt.Sprintf("test-subname-%d", rand.Int()) - producer, err := pc.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + producer, err := pc.CreateProducer(mqcommon.ProducerOptions{Topic: topic}) assert.NoError(t, err) assert.NotNil(t, producer) log.Info("Produce start") - ids := []mqwrapper.MessageID{} + ids := []mqcommon.MessageID{} arr1 := []int{1, 2, 3} arr2 := []string{"1", "2", "3"} for k, v := range arr1 { - msg := &mqwrapper.ProducerMessage{ + msg := &mqcommon.ProducerMessage{ Payload: IntToBytes(v), Properties: map[string]string{ common.TraceIDKey: arr2[k], @@ -497,7 +498,7 @@ func TestPulsarClient_SeekLatest(t *testing.T) { topic := fmt.Sprintf("test-topic-%d", rand.Int()) subName := fmt.Sprintf("test-subname-%d", rand.Int()) - producer, err := pc.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + producer, err := pc.CreateProducer(mqcommon.ProducerOptions{Topic: topic}) assert.NoError(t, err) assert.NotNil(t, producer) @@ -505,7 +506,7 @@ func TestPulsarClient_SeekLatest(t *testing.T) { arr := []int{1, 2, 3} for _, v := range arr { - msg := &mqwrapper.ProducerMessage{ + msg := &mqcommon.ProducerMessage{ Payload: IntToBytes(v), Properties: map[string]string{}, } @@ -539,7 +540,7 @@ func TestPulsarClient_SeekLatest(t *testing.T) { loop = false case <-ticker.C: log.Info("after 2 seconds") - msg := &mqwrapper.ProducerMessage{ + msg := &mqcommon.ProducerMessage{ Payload: IntToBytes(4), Properties: map[string]string{}, } @@ -681,7 +682,7 @@ func TestPulsarClient_WithTenantAndNamespace(t *testing.T) { pulsarAddress := getPulsarAddress() pc, err := NewClient(tenant, namespace, pulsar.ClientOptions{URL: pulsarAddress}) assert.NoError(t, err) - producer, err := pc.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + producer, err := pc.CreateProducer(mqcommon.ProducerOptions{Topic: topic}) defer producer.Close() assert.NoError(t, err) assert.NotNil(t, producer) @@ -694,7 +695,7 @@ func TestPulsarClient_WithTenantAndNamespace(t *testing.T) { Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) defer consumer.Close() assert.NoError(t, err) @@ -712,7 +713,7 @@ func TestPulsarCtl(t *testing.T) { Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) assert.NoError(t, err) assert.NotNil(t, consumer) @@ -722,7 +723,7 @@ func TestPulsarCtl(t *testing.T) { Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) assert.Error(t, err) @@ -731,7 +732,7 @@ func TestPulsarCtl(t *testing.T) { Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) assert.Error(t, err) @@ -761,7 +762,7 @@ func TestPulsarCtl(t *testing.T) { Topic: topic, SubscriptionName: subName, BufSize: 1024, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: mqcommon.SubscriptionPositionEarliest, }) assert.NoError(t, err) assert.NotNil(t, consumer2) diff --git a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_consumer.go b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_consumer.go index 9a644a6e36a7..9ab566476f29 100644 --- a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_consumer.go +++ b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_consumer.go @@ -27,6 +27,7 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/retry" ) @@ -34,7 +35,7 @@ import ( // Consumer consumes from pulsar type Consumer struct { c pulsar.Consumer - msgChannel chan mqwrapper.Message + msgChannel chan common.Message hasSeek bool AtLatest bool closeCh chan struct{} @@ -49,10 +50,10 @@ func (pc *Consumer) Subscription() string { } // Chan returns a message channel -func (pc *Consumer) Chan() <-chan mqwrapper.Message { +func (pc *Consumer) Chan() <-chan common.Message { if pc.msgChannel == nil { pc.once.Do(func() { - pc.msgChannel = make(chan mqwrapper.Message, 256) + pc.msgChannel = make(chan common.Message, 256) // this part handles msgstream expectation when the consumer is not seeked // pulsar's default behavior is setting postition to the earliest pointer when client of the same subscription pointer is not acked // yet, our message stream is to setting to the very start point of the topic @@ -97,7 +98,7 @@ func (pc *Consumer) Chan() <-chan mqwrapper.Message { // Seek seek consume position to the pointed messageID, // the pointed messageID will be consumed after the seek in pulsar -func (pc *Consumer) Seek(id mqwrapper.MessageID, inclusive bool) error { +func (pc *Consumer) Seek(id common.MessageID, inclusive bool) error { messageID := id.(*pulsarID).messageID err := pc.c.Seek(messageID) if err == nil { @@ -109,7 +110,7 @@ func (pc *Consumer) Seek(id mqwrapper.MessageID, inclusive bool) error { } // Ack the consumption of a single message -func (pc *Consumer) Ack(message mqwrapper.Message) { +func (pc *Consumer) Ack(message common.Message) { pm := message.(*pulsarMessage) pc.c.Ack(pm.msg) } @@ -151,7 +152,7 @@ func (pc *Consumer) Close() { }) } -func (pc *Consumer) GetLatestMsgID() (mqwrapper.MessageID, error) { +func (pc *Consumer) GetLatestMsgID() (common.MessageID, error) { msgID, err := pc.c.GetLastMessageID(pc.c.Name(), mqwrapper.DefaultPartitionIdx) return &pulsarID{messageID: msgID}, err } diff --git a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_consumer_test.go b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_consumer_test.go index cb04a7c3fd97..6f541a137f6a 100644 --- a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_consumer_test.go +++ b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_consumer_test.go @@ -28,7 +28,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + mqcommon "github.com/milvus-io/milvus/pkg/mq/common" ) func TestPulsarConsumer_Subscription(t *testing.T) { @@ -41,7 +41,7 @@ func TestPulsarConsumer_Subscription(t *testing.T) { consumer, err := pc.client.Subscribe(pulsar.ConsumerOptions{ Topic: "Topic", SubscriptionName: "SubName", - SubscriptionInitialPosition: pulsar.SubscriptionInitialPosition(mqwrapper.SubscriptionPositionEarliest), + SubscriptionInitialPosition: pulsar.SubscriptionInitialPosition(mqcommon.SubscriptionPositionEarliest), MessageChannel: receiveChannel, }) assert.NoError(t, err) @@ -74,21 +74,21 @@ func TestComsumeCompressedMessage(t *testing.T) { Topic: "TestTopics", SubscriptionName: "SubName", Type: pulsar.Exclusive, - SubscriptionInitialPosition: pulsar.SubscriptionInitialPosition(mqwrapper.SubscriptionPositionEarliest), + SubscriptionInitialPosition: pulsar.SubscriptionInitialPosition(mqcommon.SubscriptionPositionEarliest), MessageChannel: receiveChannel, }) assert.NoError(t, err) defer consumer.Close() - producer, err := pc.CreateProducer(mqwrapper.ProducerOptions{Topic: "TestTopics"}) + producer, err := pc.CreateProducer(mqcommon.ProducerOptions{Topic: "TestTopics"}) assert.NoError(t, err) - compressProducer, err := pc.CreateProducer(mqwrapper.ProducerOptions{Topic: "TestTopics", EnableCompression: true}) + compressProducer, err := pc.CreateProducer(mqcommon.ProducerOptions{Topic: "TestTopics", EnableCompression: true}) assert.NoError(t, err) msg := []byte("test message") compressedMsg := []byte("test compressed message") traceValue := "test compressed message id" - _, err = producer.Send(context.Background(), &mqwrapper.ProducerMessage{ + _, err = producer.Send(context.Background(), &mqcommon.ProducerMessage{ Payload: msg, Properties: map[string]string{}, }) @@ -98,7 +98,7 @@ func TestComsumeCompressedMessage(t *testing.T) { consumer.Ack(recvMsg) assert.Equal(t, msg, recvMsg.Payload()) - _, err = compressProducer.Send(context.Background(), &mqwrapper.ProducerMessage{ + _, err = compressProducer.Send(context.Background(), &mqcommon.ProducerMessage{ Payload: compressedMsg, Properties: map[string]string{ common.TraceIDKey: traceValue, @@ -124,7 +124,7 @@ func TestPulsarConsumer_Close(t *testing.T) { consumer, err := pc.client.Subscribe(pulsar.ConsumerOptions{ Topic: "Topic-1", SubscriptionName: "SubName-1", - SubscriptionInitialPosition: pulsar.SubscriptionInitialPosition(mqwrapper.SubscriptionPositionEarliest), + SubscriptionInitialPosition: pulsar.SubscriptionInitialPosition(mqcommon.SubscriptionPositionEarliest), MessageChannel: receiveChannel, }) assert.NoError(t, err) @@ -228,7 +228,7 @@ func TestCheckPreTopicValid(t *testing.T) { consumer, err := pc.client.Subscribe(pulsar.ConsumerOptions{ Topic: "Topic-1", SubscriptionName: "SubName-1", - SubscriptionInitialPosition: pulsar.SubscriptionInitialPosition(mqwrapper.SubscriptionPositionEarliest), + SubscriptionInitialPosition: pulsar.SubscriptionInitialPosition(mqcommon.SubscriptionPositionEarliest), MessageChannel: receiveChannel, }) assert.NoError(t, err) diff --git a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_id.go b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_id.go index b6215b409d3c..f534e17da56a 100644 --- a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_id.go +++ b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_id.go @@ -21,7 +21,7 @@ import ( "github.com/apache/pulsar-client-go/pulsar" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) type pulsarID struct { @@ -29,7 +29,7 @@ type pulsarID struct { } // Check if pulsarID implements and MessageID interface -var _ mqwrapper.MessageID = &pulsarID{} +var _ common.MessageID = &pulsarID{} func (pid *pulsarID) Serialize() []byte { return pid.messageID.Serialize() diff --git a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_message.go b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_message.go index 5a6816418f41..27cbfeb08a35 100644 --- a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_message.go +++ b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_message.go @@ -19,11 +19,11 @@ package pulsar import ( "github.com/apache/pulsar-client-go/pulsar" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) // Check pulsarMessage implements ConsumerMessage -var _ mqwrapper.Message = (*pulsarMessage)(nil) +var _ common.Message = (*pulsarMessage)(nil) type pulsarMessage struct { msg pulsar.Message @@ -41,7 +41,7 @@ func (pm *pulsarMessage) Payload() []byte { return pm.msg.Payload() } -func (pm *pulsarMessage) ID() mqwrapper.MessageID { +func (pm *pulsarMessage) ID() common.MessageID { id := pm.msg.ID() pid := &pulsarID{messageID: id} return pid diff --git a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_producer.go b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_producer.go index 0773d91483b2..fcd95ca16354 100644 --- a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_producer.go +++ b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_producer.go @@ -22,6 +22,7 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/timerecord" ) @@ -38,7 +39,7 @@ func (pp *pulsarProducer) Topic() string { return pp.p.Topic() } -func (pp *pulsarProducer) Send(ctx context.Context, message *mqwrapper.ProducerMessage) (mqwrapper.MessageID, error) { +func (pp *pulsarProducer) Send(ctx context.Context, message *common.ProducerMessage) (common.MessageID, error) { start := timerecord.NewTimeRecorder("send msg to stream") metrics.MsgStreamOpCounter.WithLabelValues(metrics.SendMsgLabel, metrics.TotalLabel).Inc() diff --git a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_producer_test.go b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_producer_test.go index 307eaf23960c..ebace99df14f 100644 --- a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_producer_test.go +++ b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_producer_test.go @@ -23,7 +23,7 @@ import ( "github.com/apache/pulsar-client-go/pulsar" "github.com/stretchr/testify/assert" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) func TestPulsarProducer(t *testing.T) { @@ -34,7 +34,7 @@ func TestPulsarProducer(t *testing.T) { assert.NotNil(t, pc) topic := "TEST" - producer, err := pc.CreateProducer(mqwrapper.ProducerOptions{Topic: topic}) + producer, err := pc.CreateProducer(common.ProducerOptions{Topic: topic}) assert.NoError(t, err) assert.NotNil(t, producer) @@ -43,7 +43,7 @@ func TestPulsarProducer(t *testing.T) { assert.NoError(t, err) assert.Equal(t, pulsarProd.Topic(), fullTopicName) - msg := &mqwrapper.ProducerMessage{ + msg := &common.ProducerMessage{ Payload: []byte{}, Properties: map[string]string{}, } diff --git a/internal/mq/msgstream/mqwrapper/rmq/rmq_client.go b/pkg/mq/msgstream/mqwrapper/rmq/rmq_client.go similarity index 88% rename from internal/mq/msgstream/mqwrapper/rmq/rmq_client.go rename to pkg/mq/msgstream/mqwrapper/rmq/rmq_client.go index 6de1d3ae6850..ce7f1d7cc95f 100644 --- a/internal/mq/msgstream/mqwrapper/rmq/rmq_client.go +++ b/pkg/mq/msgstream/mqwrapper/rmq/rmq_client.go @@ -23,10 +23,11 @@ import ( "github.com/cockroachdb/errors" "go.uber.org/zap" - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/client" - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/client" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/timerecord" ) @@ -34,6 +35,8 @@ import ( // nmqClient implements mqwrapper.Client. var _ mqwrapper.Client = &rmqClient{} +// var InitRocksMQ = server.InitRocksMQ + // rmqClient contains a rocksmq client type rmqClient struct { client client.Client @@ -55,7 +58,7 @@ func NewClient(opts client.Options) (*rmqClient, error) { } // CreateProducer creates a producer for rocksmq client -func (rc *rmqClient) CreateProducer(options mqwrapper.ProducerOptions) (mqwrapper.Producer, error) { +func (rc *rmqClient) CreateProducer(options common.ProducerOptions) (mqwrapper.Producer, error) { start := timerecord.NewTimeRecorder("create producer") metrics.MsgStreamOpCounter.WithLabelValues(metrics.CreateProducerLabel, metrics.TotalLabel).Inc() @@ -84,7 +87,7 @@ func (rc *rmqClient) Subscribe(options mqwrapper.ConsumerOptions) (mqwrapper.Con log.Warn("unexpected subscription consumer options", zap.Error(err)) return nil, err } - receiveChannel := make(chan mqwrapper.Message, options.BufSize) + receiveChannel := make(chan common.Message, options.BufSize) cli, err := rc.client.Subscribe(client.ConsumerOptions{ Topic: options.Topic, @@ -106,13 +109,13 @@ func (rc *rmqClient) Subscribe(options mqwrapper.ConsumerOptions) (mqwrapper.Con } // EarliestMessageID returns the earliest message ID for rmq client -func (rc *rmqClient) EarliestMessageID() mqwrapper.MessageID { +func (rc *rmqClient) EarliestMessageID() common.MessageID { rID := client.EarliestMessageID() return &server.RmqID{MessageID: rID} } // StringToMsgID converts string id to MessageID -func (rc *rmqClient) StringToMsgID(id string) (mqwrapper.MessageID, error) { +func (rc *rmqClient) StringToMsgID(id string) (common.MessageID, error) { rID, err := strconv.ParseInt(id, 10, 64) if err != nil { return nil, err @@ -121,7 +124,7 @@ func (rc *rmqClient) StringToMsgID(id string) (mqwrapper.MessageID, error) { } // BytesToMsgID converts a byte array to messageID -func (rc *rmqClient) BytesToMsgID(id []byte) (mqwrapper.MessageID, error) { +func (rc *rmqClient) BytesToMsgID(id []byte) (common.MessageID, error) { rID := server.DeserializeRmqID(id) return &server.RmqID{MessageID: rID}, nil } diff --git a/internal/mq/msgstream/mqwrapper/rmq/rmq_client_test.go b/pkg/mq/msgstream/mqwrapper/rmq/rmq_client_test.go similarity index 82% rename from internal/mq/msgstream/mqwrapper/rmq/rmq_client_test.go rename to pkg/mq/msgstream/mqwrapper/rmq/rmq_client_test.go index 1248914485e1..87c007e25b4b 100644 --- a/internal/mq/msgstream/mqwrapper/rmq/rmq_client_test.go +++ b/pkg/mq/msgstream/mqwrapper/rmq/rmq_client_test.go @@ -24,14 +24,13 @@ import ( "testing" "time" - "github.com/apache/pulsar-client-go/pulsar" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - rocksmqimplclient "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/client" - rocksmqimplserver "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" + "github.com/milvus-io/milvus/pkg/mq/common" + client3 "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/client" + server2 "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" - pulsarwrapper "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/pulsar" "github.com/milvus-io/milvus/pkg/util/paramtable" ) @@ -44,9 +43,9 @@ func TestMain(m *testing.M) { path := "/tmp/milvus/rdb_data" defer os.RemoveAll(path) paramtable.Get().Save("rocksmq.compressionTypes", "0,0,0,0,0") - _ = rocksmqimplserver.InitRocksMQ(path) + _ = server2.InitRocksMQ(path) exitCode := m.Run() - defer rocksmqimplserver.CloseRocksMQ() + defer server2.CloseRocksMQ() os.Exit(exitCode) } @@ -58,14 +57,14 @@ func Test_NewRmqClient(t *testing.T) { } func TestRmqClient_CreateProducer(t *testing.T) { - opts := rocksmqimplclient.Options{} + opts := client3.Options{} client, err := NewClient(opts) defer client.Close() assert.NoError(t, err) assert.NotNil(t, client) topic := "TestRmqClient_CreateProducer" - proOpts := mqwrapper.ProducerOptions{Topic: topic} + proOpts := common.ProducerOptions{Topic: topic} producer, err := client.CreateProducer(proOpts) assert.NoError(t, err) assert.NotNil(t, producer) @@ -76,14 +75,14 @@ func TestRmqClient_CreateProducer(t *testing.T) { defer rmqProducer.Close() assert.Equal(t, rmqProducer.Topic(), topic) - msg := &mqwrapper.ProducerMessage{ + msg := &common.ProducerMessage{ Payload: []byte{}, Properties: nil, } _, err = rmqProducer.Send(context.TODO(), msg) assert.NoError(t, err) - invalidOpts := mqwrapper.ProducerOptions{Topic: ""} + invalidOpts := common.ProducerOptions{Topic: ""} producer, e := client.CreateProducer(invalidOpts) assert.Nil(t, producer) assert.Error(t, e) @@ -95,13 +94,13 @@ func TestRmqClient_GetLatestMsg(t *testing.T) { defer client.Close() topic := fmt.Sprintf("t2GetLatestMsg-%d", rand.Int()) - proOpts := mqwrapper.ProducerOptions{Topic: topic} + proOpts := common.ProducerOptions{Topic: topic} producer, err := client.CreateProducer(proOpts) assert.NoError(t, err) defer producer.Close() for i := 0; i < 10; i++ { - msg := &mqwrapper.ProducerMessage{ + msg := &common.ProducerMessage{ Payload: []byte{byte(i)}, Properties: nil, } @@ -113,7 +112,7 @@ func TestRmqClient_GetLatestMsg(t *testing.T) { consumerOpts := mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: subName, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, } @@ -123,7 +122,7 @@ func TestRmqClient_GetLatestMsg(t *testing.T) { expectLastMsg, err := consumer.GetLatestMsgID() assert.NoError(t, err) - var actualLastMsg mqwrapper.Message + var actualLastMsg common.Message ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) defer cancel() for i := 0; i < 10; i++ { @@ -149,7 +148,7 @@ func TestRmqClient_Subscribe(t *testing.T) { assert.NotNil(t, client) topic := "TestRmqClient_Subscribe" - proOpts := mqwrapper.ProducerOptions{Topic: topic} + proOpts := common.ProducerOptions{Topic: topic} producer, err := client.CreateProducer(proOpts) assert.NoError(t, err) assert.NotNil(t, producer) @@ -159,7 +158,7 @@ func TestRmqClient_Subscribe(t *testing.T) { consumerOpts := mqwrapper.ConsumerOptions{ Topic: subName, SubscriptionName: subName, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 0, } consumer, err := client.Subscribe(consumerOpts) @@ -169,7 +168,7 @@ func TestRmqClient_Subscribe(t *testing.T) { consumerOpts = mqwrapper.ConsumerOptions{ Topic: "", SubscriptionName: subName, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, } @@ -184,7 +183,7 @@ func TestRmqClient_Subscribe(t *testing.T) { assert.NotNil(t, consumer) assert.Equal(t, consumer.Subscription(), subName) - msg := &mqwrapper.ProducerMessage{ + msg := &common.ProducerMessage{ Payload: []byte{1}, Properties: nil, } @@ -198,7 +197,7 @@ func TestRmqClient_Subscribe(t *testing.T) { assert.FailNow(t, "consumer failed to yield message in 100 milliseconds") case msg := <-consumer.Chan(): consumer.Ack(msg) - rmqmsg := msg.(*rocksmqimplclient.RmqMessage) + rmqmsg := msg.(*client3.RmqMessage) msgPayload := rmqmsg.Payload() assert.NotEmpty(t, msgPayload) msgTopic := rmqmsg.Topic() @@ -206,7 +205,7 @@ func TestRmqClient_Subscribe(t *testing.T) { msgProp := rmqmsg.Properties() assert.Empty(t, msgProp) msgID := rmqmsg.ID() - rID := msgID.(*rocksmqimplserver.RmqID) + rID := msgID.(*server2.RmqID) assert.NotZero(t, rID) } } @@ -238,15 +237,13 @@ func TestRmqClient_BytesToMsgID(t *testing.T) { client, _ := createRmqClient() defer client.Close() - mid := pulsar.EarliestMessageID() - binary := pulsarwrapper.SerializePulsarMsgID(mid) - + binary := server2.SerializeRmqID(0) res, err := client.BytesToMsgID(binary) assert.NoError(t, err) - assert.NotNil(t, res) + assert.Equal(t, res.(*server2.RmqID).MessageID, int64(0)) } func createRmqClient() (*rmqClient, error) { - opts := rocksmqimplclient.Options{} + opts := client3.Options{} return NewClient(opts) } diff --git a/internal/mq/msgstream/mqwrapper/rmq/rmq_consumer.go b/pkg/mq/msgstream/mqwrapper/rmq/rmq_consumer.go similarity index 82% rename from internal/mq/msgstream/mqwrapper/rmq/rmq_consumer.go rename to pkg/mq/msgstream/mqwrapper/rmq/rmq_consumer.go index d02730cdc3ba..724206e518f5 100644 --- a/internal/mq/msgstream/mqwrapper/rmq/rmq_consumer.go +++ b/pkg/mq/msgstream/mqwrapper/rmq/rmq_consumer.go @@ -20,15 +20,15 @@ import ( "sync" "sync/atomic" - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/client" - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/client" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" ) // Consumer is a client that used to consume messages from rocksmq type Consumer struct { c client.Consumer - msgChannel chan mqwrapper.Message + msgChannel chan common.Message closeCh chan struct{} once sync.Once skip int32 @@ -41,10 +41,10 @@ func (rc *Consumer) Subscription() string { } // Chan returns a channel to read messages from rocksmq -func (rc *Consumer) Chan() <-chan mqwrapper.Message { +func (rc *Consumer) Chan() <-chan common.Message { if rc.msgChannel == nil { rc.once.Do(func() { - rc.msgChannel = make(chan mqwrapper.Message, 256) + rc.msgChannel = make(chan common.Message, 256) rc.wg.Add(1) go func() { defer rc.wg.Done() @@ -78,7 +78,7 @@ func (rc *Consumer) Chan() <-chan mqwrapper.Message { } // Seek is used to seek the position in rocksmq topic -func (rc *Consumer) Seek(id mqwrapper.MessageID, inclusive bool) error { +func (rc *Consumer) Seek(id common.MessageID, inclusive bool) error { msgID := id.(*server.RmqID).MessageID // skip the first message when consume if !inclusive { @@ -88,7 +88,7 @@ func (rc *Consumer) Seek(id mqwrapper.MessageID, inclusive bool) error { } // Ack is used to ask a rocksmq message -func (rc *Consumer) Ack(message mqwrapper.Message) { +func (rc *Consumer) Ack(message common.Message) { } // Close is used to free the resources of this consumer @@ -97,7 +97,7 @@ func (rc *Consumer) Close() { rc.wg.Wait() } -func (rc *Consumer) GetLatestMsgID() (mqwrapper.MessageID, error) { +func (rc *Consumer) GetLatestMsgID() (common.MessageID, error) { msgID, err := rc.c.GetLatestMsgID() return &server.RmqID{MessageID: msgID}, err } diff --git a/internal/mq/msgstream/mqwrapper/rmq/rmq_producer.go b/pkg/mq/msgstream/mqwrapper/rmq/rmq_producer.go similarity index 86% rename from internal/mq/msgstream/mqwrapper/rmq/rmq_producer.go rename to pkg/mq/msgstream/mqwrapper/rmq/rmq_producer.go index 40d051716f20..f6bf0f3707f9 100644 --- a/internal/mq/msgstream/mqwrapper/rmq/rmq_producer.go +++ b/pkg/mq/msgstream/mqwrapper/rmq/rmq_producer.go @@ -14,9 +14,10 @@ package rmq import ( "context" - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/client" - "github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/metrics" + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/client" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/timerecord" ) @@ -34,7 +35,7 @@ func (rp *rmqProducer) Topic() string { } // Send send the producer messages to rocksmq -func (rp *rmqProducer) Send(ctx context.Context, message *mqwrapper.ProducerMessage) (mqwrapper.MessageID, error) { +func (rp *rmqProducer) Send(ctx context.Context, message *common.ProducerMessage) (common.MessageID, error) { start := timerecord.NewTimeRecorder("send msg to stream") metrics.MsgStreamOpCounter.WithLabelValues(metrics.SendMsgLabel, metrics.TotalLabel).Inc() diff --git a/pkg/mq/msgstream/msgstream.go b/pkg/mq/msgstream/msgstream.go index 62f8c8737e02..4d6e3b0a9c88 100644 --- a/pkg/mq/msgstream/msgstream.go +++ b/pkg/mq/msgstream/msgstream.go @@ -20,7 +20,7 @@ import ( "context" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/util/typeutil" ) @@ -37,7 +37,7 @@ type IntPrimaryKey = typeutil.IntPrimaryKey type MsgPosition = msgpb.MsgPosition // MessageID is an alias for short -type MessageID = mqwrapper.MessageID +type MessageID = common.MessageID // MsgPack represents a batch of msg in msgstream type MsgPack struct { @@ -61,7 +61,7 @@ type MsgStream interface { GetProduceChannels() []string Broadcast(*MsgPack) (map[string][]MessageID, error) - AsConsumer(ctx context.Context, channels []string, subName string, position mqwrapper.SubscriptionInitialPosition) error + AsConsumer(ctx context.Context, channels []string, subName string, position common.SubscriptionInitialPosition) error Chan() <-chan *MsgPack // Seek consume message from the specified position // includeCurrentMsg indicates whether to consume the current message, and in the milvus system, it should be always false diff --git a/pkg/mq/msgstream/msgstream_util.go b/pkg/mq/msgstream/msgstream_util.go index f442eac838dc..3f64c76d7492 100644 --- a/pkg/mq/msgstream/msgstream_util.go +++ b/pkg/mq/msgstream/msgstream_util.go @@ -24,7 +24,7 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) // unsubscribeChannels create consumer first, and unsubscribe channel through msgStream.close() @@ -47,7 +47,7 @@ func GetChannelLatestMsgID(ctx context.Context, factory Factory, channelName str defer dmlStream.Close() subName := fmt.Sprintf("get-latest_msg_id-%s-%d", channelName, rand.Int()) - err = dmlStream.AsConsumer(ctx, []string{channelName}, subName, mqwrapper.SubscriptionPositionUnknown) + err = dmlStream.AsConsumer(ctx, []string{channelName}, subName, common.SubscriptionPositionUnknown) if err != nil { log.Warn("fail to AsConsumer", zap.String("channelName", channelName), zap.Error(err)) return nil, err diff --git a/pkg/mq/msgstream/msgstream_util_test.go b/pkg/mq/msgstream/msgstream_util_test.go index 69fb4a8622fa..66f4bbce4515 100644 --- a/pkg/mq/msgstream/msgstream_util_test.go +++ b/pkg/mq/msgstream/msgstream_util_test.go @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/pkg/mq/common" ) func TestPulsarMsgUtil(t *testing.T) { @@ -71,7 +71,7 @@ func TestGetLatestMsgID(t *testing.T) { } { - mockMsgID := mqwrapper.NewMockMessageID(t) + mockMsgID := common.NewMockMessageID(t) mockMsgID.EXPECT().Serialize().Return([]byte("mock")).Once() stream.EXPECT().AsConsumer(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() stream.EXPECT().GetLatestMsgID(mock.Anything).Return(mockMsgID, nil).Once() diff --git a/pkg/mq/msgstream/stream_bench_test.go b/pkg/mq/msgstream/stream_bench_test.go index 823fbf637d43..ca69642244a6 100644 --- a/pkg/mq/msgstream/stream_bench_test.go +++ b/pkg/mq/msgstream/stream_bench_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/nmq" "github.com/milvus-io/milvus/pkg/util/funcutil" @@ -45,7 +46,7 @@ func benchmarkProduceAndConsume(b *testing.B, mqClient mqwrapper.Client, cases [ go func() { defer wg.Done() - p, err := mqClient.CreateProducer(mqwrapper.ProducerOptions{ + p, err := mqClient.CreateProducer(common.ProducerOptions{ Topic: topic, }) assert.NoError(b, err) @@ -57,7 +58,7 @@ func benchmarkProduceAndConsume(b *testing.B, mqClient mqwrapper.Client, cases [ c, _ := mqClient.Subscribe(mqwrapper.ConsumerOptions{ Topic: topic, SubscriptionName: topic, - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) defer c.Close() @@ -77,7 +78,7 @@ func benchmarkMQConsume(b *testing.B, c mqwrapper.Consumer, cases [][]byte) { func benchmarkMQProduce(b *testing.B, p mqwrapper.Producer, cases [][]byte) { for _, c := range cases { - p.Send(context.Background(), &mqwrapper.ProducerMessage{ + p.Send(context.Background(), &common.ProducerMessage{ Payload: c, }) } diff --git a/pkg/mq/msgstream/stream_test.go b/pkg/mq/msgstream/stream_test.go index fea2746fbe37..bcdc4004981a 100644 --- a/pkg/mq/msgstream/stream_test.go +++ b/pkg/mq/msgstream/stream_test.go @@ -13,6 +13,7 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/mq/common" "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper" "github.com/milvus-io/milvus/pkg/util/funcutil" ) @@ -39,7 +40,7 @@ func testStreamOperation(t *testing.T, mqClient mqwrapper.Client) { func testConcurrentStream(t *testing.T, mqClient mqwrapper.Client) { topics := getChannel(2) - producer, err := mqClient.CreateProducer(mqwrapper.ProducerOptions{ + producer, err := mqClient.CreateProducer(common.ProducerOptions{ Topic: topics[0], }) defer producer.Close() @@ -48,7 +49,7 @@ func testConcurrentStream(t *testing.T, mqClient mqwrapper.Client) { consumer, err := mqClient.Subscribe(mqwrapper.ConsumerOptions{ Topic: topics[0], SubscriptionName: funcutil.RandomString(8), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionEarliest, + SubscriptionInitialPosition: common.SubscriptionPositionEarliest, BufSize: 1024, }) defer consumer.Close() @@ -60,7 +61,7 @@ func testConcurrentStream(t *testing.T, mqClient mqwrapper.Client) { func testConcurrentStreamAndSubscribeLast(t *testing.T, mqClient mqwrapper.Client) { topics := getChannel(2) - producer, err := mqClient.CreateProducer(mqwrapper.ProducerOptions{ + producer, err := mqClient.CreateProducer(common.ProducerOptions{ Topic: topics[0], }) defer producer.Close() @@ -71,7 +72,7 @@ func testConcurrentStreamAndSubscribeLast(t *testing.T, mqClient mqwrapper.Clien consumer, err := mqClient.Subscribe(mqwrapper.ConsumerOptions{ Topic: topics[0], SubscriptionName: funcutil.RandomString(8), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionLatest, + SubscriptionInitialPosition: common.SubscriptionPositionLatest, BufSize: 1024, }) assert.NoError(t, err) @@ -89,7 +90,7 @@ func testConcurrentStreamAndSubscribeLast(t *testing.T, mqClient mqwrapper.Clien func testConcurrentStreamAndSeekInclusive(t *testing.T, mqClient mqwrapper.Client) { topics := getChannel(2) - producer, err := mqClient.CreateProducer(mqwrapper.ProducerOptions{ + producer, err := mqClient.CreateProducer(common.ProducerOptions{ Topic: topics[0], }) defer producer.Close() @@ -101,7 +102,7 @@ func testConcurrentStreamAndSeekInclusive(t *testing.T, mqClient mqwrapper.Clien consumer, err := mqClient.Subscribe(mqwrapper.ConsumerOptions{ Topic: topics[0], SubscriptionName: funcutil.RandomString(8), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionUnknown, + SubscriptionInitialPosition: common.SubscriptionPositionUnknown, BufSize: 1024, }) assert.NoError(t, err) @@ -123,7 +124,7 @@ func testConcurrentStreamAndSeekInclusive(t *testing.T, mqClient mqwrapper.Clien func testConcurrentStreamAndSeekNoInclusive(t *testing.T, mqClient mqwrapper.Client) { topics := getChannel(2) - producer, err := mqClient.CreateProducer(mqwrapper.ProducerOptions{ + producer, err := mqClient.CreateProducer(common.ProducerOptions{ Topic: topics[0], }) defer producer.Close() @@ -135,7 +136,7 @@ func testConcurrentStreamAndSeekNoInclusive(t *testing.T, mqClient mqwrapper.Cli consumer, err := mqClient.Subscribe(mqwrapper.ConsumerOptions{ Topic: topics[0], SubscriptionName: funcutil.RandomString(8), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionUnknown, + SubscriptionInitialPosition: common.SubscriptionPositionUnknown, BufSize: 1024, }) assert.NoError(t, err) @@ -157,7 +158,7 @@ func testConcurrentStreamAndSeekNoInclusive(t *testing.T, mqClient mqwrapper.Cli func testConcurrentStreamAndSeekToLast(t *testing.T, mqClient mqwrapper.Client) { topics := getChannel(2) - producer, err := mqClient.CreateProducer(mqwrapper.ProducerOptions{ + producer, err := mqClient.CreateProducer(common.ProducerOptions{ Topic: topics[0], }) defer producer.Close() @@ -169,7 +170,7 @@ func testConcurrentStreamAndSeekToLast(t *testing.T, mqClient mqwrapper.Client) consumer, err := mqClient.Subscribe(mqwrapper.ConsumerOptions{ Topic: topics[0], SubscriptionName: funcutil.RandomString(8), - SubscriptionInitialPosition: mqwrapper.SubscriptionPositionUnknown, + SubscriptionInitialPosition: common.SubscriptionPositionUnknown, BufSize: 1024, }) assert.NoError(t, err) @@ -189,8 +190,8 @@ func testSendAndRecv(t *testing.T, p mqwrapper.Producer, c mqwrapper.Consumer) { msg := generateRandMessage(1024*5, 10) var ( - producerIDs []mqwrapper.MessageID - consumerIDs []mqwrapper.MessageID + producerIDs []common.MessageID + consumerIDs []common.MessageID ) wg := sync.WaitGroup{} @@ -212,7 +213,7 @@ func testSendAndRecv(t *testing.T, p mqwrapper.Producer, c mqwrapper.Consumer) { assert.Empty(t, recvMessages(context.Background(), t, c, msg, time.Second)) } -func compareMultiIDs(t *testing.T, producerIDs []mqwrapper.MessageID, consumerIDs []mqwrapper.MessageID) { +func compareMultiIDs(t *testing.T, producerIDs []common.MessageID, consumerIDs []common.MessageID) { assert.Equal(t, len(producerIDs), len(consumerIDs)) for i := range producerIDs { compare, err := producerIDs[i].Equal(consumerIDs[i].Serialize()) @@ -230,10 +231,10 @@ func generateRandMessage(m int, n int) []string { return cases } -func sendMessages(ctx context.Context, t *testing.T, p mqwrapper.Producer, testCase []string) []mqwrapper.MessageID { - ids := make([]mqwrapper.MessageID, 0, len(testCase)) +func sendMessages(ctx context.Context, t *testing.T, p mqwrapper.Producer, testCase []string) []common.MessageID { + ids := make([]common.MessageID, 0, len(testCase)) for _, s := range testCase { - id, err := p.Send(ctx, &mqwrapper.ProducerMessage{ + id, err := p.Send(ctx, &common.ProducerMessage{ Payload: []byte(s), }) assert.NoError(t, err) @@ -242,8 +243,8 @@ func sendMessages(ctx context.Context, t *testing.T, p mqwrapper.Producer, testC return ids } -func recvMessages(ctx context.Context, t *testing.T, c mqwrapper.Consumer, testCase []string, timeout time.Duration) []mqwrapper.MessageID { - ids := make([]mqwrapper.MessageID, 0, len(testCase)) +func recvMessages(ctx context.Context, t *testing.T, c mqwrapper.Consumer, testCase []string, timeout time.Duration) []common.MessageID { + ids := make([]common.MessageID, 0, len(testCase)) timeoutTicker := time.NewTicker(timeout) defer timeoutTicker.Stop() for {