enhance: Enable linux code checker (milvus-io#35084)

See also milvus-io#34483 --------- Signed-off-by: Congqi Xia <congqi.xia@zilliz.com> Signed-off-by: Sumit Dubey <sumit.dubey2@ibm.com>
sumitd2 · Aug 6, 2024 · c2e3e29 · c2e3e29
1 parent a43affa
commit c2e3e29
Show file tree

Hide file tree

Showing 91 changed files with 589 additions and 570 deletions.
diff --git a/.github/mergify.yml b/.github/mergify.yml
@@ -50,7 +50,7 @@ pull_request_rules:
       - 'status-success=UT for Cpp'
       - 'status-success=UT for Go'
       - 'status-success=Integration Test'
-      # - 'status-success=Code Checker AMD64 Ubuntu 22.04'
+      - 'status-success=Code Checker AMD64 Ubuntu 22.04'
       # - 'status-success=Code Checker MacOS 12'
       # - 'status-success=Code Checker Amazonlinux 2023'
       - 'status-success=cpu-e2e'
@@ -63,7 +63,7 @@ pull_request_rules:
   - name: Test passed for code changed -2.*.*
     conditions:
       - base~=^2(\.\d+){2}$
-      # - 'status-success=Code Checker AMD64 Ubuntu 22.04'
+      - 'status-success=Code Checker AMD64 Ubuntu 22.04'
       - or: *Build_AND_TEST_STATUS_SUCESS_ON_UBUNTU_20_OR_UBUNTU_22
       - 'status-success=UT for Cpp'
       - 'status-success=UT for Go'
@@ -123,7 +123,7 @@ pull_request_rules:
           - base=sql_beta
           - base~=^2(\.\d+){1,2}$
       - or: *Build_AND_TEST_STATUS_SUCESS_ON_UBUNTU_20_OR_UBUNTU_22
-      # - 'status-success=Code Checker AMD64 Ubuntu 22.04'
+      - 'status-success=Code Checker AMD64 Ubuntu 22.04'
       # - 'status-success=Code Checker MacOS 12'
       # - 'status-success=Code Checker Amazonlinux 2023'
       - 'status-success=UT for Go'
@@ -139,7 +139,7 @@ pull_request_rules:
     conditions:
       - or: *Build_AND_TEST_STATUS_SUCESS_ON_UBUNTU_20_OR_UBUNTU_22
       - base~=^2\.2\.\d+$
-      # - 'status-success=Code Checker AMD64 Ubuntu 22.04'
+      - 'status-success=Code Checker AMD64 Ubuntu 22.04'
       # - 'status-success=Code Checker MacOS 12'
       - -files~=^(?!internal\/.*_test\.go).*$
     actions:
@@ -267,7 +267,7 @@ pull_request_rules:
           - base~=^2(\.\d+){1,2}$
       - or: *Build_AND_TEST_STATUS_SUCESS_ON_UBUNTU_20_OR_UBUNTU_22
       - title~=\[skip e2e\]
-      # - 'status-success=Code Checker AMD64 Ubuntu 22.04'
+      - 'status-success=Code Checker AMD64 Ubuntu 22.04'
       - 'status-success=UT for Cpp'
       - 'status-success=UT for Go'
       - 'status-success=Integration Test'
@@ -320,7 +320,7 @@ pull_request_rules:
       - or:
           - *failed_on_ubuntu_20
           - *failed_on_ubuntu_22
-          # - 'status-success!=Code Checker AMD64 Ubuntu 22.04'
+          - 'status-success!=Code Checker AMD64 Ubuntu 22.04'
           - 'status-success!=UT for Cpp'
           - 'status-success!=UT for Go'
           - 'status-success!=Integration Test'

diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml
@@ -61,54 +61,54 @@ jobs:
         run: |
           ./build/builder.sh /bin/bash -c "make check-proto-product && make verifiers"
 
-  amazonlinux:
-    name: Code Checker Amazonlinux 2023
-    # Run in amazonlinux docker
-    runs-on: ubuntu-latest
-    timeout-minutes: 180
-    steps:
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@master
-        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
-        with:
-          root-reserve-mb: 20480
-          swap-size-mb: 1024
-          remove-dotnet: 'true'
-          remove-android: 'true'
-          remove-haskell: 'true'
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Download Caches
-        uses: ./.github/actions/cache
-        with:
-          os: 'amazonlinux2023'
-      - name: Code Check
-        run: |
-          sed -i 's/ubuntu22.04/amazonlinux2023/g' .env
-          ./build/builder.sh /bin/bash -c "make install"
+  # amazonlinux:
+  #   name: Code Checker Amazonlinux 2023
+  #   # Run in amazonlinux docker
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 180
+  #   steps:
+  #     - name: Maximize build space
+  #       uses: easimon/maximize-build-space@master
+  #       if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
+  #       with:
+  #         root-reserve-mb: 20480
+  #         swap-size-mb: 1024
+  #         remove-dotnet: 'true'
+  #         remove-android: 'true'
+  #         remove-haskell: 'true'
+  #     - name: Checkout
+  #       uses: actions/checkout@v2
+  #     - name: Download Caches
+  #       uses: ./.github/actions/cache
+  #       with:
+  #         os: 'amazonlinux2023'
+  #     - name: Code Check
+  #       run: |
+  #         sed -i 's/ubuntu22.04/amazonlinux2023/g' .env
+  #         ./build/builder.sh /bin/bash -c "make install"
 
-  rockylinux:
-    name: Code Checker rockylinux8
-    # Run in amazonlinux docker
-    runs-on: ubuntu-latest
-    timeout-minutes: 180
-    steps:
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@master
-        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
-        with:
-          root-reserve-mb: 20480
-          swap-size-mb: 1024
-          remove-dotnet: 'true'
-          remove-android: 'true'
-          remove-haskell: 'true'
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Download Caches
-        uses: ./.github/actions/cache
-        with:
-          os: 'rockylinux8'
-      - name: Code Check
-        run: |
-          sed -i 's/ubuntu22.04/rockylinux8/g' .env
-          ./build/builder.sh /bin/bash -c "make install"
+  # rockylinux:
+  #   name: Code Checker rockylinux8
+  #   # Run in amazonlinux docker
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 180
+  #   steps:
+  #     - name: Maximize build space
+  #       uses: easimon/maximize-build-space@master
+  #       if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
+  #       with:
+  #         root-reserve-mb: 20480
+  #         swap-size-mb: 1024
+  #         remove-dotnet: 'true'
+  #         remove-android: 'true'
+  #         remove-haskell: 'true'
+  #     - name: Checkout
+  #       uses: actions/checkout@v2
+  #     - name: Download Caches
+  #       uses: ./.github/actions/cache
+  #       with:
+  #         os: 'rockylinux8'
+  #     - name: Code Check
+  #       run: |
+  #         sed -i 's/ubuntu22.04/rockylinux8/g' .env
+  #         ./build/builder.sh /bin/bash -c "make install"
diff --git a/Makefile b/Makefile
@@ -182,13 +182,13 @@ lint-fix: getdeps
 static-check: getdeps
 	@echo "Running $@ check"
 	@echo "Start check core packages"
-	@source $(PWD)/scripts/setenv.sh && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --build-tags dynamic,test --timeout=30m --config $(PWD)/.golangci.yml
+	@source $(PWD)/scripts/setenv.sh && GO111MODULE=on GOFLAGS=-buildvcs=false $(INSTALL_PATH)/golangci-lint run --build-tags dynamic,test --timeout=30m --config $(PWD)/.golangci.yml
 	@echo "Start check pkg package"
-	@source $(PWD)/scripts/setenv.sh && cd pkg && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --build-tags dynamic,test --timeout=30m --config $(PWD)/.golangci.yml
+	@source $(PWD)/scripts/setenv.sh && cd pkg && GO111MODULE=on GOFLAGS=-buildvcs=false $(INSTALL_PATH)/golangci-lint run --build-tags dynamic,test --timeout=30m --config $(PWD)/.golangci.yml
 	@echo "Start check client package"
-	@source $(PWD)/scripts/setenv.sh && cd client && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --timeout=30m --config $(PWD)/client/.golangci.yml
+	@source $(PWD)/scripts/setenv.sh && cd client && GO111MODULE=on GOFLAGS=-buildvcs=false $(INSTALL_PATH)/golangci-lint run --timeout=30m --config $(PWD)/client/.golangci.yml
 	@echo "Start check go_client e2e package"
-	@source $(PWD)/scripts/setenv.sh && cd tests/go_client && GO111MODULE=on $(INSTALL_PATH)/golangci-lint run --timeout=30m --config $(PWD)/client/.golangci.yml
+	@source $(PWD)/scripts/setenv.sh && cd tests/go_client && GO111MODULE=on GOFLAGS=-buildvcs=false $(INSTALL_PATH)/golangci-lint run --timeout=30m --config $(PWD)/client/.golangci.yml
 
 verifiers: build-cpp getdeps cppcheck fmt static-check
 

diff --git a/cmd/milvus/mck.go b/cmd/milvus/mck.go
@@ -528,40 +528,40 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e
 
 	switch header.Base.MsgType {
 	case commonpb.MsgType_LoadCollection:
-		loadReq := querypb.LoadCollectionRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.LoadCollectionRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "LoadCollectionRequest", err)
 		}
 		log.Info("LoadCollection", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "LoadCollection", emptyInt64(), emptyInt64(), nil
 	case commonpb.MsgType_LoadPartitions:
-		loadReq := querypb.LoadPartitionsRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.LoadPartitionsRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "LoadPartitionsRequest", err)
 		}
 		log.Info("LoadPartitions", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "LoadPartitions", loadReq.PartitionIDs, emptyInt64(), nil
 	case commonpb.MsgType_ReleaseCollection:
-		loadReq := querypb.ReleaseCollectionRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.ReleaseCollectionRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "ReleaseCollectionRequest", err)
 		}
 		log.Info("ReleaseCollection", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "ReleaseCollection", emptyInt64(), emptyInt64(), nil
 	case commonpb.MsgType_ReleasePartitions:
-		loadReq := querypb.ReleasePartitionsRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.ReleasePartitionsRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "ReleasePartitionsRequest", err)
 		}
 		log.Info("ReleasePartitions", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "ReleasePartitions", loadReq.PartitionIDs, emptyInt64(), nil
 	case commonpb.MsgType_LoadSegments:
-		loadReq := querypb.LoadSegmentsRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.LoadSegmentsRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "LoadSegmentsRequest", err)
 		}
@@ -584,16 +584,16 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e
 		log.Info("LoadSegments", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "LoadSegments", removeRepeatElement(partitionIDs), removeRepeatElement(segmentIDs), nil
 	case commonpb.MsgType_ReleaseSegments:
-		loadReq := querypb.ReleaseSegmentsRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.ReleaseSegmentsRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "ReleaseSegmentsRequest", err)
 		}
 		log.Info("ReleaseSegments", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "ReleaseSegments", loadReq.PartitionIDs, loadReq.SegmentIDs, nil
 	case commonpb.MsgType_WatchDmChannels:
-		loadReq := querypb.WatchDmChannelsRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.WatchDmChannelsRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "WatchDmChannelsRequest", err)
 		}
@@ -619,16 +619,16 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e
 		log.Warn("legacy WatchQueryChannels type found, ignore")
 		return "WatchQueryChannels", emptyInt64(), emptyInt64(), nil
 	case commonpb.MsgType_LoadBalanceSegments:
-		loadReq := querypb.LoadBalanceRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.LoadBalanceRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "LoadBalanceRequest", err)
 		}
 		log.Info("LoadBalanceSegments", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "LoadBalanceSegments", emptyInt64(), loadReq.SealedSegmentIDs, nil
 	case commonpb.MsgType_HandoffSegments:
-		handoffReq := querypb.HandoffSegmentsRequest{}
-		err = proto.Unmarshal([]byte(t), &handoffReq)
+		handoffReq := &querypb.HandoffSegmentsRequest{}
+		err = proto.Unmarshal([]byte(t), handoffReq)
 		if err != nil {
 			return errReturn(taskID, "HandoffSegmentsRequest", err)
 		}

diff --git a/configs/milvus.yaml b/configs/milvus.yaml
@@ -431,7 +431,7 @@ dataCoord:
     maxSize: 1024 # Maximum size of a segment in MB
     diskSegmentMaxSize: 2048 # Maximun size of a segment in MB for collection which has Disk index
     sealProportion: 0.12
-    sealProportionJitter: 0.1 # segment seal proportion jitter ratio, default value 0.1(10%), if seal propertion is 12%, with jitter=0.1, the actuall applied ratio will be 10.8~12%
+    sealProportionJitter: 0.1 # segment seal proportion jitter ratio, default value 0.1(10%), if seal proportion is 12%, with jitter=0.1, the actuall applied ratio will be 10.8~12%
     assignmentExpiration: 2000 # The time of the assignment expiration in ms
     allocLatestExpireAttempt: 200 # The time attempting to alloc latest lastExpire from rootCoord after restart
     maxLife: 86400 # The max lifetime of segment in seconds, 24*60*60

diff --git a/internal/core/src/common/QueryResult.h b/internal/core/src/common/QueryResult.h
@@ -107,8 +107,8 @@ struct VectorIterator {
         for (auto& iter : iterators_) {
             if (iter->HasNext()) {
                 auto origin_pair = iter->Next();
-                origin_pair.first = convert_to_segment_offset(
-                        origin_pair.first, idx);
+                origin_pair.first =
+                    convert_to_segment_offset(origin_pair.first, idx);
                 auto off_dis_pair =
                     std::make_shared<OffsetDisPair>(origin_pair, idx++);
                 heap_.push(off_dis_pair);

diff --git a/internal/core/src/common/Tracer.cpp b/internal/core/src/common/Tracer.cpp
@@ -64,8 +64,8 @@ initTelemetry(const TraceConfig& cfg) {
             opts.url = cfg.otlpEndpoint;
             exporter = otlp::OtlpHttpExporterFactory::Create(opts);
             LOG_INFO("init otlp http exporter, endpoint: {}", opts.url);
-        } else if (cfg.otlpMethod == "grpc" || 
-            cfg.otlpMethod == "") { // legacy configuration
+        } else if (cfg.otlpMethod == "grpc" ||
+                   cfg.otlpMethod == "") {  // legacy configuration
             auto opts = otlp::OtlpGrpcExporterOptions{};
             opts.endpoint = cfg.otlpEndpoint;
             opts.use_ssl_credentials = cfg.oltpSecure;

diff --git a/internal/core/src/exec/expression/UnaryExpr.cpp b/internal/core/src/exec/expression/UnaryExpr.cpp
@@ -364,7 +364,7 @@ PhyUnaryRangeFilterExpr::ExecArrayEqualForIndex(bool reverse) {
 
             // filtering by index, get candidates.
             auto size_per_chunk = segment_->size_per_chunk();
-            auto retrieve = [size_per_chunk, this](int64_t offset) -> auto {
+            auto retrieve = [ size_per_chunk, this ](int64_t offset) -> auto {
                 auto chunk_idx = offset / size_per_chunk;
                 auto chunk_offset = offset % size_per_chunk;
                 const auto& chunk =

diff --git a/internal/core/src/monitor/prometheus_client.cpp b/internal/core/src/monitor/prometheus_client.cpp
@@ -47,12 +47,12 @@ const prometheus::Histogram::BucketBoundaries bytesBuckets = {
     536870912,    // 512M
     1073741824};  // 1G
 
-const prometheus::Histogram::BucketBoundaries ratioBuckets =
-    {0.0,  0.05, 0.1,  0.15, 0.2,  0.25, 0.3,  0.35, 0.4,  0.45, 0.5,
-     0.55, 0.6,  0.65, 0.7,  0.75, 0.8,  0.85, 0.9,  0.95, 1.0};
+const prometheus::Histogram::BucketBoundaries ratioBuckets = {
+    0.0,  0.05, 0.1,  0.15, 0.2,  0.25, 0.3,  0.35, 0.4,  0.45, 0.5,
+    0.55, 0.6,  0.65, 0.7,  0.75, 0.8,  0.85, 0.9,  0.95, 1.0};
 
-const std::unique_ptr<PrometheusClient>
-    prometheusClient = std::make_unique<PrometheusClient>();
+const std::unique_ptr<PrometheusClient> prometheusClient =
+    std::make_unique<PrometheusClient>();
 
 /******************GetMetrics*************************************************************
  * !!! NOT use SUMMARY metrics here, because when parse SUMMARY metrics in Milvus,

diff --git a/internal/core/src/query/SubSearchResult.h b/internal/core/src/query/SubSearchResult.h
@@ -22,12 +22,11 @@
 namespace milvus::query {
 class SubSearchResult {
  public:
-    SubSearchResult(
-        int64_t num_queries,
-        int64_t topk,
-        const MetricType& metric_type,
-        int64_t round_decimal,
-        const std::vector<knowhere::IndexNode::IteratorPtr>& iters)
+    SubSearchResult(int64_t num_queries,
+                    int64_t topk,
+                    const MetricType& metric_type,
+                    int64_t round_decimal,
+                    const std::vector<knowhere::IndexNode::IteratorPtr>& iters)
         : num_queries_(num_queries),
           topk_(topk),
           round_decimal_(round_decimal),
@@ -41,12 +40,11 @@ class SubSearchResult {
                     int64_t topk,
                     const MetricType& metric_type,
                     int64_t round_decimal)
-        : SubSearchResult(
-              num_queries,
-              topk,
-              metric_type,
-              round_decimal,
-              std::vector<knowhere::IndexNode::IteratorPtr>{}) {
+        : SubSearchResult(num_queries,
+                          topk,
+                          metric_type,
+                          round_decimal,
+                          std::vector<knowhere::IndexNode::IteratorPtr>{}) {
     }
 
     SubSearchResult(SubSearchResult&& other) noexcept
@@ -130,8 +128,7 @@ class SubSearchResult {
     knowhere::MetricType metric_type_;
     std::vector<int64_t> seg_offsets_;
     std::vector<float> distances_;
-    std::vector<knowhere::IndexNode::IteratorPtr>
-        chunk_iterators_;
+    std::vector<knowhere::IndexNode::IteratorPtr> chunk_iterators_;
 };
 
 }  // namespace milvus::query
diff --git a/internal/core/src/query/groupby/SearchGroupByOperator.h b/internal/core/src/query/groupby/SearchGroupByOperator.h
@@ -206,7 +206,7 @@ struct GroupByMap {
     }
     bool
     Push(const T& t) {
-        if (group_map_.size() >= group_capacity_ && group_map_[t] == 0){
+        if (group_map_.size() >= group_capacity_ && group_map_[t] == 0) {
             return false;
         }
         if (group_map_[t] >= group_size_) {

diff --git a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
@@ -236,8 +236,7 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
     double total_cost =
         std::chrono::duration<double, std::micro>(vector_end - scalar_start)
             .count();
-    double scalar_ratio =
-        total_cost > 0.0 ? scalar_cost / total_cost : 0.0;
+    double scalar_ratio = total_cost > 0.0 ? scalar_cost / total_cost : 0.0;
     monitor::internal_core_search_latency_scalar_proportion.Observe(
         scalar_ratio);
 }