Merge pull request #434 from aerospike/stage

CLIENT-2960 merge to Stage for Go Client release v7.4.0

.github/workflows/build.yml

@@ -39,9 +39,16 @@ jobs:
         run: cd tools/cli | go build -o cli .
       - name: Build example files
         run: find examples -name "*.go" -type f -print0 | xargs -0 -n1 go build
-      #      - name: Build with Reflection code removed
-      #        run: go run github.com/onsi/ginkgo/v2/ginkgo build -tags="as_performance" .
-      #      - name: Build for Google App Engine (unsafe package removed)
-      #        run: go run github.com/onsi/ginkgo/v2/ginkgo -tags="app_engine" .
+      - name: Build with Reflection code removed
+        run: go run github.com/onsi/ginkgo/v2/ginkgo build -tags="as_performance" .
+      - name: Build for Google App Engine (unsafe package removed)
+        run: go run github.com/onsi/ginkgo/v2/ginkgo build -tags="app_engine" .
       - name: Run the tests
-        run: go run github.com/onsi/ginkgo/v2/ginkgo -cover -race -keep-going -succinct -randomize-suites -skip="HyperLogLog"
+        run: go run github.com/onsi/ginkgo/v2/ginkgo -coverprofile=./cover_native.out -covermode=atomic -coverpkg=./... -race -keep-going -succinct -randomize-suites -skip="HyperLogLog"
+      - name: Combine Cover Profiles
+        run: go run github.com/wadey/gocovmerge cover_*.out > cover_all.out
+      - name: Check Code Coverage
+        uses: vladopajic/go-test-coverage@v2
+        with:
+          # Configure action using config file (option 1)
+          config: ./.testcoverage.yml

.gitignore

@@ -26,4 +26,4 @@ Dockerfile*
 .dockerignore
 docker-compose.yml
 golangci.yml
-cover.out
+cover*.out

.testcoverage.yml

@@ -0,0 +1,69 @@
+# (mandatory)
+# Path to coverprofile file (output of `go test -coverprofile` command).
+#
+# For cases where there are many coverage profiles, such as when running
+# unit tests and integration tests separately, you can combine all those
+# profiles into one. In this case, the profile should have a comma-separated list
+# of profile files, e.g., 'cover_unit.out,cover_integration.out'.
+profile: cover_all.out
+
+# (optional; but recommended to set)
+# When specified reported file paths will not contain local prefix in the output
+local-prefix: "github.com/aerospike/aerospike-client-go/v7"
+
+# Holds coverage thresholds percentages, values should be in range [0-100]
+threshold:
+  # (optional; default 0)
+  # The minimum coverage that each file should have
+  file: 0
+
+  # (optional; default 0)
+  # The minimum coverage that each package should have
+  package: 0
+
+  # (optional; default 0)
+  # The minimum total coverage project should have
+  total: 0
+
+# Holds regexp rules which will override thresholds for matched files or packages
+# using their paths.
+#
+# First rule from this list that matches file or package is going to apply
+# new threshold to it. If project has multiple rules that match same path,
+# override rules should be listed in order from specific to more general rules.
+override:
+  # Increase coverage threshold to 100% for `foo` package
+  # (default is 80, as configured above in this example)
+  #- threshold: 100
+  #  path: ^pkg/lib/foo$
+
+# Holds regexp rules which will exclude matched files or packages
+# from coverage statistics
+exclude:
+  # Exclude files or packages matching their paths
+  paths:
+    #  - \.pb\.go$    # excludes all protobuf generated files
+    - proto/*
+    - ^pkg/* # exclude package `pkg/bar`
+    - client_builder.go
+    - info_policy.go
+    - commit_policy.go
+    - generation_policy.go
+    - privilege.go
+    - read_mode_ap.go
+    - read_mode_sc.go
+    - record_exists_action.go
+    - replica_policy.go
+    - generics.go
+    - proxy_auth_interceptor.go
+    - proxy_client.go
+    - proxy_client_reflect.go
+    - proxy_query_partition_command.go
+    - proxy_scan_command.go
+    - login_command.go
+    - types/histogram/histogram.go
+    - types/rand/xor_shift128.go
+    - internal/atomic/array.go
+# NOTES:
+# - symbol `/` in all path regexps will be replaced by current OS file path separator
+#   to properly work on Windows

CHANGELOG.md

@@ -1,7 +1,24 @@
 # Change History
 
+## May 16 2024: v7.4.0
+
+  This a minor fix release. We strongly suggest you upgrade to this version over the v7.3.0 if you use the `Client.BatchGetOperate` API.
+
+- **Improvements**
+  - Add code coverage tests to the Github Actions workflow.
+  - Call the `CancelFunc` for the `context.WithTimeout` per linter suggestions in grpc calls.
+  - Minor clean up and remove dead code.
+
+- **Fixes**
+  - [CLIENT-2943] `Client.BatchGetOperate` does not consider ops in single key transforms.
+  - [CLIENT-2704] Client dev tests failing with new server map key restrictions.
+  - Fix `as_performance` and `app_engine` build tags.
+
 ## May 3 2024: v7.3.0
 
+> [!WARNING]  
+> Do not use this version if you are using the `Client.BatchGetOperate` API.
+
 This is a major feature release of the Go client and touches some of the fundamental aspects of the inner workings of it.
 We suggest complete testing of your application before using it in production.
 

batch_command_get.go

@@ -219,15 +219,18 @@ func (cmd *batchCommandGet) transactionType() transactionType {
 func (cmd *batchCommandGet) executeSingle(client *Client) Error {
 	for _, offset := range cmd.batch.offsets {
 		var err Error
-		if cmd.objects == nil {
-			if (cmd.readAttr & _INFO1_NOBINDATA) == _INFO1_NOBINDATA {
-				cmd.records[offset], err = client.GetHeader(&cmd.policy.BasePolicy, cmd.keys[offset])
-			} else {
-				cmd.records[offset], err = client.Get(&cmd.policy.BasePolicy, cmd.keys[offset], cmd.binNames...)
+		if len(cmd.ops) > 0 {
+			// Validate that all operations are read
+			for i := range cmd.ops {
+				if cmd.ops[i].opType.isWrite {
+					return newError(types.PARAMETER_ERROR, "Write operations not allowed in batch read").setNode(cmd.node)
+				}
 			}
+			cmd.records[offset], err = client.Operate(cmd.policy.toWritePolicy(), cmd.keys[offset], cmd.ops...)
+		} else if (cmd.readAttr & _INFO1_NOBINDATA) == _INFO1_NOBINDATA {
+			cmd.records[offset], err = client.GetHeader(&cmd.policy.BasePolicy, cmd.keys[offset])
 		} else {
-			err = client.getObjectDirect(&cmd.policy.BasePolicy, cmd.keys[offset], cmd.objects[offset])
-			cmd.objectsFound[offset] = err == nil
+			cmd.records[offset], err = client.Get(&cmd.policy.BasePolicy, cmd.keys[offset], cmd.binNames...)
 		}
 		if err != nil {
 			// Key not found is NOT an error for batch requests
@@ -250,7 +253,7 @@ func (cmd *batchCommandGet) executeSingle(client *Client) Error {
 }
 
 func (cmd *batchCommandGet) Execute() Error {
-	if len(cmd.batch.offsets) == 1 {
+	if cmd.objects == nil && len(cmd.batch.offsets) == 1 {
 		return cmd.executeSingle(cmd.node.cluster.client)
 	}
 	return cmd.execute(cmd)

batch_command_operate.go

@@ -326,7 +326,8 @@ func (cmd *batchCommandOperate) ExecuteGRPC(clnt *ProxyClient) Error {
 
 	client := kvs.NewKVSClient(conn)
 
-	ctx := cmd.policy.grpcDeadlineContext()
+	ctx, cancel := cmd.policy.grpcDeadlineContext()
+	defer cancel()
 
 	streamRes, gerr := client.BatchOperate(ctx, &req)
 	if gerr != nil {

batch_test.go

@@ -1,3 +1,6 @@
+//go:build !app_engine
+// +build !app_engine
+
 // Copyright 2014-2022 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,6 +33,38 @@ import (
 // ALL tests are isolated by SetName and Key, which are 50 random characters
 var _ = gg.Describe("Aerospike", func() {
 
+	gg.Describe("BatchGetOperate operations", func() {
+		var ns = *namespace
+		var set = randString(50)
+
+		gg.It("must return the result with same ordering", func() {
+			for _, keyCount := range []int{256, 1} {
+				var keys []*as.Key
+				for i := 0; i < keyCount; i++ {
+					key, _ := as.NewKey(ns, set, i)
+					client.PutBins(nil, key, as.NewBin("i", i), as.NewBin("j", i))
+
+					keys = append(keys, key)
+				}
+
+				ops := []*as.Operation{as.GetBinOp("i"), as.PutOp(as.NewBin("h", 1))}
+				_, err := client.BatchGetOperate(nil, keys, ops...)
+				gm.Expect(err).To(gm.HaveOccurred())
+
+				ops = []*as.Operation{as.GetBinOp("i")}
+				recs, err := client.BatchGetOperate(nil, keys, ops...)
+
+				gm.Expect(err).ToNot(gm.HaveOccurred())
+				for i, rec := range recs {
+					gm.Expect(len(rec.Bins)).To(gm.Equal(1))
+					gm.Expect(rec.Bins["i"]).To(gm.Equal(i))
+				}
+
+			}
+		}) // it
+
+	}) // describe
+
 	gg.Describe("Batch Write operations", func() {
 		var ns = *namespace
 		var set = randString(50)
@@ -257,6 +292,43 @@ var _ = gg.Describe("Aerospike", func() {
 
 			})
 
+			gg.It("must successfully execute a BatchOperate for many keys", func() {
+				if *dbaas {
+					gg.Skip("Not supported in DBAAS environment")
+				}
+
+				gm.Expect(err).ToNot(gm.HaveOccurred())
+				bwPolicy := as.NewBatchWritePolicy()
+				bdPolicy := as.NewBatchDeletePolicy()
+
+				var keys []*as.Key
+				for i := 0; i < 64; i++ {
+					key, _ := as.NewKey(ns, set, i)
+					if i == 0 {
+						keys = append(keys, key)
+					}
+					bin0 := as.NewBin("count", i)
+					err := client.PutBins(nil, key, bin0)
+					gm.Expect(err).ToNot(gm.HaveOccurred())
+				}
+
+				for _, sendKey := range []bool{true, false} {
+					bwPolicy.SendKey = sendKey
+					bdPolicy.SendKey = sendKey
+					bpolicy.SendKey = !sendKey
+
+					var brecs []as.BatchRecordIfc
+					for _, key := range keys {
+						brecs = append(brecs, as.NewBatchWrite(bwPolicy, key, as.PutOp(as.NewBin("bin1", "a")), as.PutOp(as.NewBin("bin2", "b"))))
+						brecs = append(brecs, as.NewBatchDelete(bdPolicy, key))
+						brecs = append(brecs, as.NewBatchRead(nil, key, []string{"bin2"}))
+					}
+
+					err := client.BatchOperate(bpolicy, brecs)
+					gm.Expect(err).ToNot(gm.HaveOccurred())
+				}
+			})
+
 			gg.It("must successfully execute a delete op", func() {
 				if *dbaas {
 					gg.Skip("Not supported in DBAAS environment")

cdt_map_test.go

@@ -1,3 +1,6 @@
+//go:build !as_performance && !app_engine
+// +build !as_performance,!app_engine
+
 // Copyright 2014-2022 Aerospike, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");

client.go

@@ -137,6 +137,11 @@ func (clnt *Client) GetDefaultBatchWritePolicy() *BatchWritePolicy {
 	return clnt.DefaultBatchWritePolicy
 }
 
+// DefaultBatchReadPolicy returns corresponding default policy from the client
+func (clnt *Client) GetDefaultBatchReadPolicy() *BatchReadPolicy {
+	return clnt.DefaultBatchReadPolicy
+}
+
 // DefaultBatchDeletePolicy returns corresponding default policy from the client
 func (clnt *Client) GetDefaultBatchDeletePolicy() *BatchDeletePolicy {
 	return clnt.DefaultBatchDeletePolicy
@@ -187,6 +192,11 @@ func (clnt *Client) SetDefaultBatchWritePolicy(policy *BatchWritePolicy) {
 	clnt.DefaultBatchWritePolicy = policy
 }
 
+// DefaultBatchReadPolicy returns corresponding default policy from the client
+func (clnt *Client) SetDefaultBatchReadPolicy(policy *BatchReadPolicy) {
+	clnt.DefaultBatchReadPolicy = policy
+}
+
 // DefaultBatchDeletePolicy returns corresponding default policy from the client
 func (clnt *Client) SetDefaultBatchDeletePolicy(policy *BatchDeletePolicy) {
 	clnt.DefaultBatchDeletePolicy = policy

client_ifc_app_engine.go

@@ -103,8 +103,22 @@ type ClientIfc interface {
 
 	// TODO: Synchronization here for the sake of dynamic config in the future
 
+	getUsablePolicy(*BasePolicy) *BasePolicy
+	getUsableWritePolicy(*WritePolicy) *WritePolicy
+	getUsableScanPolicy(*ScanPolicy) *ScanPolicy
+	getUsableQueryPolicy(*QueryPolicy) *QueryPolicy
+	getUsableAdminPolicy(*AdminPolicy) *AdminPolicy
+	getUsableInfoPolicy(*InfoPolicy) *InfoPolicy
+
+	getUsableBatchPolicy(*BatchPolicy) *BatchPolicy
+	getUsableBatchReadPolicy(*BatchReadPolicy) *BatchReadPolicy
+	getUsableBatchWritePolicy(*BatchWritePolicy) *BatchWritePolicy
+	getUsableBatchDeletePolicy(*BatchDeletePolicy) *BatchDeletePolicy
+	getUsableBatchUDFPolicy(*BatchUDFPolicy) *BatchUDFPolicy
+
 	GetDefaultPolicy() *BasePolicy
 	GetDefaultBatchPolicy() *BatchPolicy
+	GetDefaultBatchReadPolicy() *BatchReadPolicy
 	GetDefaultBatchWritePolicy() *BatchWritePolicy
 	GetDefaultBatchDeletePolicy() *BatchDeletePolicy
 	GetDefaultBatchUDFPolicy() *BatchUDFPolicy
@@ -116,6 +130,7 @@ type ClientIfc interface {
 
 	SetDefaultPolicy(*BasePolicy)
 	SetDefaultBatchPolicy(*BatchPolicy)
+	SetDefaultBatchReadPolicy(*BatchReadPolicy)
 	SetDefaultBatchWritePolicy(*BatchWritePolicy)
 	SetDefaultBatchDeletePolicy(*BatchDeletePolicy)
 	SetDefaultBatchUDFPolicy(*BatchUDFPolicy)

client_ifc_as_performance.go

@@ -103,8 +103,22 @@ type ClientIfc interface {
 
 	// TODO: Synchronization here for the sake of dynamic config in the future
 
+	getUsablePolicy(*BasePolicy) *BasePolicy
+	getUsableWritePolicy(*WritePolicy) *WritePolicy
+	getUsableScanPolicy(*ScanPolicy) *ScanPolicy
+	getUsableQueryPolicy(*QueryPolicy) *QueryPolicy
+	getUsableAdminPolicy(*AdminPolicy) *AdminPolicy
+	getUsableInfoPolicy(*InfoPolicy) *InfoPolicy
+
+	getUsableBatchPolicy(*BatchPolicy) *BatchPolicy
+	getUsableBatchReadPolicy(*BatchReadPolicy) *BatchReadPolicy
+	getUsableBatchWritePolicy(*BatchWritePolicy) *BatchWritePolicy
+	getUsableBatchDeletePolicy(*BatchDeletePolicy) *BatchDeletePolicy
+	getUsableBatchUDFPolicy(*BatchUDFPolicy) *BatchUDFPolicy
+
 	GetDefaultPolicy() *BasePolicy
 	GetDefaultBatchPolicy() *BatchPolicy
+	GetDefaultBatchReadPolicy() *BatchReadPolicy
 	GetDefaultBatchWritePolicy() *BatchWritePolicy
 	GetDefaultBatchDeletePolicy() *BatchDeletePolicy
 	GetDefaultBatchUDFPolicy() *BatchUDFPolicy
@@ -116,6 +130,7 @@ type ClientIfc interface {
 
 	SetDefaultPolicy(*BasePolicy)
 	SetDefaultBatchPolicy(*BatchPolicy)
+	SetDefaultBatchReadPolicy(*BatchReadPolicy)
 	SetDefaultBatchWritePolicy(*BatchWritePolicy)
 	SetDefaultBatchDeletePolicy(*BatchDeletePolicy)
 	SetDefaultBatchUDFPolicy(*BatchUDFPolicy)

client_object_test.go

@@ -1,5 +1,5 @@
-//go:build !as_performance
-// +build !as_performance
+//go:build !as_performance && !app_engine
+// +build !as_performance,!app_engine
 
 // Copyright 2014-2022 Aerospike, Inc.
 //
@@ -431,7 +431,7 @@ var _ = gg.Describe("Aerospike", func() {
 					ArrayOfStructs: [1]SomeStruct{{A: 1, Self: &SomeStruct{A: 1}}},
 					SliceOfStructs: []SomeStruct{{A: 1, Self: &SomeStruct{A: 1}}},
 
-					Map:           map[interface{}]interface{}{1: "string", "string": nil, nil: map[interface{}]interface{}{"1": ip}, true: false},
+					Map:           map[interface{}]interface{}{1: "string", "string": nil /*nil: map[interface{}]interface{}{"1": ip}, true: false*/},
 					MapOfMaps:     map[string]map[int64]byte{"1": {1: 1, 2: 2}},
 					MapOfSlices:   map[string][]byte{"1": {1, 2}, "2": {3, 4}},
 					MapOfArrays:   map[string][3]byte{"1": {1, 2, 3}, "2": {3, 4, 5}},
@@ -581,7 +581,7 @@ var _ = gg.Describe("Aerospike", func() {
 					ArrayOfStructs: [1]SomeStruct{{A: 1, Self: &SomeStruct{A: 1}}},
 					SliceOfStructs: []SomeStruct{{A: 1, Self: &SomeStruct{A: 1}}},
 
-					Map:           map[interface{}]interface{}{1: "string", "string": nil, nil: map[interface{}]interface{}{"1": ip}, true: false},
+					Map:           map[interface{}]interface{}{1: "string", "string": nil /*nil: map[interface{}]interface{}{"1": ip}, true: false*/},
 					MapOfMaps:     map[string]map[int64]byte{"1": {1: 1, 2: 2}},
 					MapOfSlices:   map[string][]byte{"1": {1, 2}, "2": {3, 4}},
 					MapOfArrays:   map[string][3]byte{"1": {1, 2, 3}, "2": {3, 4, 5}},
@@ -712,7 +712,7 @@ var _ = gg.Describe("Aerospike", func() {
 					gm.Expect(rec.Bins).To(gm.Equal(as.BinMap{"name": t.Name, "inner": map[interface{}]interface{}{"v1": 0, "v2": ""}}))
 				})
 
-				gg.It("must save an object with the most complex structure possible", func() {
+				gg.It("must save an object with the most complex structure possible and retrieve it via BatchGetObject", func() {
 
 					testObj := makeTestObject()
 					err := client.PutObject(nil, key, &testObj)