From d3c343df13a235268942b4e87284541f06cc3663 Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Wed, 17 Sep 2025 23:07:24 -0400 Subject: [PATCH 1/5] fix: Fix the Makefile for Golang unit test. Signed-off-by: Shuchu Han --- Makefile | 19 ++++++++++++------- go.mod | 4 ++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 20220164e87..d049439d504 100644 --- a/Makefile +++ b/Makefile @@ -713,20 +713,25 @@ compile-protos-go: install-go-proto-dependencies ## Compile Go protobuf files --go-grpc_out=$(ROOT_DIR)/go/protos \ --go-grpc_opt=module=github.com/feast-dev/feast/go/protos $(ROOT_DIR)/protos/feast/$(folder)/*.proto; ) true -#install-go-ci-dependencies: - # go install golang.org/x/tools/cmd/goimports - # python -m pip install "pybindgen==0.22.1" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1" +install-go-ci-dependencies: + go install golang.org/x/tools/cmd/goimports + uv pip install "pybindgen==0.22.1" "grpcio-tools>=1.56.2,<2" "mypy-protobuf>=3.1" .PHONY: build-go build-go: compile-protos-go ## Build Go code go build -o feast ./go/main.go -.PHONY: install-feast-ci-locally -install-feast-ci-locally: ## Install Feast CI dependencies locally - uv pip install -e ".[ci]" + +## Assume the uv will create an .venv folder for itself. +## The unit test funcions will call the Python "feast" command to initialze a feast repo. +.PHONY: install-feast-locally +install-feast-locally: ## Install Feast locally + uv pip install -e "." + @export PATH=$(ROOT_DIR)/.venv/bin:$$PATH + @echo $$PATH .PHONY: test-go -test-go: compile-protos-go install-feast-ci-locally compile-protos-python ## Run Go tests +test-go: compile-protos-python compile-protos-go install-go-ci-dependencies install-feast-locally ## Run Go tests CGO_ENABLED=1 go test -coverprofile=coverage.out ./... && go tool cover -html=coverage.out -o coverage.html .PHONY: format-go diff --git a/go.mod b/go.mod index 7a65f5b744b..91ab0c05559 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/feast-dev/feast -go 1.22.0 +go 1.23 -toolchain go1.22.5 +toolchain go1.23.12 require ( github.com/apache/arrow/go/v17 v17.0.0 From 262e840db88b75d6c2f04da0eba2f2bfd61ecfc2 Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Fri, 19 Sep 2025 22:04:31 -0400 Subject: [PATCH 2/5] fix: Update the serialization of EntityKey to version 3 in Go Feature server. Signed-off-by: Shuchu Han --- go/internal/feast/onlinestore/onlinestore.go | 68 ++++++++++++++++--- .../feast/onlinestore/onlinestore_test.go | 46 +++++++++++++ .../feast/onlinestore/redisonlinestore.go | 23 ------- 3 files changed, 103 insertions(+), 34 deletions(-) create mode 100644 go/internal/feast/onlinestore/onlinestore_test.go diff --git a/go/internal/feast/onlinestore/onlinestore.go b/go/internal/feast/onlinestore/onlinestore.go index 5fd23c52ee9..55bbf2d4a5a 100644 --- a/go/internal/feast/onlinestore/onlinestore.go +++ b/go/internal/feast/onlinestore/onlinestore.go @@ -87,25 +87,42 @@ func serializeEntityKey(entityKey *types.EntityKey, entityKeySerializationVersio } keys := make([]string, 0, len(m)) - for k := range entityKey.JoinKeys { - keys = append(keys, entityKey.JoinKeys[k]) - } - sort.Strings(keys) + keys = append(keys, entityKey.JoinKeys...) + sort.Strings(keys) // Sort the keysgiut // Build the key - length := 5 * len(keys) + length := 7 * len(keys) bufferList := make([][]byte, length) + offset := 0 + + // For entityKeySerializationVersion 3 and above, we add the number of join keys + // as the first 4 bytes of the serialized key. + if entityKeySerializationVersion >= 3 { + byteBuffer := make([]byte, 4) + binary.LittleEndian.PutUint32(byteBuffer, uint32(len(keys))) + bufferList[offset] = byteBuffer // First buffer is always the length of the keys + offset++ + } for i := 0; i < len(keys); i++ { - offset := i * 2 + // Add the key type STRING info byteBuffer := make([]byte, 4) binary.LittleEndian.PutUint32(byteBuffer, uint32(types.ValueType_Enum_value["STRING"])) bufferList[offset] = byteBuffer - bufferList[offset+1] = []byte(keys[i]) + offset++ + + // Add the size of current "key" string + keyLenByteBuffer := make([]byte, 4) + binary.LittleEndian.PutUint32(keyLenByteBuffer, uint32(len(keys[i]))) + bufferList[offset] = keyLenByteBuffer + offset++ + + // Add value + bufferList[offset] = []byte(keys[i]) + offset++ } for i := 0; i < len(keys); i++ { - offset := (2 * len(keys)) + (i * 3) value := m[keys[i]].GetVal() valueBytes, valueTypeBytes, err := serializeValue(value, entityKeySerializationVersion) @@ -113,15 +130,20 @@ func serializeEntityKey(entityKey *types.EntityKey, entityKeySerializationVersio return valueBytes, err } + // Add value type info typeBuffer := make([]byte, 4) binary.LittleEndian.PutUint32(typeBuffer, uint32(valueTypeBytes)) + bufferList[offset] = typeBuffer + offset++ + // Add length info lenBuffer := make([]byte, 4) binary.LittleEndian.PutUint32(lenBuffer, uint32(len(*valueBytes))) + bufferList[offset] = lenBuffer + offset++ - bufferList[offset+0] = typeBuffer - bufferList[offset+1] = lenBuffer - bufferList[offset+2] = *valueBytes + bufferList[offset] = *valueBytes + offset++ } // Convert from an array of byte arrays to a single byte array @@ -132,3 +154,27 @@ func serializeEntityKey(entityKey *types.EntityKey, entityKeySerializationVersio return &entityKeyBuffer, nil } + + +func serializeValue(value interface{}, entityKeySerializationVersion int64) (*[]byte, types.ValueType_Enum, error) { + // TODO: Implement support for other types (at least the major types like ints, strings, bytes) + switch x := (value).(type) { + case *types.Value_StringVal: + valueString := []byte(x.StringVal) + return &valueString, types.ValueType_STRING, nil + case *types.Value_BytesVal: + return &x.BytesVal, types.ValueType_BYTES, nil + case *types.Value_Int32Val: + valueBuffer := make([]byte, 4) + binary.LittleEndian.PutUint32(valueBuffer, uint32(x.Int32Val)) + return &valueBuffer, types.ValueType_INT32, nil + case *types.Value_Int64Val: + valueBuffer := make([]byte, 8) + binary.LittleEndian.PutUint64(valueBuffer, uint64(x.Int64Val)) + return &valueBuffer, types.ValueType_INT64, nil + case nil: + return nil, types.ValueType_INVALID, fmt.Errorf("could not detect type for %v", x) + default: + return nil, types.ValueType_INVALID, fmt.Errorf("could not detect type for %v", x) + } +} diff --git a/go/internal/feast/onlinestore/onlinestore_test.go b/go/internal/feast/onlinestore/onlinestore_test.go new file mode 100644 index 00000000000..6e9cb0f07e4 --- /dev/null +++ b/go/internal/feast/onlinestore/onlinestore_test.go @@ -0,0 +1,46 @@ +package onlinestore + +import ( + "testing" + "reflect" + "github.com/stretchr/testify/assert" + "github.com/feast-dev/feast/go/protos/feast/types" +) + +func Test_serializeEntityKey(t *testing.T) { + expect_res := []byte{1, 0, 0, 0, 2, 0, 0, 0, 9, 0, 0, 0, 100, 114, 105, 118, 101, 114, 95, 105, 100, 4, 0, 0, 0, 8, 0, 0, 0, 233, 3, 0, 0, 0, 0, 0, 0} + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + entityKey *types.EntityKey + entityKeySerializationVersion int64 + want []byte + wantErr bool + }{ + { + "test a specific key", + &types.EntityKey{ + JoinKeys: []string{"driver_id"}, + EntityValues: []*types.Value{{Val: &types.Value_Int64Val{Int64Val: 1001}}}, + }, + 3, + expect_res, + false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := serializeEntityKey(tt.entityKey, tt.entityKeySerializationVersion) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("serializeEntityKey() failed: %v", gotErr) + } + return + } + if tt.wantErr { + t.Fatal("serializeEntityKey() succeeded unexpectedly") + } + assert.True(t, reflect.DeepEqual(*got, tt.want)) + }) + } +} diff --git a/go/internal/feast/onlinestore/redisonlinestore.go b/go/internal/feast/onlinestore/redisonlinestore.go index d5713d5e4d0..3fa6cf580c7 100644 --- a/go/internal/feast/onlinestore/redisonlinestore.go +++ b/go/internal/feast/onlinestore/redisonlinestore.go @@ -338,26 +338,3 @@ func buildRedisKey(project string, entityKey *types.EntityKey, entityKeySerializ fullKey := append(*serKey, []byte(project)...) return &fullKey, nil } - -func serializeValue(value interface{}, entityKeySerializationVersion int64) (*[]byte, types.ValueType_Enum, error) { - // TODO: Implement support for other types (at least the major types like ints, strings, bytes) - switch x := (value).(type) { - case *types.Value_StringVal: - valueString := []byte(x.StringVal) - return &valueString, types.ValueType_STRING, nil - case *types.Value_BytesVal: - return &x.BytesVal, types.ValueType_BYTES, nil - case *types.Value_Int32Val: - valueBuffer := make([]byte, 4) - binary.LittleEndian.PutUint32(valueBuffer, uint32(x.Int32Val)) - return &valueBuffer, types.ValueType_INT32, nil - case *types.Value_Int64Val: - valueBuffer := make([]byte, 8) - binary.LittleEndian.PutUint64(valueBuffer, uint64(x.Int64Val)) - return &valueBuffer, types.ValueType_INT64, nil - case nil: - return nil, types.ValueType_INVALID, fmt.Errorf("could not detect type for %v", x) - default: - return nil, types.ValueType_INVALID, fmt.Errorf("could not detect type for %v", x) - } -} From 1bc80850b9eb13c7f5ec31ba85a9a3664cde4faa Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Fri, 19 Sep 2025 22:28:09 -0400 Subject: [PATCH 3/5] fix: Add serialization verstion number in the feature_repo.yaml for testing. Signed-off-by: Shuchu Han --- go/internal/test/feature_repo/feature_store.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/internal/test/feature_repo/feature_store.yaml b/go/internal/test/feature_repo/feature_store.yaml index 3b48f432875..e6856e1aab9 100644 --- a/go/internal/test/feature_repo/feature_store.yaml +++ b/go/internal/test/feature_repo/feature_store.yaml @@ -2,4 +2,5 @@ project: feature_repo registry: data/registry.db provider: local online_store: - path: data/online_store.db \ No newline at end of file + path: data/online_store.db +entity_key_serialization_version: 3 \ No newline at end of file From d0e7ff6d1ce148c2ab95103e4188b3bea94f94b4 Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Fri, 19 Sep 2025 22:35:25 -0400 Subject: [PATCH 4/5] fix: Fix a typo in one comment. Signed-off-by: Shuchu Han --- go/internal/feast/onlinestore/onlinestore.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/internal/feast/onlinestore/onlinestore.go b/go/internal/feast/onlinestore/onlinestore.go index 55bbf2d4a5a..e152eafd24b 100644 --- a/go/internal/feast/onlinestore/onlinestore.go +++ b/go/internal/feast/onlinestore/onlinestore.go @@ -88,7 +88,7 @@ func serializeEntityKey(entityKey *types.EntityKey, entityKeySerializationVersio keys := make([]string, 0, len(m)) keys = append(keys, entityKey.JoinKeys...) - sort.Strings(keys) // Sort the keysgiut + sort.Strings(keys) // Sort the keys // Build the key length := 7 * len(keys) From 567e8b09c69dc530a09e1866266a507953fa099c Mon Sep 17 00:00:00 2001 From: Shuchu Han Date: Fri, 19 Sep 2025 22:47:23 -0400 Subject: [PATCH 5/5] fix: Format the code. Signed-off-by: Shuchu Han --- go/internal/feast/onlinestore/dynamodbonlinestore.go | 2 +- go/internal/feast/onlinestore/onlinestore.go | 5 ++--- go/internal/feast/onlinestore/onlinestore_test.go | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go/internal/feast/onlinestore/dynamodbonlinestore.go b/go/internal/feast/onlinestore/dynamodbonlinestore.go index 6cea79b5067..0a8ccbc855d 100644 --- a/go/internal/feast/onlinestore/dynamodbonlinestore.go +++ b/go/internal/feast/onlinestore/dynamodbonlinestore.go @@ -278,7 +278,7 @@ func (d *DynamodbOnlineStore) OnlineRead(ctx context.Context, entityKeys []*type // process null imputation for entity ids that don't exist in dynamodb currentTime := timestamppb.Now() // TODO: should use a different timestamp? - for entityId, _ := range unprocessedEntityIdsFeatureView { + for entityId := range unprocessedEntityIdsFeatureView { entityIndex := entityIndexMap[entityId] for _, featureName := range featureNames { featureIndex := featureNamesIndex[featureName] diff --git a/go/internal/feast/onlinestore/onlinestore.go b/go/internal/feast/onlinestore/onlinestore.go index e152eafd24b..a2652af2e1b 100644 --- a/go/internal/feast/onlinestore/onlinestore.go +++ b/go/internal/feast/onlinestore/onlinestore.go @@ -88,7 +88,7 @@ func serializeEntityKey(entityKey *types.EntityKey, entityKeySerializationVersio keys := make([]string, 0, len(m)) keys = append(keys, entityKey.JoinKeys...) - sort.Strings(keys) // Sort the keys + sort.Strings(keys) // Sort the keys // Build the key length := 7 * len(keys) @@ -100,7 +100,7 @@ func serializeEntityKey(entityKey *types.EntityKey, entityKeySerializationVersio if entityKeySerializationVersion >= 3 { byteBuffer := make([]byte, 4) binary.LittleEndian.PutUint32(byteBuffer, uint32(len(keys))) - bufferList[offset] = byteBuffer // First buffer is always the length of the keys + bufferList[offset] = byteBuffer // First buffer is always the length of the keys offset++ } @@ -155,7 +155,6 @@ func serializeEntityKey(entityKey *types.EntityKey, entityKeySerializationVersio return &entityKeyBuffer, nil } - func serializeValue(value interface{}, entityKeySerializationVersion int64) (*[]byte, types.ValueType_Enum, error) { // TODO: Implement support for other types (at least the major types like ints, strings, bytes) switch x := (value).(type) { diff --git a/go/internal/feast/onlinestore/onlinestore_test.go b/go/internal/feast/onlinestore/onlinestore_test.go index 6e9cb0f07e4..d22788639f5 100644 --- a/go/internal/feast/onlinestore/onlinestore_test.go +++ b/go/internal/feast/onlinestore/onlinestore_test.go @@ -1,10 +1,10 @@ package onlinestore import ( - "testing" - "reflect" - "github.com/stretchr/testify/assert" "github.com/feast-dev/feast/go/protos/feast/types" + "github.com/stretchr/testify/assert" + "reflect" + "testing" ) func Test_serializeEntityKey(t *testing.T) {