diff --git a/go.mod b/go.mod index d5917cb33e3..f116fffa831 100644 --- a/go.mod +++ b/go.mod @@ -25,8 +25,7 @@ require ( golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.0.0-20200513185701-a91f0712d120 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect - golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc // indirect - google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587 // indirect + golang.org/x/tools v0.0.0-20200604042327-9b20fe4cabe8 // indirect google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.24.0 // indirect gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 90a517da7e4..4be66277bcf 100644 --- a/go.sum +++ b/go.sum @@ -162,6 +162,7 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= @@ -319,6 +320,7 @@ github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -466,8 +468,12 @@ golang.org/x/tools v0.0.0-20200519205726-57a9e4404bf7 h1:nm4zDh9WvH4jiuUpMY5RUsv golang.org/x/tools v0.0.0-20200519205726-57a9e4404bf7/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc h1:6m2YO+AmBApbUOmhsghW+IfRyZOY4My4UYvQQrEpHfY= golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200601175630-2caf76543d99/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200604042327-9b20fe4cabe8 h1:8Xr1qwxn90MXYKftwNxIO2g4J+26naghxFS5rYiTZww= +golang.org/x/tools v0.0.0-20200604042327-9b20fe4cabe8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= @@ -491,6 +497,7 @@ google.golang.org/genproto v0.0.0-20200319113533-08878b785e9c h1:5aI3/f/3eCZps9x google.golang.org/genproto v0.0.0-20200319113533-08878b785e9c/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587 h1:1Ym+vvUpq1ZHvxzn34gENJX8U4aKO+vhy2P/2+Xl6qQ= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -540,6 +547,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto index cd7d51bd59c..8e36d459d66 100644 --- a/protos/feast/serving/ServingService.proto +++ b/protos/feast/serving/ServingService.proto @@ -91,8 +91,8 @@ message GetOnlineFeaturesRequest { bool omit_entities_in_response = 3; message EntityRow { - // Request timestamp of this row. This value will be used, together with maxAge, - // to determine feature staleness. + // Request timestamp of this row. This value will be used, + // together with maxAge, to determine feature staleness. google.protobuf.Timestamp entity_timestamp = 1; // Map containing mapping of entity name to entity value. @@ -100,15 +100,6 @@ message GetOnlineFeaturesRequest { } } -message GetBatchFeaturesRequest { - // List of features that are being retrieved - repeated FeatureReference features = 3; - - // Source of the entity dataset containing the timestamps and entity keys to retrieve - // features for. - DatasetSource dataset_source = 2; -} - message GetOnlineFeaturesResponse { // Feature values retrieved from feast. repeated FieldValues field_values = 1; @@ -116,10 +107,42 @@ message GetOnlineFeaturesResponse { message FieldValues { // Map of feature or entity name to feature/entity values. // Timestamps are not returned in this response. - map fields = 1; + map fields = 1; + // Map of feature or entity name to feature/entity statuses/metadata. + map statuses = 2; + } + + enum FieldStatus { + // Status is unset for this field. + INVALID = 0; + + // Field value is present for this field and age is within max age. + PRESENT = 1; + + // Values could be found for entity key and age is within max age, but + // this field value is assigned a value on ingestion into feast. + NULL_VALUE = 2; + + // Entity key did not return any values as they do not exist in Feast. + // This could suggest that the feature values have not yet been ingested + // into feast or the ingestion failed. + NOT_FOUND = 3; + + // Values could be found for entity key, but field values are outside the maximum + // allowable range. + OUTSIDE_MAX_AGE = 4; } } +message GetBatchFeaturesRequest { + // List of features that are being retrieved + repeated FeatureReference features = 3; + + // Source of the entity dataset containing the timestamps and entity keys to retrieve + // features for. + DatasetSource dataset_source = 2; +} + message GetBatchFeaturesResponse { Job job = 1; } diff --git a/sdk/go/client.go b/sdk/go/client.go index 30b3dc4bf09..617c37a6f10 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -59,21 +59,25 @@ func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeatures } } - // strip projects from to projects - for _, fieldValue := range resp.GetFieldValues() { - stripFields := make(map[string]*types.Value, len(fieldValue.Fields)) - for refStr, value := range fieldValue.Fields { + // strip projects from feature refs recieved + for _, fieldValues := range resp.GetFieldValues() { + stripFields := make(map[string]*types.Value, len(fieldValues.Fields)) + stripStatuses := make(map[string]serving.GetOnlineFeaturesResponse_FieldStatus, len(fieldValues.Statuses)) + for refStr, _ := range fieldValues.Fields { _, isEntity := entityRefs[refStr] + stripRefStr := refStr if !isEntity { // is feature ref featureRef, err := parseFeatureRef(refStr, true) if err != nil { return nil, err } - refStr = toFeatureRefStr(featureRef) + stripRefStr = toFeatureRefStr(featureRef) } - stripFields[refStr] = value + + stripFields[stripRefStr] = fieldValues.Fields[refStr] + stripStatuses[stripRefStr] = fieldValues.Statuses[refStr] } - fieldValue.Fields = stripFields + fieldValues.Fields, fieldValues.Statuses = stripFields, stripStatuses } return &OnlineFeaturesResponse{RawResponse: resp}, err diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 1ad6629e9af..49fbe83266c 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -27,6 +27,7 @@ func TestGetOnlineFeatures(t *testing.T) { Features: []string{ "driver:rating", "rating", + "null_value", }, Entities: []Row{ {"driver_id": Int64Val(1)}, @@ -41,6 +42,12 @@ func TestGetOnlineFeatures(t *testing.T) { Fields: map[string]*types.Value{ "driver_project/driver:rating": Int64Val(1), "driver_project/rating": Int64Val(1), + "driver_project/null_value": {}, + }, + Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{ + "driver_project/driver:rating": serving.GetOnlineFeaturesResponse_PRESENT, + "driver_project/rating": serving.GetOnlineFeaturesResponse_PRESENT, + "driver_project/null_value": serving.GetOnlineFeaturesResponse_NULL_VALUE, }, }, }, @@ -53,6 +60,12 @@ func TestGetOnlineFeatures(t *testing.T) { Fields: map[string]*types.Value{ "driver:rating": Int64Val(1), "rating": Int64Val(1), + "null_value": {}, + }, + Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{ + "driver:rating": serving.GetOnlineFeaturesResponse_PRESENT, + "rating": serving.GetOnlineFeaturesResponse_PRESENT, + "null_value": serving.GetOnlineFeaturesResponse_NULL_VALUE, }, }, }, diff --git a/sdk/go/protos/feast/core/CoreService.pb.go b/sdk/go/protos/feast/core/CoreService.pb.go index 4f22ae4c373..a6d1dee6651 100644 --- a/sdk/go/protos/feast/core/CoreService.pb.go +++ b/sdk/go/protos/feast/core/CoreService.pb.go @@ -24,7 +24,9 @@ package core import ( context "context" + v0 "github.com/feast-dev/feast/sdk/go/protos/tensorflow_metadata/proto/v0" proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -1273,6 +1275,168 @@ func (*StopIngestionJobResponse) Descriptor() ([]byte, []int) { return file_feast_core_CoreService_proto_rawDescGZIP(), []int{23} } +type GetFeatureStatisticsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Feature set to retrieve the statistics for. A fully qualified feature set + // id in the format of project/feature_set must be provided. + FeatureSetId string `protobuf:"bytes,1,opt,name=feature_set_id,json=featureSetId,proto3" json:"feature_set_id,omitempty"` + // Optional filter which filters returned statistics by selected features. These + // features must be present in the data that is being processed. + Features []string `protobuf:"bytes,2,rep,name=features,proto3" json:"features,omitempty"` + // Optional filter to select store over which the statistics will retrieved. + // Only historical stores are allowed. + Store string `protobuf:"bytes,3,opt,name=store,proto3" json:"store,omitempty"` + // Optional start and end dates over which to filter statistical data + // Start date is inclusive, but end date is not. + // Only dates are supported, not times. + // Cannot be used with dataset_ids. + // If this period spans multiple days, unaggregatable statistics will be dropped. + StartDate *timestamp.Timestamp `protobuf:"bytes,4,opt,name=start_date,json=startDate,proto3" json:"start_date,omitempty"` + EndDate *timestamp.Timestamp `protobuf:"bytes,5,opt,name=end_date,json=endDate,proto3" json:"end_date,omitempty"` + // Optional list of ingestion Ids by which to filter data before + // retrieving statistics. + // Cannot be used with the date ranges + // If multiple dataset ids are provided, unaggregatable statistics will be dropped. + IngestionIds []string `protobuf:"bytes,6,rep,name=ingestion_ids,json=ingestionIds,proto3" json:"ingestion_ids,omitempty"` + // Setting this flag to true will force a recalculation of statistics and overwrite results currently in the + // cache, if any. + ForceRefresh bool `protobuf:"varint,7,opt,name=force_refresh,json=forceRefresh,proto3" json:"force_refresh,omitempty"` +} + +func (x *GetFeatureStatisticsRequest) Reset() { + *x = GetFeatureStatisticsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_CoreService_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetFeatureStatisticsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetFeatureStatisticsRequest) ProtoMessage() {} + +func (x *GetFeatureStatisticsRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[24] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetFeatureStatisticsRequest.ProtoReflect.Descriptor instead. +func (*GetFeatureStatisticsRequest) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{24} +} + +func (x *GetFeatureStatisticsRequest) GetFeatureSetId() string { + if x != nil { + return x.FeatureSetId + } + return "" +} + +func (x *GetFeatureStatisticsRequest) GetFeatures() []string { + if x != nil { + return x.Features + } + return nil +} + +func (x *GetFeatureStatisticsRequest) GetStore() string { + if x != nil { + return x.Store + } + return "" +} + +func (x *GetFeatureStatisticsRequest) GetStartDate() *timestamp.Timestamp { + if x != nil { + return x.StartDate + } + return nil +} + +func (x *GetFeatureStatisticsRequest) GetEndDate() *timestamp.Timestamp { + if x != nil { + return x.EndDate + } + return nil +} + +func (x *GetFeatureStatisticsRequest) GetIngestionIds() []string { + if x != nil { + return x.IngestionIds + } + return nil +} + +func (x *GetFeatureStatisticsRequest) GetForceRefresh() bool { + if x != nil { + return x.ForceRefresh + } + return false +} + +type GetFeatureStatisticsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Contains statistics for the requested data. + // Due to the limitations of TFDV and Facets, only a single dataset can be returned in, + // despite the message being of list type. + DatasetFeatureStatisticsList *v0.DatasetFeatureStatisticsList `protobuf:"bytes,1,opt,name=dataset_feature_statistics_list,json=datasetFeatureStatisticsList,proto3" json:"dataset_feature_statistics_list,omitempty"` +} + +func (x *GetFeatureStatisticsResponse) Reset() { + *x = GetFeatureStatisticsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_CoreService_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetFeatureStatisticsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetFeatureStatisticsResponse) ProtoMessage() {} + +func (x *GetFeatureStatisticsResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[25] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetFeatureStatisticsResponse.ProtoReflect.Descriptor instead. +func (*GetFeatureStatisticsResponse) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{25} +} + +func (x *GetFeatureStatisticsResponse) GetDatasetFeatureStatisticsList() *v0.DatasetFeatureStatisticsList { + if x != nil { + return x.DatasetFeatureStatisticsList + } + return nil +} + type ListFeatureSetsRequest_Filter struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1299,7 +1463,7 @@ type ListFeatureSetsRequest_Filter struct { func (x *ListFeatureSetsRequest_Filter) Reset() { *x = ListFeatureSetsRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[24] + mi := &file_feast_core_CoreService_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1312,7 +1476,7 @@ func (x *ListFeatureSetsRequest_Filter) String() string { func (*ListFeatureSetsRequest_Filter) ProtoMessage() {} func (x *ListFeatureSetsRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[24] + mi := &file_feast_core_CoreService_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1354,7 +1518,7 @@ type ListStoresRequest_Filter struct { func (x *ListStoresRequest_Filter) Reset() { *x = ListStoresRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[25] + mi := &file_feast_core_CoreService_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1367,7 +1531,7 @@ func (x *ListStoresRequest_Filter) String() string { func (*ListStoresRequest_Filter) ProtoMessage() {} func (x *ListStoresRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[25] + mi := &file_feast_core_CoreService_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1406,7 +1570,7 @@ type ListIngestionJobsRequest_Filter struct { func (x *ListIngestionJobsRequest_Filter) Reset() { *x = ListIngestionJobsRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[26] + mi := &file_feast_core_CoreService_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1419,7 +1583,7 @@ func (x *ListIngestionJobsRequest_Filter) String() string { func (*ListIngestionJobsRequest_Filter) ProtoMessage() {} func (x *ListIngestionJobsRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[26] + mi := &file_feast_core_CoreService_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1461,7 +1625,12 @@ var File_feast_core_CoreService_proto protoreflect.FileDescriptor var file_feast_core_CoreService_proto_rawDesc = []byte{ 0x0a, 0x1c, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x1a, 0x1b, 0x66, 0x65, 0x61, 0x73, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x2d, 0x74, 0x65, 0x6e, + 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x69, 0x73, + 0x74, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x16, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, @@ -1586,82 +1755,118 @@ var file_feast_core_CoreService_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x1a, 0x0a, 0x18, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, - 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xcb, 0x08, 0x0a, - 0x0b, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x66, 0x0a, 0x13, - 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, - 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xb1, 0x02, 0x0a, + 0x1b, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, + 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0e, + 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, + 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x64, 0x61, + 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x44, 0x61, 0x74, 0x65, 0x12, + 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x65, + 0x6e, 0x64, 0x44, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x67, 0x65, 0x73, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, + 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x66, + 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0c, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, + 0x22, 0x9b, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, + 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x7b, 0x0a, 0x1f, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x66, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x5f, + 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x74, 0x65, 0x6e, + 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x76, 0x30, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x4c, 0x69, 0x73, 0x74, + 0x52, 0x1c, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x32, 0xb6, + 0x09, 0x0a, 0x0b, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x66, + 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, + 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, - 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x4c, 0x69, - 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x12, 0x22, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, - 0x6f, 0x72, 0x65, 0x73, 0x12, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4e, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1e, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x54, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, - 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, + 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x12, + 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, + 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, + 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, + 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, + 0x73, 0x12, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x5a, 0x0a, 0x0f, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x53, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0b, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1e, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, + 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, + 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0d, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x20, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x12, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, - 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x60, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, - 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, - 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, - 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x13, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, - 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, - 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, 0x53, - 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, - 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, - 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, - 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x59, 0x0a, 0x10, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x10, - 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, - 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x4c, + 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, + 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, + 0x6f, 0x62, 0x73, 0x12, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, + 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, + 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x66, 0x0a, 0x13, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, + 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, + 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, 0x53, 0x74, 0x6f, 0x70, + 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x23, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, + 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, + 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x59, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x10, 0x43, 0x6f, 0x72, + 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, + 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, + 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1677,7 +1882,7 @@ func file_feast_core_CoreService_proto_rawDescGZIP() []byte { } var file_feast_core_CoreService_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_feast_core_CoreService_proto_msgTypes = make([]protoimpl.MessageInfo, 27) +var file_feast_core_CoreService_proto_msgTypes = make([]protoimpl.MessageInfo, 29) var file_feast_core_CoreService_proto_goTypes = []interface{}{ (ApplyFeatureSetResponse_Status)(0), // 0: feast.core.ApplyFeatureSetResponse.Status (UpdateStoreResponse_Status)(0), // 1: feast.core.UpdateStoreResponse.Status @@ -1705,58 +1910,67 @@ var file_feast_core_CoreService_proto_goTypes = []interface{}{ (*RestartIngestionJobResponse)(nil), // 23: feast.core.RestartIngestionJobResponse (*StopIngestionJobRequest)(nil), // 24: feast.core.StopIngestionJobRequest (*StopIngestionJobResponse)(nil), // 25: feast.core.StopIngestionJobResponse - (*ListFeatureSetsRequest_Filter)(nil), // 26: feast.core.ListFeatureSetsRequest.Filter - (*ListStoresRequest_Filter)(nil), // 27: feast.core.ListStoresRequest.Filter - (*ListIngestionJobsRequest_Filter)(nil), // 28: feast.core.ListIngestionJobsRequest.Filter - (*FeatureSet)(nil), // 29: feast.core.FeatureSet - (*Store)(nil), // 30: feast.core.Store - (*IngestionJob)(nil), // 31: feast.core.IngestionJob - (*FeatureSetReference)(nil), // 32: feast.core.FeatureSetReference + (*GetFeatureStatisticsRequest)(nil), // 26: feast.core.GetFeatureStatisticsRequest + (*GetFeatureStatisticsResponse)(nil), // 27: feast.core.GetFeatureStatisticsResponse + (*ListFeatureSetsRequest_Filter)(nil), // 28: feast.core.ListFeatureSetsRequest.Filter + (*ListStoresRequest_Filter)(nil), // 29: feast.core.ListStoresRequest.Filter + (*ListIngestionJobsRequest_Filter)(nil), // 30: feast.core.ListIngestionJobsRequest.Filter + (*FeatureSet)(nil), // 31: feast.core.FeatureSet + (*Store)(nil), // 32: feast.core.Store + (*IngestionJob)(nil), // 33: feast.core.IngestionJob + (*timestamp.Timestamp)(nil), // 34: google.protobuf.Timestamp + (*v0.DatasetFeatureStatisticsList)(nil), // 35: tensorflow.metadata.v0.DatasetFeatureStatisticsList + (*FeatureSetReference)(nil), // 36: feast.core.FeatureSetReference } var file_feast_core_CoreService_proto_depIdxs = []int32{ - 29, // 0: feast.core.GetFeatureSetResponse.feature_set:type_name -> feast.core.FeatureSet - 26, // 1: feast.core.ListFeatureSetsRequest.filter:type_name -> feast.core.ListFeatureSetsRequest.Filter - 29, // 2: feast.core.ListFeatureSetsResponse.feature_sets:type_name -> feast.core.FeatureSet - 27, // 3: feast.core.ListStoresRequest.filter:type_name -> feast.core.ListStoresRequest.Filter - 30, // 4: feast.core.ListStoresResponse.store:type_name -> feast.core.Store - 29, // 5: feast.core.ApplyFeatureSetRequest.feature_set:type_name -> feast.core.FeatureSet - 29, // 6: feast.core.ApplyFeatureSetResponse.feature_set:type_name -> feast.core.FeatureSet + 31, // 0: feast.core.GetFeatureSetResponse.feature_set:type_name -> feast.core.FeatureSet + 28, // 1: feast.core.ListFeatureSetsRequest.filter:type_name -> feast.core.ListFeatureSetsRequest.Filter + 31, // 2: feast.core.ListFeatureSetsResponse.feature_sets:type_name -> feast.core.FeatureSet + 29, // 3: feast.core.ListStoresRequest.filter:type_name -> feast.core.ListStoresRequest.Filter + 32, // 4: feast.core.ListStoresResponse.store:type_name -> feast.core.Store + 31, // 5: feast.core.ApplyFeatureSetRequest.feature_set:type_name -> feast.core.FeatureSet + 31, // 6: feast.core.ApplyFeatureSetResponse.feature_set:type_name -> feast.core.FeatureSet 0, // 7: feast.core.ApplyFeatureSetResponse.status:type_name -> feast.core.ApplyFeatureSetResponse.Status - 30, // 8: feast.core.UpdateStoreRequest.store:type_name -> feast.core.Store - 30, // 9: feast.core.UpdateStoreResponse.store:type_name -> feast.core.Store + 32, // 8: feast.core.UpdateStoreRequest.store:type_name -> feast.core.Store + 32, // 9: feast.core.UpdateStoreResponse.store:type_name -> feast.core.Store 1, // 10: feast.core.UpdateStoreResponse.status:type_name -> feast.core.UpdateStoreResponse.Status - 28, // 11: feast.core.ListIngestionJobsRequest.filter:type_name -> feast.core.ListIngestionJobsRequest.Filter - 31, // 12: feast.core.ListIngestionJobsResponse.jobs:type_name -> feast.core.IngestionJob - 32, // 13: feast.core.ListIngestionJobsRequest.Filter.feature_set_reference:type_name -> feast.core.FeatureSetReference - 10, // 14: feast.core.CoreService.GetFeastCoreVersion:input_type -> feast.core.GetFeastCoreVersionRequest - 2, // 15: feast.core.CoreService.GetFeatureSet:input_type -> feast.core.GetFeatureSetRequest - 4, // 16: feast.core.CoreService.ListFeatureSets:input_type -> feast.core.ListFeatureSetsRequest - 6, // 17: feast.core.CoreService.ListStores:input_type -> feast.core.ListStoresRequest - 8, // 18: feast.core.CoreService.ApplyFeatureSet:input_type -> feast.core.ApplyFeatureSetRequest - 12, // 19: feast.core.CoreService.UpdateStore:input_type -> feast.core.UpdateStoreRequest - 14, // 20: feast.core.CoreService.CreateProject:input_type -> feast.core.CreateProjectRequest - 16, // 21: feast.core.CoreService.ArchiveProject:input_type -> feast.core.ArchiveProjectRequest - 18, // 22: feast.core.CoreService.ListProjects:input_type -> feast.core.ListProjectsRequest - 20, // 23: feast.core.CoreService.ListIngestionJobs:input_type -> feast.core.ListIngestionJobsRequest - 22, // 24: feast.core.CoreService.RestartIngestionJob:input_type -> feast.core.RestartIngestionJobRequest - 24, // 25: feast.core.CoreService.StopIngestionJob:input_type -> feast.core.StopIngestionJobRequest - 11, // 26: feast.core.CoreService.GetFeastCoreVersion:output_type -> feast.core.GetFeastCoreVersionResponse - 3, // 27: feast.core.CoreService.GetFeatureSet:output_type -> feast.core.GetFeatureSetResponse - 5, // 28: feast.core.CoreService.ListFeatureSets:output_type -> feast.core.ListFeatureSetsResponse - 7, // 29: feast.core.CoreService.ListStores:output_type -> feast.core.ListStoresResponse - 9, // 30: feast.core.CoreService.ApplyFeatureSet:output_type -> feast.core.ApplyFeatureSetResponse - 13, // 31: feast.core.CoreService.UpdateStore:output_type -> feast.core.UpdateStoreResponse - 15, // 32: feast.core.CoreService.CreateProject:output_type -> feast.core.CreateProjectResponse - 17, // 33: feast.core.CoreService.ArchiveProject:output_type -> feast.core.ArchiveProjectResponse - 19, // 34: feast.core.CoreService.ListProjects:output_type -> feast.core.ListProjectsResponse - 21, // 35: feast.core.CoreService.ListIngestionJobs:output_type -> feast.core.ListIngestionJobsResponse - 23, // 36: feast.core.CoreService.RestartIngestionJob:output_type -> feast.core.RestartIngestionJobResponse - 25, // 37: feast.core.CoreService.StopIngestionJob:output_type -> feast.core.StopIngestionJobResponse - 26, // [26:38] is the sub-list for method output_type - 14, // [14:26] is the sub-list for method input_type - 14, // [14:14] is the sub-list for extension type_name - 14, // [14:14] is the sub-list for extension extendee - 0, // [0:14] is the sub-list for field type_name + 30, // 11: feast.core.ListIngestionJobsRequest.filter:type_name -> feast.core.ListIngestionJobsRequest.Filter + 33, // 12: feast.core.ListIngestionJobsResponse.jobs:type_name -> feast.core.IngestionJob + 34, // 13: feast.core.GetFeatureStatisticsRequest.start_date:type_name -> google.protobuf.Timestamp + 34, // 14: feast.core.GetFeatureStatisticsRequest.end_date:type_name -> google.protobuf.Timestamp + 35, // 15: feast.core.GetFeatureStatisticsResponse.dataset_feature_statistics_list:type_name -> tensorflow.metadata.v0.DatasetFeatureStatisticsList + 36, // 16: feast.core.ListIngestionJobsRequest.Filter.feature_set_reference:type_name -> feast.core.FeatureSetReference + 10, // 17: feast.core.CoreService.GetFeastCoreVersion:input_type -> feast.core.GetFeastCoreVersionRequest + 2, // 18: feast.core.CoreService.GetFeatureSet:input_type -> feast.core.GetFeatureSetRequest + 4, // 19: feast.core.CoreService.ListFeatureSets:input_type -> feast.core.ListFeatureSetsRequest + 26, // 20: feast.core.CoreService.GetFeatureStatistics:input_type -> feast.core.GetFeatureStatisticsRequest + 6, // 21: feast.core.CoreService.ListStores:input_type -> feast.core.ListStoresRequest + 8, // 22: feast.core.CoreService.ApplyFeatureSet:input_type -> feast.core.ApplyFeatureSetRequest + 12, // 23: feast.core.CoreService.UpdateStore:input_type -> feast.core.UpdateStoreRequest + 14, // 24: feast.core.CoreService.CreateProject:input_type -> feast.core.CreateProjectRequest + 16, // 25: feast.core.CoreService.ArchiveProject:input_type -> feast.core.ArchiveProjectRequest + 18, // 26: feast.core.CoreService.ListProjects:input_type -> feast.core.ListProjectsRequest + 20, // 27: feast.core.CoreService.ListIngestionJobs:input_type -> feast.core.ListIngestionJobsRequest + 22, // 28: feast.core.CoreService.RestartIngestionJob:input_type -> feast.core.RestartIngestionJobRequest + 24, // 29: feast.core.CoreService.StopIngestionJob:input_type -> feast.core.StopIngestionJobRequest + 11, // 30: feast.core.CoreService.GetFeastCoreVersion:output_type -> feast.core.GetFeastCoreVersionResponse + 3, // 31: feast.core.CoreService.GetFeatureSet:output_type -> feast.core.GetFeatureSetResponse + 5, // 32: feast.core.CoreService.ListFeatureSets:output_type -> feast.core.ListFeatureSetsResponse + 27, // 33: feast.core.CoreService.GetFeatureStatistics:output_type -> feast.core.GetFeatureStatisticsResponse + 7, // 34: feast.core.CoreService.ListStores:output_type -> feast.core.ListStoresResponse + 9, // 35: feast.core.CoreService.ApplyFeatureSet:output_type -> feast.core.ApplyFeatureSetResponse + 13, // 36: feast.core.CoreService.UpdateStore:output_type -> feast.core.UpdateStoreResponse + 15, // 37: feast.core.CoreService.CreateProject:output_type -> feast.core.CreateProjectResponse + 17, // 38: feast.core.CoreService.ArchiveProject:output_type -> feast.core.ArchiveProjectResponse + 19, // 39: feast.core.CoreService.ListProjects:output_type -> feast.core.ListProjectsResponse + 21, // 40: feast.core.CoreService.ListIngestionJobs:output_type -> feast.core.ListIngestionJobsResponse + 23, // 41: feast.core.CoreService.RestartIngestionJob:output_type -> feast.core.RestartIngestionJobResponse + 25, // 42: feast.core.CoreService.StopIngestionJob:output_type -> feast.core.StopIngestionJobResponse + 30, // [30:43] is the sub-list for method output_type + 17, // [17:30] is the sub-list for method input_type + 17, // [17:17] is the sub-list for extension type_name + 17, // [17:17] is the sub-list for extension extendee + 0, // [0:17] is the sub-list for field type_name } func init() { file_feast_core_CoreService_proto_init() } @@ -2058,7 +2272,7 @@ func file_feast_core_CoreService_proto_init() { } } file_feast_core_CoreService_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListFeatureSetsRequest_Filter); i { + switch v := v.(*GetFeatureStatisticsRequest); i { case 0: return &v.state case 1: @@ -2070,7 +2284,7 @@ func file_feast_core_CoreService_proto_init() { } } file_feast_core_CoreService_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListStoresRequest_Filter); i { + switch v := v.(*GetFeatureStatisticsResponse); i { case 0: return &v.state case 1: @@ -2082,6 +2296,30 @@ func file_feast_core_CoreService_proto_init() { } } file_feast_core_CoreService_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListFeatureSetsRequest_Filter); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_CoreService_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListStoresRequest_Filter); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_CoreService_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListIngestionJobsRequest_Filter); i { case 0: return &v.state @@ -2100,7 +2338,7 @@ func file_feast_core_CoreService_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_CoreService_proto_rawDesc, NumEnums: 2, - NumMessages: 27, + NumMessages: 29, NumExtensions: 0, NumServices: 1, }, @@ -2138,6 +2376,10 @@ type CoreServiceClient interface { // If no filter is provided in the request, the response will contain all the feature // sets currently stored in the registry. ListFeatureSets(ctx context.Context, in *ListFeatureSetsRequest, opts ...grpc.CallOption) (*ListFeatureSetsResponse, error) + // Get feature statistics computed over the data in the batch stores. + // + // Returns a dataset containing TFDV statistics mapped to each valid historical store. + GetFeatureStatistics(ctx context.Context, in *GetFeatureStatisticsRequest, opts ...grpc.CallOption) (*GetFeatureStatisticsResponse, error) // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. @@ -2220,6 +2462,15 @@ func (c *coreServiceClient) ListFeatureSets(ctx context.Context, in *ListFeature return out, nil } +func (c *coreServiceClient) GetFeatureStatistics(ctx context.Context, in *GetFeatureStatisticsRequest, opts ...grpc.CallOption) (*GetFeatureStatisticsResponse, error) { + out := new(GetFeatureStatisticsResponse) + err := c.cc.Invoke(ctx, "/feast.core.CoreService/GetFeatureStatistics", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *coreServiceClient) ListStores(ctx context.Context, in *ListStoresRequest, opts ...grpc.CallOption) (*ListStoresResponse, error) { out := new(ListStoresResponse) err := c.cc.Invoke(ctx, "/feast.core.CoreService/ListStores", in, out, opts...) @@ -2314,6 +2565,10 @@ type CoreServiceServer interface { // If no filter is provided in the request, the response will contain all the feature // sets currently stored in the registry. ListFeatureSets(context.Context, *ListFeatureSetsRequest) (*ListFeatureSetsResponse, error) + // Get feature statistics computed over the data in the batch stores. + // + // Returns a dataset containing TFDV statistics mapped to each valid historical store. + GetFeatureStatistics(context.Context, *GetFeatureStatisticsRequest) (*GetFeatureStatisticsResponse, error) // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. @@ -2374,6 +2629,9 @@ func (*UnimplementedCoreServiceServer) GetFeatureSet(context.Context, *GetFeatur func (*UnimplementedCoreServiceServer) ListFeatureSets(context.Context, *ListFeatureSetsRequest) (*ListFeatureSetsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListFeatureSets not implemented") } +func (*UnimplementedCoreServiceServer) GetFeatureStatistics(context.Context, *GetFeatureStatisticsRequest) (*GetFeatureStatisticsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetFeatureStatistics not implemented") +} func (*UnimplementedCoreServiceServer) ListStores(context.Context, *ListStoresRequest) (*ListStoresResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListStores not implemented") } @@ -2460,6 +2718,24 @@ func _CoreService_ListFeatureSets_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _CoreService_GetFeatureStatistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetFeatureStatisticsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CoreServiceServer).GetFeatureStatistics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.CoreService/GetFeatureStatistics", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CoreServiceServer).GetFeatureStatistics(ctx, req.(*GetFeatureStatisticsRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _CoreService_ListStores_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ListStoresRequest) if err := dec(in); err != nil { @@ -2638,6 +2914,10 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ MethodName: "ListFeatureSets", Handler: _CoreService_ListFeatureSets_Handler, }, + { + MethodName: "GetFeatureStatistics", + Handler: _CoreService_GetFeatureStatistics_Handler, + }, { MethodName: "ListStores", Handler: _CoreService_ListStores_Handler, diff --git a/sdk/go/protos/feast/core/Runner.pb.go b/sdk/go/protos/feast/core/Runner.pb.go index b1255e52ba6..74dcf9fad77 100644 --- a/sdk/go/protos/feast/core/Runner.pb.go +++ b/sdk/go/protos/feast/core/Runner.pb.go @@ -53,6 +53,9 @@ type DirectRunnerConfigOptions struct { TargetParallelism int32 `protobuf:"varint,1,opt,name=targetParallelism,proto3" json:"targetParallelism,omitempty"` // BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID DeadLetterTableSpec string `protobuf:"bytes,2,opt,name=deadLetterTableSpec,proto3" json:"deadLetterTableSpec,omitempty"` + // A pipeline level default location for storing temporary files. + // Support Google Cloud Storage locations or local path + TempLocation string `protobuf:"bytes,3,opt,name=tempLocation,proto3" json:"tempLocation,omitempty"` } func (x *DirectRunnerConfigOptions) Reset() { @@ -101,6 +104,13 @@ func (x *DirectRunnerConfigOptions) GetDeadLetterTableSpec() string { return "" } +func (x *DirectRunnerConfigOptions) GetTempLocation() string { + if x != nil { + return x.TempLocation + } + return "" +} + type DataflowRunnerConfigOptions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -131,6 +141,8 @@ type DataflowRunnerConfigOptions struct { MaxNumWorkers int32 `protobuf:"varint,11,opt,name=maxNumWorkers,proto3" json:"maxNumWorkers,omitempty"` // BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID DeadLetterTableSpec string `protobuf:"bytes,12,opt,name=deadLetterTableSpec,proto3" json:"deadLetterTableSpec,omitempty"` + // Labels to apply to the dataflow job + Labels map[string]string `protobuf:"bytes,13,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *DataflowRunnerConfigOptions) Reset() { @@ -249,54 +261,72 @@ func (x *DataflowRunnerConfigOptions) GetDeadLetterTableSpec() string { return "" } +func (x *DataflowRunnerConfigOptions) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + var File_feast_core_Runner_proto protoreflect.FileDescriptor var file_feast_core_Runner_proto_rawDesc = []byte{ 0x0a, 0x17, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x7b, 0x0a, 0x19, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x52, - 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x72, 0x61, - 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, 0x6d, - 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, - 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, - 0x65, 0x63, 0x22, 0xc7, 0x03, 0x0a, 0x1b, 0x44, 0x61, 0x74, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x52, - 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, - 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, - 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x75, - 0x62, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x2c, 0x0a, 0x11, 0x77, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x32, 0x0a, 0x14, 0x61, 0x75, 0x74, 0x6f, - 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x69, 0x6e, 0x67, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x22, 0x0a, 0x0c, - 0x75, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x70, 0x73, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0c, 0x75, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x70, 0x73, - 0x12, 0x22, 0x0a, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x61, 0x78, 0x4e, 0x75, 0x6d, 0x57, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x6d, 0x61, 0x78, - 0x4e, 0x75, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x9f, 0x01, 0x0a, 0x19, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x72, + 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, + 0x6d, 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, + 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, + 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, + 0x70, 0x65, 0x63, 0x12, 0x22, 0x0a, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xcf, 0x04, 0x0a, 0x1b, 0x44, 0x61, 0x74, 0x61, + 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x26, 0x0a, + 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, + 0x1e, 0x0a, 0x0a, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, + 0x2c, 0x0a, 0x11, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x77, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x32, 0x0a, + 0x14, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x41, 0x6c, 0x67, 0x6f, + 0x72, 0x69, 0x74, 0x68, 0x6d, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x61, 0x75, 0x74, + 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, + 0x6d, 0x12, 0x22, 0x0a, 0x0c, 0x75, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x70, + 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x75, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, + 0x69, 0x63, 0x49, 0x70, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x74, 0x65, 0x6d, + 0x70, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x61, 0x78, + 0x4e, 0x75, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0d, 0x6d, 0x61, 0x78, 0x4e, 0x75, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, + 0x30, 0x0a, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, + 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, - 0x63, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, - 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x42, 0x54, 0x0a, 0x10, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x42, 0x0b, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, - 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, - 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x63, 0x12, 0x4b, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x33, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, + 0x61, 0x74, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, + 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x54, 0x0a, 0x10, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0b, 0x52, + 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, + 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -311,17 +341,19 @@ func file_feast_core_Runner_proto_rawDescGZIP() []byte { return file_feast_core_Runner_proto_rawDescData } -var file_feast_core_Runner_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_feast_core_Runner_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_feast_core_Runner_proto_goTypes = []interface{}{ (*DirectRunnerConfigOptions)(nil), // 0: feast.core.DirectRunnerConfigOptions (*DataflowRunnerConfigOptions)(nil), // 1: feast.core.DataflowRunnerConfigOptions + nil, // 2: feast.core.DataflowRunnerConfigOptions.LabelsEntry } var file_feast_core_Runner_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 2, // 0: feast.core.DataflowRunnerConfigOptions.labels:type_name -> feast.core.DataflowRunnerConfigOptions.LabelsEntry + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_feast_core_Runner_proto_init() } @@ -361,7 +393,7 @@ func file_feast_core_Runner_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_Runner_proto_rawDesc, NumEnums: 0, - NumMessages: 2, + NumMessages: 3, NumExtensions: 0, NumServices: 0, }, diff --git a/sdk/go/protos/feast/core/Store.pb.go b/sdk/go/protos/feast/core/Store.pb.go index a36c6afc6ab..829e2e7c180 100644 --- a/sdk/go/protos/feast/core/Store.pb.go +++ b/sdk/go/protos/feast/core/Store.pb.go @@ -377,11 +377,12 @@ type Store_BigQueryConfig struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` - DatasetId string `protobuf:"bytes,2,opt,name=dataset_id,json=datasetId,proto3" json:"dataset_id,omitempty"` - StagingLocation string `protobuf:"bytes,3,opt,name=staging_location,json=stagingLocation,proto3" json:"staging_location,omitempty"` - InitialRetryDelaySeconds int32 `protobuf:"varint,4,opt,name=initial_retry_delay_seconds,json=initialRetryDelaySeconds,proto3" json:"initial_retry_delay_seconds,omitempty"` - TotalTimeoutSeconds int32 `protobuf:"varint,5,opt,name=total_timeout_seconds,json=totalTimeoutSeconds,proto3" json:"total_timeout_seconds,omitempty"` + ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` + DatasetId string `protobuf:"bytes,2,opt,name=dataset_id,json=datasetId,proto3" json:"dataset_id,omitempty"` + StagingLocation string `protobuf:"bytes,3,opt,name=staging_location,json=stagingLocation,proto3" json:"staging_location,omitempty"` + InitialRetryDelaySeconds int32 `protobuf:"varint,4,opt,name=initial_retry_delay_seconds,json=initialRetryDelaySeconds,proto3" json:"initial_retry_delay_seconds,omitempty"` + TotalTimeoutSeconds int32 `protobuf:"varint,5,opt,name=total_timeout_seconds,json=totalTimeoutSeconds,proto3" json:"total_timeout_seconds,omitempty"` + WriteTriggeringFrequencySeconds int32 `protobuf:"varint,6,opt,name=write_triggering_frequency_seconds,json=writeTriggeringFrequencySeconds,proto3" json:"write_triggering_frequency_seconds,omitempty"` } func (x *Store_BigQueryConfig) Reset() { @@ -451,6 +452,13 @@ func (x *Store_BigQueryConfig) GetTotalTimeoutSeconds() int32 { return 0 } +func (x *Store_BigQueryConfig) GetWriteTriggeringFrequencySeconds() int32 { + if x != nil { + return x.WriteTriggeringFrequencySeconds + } + return 0 +} + type Store_CassandraConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -643,7 +651,7 @@ var File_feast_core_Store_proto protoreflect.FileDescriptor var file_feast_core_Store_proto_rawDesc = []byte{ 0x0a, 0x16, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x22, 0xb4, 0x09, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, + 0x63, 0x6f, 0x72, 0x65, 0x22, 0x81, 0x0a, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, @@ -680,7 +688,7 @@ var file_feast_core_Store_proto_rawDesc = []byte{ 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0xec, 0x01, 0x0a, 0x0e, + 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0xb9, 0x02, 0x0a, 0x0e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, @@ -695,36 +703,41 @@ var file_feast_core_Store_proto_rawDesc = []byte{ 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x1a, 0x39, 0x0a, 0x0f, 0x43, 0x61, - 0x73, 0x73, 0x61, 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, - 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, - 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x04, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x90, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x64, 0x69, 0x73, 0x43, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2b, 0x0a, 0x11, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x69, - 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, - 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, - 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6d, 0x61, - 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0x42, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0x53, 0x0a, 0x09, - 0x53, 0x74, 0x6f, 0x72, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, - 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x44, 0x49, 0x53, 0x10, - 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x49, 0x47, 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, - 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x53, 0x53, 0x41, 0x4e, 0x44, 0x52, 0x41, 0x10, 0x03, 0x12, 0x11, - 0x0a, 0x0d, 0x52, 0x45, 0x44, 0x49, 0x53, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x10, - 0x04, 0x42, 0x08, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x53, 0x0a, 0x10, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, - 0x0a, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, - 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x4b, 0x0a, 0x22, 0x77, 0x72, + 0x69, 0x74, 0x65, 0x5f, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x66, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x1f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x54, 0x72, 0x69, + 0x67, 0x67, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, + 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x1a, 0x39, 0x0a, 0x0f, 0x43, 0x61, 0x73, 0x73, 0x61, + 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, + 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, + 0x72, 0x74, 0x1a, 0x90, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, + 0x6c, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x6f, + 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, + 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, + 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0x42, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0x53, 0x0a, 0x09, 0x53, 0x74, 0x6f, + 0x72, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, + 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x44, 0x49, 0x53, 0x10, 0x01, 0x12, 0x0c, + 0x0a, 0x08, 0x42, 0x49, 0x47, 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, + 0x43, 0x41, 0x53, 0x53, 0x41, 0x4e, 0x44, 0x52, 0x41, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, + 0x45, 0x44, 0x49, 0x53, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x10, 0x04, 0x42, 0x08, + 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x53, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0a, 0x53, 0x74, + 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index fc2748d5d84..c8b09a352b5 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -243,6 +243,70 @@ func (DataFormat) EnumDescriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{3} } +type GetOnlineFeaturesResponse_FieldStatus int32 + +const ( + // Status is unset for this field. + GetOnlineFeaturesResponse_INVALID GetOnlineFeaturesResponse_FieldStatus = 0 + // Field value is present for this field and age is within max age. + GetOnlineFeaturesResponse_PRESENT GetOnlineFeaturesResponse_FieldStatus = 1 + // Values could be found for entity key and age is within max age, but + // this field value is assigned a value on ingestion into feast. + GetOnlineFeaturesResponse_NULL_VALUE GetOnlineFeaturesResponse_FieldStatus = 2 + // Entity key did not return any values as they do not exist in Feast. + // This could suggest that the feature values have not yet been ingested + // into feast or the ingestion failed. + GetOnlineFeaturesResponse_NOT_FOUND GetOnlineFeaturesResponse_FieldStatus = 3 + // Values could be found for entity key, but field values are outside the maximum + // allowable range. + GetOnlineFeaturesResponse_OUTSIDE_MAX_AGE GetOnlineFeaturesResponse_FieldStatus = 4 +) + +// Enum value maps for GetOnlineFeaturesResponse_FieldStatus. +var ( + GetOnlineFeaturesResponse_FieldStatus_name = map[int32]string{ + 0: "INVALID", + 1: "PRESENT", + 2: "NULL_VALUE", + 3: "NOT_FOUND", + 4: "OUTSIDE_MAX_AGE", + } + GetOnlineFeaturesResponse_FieldStatus_value = map[string]int32{ + "INVALID": 0, + "PRESENT": 1, + "NULL_VALUE": 2, + "NOT_FOUND": 3, + "OUTSIDE_MAX_AGE": 4, + } +) + +func (x GetOnlineFeaturesResponse_FieldStatus) Enum() *GetOnlineFeaturesResponse_FieldStatus { + p := new(GetOnlineFeaturesResponse_FieldStatus) + *p = x + return p +} + +func (x GetOnlineFeaturesResponse_FieldStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GetOnlineFeaturesResponse_FieldStatus) Descriptor() protoreflect.EnumDescriptor { + return file_feast_serving_ServingService_proto_enumTypes[4].Descriptor() +} + +func (GetOnlineFeaturesResponse_FieldStatus) Type() protoreflect.EnumType { + return &file_feast_serving_ServingService_proto_enumTypes[4] +} + +func (x GetOnlineFeaturesResponse_FieldStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GetOnlineFeaturesResponse_FieldStatus.Descriptor instead. +func (GetOnlineFeaturesResponse_FieldStatus) EnumDescriptor() ([]byte, []int) { + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 0} +} + type GetFeastServingInfoRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -486,20 +550,17 @@ func (x *GetOnlineFeaturesRequest) GetOmitEntitiesInResponse() bool { return false } -type GetBatchFeaturesRequest struct { +type GetOnlineFeaturesResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // List of features that are being retrieved - Features []*FeatureReference `protobuf:"bytes,3,rep,name=features,proto3" json:"features,omitempty"` - // Source of the entity dataset containing the timestamps and entity keys to retrieve - // features for. - DatasetSource *DatasetSource `protobuf:"bytes,2,opt,name=dataset_source,json=datasetSource,proto3" json:"dataset_source,omitempty"` + // Feature values retrieved from feast. + FieldValues []*GetOnlineFeaturesResponse_FieldValues `protobuf:"bytes,1,rep,name=field_values,json=fieldValues,proto3" json:"field_values,omitempty"` } -func (x *GetBatchFeaturesRequest) Reset() { - *x = GetBatchFeaturesRequest{} +func (x *GetOnlineFeaturesResponse) Reset() { + *x = GetOnlineFeaturesResponse{} if protoimpl.UnsafeEnabled { mi := &file_feast_serving_ServingService_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -507,13 +568,13 @@ func (x *GetBatchFeaturesRequest) Reset() { } } -func (x *GetBatchFeaturesRequest) String() string { +func (x *GetOnlineFeaturesResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetBatchFeaturesRequest) ProtoMessage() {} +func (*GetOnlineFeaturesResponse) ProtoMessage() {} -func (x *GetBatchFeaturesRequest) ProtoReflect() protoreflect.Message { +func (x *GetOnlineFeaturesResponse) ProtoReflect() protoreflect.Message { mi := &file_feast_serving_ServingService_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -525,36 +586,32 @@ func (x *GetBatchFeaturesRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetBatchFeaturesRequest.ProtoReflect.Descriptor instead. -func (*GetBatchFeaturesRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use GetOnlineFeaturesResponse.ProtoReflect.Descriptor instead. +func (*GetOnlineFeaturesResponse) Descriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4} } -func (x *GetBatchFeaturesRequest) GetFeatures() []*FeatureReference { - if x != nil { - return x.Features - } - return nil -} - -func (x *GetBatchFeaturesRequest) GetDatasetSource() *DatasetSource { +func (x *GetOnlineFeaturesResponse) GetFieldValues() []*GetOnlineFeaturesResponse_FieldValues { if x != nil { - return x.DatasetSource + return x.FieldValues } return nil } -type GetOnlineFeaturesResponse struct { +type GetBatchFeaturesRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Feature values retrieved from feast. - FieldValues []*GetOnlineFeaturesResponse_FieldValues `protobuf:"bytes,1,rep,name=field_values,json=fieldValues,proto3" json:"field_values,omitempty"` + // List of features that are being retrieved + Features []*FeatureReference `protobuf:"bytes,3,rep,name=features,proto3" json:"features,omitempty"` + // Source of the entity dataset containing the timestamps and entity keys to retrieve + // features for. + DatasetSource *DatasetSource `protobuf:"bytes,2,opt,name=dataset_source,json=datasetSource,proto3" json:"dataset_source,omitempty"` } -func (x *GetOnlineFeaturesResponse) Reset() { - *x = GetOnlineFeaturesResponse{} +func (x *GetBatchFeaturesRequest) Reset() { + *x = GetBatchFeaturesRequest{} if protoimpl.UnsafeEnabled { mi := &file_feast_serving_ServingService_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -562,13 +619,13 @@ func (x *GetOnlineFeaturesResponse) Reset() { } } -func (x *GetOnlineFeaturesResponse) String() string { +func (x *GetBatchFeaturesRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetOnlineFeaturesResponse) ProtoMessage() {} +func (*GetBatchFeaturesRequest) ProtoMessage() {} -func (x *GetOnlineFeaturesResponse) ProtoReflect() protoreflect.Message { +func (x *GetBatchFeaturesRequest) ProtoReflect() protoreflect.Message { mi := &file_feast_serving_ServingService_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -580,14 +637,21 @@ func (x *GetOnlineFeaturesResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetOnlineFeaturesResponse.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesResponse) Descriptor() ([]byte, []int) { +// Deprecated: Use GetBatchFeaturesRequest.ProtoReflect.Descriptor instead. +func (*GetBatchFeaturesRequest) Descriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{5} } -func (x *GetOnlineFeaturesResponse) GetFieldValues() []*GetOnlineFeaturesResponse_FieldValues { +func (x *GetBatchFeaturesRequest) GetFeatures() []*FeatureReference { if x != nil { - return x.FieldValues + return x.Features + } + return nil +} + +func (x *GetBatchFeaturesRequest) GetDatasetSource() *DatasetSource { + if x != nil { + return x.DatasetSource } return nil } @@ -899,8 +963,8 @@ type GetOnlineFeaturesRequest_EntityRow struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Request timestamp of this row. This value will be used, together with maxAge, - // to determine feature staleness. + // Request timestamp of this row. This value will be used, + // together with maxAge, to determine feature staleness. EntityTimestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=entity_timestamp,json=entityTimestamp,proto3" json:"entity_timestamp,omitempty"` // Map containing mapping of entity name to entity value. Fields map[string]*types.Value `protobuf:"bytes,2,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -960,6 +1024,8 @@ type GetOnlineFeaturesResponse_FieldValues struct { // Map of feature or entity name to feature/entity values. // Timestamps are not returned in this response. Fields map[string]*types.Value `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Map of feature or entity name to feature/entity statuses/metadata. + Statuses map[string]GetOnlineFeaturesResponse_FieldStatus `protobuf:"bytes,2,rep,name=statuses,proto3" json:"statuses,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=feast.serving.GetOnlineFeaturesResponse_FieldStatus"` } func (x *GetOnlineFeaturesResponse_FieldValues) Reset() { @@ -991,7 +1057,7 @@ func (x *GetOnlineFeaturesResponse_FieldValues) ProtoReflect() protoreflect.Mess // Deprecated: Use GetOnlineFeaturesResponse_FieldValues.ProtoReflect.Descriptor instead. func (*GetOnlineFeaturesResponse_FieldValues) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{5, 0} + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 0} } func (x *GetOnlineFeaturesResponse_FieldValues) GetFields() map[string]*types.Value { @@ -1001,6 +1067,13 @@ func (x *GetOnlineFeaturesResponse_FieldValues) GetFields() map[string]*types.Va return nil } +func (x *GetOnlineFeaturesResponse_FieldValues) GetStatuses() map[string]GetOnlineFeaturesResponse_FieldStatus { + if x != nil { + return x.Statuses + } + return nil +} + type DatasetSource_FileSource struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1017,7 +1090,7 @@ type DatasetSource_FileSource struct { func (x *DatasetSource_FileSource) Reset() { *x = DatasetSource_FileSource{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[15] + mi := &file_feast_serving_ServingService_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1030,7 +1103,7 @@ func (x *DatasetSource_FileSource) String() string { func (*DatasetSource_FileSource) ProtoMessage() {} func (x *DatasetSource_FileSource) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[15] + mi := &file_feast_serving_ServingService_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1119,6 +1192,44 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0xdd, 0x04, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, + 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x89, 0x03, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x12, 0x5e, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, + 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, + 0x71, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x4a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x5b, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0b, + 0x0a, 0x07, 0x50, 0x52, 0x45, 0x53, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, + 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x4e, + 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x4f, 0x55, + 0x54, 0x53, 0x49, 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, 0x10, 0x04, 0x22, 0x9b, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, @@ -1128,26 +1239,7 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x0d, - 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xad, 0x02, - 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0c, 0x66, - 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x73, 0x1a, 0xb6, 0x01, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x4d, - 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x40, 0x0a, + 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x40, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, @@ -1254,65 +1346,69 @@ func file_feast_serving_ServingService_proto_rawDescGZIP() []byte { return file_feast_serving_ServingService_proto_rawDescData } -var file_feast_serving_ServingService_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_feast_serving_ServingService_proto_msgTypes = make([]protoimpl.MessageInfo, 16) +var file_feast_serving_ServingService_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_feast_serving_ServingService_proto_msgTypes = make([]protoimpl.MessageInfo, 17) var file_feast_serving_ServingService_proto_goTypes = []interface{}{ - (FeastServingType)(0), // 0: feast.serving.FeastServingType - (JobType)(0), // 1: feast.serving.JobType - (JobStatus)(0), // 2: feast.serving.JobStatus - (DataFormat)(0), // 3: feast.serving.DataFormat - (*GetFeastServingInfoRequest)(nil), // 4: feast.serving.GetFeastServingInfoRequest - (*GetFeastServingInfoResponse)(nil), // 5: feast.serving.GetFeastServingInfoResponse - (*FeatureReference)(nil), // 6: feast.serving.FeatureReference - (*GetOnlineFeaturesRequest)(nil), // 7: feast.serving.GetOnlineFeaturesRequest - (*GetBatchFeaturesRequest)(nil), // 8: feast.serving.GetBatchFeaturesRequest + (FeastServingType)(0), // 0: feast.serving.FeastServingType + (JobType)(0), // 1: feast.serving.JobType + (JobStatus)(0), // 2: feast.serving.JobStatus + (DataFormat)(0), // 3: feast.serving.DataFormat + (GetOnlineFeaturesResponse_FieldStatus)(0), // 4: feast.serving.GetOnlineFeaturesResponse.FieldStatus + (*GetFeastServingInfoRequest)(nil), // 5: feast.serving.GetFeastServingInfoRequest + (*GetFeastServingInfoResponse)(nil), // 6: feast.serving.GetFeastServingInfoResponse + (*FeatureReference)(nil), // 7: feast.serving.FeatureReference + (*GetOnlineFeaturesRequest)(nil), // 8: feast.serving.GetOnlineFeaturesRequest (*GetOnlineFeaturesResponse)(nil), // 9: feast.serving.GetOnlineFeaturesResponse - (*GetBatchFeaturesResponse)(nil), // 10: feast.serving.GetBatchFeaturesResponse - (*GetJobRequest)(nil), // 11: feast.serving.GetJobRequest - (*GetJobResponse)(nil), // 12: feast.serving.GetJobResponse - (*Job)(nil), // 13: feast.serving.Job - (*DatasetSource)(nil), // 14: feast.serving.DatasetSource - (*GetOnlineFeaturesRequest_EntityRow)(nil), // 15: feast.serving.GetOnlineFeaturesRequest.EntityRow - nil, // 16: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry - (*GetOnlineFeaturesResponse_FieldValues)(nil), // 17: feast.serving.GetOnlineFeaturesResponse.FieldValues - nil, // 18: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry - (*DatasetSource_FileSource)(nil), // 19: feast.serving.DatasetSource.FileSource - (*timestamp.Timestamp)(nil), // 20: google.protobuf.Timestamp - (*types.Value)(nil), // 21: feast.types.Value + (*GetBatchFeaturesRequest)(nil), // 10: feast.serving.GetBatchFeaturesRequest + (*GetBatchFeaturesResponse)(nil), // 11: feast.serving.GetBatchFeaturesResponse + (*GetJobRequest)(nil), // 12: feast.serving.GetJobRequest + (*GetJobResponse)(nil), // 13: feast.serving.GetJobResponse + (*Job)(nil), // 14: feast.serving.Job + (*DatasetSource)(nil), // 15: feast.serving.DatasetSource + (*GetOnlineFeaturesRequest_EntityRow)(nil), // 16: feast.serving.GetOnlineFeaturesRequest.EntityRow + nil, // 17: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry + (*GetOnlineFeaturesResponse_FieldValues)(nil), // 18: feast.serving.GetOnlineFeaturesResponse.FieldValues + nil, // 19: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry + nil, // 20: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry + (*DatasetSource_FileSource)(nil), // 21: feast.serving.DatasetSource.FileSource + (*timestamp.Timestamp)(nil), // 22: google.protobuf.Timestamp + (*types.Value)(nil), // 23: feast.types.Value } var file_feast_serving_ServingService_proto_depIdxs = []int32{ 0, // 0: feast.serving.GetFeastServingInfoResponse.type:type_name -> feast.serving.FeastServingType - 6, // 1: feast.serving.GetOnlineFeaturesRequest.features:type_name -> feast.serving.FeatureReference - 15, // 2: feast.serving.GetOnlineFeaturesRequest.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow - 6, // 3: feast.serving.GetBatchFeaturesRequest.features:type_name -> feast.serving.FeatureReference - 14, // 4: feast.serving.GetBatchFeaturesRequest.dataset_source:type_name -> feast.serving.DatasetSource - 17, // 5: feast.serving.GetOnlineFeaturesResponse.field_values:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues - 13, // 6: feast.serving.GetBatchFeaturesResponse.job:type_name -> feast.serving.Job - 13, // 7: feast.serving.GetJobRequest.job:type_name -> feast.serving.Job - 13, // 8: feast.serving.GetJobResponse.job:type_name -> feast.serving.Job + 7, // 1: feast.serving.GetOnlineFeaturesRequest.features:type_name -> feast.serving.FeatureReference + 16, // 2: feast.serving.GetOnlineFeaturesRequest.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow + 18, // 3: feast.serving.GetOnlineFeaturesResponse.field_values:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues + 7, // 4: feast.serving.GetBatchFeaturesRequest.features:type_name -> feast.serving.FeatureReference + 15, // 5: feast.serving.GetBatchFeaturesRequest.dataset_source:type_name -> feast.serving.DatasetSource + 14, // 6: feast.serving.GetBatchFeaturesResponse.job:type_name -> feast.serving.Job + 14, // 7: feast.serving.GetJobRequest.job:type_name -> feast.serving.Job + 14, // 8: feast.serving.GetJobResponse.job:type_name -> feast.serving.Job 1, // 9: feast.serving.Job.type:type_name -> feast.serving.JobType 2, // 10: feast.serving.Job.status:type_name -> feast.serving.JobStatus 3, // 11: feast.serving.Job.data_format:type_name -> feast.serving.DataFormat - 19, // 12: feast.serving.DatasetSource.file_source:type_name -> feast.serving.DatasetSource.FileSource - 20, // 13: feast.serving.GetOnlineFeaturesRequest.EntityRow.entity_timestamp:type_name -> google.protobuf.Timestamp - 16, // 14: feast.serving.GetOnlineFeaturesRequest.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry - 21, // 15: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry.value:type_name -> feast.types.Value - 18, // 16: feast.serving.GetOnlineFeaturesResponse.FieldValues.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry - 21, // 17: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry.value:type_name -> feast.types.Value - 3, // 18: feast.serving.DatasetSource.FileSource.data_format:type_name -> feast.serving.DataFormat - 4, // 19: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest - 7, // 20: feast.serving.ServingService.GetOnlineFeatures:input_type -> feast.serving.GetOnlineFeaturesRequest - 8, // 21: feast.serving.ServingService.GetBatchFeatures:input_type -> feast.serving.GetBatchFeaturesRequest - 11, // 22: feast.serving.ServingService.GetJob:input_type -> feast.serving.GetJobRequest - 5, // 23: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse - 9, // 24: feast.serving.ServingService.GetOnlineFeatures:output_type -> feast.serving.GetOnlineFeaturesResponse - 10, // 25: feast.serving.ServingService.GetBatchFeatures:output_type -> feast.serving.GetBatchFeaturesResponse - 12, // 26: feast.serving.ServingService.GetJob:output_type -> feast.serving.GetJobResponse - 23, // [23:27] is the sub-list for method output_type - 19, // [19:23] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 21, // 12: feast.serving.DatasetSource.file_source:type_name -> feast.serving.DatasetSource.FileSource + 22, // 13: feast.serving.GetOnlineFeaturesRequest.EntityRow.entity_timestamp:type_name -> google.protobuf.Timestamp + 17, // 14: feast.serving.GetOnlineFeaturesRequest.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry + 23, // 15: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry.value:type_name -> feast.types.Value + 19, // 16: feast.serving.GetOnlineFeaturesResponse.FieldValues.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry + 20, // 17: feast.serving.GetOnlineFeaturesResponse.FieldValues.statuses:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry + 23, // 18: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry.value:type_name -> feast.types.Value + 4, // 19: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry.value:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldStatus + 3, // 20: feast.serving.DatasetSource.FileSource.data_format:type_name -> feast.serving.DataFormat + 5, // 21: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest + 8, // 22: feast.serving.ServingService.GetOnlineFeatures:input_type -> feast.serving.GetOnlineFeaturesRequest + 10, // 23: feast.serving.ServingService.GetBatchFeatures:input_type -> feast.serving.GetBatchFeaturesRequest + 12, // 24: feast.serving.ServingService.GetJob:input_type -> feast.serving.GetJobRequest + 6, // 25: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse + 9, // 26: feast.serving.ServingService.GetOnlineFeatures:output_type -> feast.serving.GetOnlineFeaturesResponse + 11, // 27: feast.serving.ServingService.GetBatchFeatures:output_type -> feast.serving.GetBatchFeaturesResponse + 13, // 28: feast.serving.ServingService.GetJob:output_type -> feast.serving.GetJobResponse + 25, // [25:29] is the sub-list for method output_type + 21, // [21:25] is the sub-list for method input_type + 21, // [21:21] is the sub-list for extension type_name + 21, // [21:21] is the sub-list for extension extendee + 0, // [0:21] is the sub-list for field type_name } func init() { file_feast_serving_ServingService_proto_init() } @@ -1370,7 +1466,7 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetBatchFeaturesRequest); i { + switch v := v.(*GetOnlineFeaturesResponse); i { case 0: return &v.state case 1: @@ -1382,7 +1478,7 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesResponse); i { + switch v := v.(*GetBatchFeaturesRequest); i { case 0: return &v.state case 1: @@ -1477,7 +1573,7 @@ func file_feast_serving_ServingService_proto_init() { return nil } } - file_feast_serving_ServingService_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + file_feast_serving_ServingService_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DatasetSource_FileSource); i { case 0: return &v.state @@ -1498,8 +1594,8 @@ func file_feast_serving_ServingService_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_serving_ServingService_proto_rawDesc, - NumEnums: 4, - NumMessages: 16, + NumEnums: 5, + NumMessages: 17, NumExtensions: 0, NumServices: 1, }, diff --git a/sdk/go/request.go b/sdk/go/request.go index 683f17c16ed..a11999ae890 100644 --- a/sdk/go/request.go +++ b/sdk/go/request.go @@ -25,6 +25,9 @@ type OnlineFeaturesRequest struct { // Project specifies the project would contain the feature sets where the requested features belong to. Project string + + // whether to omit the entities fields in the response. + OmitEntities bool } // Builds the feast-specified request payload from the wrapper. @@ -34,16 +37,18 @@ func (r OnlineFeaturesRequest) buildRequest() (*serving.GetOnlineFeaturesRequest return nil, err } + // build request entity rows from native entities entityRows := make([]*serving.GetOnlineFeaturesRequest_EntityRow, len(r.Entities)) - - for i := range r.Entities { + for i, entity := range r.Entities { entityRows[i] = &serving.GetOnlineFeaturesRequest_EntityRow{ - Fields: r.Entities[i], + Fields: entity, } } + return &serving.GetOnlineFeaturesRequest{ - Features: featureRefs, - EntityRows: entityRows, + Features: featureRefs, + EntityRows: entityRows, + OmitEntitiesInResponse: r.OmitEntities, }, nil } diff --git a/sdk/go/response.go b/sdk/go/response.go index 1bc80af9fbb..7fa50761b69 100644 --- a/sdk/go/response.go +++ b/sdk/go/response.go @@ -25,12 +25,22 @@ type OnlineFeaturesResponse struct { // Rows retrieves the result of the request as a list of Rows. func (r OnlineFeaturesResponse) Rows() []Row { rows := make([]Row, len(r.RawResponse.FieldValues)) - for i, val := range r.RawResponse.FieldValues { - rows[i] = val.Fields + for i, fieldValues := range r.RawResponse.FieldValues { + rows[i] = fieldValues.Fields } return rows } +// Statuses retrieves field level status metadata for each row in Rows(). +// Each status map returned maps status 1:1 to each returned row from Rows() +func (r OnlineFeaturesResponse) Statuses() []map[string]serving.GetOnlineFeaturesResponse_FieldStatus { + statuses := make([]map[string]serving.GetOnlineFeaturesResponse_FieldStatus, len(r.RawResponse.FieldValues)) + for i, fieldValues := range r.RawResponse.FieldValues { + statuses[i] = fieldValues.Statuses + } + return statuses +} + // Int64Arrays retrieves the result of the request as a list of int64 slices. Any missing values will be filled // with the missing values provided. func (r OnlineFeaturesResponse) Int64Arrays(order []string, fillNa []int64) ([][]int64, error) { @@ -38,17 +48,17 @@ func (r OnlineFeaturesResponse) Int64Arrays(order []string, fillNa []int64) ([][ if len(fillNa) != len(order) { return nil, fmt.Errorf(ErrLengthMismatch, len(fillNa), len(order)) } - for i, val := range r.RawResponse.FieldValues { + for i, fieldValues := range r.RawResponse.FieldValues { rows[i] = make([]int64, len(order)) for j, fname := range order { - fValue, exists := val.Fields[fname] + value, exists := fieldValues.Fields[fname] if !exists { return nil, fmt.Errorf(ErrFeatureNotFound, fname) } - val := fValue.GetVal() - if val == nil { + valType := value.GetVal() + if valType == nil { rows[i][j] = fillNa[j] - } else if int64Val, ok := val.(*types.Value_Int64Val); ok { + } else if int64Val, ok := valType.(*types.Value_Int64Val); ok { rows[i][j] = int64Val.Int64Val } else { return nil, fmt.Errorf(ErrTypeMismatch, "int64") @@ -65,17 +75,17 @@ func (r OnlineFeaturesResponse) Float64Arrays(order []string, fillNa []float64) if len(fillNa) != len(order) { return nil, fmt.Errorf(ErrLengthMismatch, len(fillNa), len(order)) } - for i, val := range r.RawResponse.FieldValues { + for i, records := range r.RawResponse.FieldValues { rows[i] = make([]float64, len(order)) for j, fname := range order { - fValue, exists := val.Fields[fname] + value, exists := records.Fields[fname] if !exists { return nil, fmt.Errorf(ErrFeatureNotFound, fname) } - val := fValue.GetVal() - if val == nil { + valType := value.GetVal() + if valType == nil { rows[i][j] = fillNa[j] - } else if doubleVal, ok := val.(*types.Value_DoubleVal); ok { + } else if doubleVal, ok := valType.(*types.Value_DoubleVal); ok { rows[i][j] = doubleVal.DoubleVal } else { return nil, fmt.Errorf(ErrTypeMismatch, "float64") diff --git a/sdk/go/response_test.go b/sdk/go/response_test.go index 6a7c4211187..0949f24d679 100644 --- a/sdk/go/response_test.go +++ b/sdk/go/response_test.go @@ -16,12 +16,20 @@ var response = OnlineFeaturesResponse{ "project1/feature1": Int64Val(1), "project1/feature2": {}, }, + Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{ + "project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT, + "project1/feature2": serving.GetOnlineFeaturesResponse_NULL_VALUE, + }, }, { Fields: map[string]*types.Value{ "project1/feature1": Int64Val(2), "project1/feature2": Int64Val(2), }, + Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{ + "project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT, + "project1/feature2": serving.GetOnlineFeaturesResponse_PRESENT, + }, }, }, }, @@ -43,6 +51,28 @@ func TestOnlineFeaturesResponseToRow(t *testing.T) { } } +func TestOnlineFeaturesResponseoToStatuses(t *testing.T) { + actual := response.Statuses() + expected := []map[string]serving.GetOnlineFeaturesResponse_FieldStatus{ + { + "project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT, + "project1/feature2": serving.GetOnlineFeaturesResponse_NULL_VALUE, + }, + { + "project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT, + "project1/feature2": serving.GetOnlineFeaturesResponse_PRESENT, + }, + } + if len(expected) != len(actual) { + t.Errorf("expected: %v, got: %v", expected, actual) + } + for i := range expected { + if !cmp.Equal(expected[i], actual[i]) { + t.Errorf("expected: %v, got: %v", expected, actual) + } + } +} + func TestOnlineFeaturesResponseToInt64Array(t *testing.T) { type args struct { order []string diff --git a/sdk/go/types.go b/sdk/go/types.go index dd6553163bd..600af0b658a 100644 --- a/sdk/go/types.go +++ b/sdk/go/types.go @@ -5,7 +5,7 @@ import ( "github.com/golang/protobuf/proto" ) -// Row map of entity values +// Row is a map of fields type Row map[string]*types.Value func (r Row) equalTo(other Row) bool { diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index a81fdab21a2..11b4f352f64 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -24,7 +24,6 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc; import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import feast.proto.types.ValueProto.Value; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import java.util.HashSet; @@ -143,20 +142,21 @@ public List getOnlineFeatures( return response.getFieldValuesList().stream() .map( - field -> { + fieldValues -> { Row row = Row.create(); - field - .getFieldsMap() - .forEach( - (String name, Value value) -> { - // Strip project from string Feature References from returned from serving - if (!entityRefs.contains(name)) { - FeatureReference featureRef = - RequestUtil.parseFeatureRef(name, true).build(); - name = RequestUtil.renderFeatureRef(featureRef); - } - row.set(name, value); - }); + for (String fieldName : fieldValues.getFieldsMap().keySet()) { + String stripFieldName = fieldName; + if (!entityRefs.contains(fieldName)) { + // Strip project from string Feature References from returned from serving + FeatureReference featureRef = + RequestUtil.parseFeatureRef(fieldName, true).build(); + stripFieldName = RequestUtil.renderFeatureRef(featureRef); + row.set( + stripFieldName, + fieldValues.getFieldsMap().get(fieldName), + fieldValues.getStatusesMap().get(fieldName)); + } + } return row; }) .collect(Collectors.toList()); diff --git a/sdk/java/src/main/java/com/gojek/feast/Row.java b/sdk/java/src/main/java/com/gojek/feast/Row.java index 4a3035f8f19..51f820e320a 100644 --- a/sdk/java/src/main/java/com/gojek/feast/Row.java +++ b/sdk/java/src/main/java/com/gojek/feast/Row.java @@ -19,6 +19,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.types.ValueProto.Value; import feast.proto.types.ValueProto.Value.ValCase; import java.time.Instant; @@ -32,11 +33,13 @@ public class Row { private Timestamp entity_timestamp; private HashMap fields; + private HashMap fieldStatuses; public static Row create() { Row row = new Row(); row.entity_timestamp = Timestamps.fromMillis(System.currentTimeMillis()); row.fields = new HashMap<>(); + row.fieldStatuses = new HashMap<>(); return row; } @@ -55,6 +58,10 @@ public Row setEntityTimestamp(String dateTime) { } public Row set(String fieldName, Object value) { + return this.set(fieldName, value, FieldStatus.PRESENT); + } + + public Row set(String fieldName, Object value, FieldStatus status) { String valueType = value.getClass().getCanonicalName(); switch (valueType) { case "java.lang.Integer": @@ -85,6 +92,8 @@ public Row set(String fieldName, Object value) { "Type '%s' is unsupported in Feast. Please use one of these value types: Integer, Long, Float, Double, String, byte[].", valueType)); } + + fieldStatuses.put(fieldName, status); return this; } @@ -116,6 +125,14 @@ public byte[] getByte(String fieldName) { return getValue(fieldName).map(Value::getBytesVal).map(ByteString::toByteArray).orElse(null); } + public Map getStatuses() { + return fieldStatuses; + } + + public FieldStatus getStatus(String fieldName) { + return fieldStatuses.get(fieldName); + } + @Override public String toString() { List parts = new ArrayList<>(); diff --git a/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java b/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java index 717792244ee..5cb756f8de6 100644 --- a/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java +++ b/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java @@ -25,6 +25,7 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase; import feast.proto.types.ValueProto.Value; @@ -53,8 +54,9 @@ public class FeastClientTest { public void getOnlineFeatures( GetOnlineFeaturesRequest request, StreamObserver responseObserver) { + if (!request.equals(FeastClientTest.getFakeRequest())) { - responseObserver.onError(Status.UNKNOWN.asRuntimeException()); + responseObserver.onError(Status.FAILED_PRECONDITION.asRuntimeException()); } responseObserver.onNext(FeastClientTest.getFakeResponse()); @@ -86,7 +88,7 @@ public void setup() throws Exception { public void shouldGetOnlineFeatures() { List rows = this.client.getOnlineFeatures( - Arrays.asList("driver:name", "rating"), + Arrays.asList("driver:name", "rating", "null_value"), Arrays.asList( Row.create().set("driver_id", 1).setEntityTimestamp(Instant.ofEpochSecond(100))), "driver_project"); @@ -98,6 +100,17 @@ public void shouldGetOnlineFeatures() { put("driver_id", intValue(1)); put("driver:name", strValue("david")); put("rating", intValue(3)); + put("null_value", Value.newBuilder().build()); + } + }); + assertEquals( + rows.get(0).getStatuses(), + new HashMap() { + { + put("driver_id", FieldStatus.PRESENT); + put("driver:name", FieldStatus.PRESENT); + put("rating", FieldStatus.PRESENT); + put("null_value", FieldStatus.NULL_VALUE); } }); } @@ -113,6 +126,11 @@ private static GetOnlineFeaturesRequest getFakeRequest() { .build()) .addFeatures( FeatureReference.newBuilder().setProject("driver_project").setName("rating").build()) + .addFeatures( + FeatureReference.newBuilder() + .setProject("driver_project") + .setName("null_value") + .build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -124,14 +142,14 @@ private static GetOnlineFeaturesResponse getFakeResponse() { return GetOnlineFeaturesResponse.newBuilder() .addFieldValues( FieldValues.newBuilder() - .putAllFields( - new HashMap() { - { - put("driver_id", intValue(1)); - put("driver:name", strValue("david")); - put("rating", intValue(3)); - } - }) + .putFields("driver_project/driver_id", intValue(1)) + .putStatuses("driver_project/driver_id", FieldStatus.PRESENT) + .putFields("driver_project/driver:name", strValue("david")) + .putStatuses("driver_project/driver:name", FieldStatus.PRESENT) + .putFields("driver_project/rating", intValue(3)) + .putStatuses("driver_project/rating", FieldStatus.PRESENT) + .putFields("driver_project/null_value", Value.newBuilder().build()) + .putStatuses("driver_project/null_value", FieldStatus.NULL_VALUE) .build()) .build(); } diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 68fee30c9b1..9eb356034a8 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -18,8 +18,8 @@ import click import pkg_resources - import yaml + from feast.client import Client from feast.config import Config from feast.core.IngestionJob_pb2 import IngestionJobStatus diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 8d85f07cd33..c30e4265df7 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -26,10 +26,10 @@ import grpc import pandas as pd -from google.protobuf.timestamp_pb2 import Timestamp - import pyarrow as pa import pyarrow.parquet as pq +from google.protobuf.timestamp_pb2 import Timestamp + from feast.config import Config from feast.constants import ( CONFIG_CORE_SECURE_KEY, @@ -671,6 +671,7 @@ def get_online_features( feature_refs: List[str], entity_rows: List[GetOnlineFeaturesRequest.EntityRow], project: Optional[str] = None, + omit_entities: bool = False, ) -> GetOnlineFeaturesResponse: """ Retrieves the latest online feature data from Feast Serving @@ -686,16 +687,19 @@ def get_online_features( retrieval. All entity types within a feature project: Specifies the project which contain the FeatureSets which the requested features belong to. + omit_entities: If true will omit entity values in the returned feature data. Returns: - Returns a list of maps where each item in the list contains the - latest feature values for the provided entities + GetOnlineFeaturesResponse containing the feature data in records. + Each EntityRow provided will yield one record, which contains + data fields with data value and field status metadata (if included). """ self._connect_serving() try: response = self._serving_service_stub.GetOnlineFeatures( GetOnlineFeaturesRequest( + omit_entities_in_response=omit_entities, features=_build_feature_references( feature_ref_strs=feature_refs, project=project if project is not None else self.project, @@ -703,25 +707,33 @@ def get_online_features( entity_rows=entity_rows, ) ) - # collect entity row refs - entity_refs = set() - for entity_row in entity_rows: - entity_refs.update(entity_row.fields.keys()) + entity_refs = { + key for entity_row in entity_rows for key in entity_row.fields.keys() + } + # strip the project part the string feature references returned from serving + strip = ( + lambda ref: repr(FeatureRef.from_str(ref, ignore_project=True)) + if ref not in entity_refs + else ref + ) strip_field_values = [] for field_value in response.field_values: - # strip the project part the string feature references returned from serving - strip_fields = {} - for ref_str, value in field_value.fields.items(): - if ref_str not in entity_refs: - ref_str = repr( - FeatureRef.from_str(ref_str, ignore_project=True) - ) - strip_fields[ref_str] = value + keys, fields, statuses = ( + field_value.fields.keys(), + field_value.fields, + field_value.statuses, + ) + fields_and_statuses = [ + (strip(key), fields[key], statuses[key]) for key in keys + ] + keys, fields, statuses = zip(*fields_and_statuses) strip_field_values.append( - GetOnlineFeaturesResponse.FieldValues(fields=strip_fields) + GetOnlineFeaturesResponse.FieldValues( + fields=dict(zip(keys, fields)), + statuses=dict(zip(keys, statuses)), + ) ) - del response.field_values[:] response.field_values.extend(strip_field_values) diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index c4bde75404a..4b22a8e975d 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -35,6 +35,9 @@ "batch_feature_request_wait_time_seconds" ) +CONFIG_TIMEOUT_KEY = "timeout" +CONFIG_MAX_WAIT_INTERVAL_KEY = "max_wait_interval" + # Configuration option default values FEAST_DEFAULT_OPTIONS = { CONFIG_PROJECT_KEY: "default", @@ -45,4 +48,6 @@ CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY: "3", CONFIG_GRPC_CONNECTION_TIMEOUT_APPLY_KEY: "600", CONFIG_BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS_KEY: "600", + CONFIG_TIMEOUT_KEY: "21600", + CONFIG_MAX_WAIT_INTERVAL_KEY: "60", } diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index f4397f6cbd5..ee06463bc6f 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -16,14 +16,15 @@ from typing import Dict, List, MutableMapping, Optional import pandas as pd +import pyarrow as pa +import yaml from google.protobuf import json_format from google.protobuf.duration_pb2 import Duration from google.protobuf.json_format import MessageToDict, MessageToJson from google.protobuf.message import Message from pandas.api.types import is_datetime64_ns_dtype +from pyarrow.lib import TimestampType -import pyarrow as pa -import yaml from feast.core.FeatureSet_pb2 import FeatureSet as FeatureSetProto from feast.core.FeatureSet_pb2 import FeatureSetMeta as FeatureSetMetaProto from feast.core.FeatureSet_pb2 import FeatureSetSpec as FeatureSetSpecProto @@ -39,7 +40,6 @@ pa_to_feast_value_type, python_type_to_feast_value_type, ) -from pyarrow.lib import TimestampType from tensorflow_metadata.proto.v0 import schema_pb2 diff --git a/sdk/python/feast/job.py b/sdk/python/feast/job.py index 21b08224bad..829b4f79319 100644 --- a/sdk/python/feast/job.py +++ b/sdk/python/feast/job.py @@ -1,6 +1,4 @@ import tempfile -import time -from datetime import datetime, timedelta from typing import List from urllib.parse import urlparse @@ -9,6 +7,8 @@ from google.cloud import storage from google.protobuf.json_format import MessageToJson +from feast.constants import CONFIG_TIMEOUT_KEY +from feast.constants import FEAST_DEFAULT_OPTIONS as defaults from feast.core.CoreService_pb2 import ListIngestionJobsRequest from feast.core.CoreService_pb2_grpc import CoreServiceStub from feast.core.IngestionJob_pb2 import IngestionJob as IngestJobProto @@ -23,13 +23,7 @@ from feast.serving.ServingService_pb2 import Job as JobProto from feast.serving.ServingService_pb2_grpc import ServingServiceStub from feast.source import Source - -# Maximum no of seconds to wait until the retrieval jobs status is DONE in Feast -# Currently set to the maximum query execution time limit in BigQuery -DEFAULT_TIMEOUT_SEC: int = 21600 - -# Maximum no of seconds to wait before reloading the job status in Feast -MAX_WAIT_INTERVAL_SEC: int = 60 +from feast.wait import wait_retry_backoff class RetrievalJob: @@ -71,7 +65,7 @@ def reload(self): """ self.job_proto = self.serving_stub.GetJob(GetJobRequest(job=self.job_proto)).job - def get_avro_files(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): + def get_avro_files(self, timeout_sec: int = int(defaults[CONFIG_TIMEOUT_KEY])): """ Wait until job is done to get the file uri to Avro result files on Google Cloud Storage. @@ -84,21 +78,17 @@ def get_avro_files(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): Returns: str: Google Cloud Storage file uris of the returned Avro files. """ - max_wait_datetime = datetime.now() + timedelta(seconds=timeout_sec) - wait_duration_sec = 2 - - while self.status != JOB_STATUS_DONE: - if datetime.now() > max_wait_datetime: - raise Exception( - "Timeout exceeded while waiting for result. Please retry " - "this method or use a longer timeout value." - ) + def try_retrieve(): self.reload() - time.sleep(wait_duration_sec) + return None, self.status == JOB_STATUS_DONE - # Backoff the wait duration exponentially up till MAX_WAIT_INTERVAL_SEC - wait_duration_sec = min(wait_duration_sec * 2, MAX_WAIT_INTERVAL_SEC) + wait_retry_backoff( + retry_fn=try_retrieve, + timeout_secs=timeout_sec, + timeout_msg="Timeout exceeded while waiting for result. Please retry " + "this method or use a longer timeout value.", + ) if self.job_proto.error: raise Exception(self.job_proto.error) @@ -111,7 +101,7 @@ def get_avro_files(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): return [urlparse(uri) for uri in self.job_proto.file_uris] - def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): + def result(self, timeout_sec: int = int(defaults[CONFIG_TIMEOUT_KEY])): """ Wait until job is done to get an iterable rows of result. The row can only represent an Avro row in Feast 0.3. @@ -142,7 +132,9 @@ def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): for record in avro_reader: yield record - def to_dataframe(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC) -> pd.DataFrame: + def to_dataframe( + self, timeout_sec: int = int(defaults[CONFIG_TIMEOUT_KEY]) + ) -> pd.DataFrame: """ Wait until a job is done to get an iterable rows of result. This method will split the response into chunked DataFrame of a specified size to @@ -164,7 +156,9 @@ def to_dataframe(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC) -> pd.DataFrame: return pd.DataFrame.from_records(records) def to_chunked_dataframe( - self, max_chunk_size: int = -1, timeout_sec: int = DEFAULT_TIMEOUT_SEC + self, + max_chunk_size: int = -1, + timeout_sec: int = int(defaults[CONFIG_TIMEOUT_KEY]), ) -> pd.DataFrame: """ Wait until a job is done to get an iterable rows of result. This method @@ -281,18 +275,11 @@ def wait(self, status: IngestionJobStatus, timeout_secs: float = 300): timeout_secs: Maximum seconds to wait before timing out. """ # poll & wait for job status to transition - wait_begin = time.time() - wait_secs = 2 - elapsed_secs = 0 - while self.status != status and elapsed_secs <= timeout_secs: - time.sleep(wait_secs) - # back off wait duration exponentially, capped at MAX_WAIT_INTERVAL_SEC - wait_secs = min(wait_secs * 2, MAX_WAIT_INTERVAL_SEC) - elapsed_secs = time.time() - wait_begin - - # raise error if timeout - if elapsed_secs > timeout_secs: - raise TimeoutError("Wait for IngestJob's status to transition timed out") + wait_retry_backoff( + retry_fn=(lambda: (None, self.status == status)), + timeout_secs=timeout_secs, + timeout_msg="Wait for IngestJob's status to transition timed out", + ) def __str__(self): # render the contents of ingest job as human readable string diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index 5e8e5da4d41..b439dbd3027 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -4,8 +4,8 @@ from typing import Iterable, List import pandas as pd - import pyarrow.parquet as pq + from feast.constants import DATETIME_COLUMN from feast.feature_set import FeatureSet from feast.type_map import ( diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 7a483a6c2a0..85def25fcb9 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -17,9 +17,10 @@ import numpy as np import pandas as pd +import pyarrow as pa from google.protobuf.timestamp_pb2 import Timestamp +from pyarrow.lib import TimestampType -import pyarrow as pa from feast.constants import DATETIME_COLUMN from feast.types import FeatureRow_pb2 as FeatureRowProto from feast.types import Field_pb2 as FieldProto @@ -35,7 +36,6 @@ from feast.types.Value_pb2 import Value as ProtoValue from feast.types.Value_pb2 import ValueType as ProtoValueType from feast.value_type import ValueType -from pyarrow.lib import TimestampType def python_type_to_feast_value_type( diff --git a/sdk/python/feast/wait.py b/sdk/python/feast/wait.py new file mode 100644 index 00000000000..f25036d76ea --- /dev/null +++ b/sdk/python/feast/wait.py @@ -0,0 +1,56 @@ +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from typing import Any, Callable, Optional, Tuple + +from feast.constants import CONFIG_MAX_WAIT_INTERVAL_KEY +from feast.constants import FEAST_DEFAULT_OPTIONS as defaults + + +def wait_retry_backoff( + retry_fn: Callable[[], Tuple[Any, bool]], + timeout_secs: Optional[int] = None, + timeout_msg: Optional[str] = "Timeout while waiting for retry_fn() to return True", + max_interval_secs: Optional[int] = int(defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]), +) -> Any: + """ + Repeatedly try calling given retry_fn until it returns a True boolean success flag. + Waits with a exponential backoff between retries until timeout when it throws TimeoutError. + Args: + retry_fn: Callable that returns a result and a boolean success flag. + timeout_secs: timeout in seconds to give up retrying and throw TimeoutError, + or None to retry perpectually. + timeout_msg: Message to use when throwing TimeoutError. + max_interval_secs: max wait in seconds to wait between retries. + Returns: + Returned Result from retry_fn() if success flag is True. + """ + wait_secs, elapsed_secs = 1, 0 + result, is_success = retry_fn() + wait_begin = time.time() + while not is_success and elapsed_secs <= timeout_secs: + # back off wait duration exponentially, capped at MAX_WAIT_INTERVAL_SEC + elapsed_secs = time.time() - wait_begin + till_timeout_secs = timeout_secs - elapsed_secs + wait_secs = min(wait_secs * 2, max_interval_secs, till_timeout_secs) + time.sleep(wait_secs) + # retry call + result, is_success = retry_fn() + elapsed_secs = time.time() - wait_begin + + if not is_success and elapsed_secs > timeout_secs: + raise TimeoutError(timeout_msg) + + return result diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 04626f73cd5..9437ef735d1 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -220,6 +220,7 @@ def int_val(x): project="driver_project", feature_set="driver", name="age" ), FeatureRefProto(project="driver_project", name="rating"), + FeatureRefProto(project="driver_project", name="null_value"), ] ) recieve_response = GetOnlineFeaturesResponse() @@ -234,7 +235,14 @@ def int_val(x): "driver_id": int_val(row_number), "driver_project/driver:age": int_val(1), "driver_project/rating": int_val(9), - } + "driver_project/null_value": ValueProto.Value(), + }, + statuses={ + "driver_id": GetOnlineFeaturesResponse.FieldStatus.PRESENT, + "driver_project/driver:age": GetOnlineFeaturesResponse.FieldStatus.PRESENT, + "driver_project/rating": GetOnlineFeaturesResponse.FieldStatus.PRESENT, + "driver_project/null_value": GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE, + }, ) recieve_response.field_values.append(field_values) @@ -245,7 +253,7 @@ def int_val(x): ) got_response = mocked_client.get_online_features( entity_rows=request.entity_rows, - feature_refs=["driver:age", "rating"], + feature_refs=["driver:age", "rating", "null_value"], project="driver_project", ) # type: GetOnlineFeaturesResponse mocked_client._serving_service_stub.GetOnlineFeatures.assert_called_with( @@ -253,10 +261,19 @@ def int_val(x): ) got_fields = got_response.field_values[0].fields + got_statuses = got_response.field_values[0].statuses assert ( got_fields["driver_id"] == int_val(1) + and got_statuses["driver_id"] + == GetOnlineFeaturesResponse.FieldStatus.PRESENT and got_fields["driver:age"] == int_val(1) + and got_statuses["driver:age"] + == GetOnlineFeaturesResponse.FieldStatus.PRESENT and got_fields["rating"] == int_val(9) + and got_statuses["rating"] == GetOnlineFeaturesResponse.FieldStatus.PRESENT + and got_fields["null_value"] == ValueProto.Value() + and got_statuses["null_value"] + == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE ) @pytest.mark.parametrize( diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index bb73e34f51c..6d581f6c710 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -17,12 +17,13 @@ package feast.serving.service; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; import com.google.protobuf.Duration; import feast.proto.serving.ServingAPIProto.*; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.FeatureRowProto.FeatureRow; +import feast.proto.types.FieldProto.Field; import feast.proto.types.ValueProto.Value; import feast.serving.specs.CachedSpecService; import feast.serving.util.Metrics; @@ -32,9 +33,10 @@ import io.grpc.Status; import io.opentracing.Scope; import io.opentracing.Tracer; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; +import org.apache.beam.vendor.grpc.v1p21p0.com.google.common.collect.Streams; +import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; public class OnlineServingService implements ServingService { @@ -64,79 +66,239 @@ public GetFeastServingInfoResponse getFeastServingInfo( @Override public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { try (Scope scope = tracer.buildSpan("getOnlineFeatures").startActive(true)) { - GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder = - GetOnlineFeaturesResponse.newBuilder(); - List featureSetRequests = - specService.getFeatureSets(request.getFeaturesList()); List entityRows = request.getEntityRowsList(); - Map> featureValuesMap = - entityRows.stream() - .collect(Collectors.toMap(row -> row, row -> Maps.newHashMap(row.getFieldsMap()))); - // Get all feature rows from the retriever. Each feature row list corresponds to a single - // feature set request. - List> featureRows = - retriever.getOnlineFeatures(entityRows, featureSetRequests); - if (scope != null) { - scope.span().log(ImmutableMap.of("event", "featureRows", "value", featureRows)); + // Collect the feature/entity value for each entity row in entityValueMap + Map> entityValuesMap = + entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); + // Collect the feature/entity status metadata for each entity row in entityValueMap + Map> entityStatusesMap = + entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); + // Collect featureRows retrieved for logging/tracing + List>> logFeatureRows = new LinkedList<>(); + + if (!request.getOmitEntitiesInResponse()) { + // Add entity row's fields as response fields + entityRows.forEach( + entityRow -> { + Map valueMap = entityRow.getFieldsMap(); + entityValuesMap.get(entityRow).putAll(valueMap); + entityStatusesMap.get(entityRow).putAll(getMetadataMap(valueMap, false, false)); + }); } - // For each feature set request, read the feature rows returned by the retriever, and - // populate the featureValuesMap with the feature values corresponding to that entity row. - for (var fsIdx = 0; fsIdx < featureRows.size(); fsIdx++) { - List featureRowsForFs = featureRows.get(fsIdx); - FeatureSetRequest featureSetRequest = featureSetRequests.get(fsIdx); - - String project = featureSetRequest.getSpec().getProject(); - - // In order to return values containing the same feature references provided by the user, - // we reuse the feature references in the request as the keys in the featureValuesMap - Map refsByName = featureSetRequest.getFeatureRefsByName(); - - // Each feature row returned (per feature set request) corresponds to a given entity row. - // For each feature row, update the featureValuesMap. - for (var entityRowIdx = 0; entityRowIdx < entityRows.size(); entityRowIdx++) { - FeatureRow featureRow = featureRowsForFs.get(entityRowIdx); - EntityRow entityRow = entityRows.get(entityRowIdx); - - // If the row is stale, put an empty value into the featureValuesMap. - if (isStale(featureSetRequest, entityRow, featureRow)) { - featureSetRequest - .getFeatureReferences() - .forEach( - ref -> { - populateStaleKeyCountMetrics(project, ref); - featureValuesMap - .get(entityRow) - .put(RefUtil.generateFeatureStringRef(ref), Value.newBuilder().build()); - }); - - } else { - populateRequestCountMetrics(featureSetRequest); - - // Else populate the featureValueMap at this entityRow with the values in the feature - // row. - featureRow.getFieldsList().stream() - .filter(field -> refsByName.containsKey(field.getName())) - .forEach( - field -> { - FeatureReference ref = refsByName.get(field.getName()); - String id = RefUtil.generateFeatureStringRef(ref); - featureValuesMap.get(entityRow).put(id, field.getValue()); - }); - } + List featureSetRequests = + specService.getFeatureSets(request.getFeaturesList()); + for (FeatureSetRequest featureSetRequest : featureSetRequests) { + // Pull feature rows for given entity rows from the feature/featureset specified in feature + // set request. + // from the configured online + List> featureRows = + retriever.getOnlineFeatures(entityRows, featureSetRequest); + // Check that feature row returned corresponds to a given entity row. + if (featureRows.size() != entityRows.size()) { + throw Status.INTERNAL + .withDescription( + "The no. of FeatureRow obtained from OnlineRetriever" + + "does not match no. of entityRow passed.") + .asRuntimeException(); } + + Streams.zip(entityRows.stream(), featureRows.stream(), Pair::of) + .forEach( + entityFeaturePair -> { + EntityRow entityRow = entityFeaturePair.getLeft(); + Optional featureRow = entityFeaturePair.getRight(); + // Unpack feature field values and merge into entityValueMap + boolean isOutsideMaxAge = + checkOutsideMaxAge(featureSetRequest, entityRow, featureRow); + Map valueMap = + unpackValueMap(featureRow, featureSetRequest, isOutsideMaxAge); + entityValuesMap.get(entityRow).putAll(valueMap); + + // Generate metadata for feature values and merge into entityFieldsMap + boolean isNotFound = featureRow.isEmpty(); + Map statusMap = + getMetadataMap(valueMap, isNotFound, isOutsideMaxAge); + entityStatusesMap.get(entityRow).putAll(statusMap); + + // Populate metrics/log request + populateStaleKeyCountMetrics(statusMap, featureSetRequest); + }); + populateRequestCountMetrics(featureSetRequest); + logFeatureRows.add(featureRows); + } + if (scope != null) { + logFeatureRowsTrace(scope, logFeatureRows, featureSetRequests); } - List fieldValues = - featureValuesMap.values().stream() - .map(valueMap -> FieldValues.newBuilder().putAllFields(valueMap).build()) + // Build response field values from entityValuesMap and entityStatusesMap + // Reponse field values should be in the same order as the entityRows provided by the user. + List fieldValuesList = + entityRows.stream() + .map( + entityRow -> { + return FieldValues.newBuilder() + .putAllFields(entityValuesMap.get(entityRow)) + .putAllStatuses(entityStatusesMap.get(entityRow)) + .build(); + }) .collect(Collectors.toList()); - return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build(); + return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); + } + } + + /** + * Unpack feature values using data from the given feature row for features specified in the given + * feature set request. + * + * @param featureRow optional to unpack for feature values. + * @param featureSetRequest for which the feature row was retrieved. + * @param isOutsideMaxAge whether which the feature row contains values that is outside max age. + * @return valueMap mapping string feature name to feature value for the given feature set + * request. + */ + private static Map unpackValueMap( + Optional featureRow, + FeatureSetRequest featureSetRequest, + boolean isOutsideMaxAge) { + Map valueMap = new HashMap<>(); + // In order to return values containing the same feature references provided by the user, + // we reuse the feature references in the request as the keys in field builder map + Map nameRefMap = featureSetRequest.getFeatureRefsByName(); + if (featureRow.isPresent()) { + // unpack feature row's feature values and populate value map + Map featureValueMap = + featureRow.get().getFieldsList().stream() + .filter(featureRowField -> nameRefMap.containsKey(featureRowField.getName())) + .collect( + Collectors.toMap( + featureRowField -> { + FeatureReference featureRef = nameRefMap.get(featureRowField.getName()); + return RefUtil.generateFeatureStringRef(featureRef); + }, + featureRowField -> { + // drop feature values with an age outside feature set's max age. + return (isOutsideMaxAge) + ? Value.newBuilder().build() + : featureRowField.getValue(); + })); + valueMap.putAll(featureValueMap); } + // create empty values for features specified in request but not present in feature row. + Set missingFeatures = + nameRefMap.values().stream() + .map(ref -> RefUtil.generateFeatureStringRef(ref)) + .collect(Collectors.toSet()); + missingFeatures.removeAll(valueMap.keySet()); + missingFeatures.forEach(refString -> valueMap.put(refString, Value.newBuilder().build())); + + return valueMap; } - private void populateStaleKeyCountMetrics(String project, FeatureReference ref) { - Metrics.staleKeyCount.labels(project, ref.getName()).inc(); + /** + * Generate Field level Status metadata for the given valueMap. + * + * @param valueMap map of field name to value to generate metadata for. + * @param isNotFound whether the given valueMap represents values that were not found in the + * online retriever. + * @param isOutsideMaxAge whether the given valueMap contains values with age outside feature + * set's max age. + * @return a 1:1 map keyed by field name containing field status metadata instead of values in the + * given valueMap. + */ + private static Map getMetadataMap( + Map valueMap, boolean isNotFound, boolean isOutsideMaxAge) { + return valueMap.entrySet().stream() + .collect( + Collectors.toMap( + es -> es.getKey(), + es -> { + Value fieldValue = es.getValue(); + if (isNotFound) { + return FieldStatus.NOT_FOUND; + } else if (isOutsideMaxAge) { + return FieldStatus.OUTSIDE_MAX_AGE; + } else if (fieldValue.getValCase().equals(Value.ValCase.VAL_NOT_SET)) { + return FieldStatus.NULL_VALUE; + } + return FieldStatus.PRESENT; + })); + } + + /** + * Determine if the feature data in the given feature row is outside maxAge. Data is outside + * maxAge to be when the difference ingestion time set in feature row and the retrieval time set + * in entity row exceeds featureset max age. + * + * @param featureSetRequest contains the spec where feature's max age is extracted. + * @param entityRow contains the retrieval timing of when features are pulled. + * @param featureRow contains the ingestion timing and feature data. + */ + private static boolean checkOutsideMaxAge( + FeatureSetRequest featureSetRequest, EntityRow entityRow, Optional featureRow) { + Duration maxAge = featureSetRequest.getSpec().getMaxAge(); + if (featureRow.isEmpty()) { // no data to consider + return false; + } + if (maxAge.equals(Duration.getDefaultInstance())) { // max age is not set + return false; + } + + long givenTimestamp = entityRow.getEntityTimestamp().getSeconds(); + if (givenTimestamp == 0) { + givenTimestamp = System.currentTimeMillis() / 1000; + } + long timeDifference = givenTimestamp - featureRow.get().getEventTimestamp().getSeconds(); + return timeDifference > maxAge.getSeconds(); + } + + private void logFeatureRowsTrace( + Scope scope, + List>> logFeatureRows, + List featureSetRequests) { + List> loggableFeatureRows = + Streams.zip( + logFeatureRows.stream(), + featureSetRequests.stream(), + (featureRows, featureSetRequest) -> { + FeatureRow.Builder nullFeatureRowBuilder = + FeatureRow.newBuilder() + .setFeatureSet( + RefUtil.generateFeatureSetStringRef(featureSetRequest.getSpec())); + for (FeatureReference featureReference : + featureSetRequest.getFeatureReferences()) { + nullFeatureRowBuilder.addFields( + Field.newBuilder().setName(featureReference.getName())); + } + + // log null feature row when feature row is empty + return featureRows.stream() + .map( + featureRow -> { + return (featureRow.isEmpty()) + ? nullFeatureRowBuilder.build() + : featureRow.get(); + }) + .collect(Collectors.toList()); + }) + .collect(Collectors.toList()); + + scope.span().log(ImmutableMap.of("event", "featureRows", "value", loggableFeatureRows)); + } + + private void populateStaleKeyCountMetrics( + Map statusMap, FeatureSetRequest featureSetRequest) { + String project = featureSetRequest.getSpec().getProject(); + statusMap + .entrySet() + .forEach( + es -> { + String featureRefString = es.getKey(); + FieldStatus status = es.getValue(); + if (status == FieldStatus.OUTSIDE_MAX_AGE) { + Metrics.staleKeyCount.labels(project, featureRefString).inc(); + } + }); } private void populateRequestCountMetrics(FeatureSetRequest featureSetRequest) { @@ -156,18 +318,4 @@ public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeat public GetJobResponse getJob(GetJobRequest getJobRequest) { throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); } - - private boolean isStale( - FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { - Duration maxAge = featureSetRequest.getSpec().getMaxAge(); - if (maxAge.equals(Duration.getDefaultInstance())) { - return false; - } - long givenTimestamp = entityRow.getEntityTimestamp().getSeconds(); - if (givenTimestamp == 0) { - givenTimestamp = System.currentTimeMillis() / 1000; - } - long timeDifference = givenTimestamp - featureRow.getEventTimestamp().getSeconds(); - return timeDifference > maxAge.getSeconds(); - } } diff --git a/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java b/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java index 6aba17ac431..238df549513 100644 --- a/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java +++ b/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java @@ -29,7 +29,7 @@ public class ResponseJSONMapper { public static List> mapGetOnlineFeaturesResponse( GetOnlineFeaturesResponse response) { return response.getFieldValuesList().stream() - .map(fieldValue -> convertFieldValuesToMap(fieldValue)) + .map(fieldValues -> convertFieldValuesToMap(fieldValues)) .collect(Collectors.toList()); } diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index b5eff65391b..2399d132ef9 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -10,10 +10,9 @@ feast: # List of store configurations stores: - # Below are two store configurations. One for Redis and one for BigQuery. # Please see https://api.docs.feast.dev/grpc/feast.core.pb.html#Store for configuration options - name: online # Name of the store (referenced by active_store) - type: REDIS # Type of the store. REDIS, BIGQUERY are available options + type: REDIS # Type of the store. REDIS, REDIS_CLUSTER, BIGQUERY are available options config: # Store specific configuration. See host: localhost port: 6379 @@ -22,7 +21,15 @@ feast: # Wildcards match all options. No filtering is done. - name: "*" project: "*" - + - name: online_cluster + type: REDIS_CLUSTER + config: # Store specific configuration. + # Connection string specifies the host:port of Redis instances in the redis cluster. + connection_string: "localhost:7000,localhost:7001,localhost:7002,localhost:7003,localhost:7004,localhost:7005" + subscriptions: + - name: "*" + project: "*" + version: "*" - name: historical type: BIGQUERY config: # Store specific configuration. @@ -77,4 +84,4 @@ server: # The port number on which the Tomcat webserver that serves REST API endpoints should listen # It is set by default to 8081 so it does not conflict with Tomcat webserver on Feast Core # if both Feast Core and Serving are running on the same machine - port: ${SERVER_PORT:8081} \ No newline at end of file + port: ${SERVER_PORT:8081} diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 6358460a070..9ab892a0c44 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -17,7 +17,7 @@ package feast.serving.service; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -30,9 +30,10 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; +import feast.proto.types.FieldProto; import feast.proto.types.ValueProto.Value; import feast.serving.specs.CachedSpecService; import feast.storage.api.retriever.FeatureSetRequest; @@ -41,8 +42,7 @@ import io.opentracing.Tracer.SpanBuilder; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; +import java.util.Optional; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatchers; @@ -66,11 +66,13 @@ public void setUp() { } @Test - public void shouldReturnResponseWithValuesIfKeysPresent() { + public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() + .setOmitEntitiesInResponse(false) .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) - .addFeatures(FeatureReference.newBuilder().setName("feature2").build()) + .addFeatures( + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -83,26 +85,54 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { .putFields("entity2", strValue("b"))) .build(); - List featureRows = + List> featureRows = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build()); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .setFeatureSet("featureSet") + .build()), + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(2)) + .build())) + .setFeatureSet("featureSet") + .build())); FeatureSetRequest featureSetRequest = FeatureSetRequest.newBuilder() @@ -112,9 +142,8 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures( - request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) - .thenReturn(Collections.singletonList(featureRows)); + when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) + .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -122,28 +151,38 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(1)) + .putStatuses("entity1", FieldStatus.PRESENT) .putFields("entity2", strValue("a")) + .putStatuses("entity2", FieldStatus.PRESENT) .putFields("feature1", intValue(1)) - .putFields("feature2", intValue(1))) + .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("project/feature2", intValue(1)) + .putStatuses("project/feature2", FieldStatus.PRESENT) + .build()) .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(2)) + .putStatuses("entity1", FieldStatus.PRESENT) .putFields("entity2", strValue("b")) + .putStatuses("entity2", FieldStatus.PRESENT) .putFields("feature1", intValue(2)) - .putFields("feature2", intValue(2))) + .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("project/feature2", intValue(2)) + .putStatuses("project/feature2", FieldStatus.PRESENT) + .build()) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat( - responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); + assertThat(actual, equalTo(expected)); } @Test - public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { + public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { // some keys not present, should have empty values GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) - .addFeatures(FeatureReference.newBuilder().setName("feature2").build()) + .addFeatures( + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -162,29 +201,29 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { .setSpec(getFeatureSetSpec()) .build(); - List featureRows = + List> featureRows = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").build(), - Field.newBuilder().setName("feature2").build())) - .build()); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .build()), + Optional.empty()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures( - request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) - .thenReturn(Collections.singletonList(featureRows)); + when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) + .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -192,28 +231,39 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(1)) + .putStatuses("entity1", FieldStatus.PRESENT) .putFields("entity2", strValue("a")) + .putStatuses("entity2", FieldStatus.PRESENT) .putFields("feature1", intValue(1)) - .putFields("feature2", intValue(1))) + .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("project/feature2", intValue(1)) + .putStatuses("project/feature2", FieldStatus.PRESENT) + .build()) .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(2)) + .putStatuses("entity1", FieldStatus.PRESENT) .putFields("entity2", strValue("b")) + .putStatuses("entity2", FieldStatus.PRESENT) .putFields("feature1", Value.newBuilder().build()) - .putFields("feature2", Value.newBuilder().build())) + .putStatuses("feature1", FieldStatus.NOT_FOUND) + .putFields("project/feature2", Value.newBuilder().build()) + .putStatuses("project/feature2", FieldStatus.NOT_FOUND) + .build()) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat( - responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); + assertThat(actual, equalTo(expected)); } @Test - public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { - // keys present, but too stale comp. to maxAge + public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { + // keys present, but considered stale when compared to maxAge GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) - .addFeatures(FeatureReference.newBuilder().setName("feature2").build()) + .addFeatures( + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) + .setOmitEntitiesInResponse(false) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -226,29 +276,55 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { .putFields("entity2", strValue("b"))) .build(); - List featureRows = + List> featureRows = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .setFeatureSet("project/featureSet") - .build(), - FeatureRow.newBuilder() - .setEventTimestamp( - Timestamp.newBuilder().setSeconds(50)) // this value should be nulled - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .setFeatureSet("project/featureSet") - .build()); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .setFeatureSet("project/featureSet") + .build()), + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp( + Timestamp.newBuilder().setSeconds(50)) // this value should be nulled + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("project/feature2") + .setValue(intValue(2)) + .build())) + .setFeatureSet("project/featureSet") + .build())); FeatureSetSpec spec = getFeatureSetSpec().toBuilder().setMaxAge(Duration.newBuilder().setSeconds(1)).build(); @@ -260,9 +336,8 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures( - request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) - .thenReturn(Collections.singletonList(featureRows)); + when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) + .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -270,19 +345,28 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(1)) + .putStatuses("entity1", FieldStatus.PRESENT) .putFields("entity2", strValue("a")) + .putStatuses("entity2", FieldStatus.PRESENT) .putFields("feature1", intValue(1)) - .putFields("feature2", intValue(1))) + .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("project/feature2", intValue(1)) + .putStatuses("project/feature2", FieldStatus.PRESENT) + .build()) .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(2)) + .putStatuses("entity1", FieldStatus.PRESENT) .putFields("entity2", strValue("b")) + .putStatuses("entity2", FieldStatus.PRESENT) .putFields("feature1", Value.newBuilder().build()) - .putFields("feature2", Value.newBuilder().build())) + .putStatuses("feature1", FieldStatus.OUTSIDE_MAX_AGE) + .putFields("project/feature2", Value.newBuilder().build()) + .putStatuses("project/feature2", FieldStatus.OUTSIDE_MAX_AGE) + .build()) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat( - responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); + assertThat(actual, equalTo(expected)); } @Test @@ -290,6 +374,7 @@ public void shouldFilterOutUndesiredRows() { // requested rows less than the rows available in the featureset GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() + .setOmitEntitiesInResponse(false) .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) .addEntityRows( EntityRow.newBuilder() @@ -303,26 +388,54 @@ public void shouldFilterOutUndesiredRows() { .putFields("entity2", strValue("b"))) .build(); - List featureRows = + List> featureRows = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build()); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .setFeatureSet("featureSet") + .build()), + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(2)) + .build())) + .setFeatureSet("featureSet") + .build())); FeatureSetRequest featureSetRequest = FeatureSetRequest.newBuilder() @@ -332,9 +445,8 @@ public void shouldFilterOutUndesiredRows() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures( - request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) - .thenReturn(Collections.singletonList(featureRows)); + when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) + .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -342,27 +454,28 @@ public void shouldFilterOutUndesiredRows() { .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(1)) + .putStatuses("entity1", FieldStatus.PRESENT) .putFields("entity2", strValue("a")) - .putFields("feature1", intValue(1))) + .putStatuses("entity2", FieldStatus.PRESENT) + .putFields("feature1", intValue(1)) + .putStatuses("feature1", FieldStatus.PRESENT) + .build()) .addFieldValues( FieldValues.newBuilder() .putFields("entity1", intValue(2)) + .putStatuses("entity1", FieldStatus.PRESENT) .putFields("entity2", strValue("b")) - .putFields("feature1", intValue(2))) + .putStatuses("entity2", FieldStatus.PRESENT) + .putFields("feature1", intValue(2)) + .putStatuses("feature1", FieldStatus.PRESENT) + .build()) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat( - responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); - } - - private List> responseToMapList(GetOnlineFeaturesResponse response) { - return response.getFieldValuesList().stream() - .map(FieldValues::getFieldsMap) - .collect(Collectors.toList()); + assertThat(actual, equalTo(expected)); } private Value intValue(int val) { - return Value.newBuilder().setInt64Val(val).build(); + return Value.newBuilder().setInt32Val(val).build(); } private Value strValue(String val) { diff --git a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java index 45d6b352632..f56cf6ca022 100644 --- a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java +++ b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java @@ -19,22 +19,25 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.types.FeatureRowProto.FeatureRow; import java.util.List; +import java.util.Optional; -/** - * An online retriever is a feature retriever that retrieves the latest feature data corresponding - * to provided entities. - */ +/** An online retriever is a feature retriever that retrieves the latest feature data. */ public interface OnlineRetriever { /** - * Get all values corresponding to the request. + * Get online features for the given entity rows using data retrieved from the feature/featureset + * specified in feature set request. + * + *

Each {@link FeatureRow} optional in the returned list then corresponds to an {@link + * EntityRow} provided by the user. If feature for a given entity row is not found, will return an + * empty optional instead. The no. of {@link FeatureRow} returned should match the no. of given + * {@link EntityRow}s * - * @param entityRows list of entity rows in the feature request - * @param featureSetRequests List of {@link FeatureSetRequest} to feature references in the - * request tied to that feature set. - * @return list of lists of {@link FeatureRow}s corresponding to each feature set request and - * entity row. + * @param entityRows list of entity rows to request features for. + * @param featureSetRequest specifies the features/feature set to retrieve data from + * @return list of {@link FeatureRow}s corresponding to data retrieved for each entity row from + * feature/featureset specified in featureset request. */ - List> getOnlineFeatures( - List entityRows, List featureSetRequests); + List> getOnlineFeatures( + List entityRows, FeatureSetRequest featureSetRequest); } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java index fd9556841e4..aad3147f710 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java @@ -43,7 +43,7 @@ public FeatureRowDecoder(String featureSetRef, FeatureSetSpec spec) { * @param featureRow Feature row * @return boolean */ - public Boolean isEncoded(FeatureRow featureRow) { + public boolean isEncoded(FeatureRow featureRow) { return featureRow.getFeatureSet().isEmpty() && featureRow.getFieldsList().stream().allMatch(field -> field.getName().isEmpty()); } @@ -54,7 +54,7 @@ public Boolean isEncoded(FeatureRow featureRow) { * @param featureRow Feature row * @return boolean */ - public Boolean isEncodingValid(FeatureRow featureRow) { + public boolean isEncodingValid(FeatureRow featureRow) { return featureRow.getFieldsList().size() == spec.getFeaturesList().size(); } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java index 18619252c27..c006149cd51 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java @@ -20,7 +20,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import feast.proto.core.FeatureSetProto.EntitySpec; import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.storage.RedisProto.RedisKey; import feast.proto.types.FeatureRowProto.FeatureRow; @@ -38,9 +37,11 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +/** Defines a storage retriever */ public class RedisClusterOnlineRetriever implements OnlineRetriever { private final RedisAdvancedClusterCommands syncCommands; @@ -69,36 +70,24 @@ public static OnlineRetriever create(StatefulRedisClusterConnection> getOnlineFeatures( - List entityRows, List featureSetRequests) { - - List> featureRows = new ArrayList<>(); - for (FeatureSetRequest featureSetRequest : featureSetRequests) { - List redisKeys = buildRedisKeys(entityRows, featureSetRequest.getSpec()); - try { - List featureRowsForFeatureSet = - sendAndProcessMultiGet( - redisKeys, - featureSetRequest.getSpec(), - featureSetRequest.getFeatureReferences().asList()); - featureRows.add(featureRowsForFeatureSet); - } catch (InvalidProtocolBufferException | ExecutionException e) { - throw Status.INTERNAL - .withDescription("Unable to parse protobuf while retrieving feature") - .withCause(e) - .asRuntimeException(); - } + public List> getOnlineFeatures( + List entityRows, FeatureSetRequest featureSetRequest) { + + // get features for this features/featureset in featureset request + FeatureSetSpec featureSetSpec = featureSetRequest.getSpec(); + List redisKeys = buildRedisKeys(entityRows, featureSetSpec); + FeatureRowDecoder decoder = + new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); + List> featureRows = new ArrayList<>(); + try { + featureRows = getFeaturesFromRedis(redisKeys, decoder); + } catch (InvalidProtocolBufferException | ExecutionException e) { + throw Status.INTERNAL + .withDescription("Unable to parse protobuf while retrieving feature") + .withCause(e) + .asRuntimeException(); } return featureRows; } @@ -147,49 +136,51 @@ private RedisKey makeRedisKey( return builder.build(); } - private List sendAndProcessMultiGet( - List redisKeys, - FeatureSetSpec featureSetSpec, - List featureReferences) + /** + * Get features from data pulled from the Redis for a specific featureset. + * + * @param redisKeys keys used to retrieve data from Redis for a specific featureset. + * @param decoder used to decode the data retrieved from Redis for a specific featureset. + * @return List of {@link FeatureRow} optionals + */ + private List> getFeaturesFromRedis( + List redisKeys, FeatureRowDecoder decoder) throws InvalidProtocolBufferException, ExecutionException { + // pull feature row data bytes from redis using given redis keys + List featureRowsBytes = sendMultiGet(redisKeys); + List> featureRows = new ArrayList<>(); - List values = sendMultiGet(redisKeys); - List featureRows = new ArrayList<>(); - - FeatureRow.Builder nullFeatureRowBuilder = - FeatureRow.newBuilder().setFeatureSet(generateFeatureSetStringRef(featureSetSpec)); - for (FeatureReference featureReference : featureReferences) { - nullFeatureRowBuilder.addFields(Field.newBuilder().setName(featureReference.getName())); - } - - for (int i = 0; i < values.size(); i++) { - - byte[] value = values.get(i); - if (value == null) { - featureRows.add(nullFeatureRowBuilder.build()); + for (byte[] featureRowBytes : featureRowsBytes) { + if (featureRowBytes == null) { + featureRows.add(Optional.empty()); continue; } - FeatureRow featureRow = FeatureRow.parseFrom(value); - String featureSetRef = redisKeys.get(i).getFeatureSet(); - FeatureRowDecoder decoder = new FeatureRowDecoder(featureSetRef, featureSetSpec); - if (decoder.isEncodingValid(featureRow)) { - featureRow = decoder.decode(featureRow); - } else { - featureRows.add(nullFeatureRowBuilder.build()); - continue; + // decode feature rows from data bytes using decoder. + FeatureRow featureRow = FeatureRow.parseFrom(featureRowBytes); + if (decoder.isEncoded(featureRow)) { + if (decoder.isEncodingValid(featureRow)) { + featureRow = decoder.decode(featureRow); + } else { + // decoding feature row failed: data corruption could have occurred + throw Status.DATA_LOSS + .withDescription( + "Failed to decode FeatureRow from bytes retrieved from redis" + + ": Possible data corruption") + .asRuntimeException(); + } } - - featureRows.add(featureRow); + featureRows.add(Optional.of(featureRow)); } return featureRows; } /** - * Send a list of get request as an mget + * Pull the data stored in Redis at the given keys as bytes using the mget command. If no data is + * stored at a given key in Redis, will subsitute the data with null. * - * @param keys list of {@link RedisKey} - * @return list of {@link FeatureRow} in primitive byte representation for each {@link RedisKey} + * @param keys list of {@link RedisKey} to pull from redis. + * @return list of data bytes or null pulled from redis for each given key. */ private List sendMultiGet(List keys) { try { diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index 0db4837c069..2b09f7346bb 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -20,7 +20,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import feast.proto.core.FeatureSetProto.EntitySpec; import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.storage.RedisProto.RedisKey; import feast.proto.types.FeatureRowProto.FeatureRow; @@ -37,6 +36,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -62,37 +62,26 @@ public static OnlineRetriever create(StatefulRedisConnection con return new RedisOnlineRetriever(connection); } - /** - * Gets online features from redis. This method returns a list of {@link FeatureRow}s - * corresponding to each feature set spec. Each feature row in the list then corresponds to an - * {@link EntityRow} provided by the user. - * - * @param entityRows list of entity rows in the feature request - * @param featureSetRequests Map of {@link feast.proto.core.FeatureSetProto.FeatureSetSpec} to - * feature references in the request tied to that feature set. - * @return List of List of {@link FeatureRow} - */ + /** {@inheritDoc} */ @Override - public List> getOnlineFeatures( - List entityRows, List featureSetRequests) { - - List> featureRows = new ArrayList<>(); - for (FeatureSetRequest featureSetRequest : featureSetRequests) { - List redisKeys = buildRedisKeys(entityRows, featureSetRequest.getSpec()); - try { - List featureRowsForFeatureSet = - sendAndProcessMultiGet( - redisKeys, - featureSetRequest.getSpec(), - featureSetRequest.getFeatureReferences().asList()); - featureRows.add(featureRowsForFeatureSet); - } catch (InvalidProtocolBufferException | ExecutionException e) { - throw Status.INTERNAL - .withDescription("Unable to parse protobuf while retrieving feature") - .withCause(e) - .asRuntimeException(); - } + public List> getOnlineFeatures( + List entityRows, FeatureSetRequest featureSetRequest) { + + // get features for this features/featureset in featureset request + FeatureSetSpec featureSetSpec = featureSetRequest.getSpec(); + List redisKeys = buildRedisKeys(entityRows, featureSetSpec); + FeatureRowDecoder decoder = + new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); + List> featureRows = new ArrayList<>(); + try { + featureRows = getFeaturesFromRedis(redisKeys, decoder); + } catch (InvalidProtocolBufferException | ExecutionException e) { + throw Status.INTERNAL + .withDescription("Unable to parse protobuf while retrieving feature") + .withCause(e) + .asRuntimeException(); } + return featureRows; } @@ -140,51 +129,51 @@ private RedisKey makeRedisKey( return builder.build(); } - private List sendAndProcessMultiGet( - List redisKeys, - FeatureSetSpec featureSetSpec, - List featureReferences) + /** + * Get features from data pulled from the Redis for a specific featureset. + * + * @param redisKeys keys used to retrieve data from Redis for a specific featureset. + * @param decoder used to decode the data retrieved from Redis for a specific featureset. + * @return List of {@link FeatureRow} optionals + */ + private List> getFeaturesFromRedis( + List redisKeys, FeatureRowDecoder decoder) throws InvalidProtocolBufferException, ExecutionException { + // pull feature row data bytes from redis using given redis keys + List featureRowsBytes = sendMultiGet(redisKeys); + List> featureRows = new ArrayList<>(); - List values = sendMultiGet(redisKeys); - List featureRows = new ArrayList<>(); - - FeatureRow.Builder nullFeatureRowBuilder = - FeatureRow.newBuilder().setFeatureSet(generateFeatureSetStringRef(featureSetSpec)); - for (FeatureReference featureReference : featureReferences) { - nullFeatureRowBuilder.addFields(Field.newBuilder().setName(featureReference.getName())); - } - - for (int i = 0; i < values.size(); i++) { - - byte[] value = values.get(i); - if (value == null) { - featureRows.add(nullFeatureRowBuilder.build()); + for (byte[] featureRowBytes : featureRowsBytes) { + if (featureRowBytes == null) { + featureRows.add(Optional.empty()); continue; } - FeatureRow featureRow = FeatureRow.parseFrom(value); - String featureSetRef = redisKeys.get(i).getFeatureSet(); - FeatureRowDecoder decoder = new FeatureRowDecoder(featureSetRef, featureSetSpec); + // decode feature rows from data bytes using decoder. + FeatureRow featureRow = FeatureRow.parseFrom(featureRowBytes); if (decoder.isEncoded(featureRow)) { if (decoder.isEncodingValid(featureRow)) { featureRow = decoder.decode(featureRow); } else { - featureRows.add(nullFeatureRowBuilder.build()); - continue; + // decoding feature row failed: data corruption could have occurred + throw Status.DATA_LOSS + .withDescription( + "Failed to decode FeatureRow from bytes retrieved from redis" + + ": Possible data corruption") + .asRuntimeException(); } } - - featureRows.add(featureRow); + featureRows.add(Optional.of(featureRow)); } return featureRows; } /** - * Send a list of get request as an mget + * Pull the data stored in Redis at the given keys as bytes using the mget command. If no data is + * stored at a given key in Redis, will subsitute the data with null. * - * @param keys list of {@link RedisKey} - * @return list of {@link FeatureRow} in primitive byte representation for each {@link RedisKey} + * @param keys list of {@link RedisKey} to pull from redis. + * @return list of data bytes or null pulled from redis for each given key. */ private List sendMultiGet(List keys) { try { @@ -203,14 +192,14 @@ private List sendMultiGet(List keys) { }) .collect(Collectors.toList()); } catch (Exception e) { - throw Status.NOT_FOUND - .withDescription("Unable to retrieve feature from Redis") + throw Status.UNKNOWN + .withDescription("Unexpected error when pulling data from from Redis.") .withCause(e) .asRuntimeException(); } } - // TODO: Refactor this out to common package? + // TODO: Refactor this out to common package private static String generateFeatureSetStringRef(FeatureSetSpec featureSetSpec) { String ref = String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName()); return ref; diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java index 45842cef6ee..76e22d07c54 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java @@ -132,9 +132,9 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List> expected = - ImmutableList.of( - Lists.newArrayList( + List> expected = + Lists.newArrayList( + Optional.of( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .setFeatureSet("project/featureSet") @@ -142,7 +142,8 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), + .build()), + Optional.of( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .setFeatureSet("project/featureSet") @@ -152,14 +153,13 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) .build())); - List> actual = - redisClusterOnlineRetriever.getOnlineFeatures( - entityRows, ImmutableList.of(featureSetRequest)); + List> actual = + redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } @Test - public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { + public void shouldReturnNullIfKeysNotPresent() { FeatureSetRequest featureSetRequest = FeatureSetRequest.newBuilder() .setSpec(getFeatureSetSpec()) @@ -201,9 +201,9 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List> expected = - ImmutableList.of( - Lists.newArrayList( + List> expected = + Lists.newArrayList( + Optional.of( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .setFeatureSet("project/featureSet") @@ -211,18 +211,11 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").build(), - Field.newBuilder().setName("feature2").build())) - .build())); + .build()), + Optional.empty()); - List> actual = - redisClusterOnlineRetriever.getOnlineFeatures( - entityRows, ImmutableList.of(featureSetRequest)); + List> actual = + redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java index adacacb941e..1292f4ab0dc 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java @@ -132,9 +132,9 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List> expected = - ImmutableList.of( - Lists.newArrayList( + List> expected = + Lists.newArrayList( + Optional.of( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .setFeatureSet("project/featureSet") @@ -142,7 +142,8 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), + .build()), + Optional.of( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .setFeatureSet("project/featureSet") @@ -152,13 +153,13 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) .build())); - List> actual = - redisOnlineRetriever.getOnlineFeatures(entityRows, ImmutableList.of(featureSetRequest)); + List> actual = + redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } @Test - public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { + public void shouldReturnNullIfKeysNotPresent() { FeatureSetRequest featureSetRequest = FeatureSetRequest.newBuilder() .setSpec(getFeatureSetSpec()) @@ -192,7 +193,7 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { List> featureRowBytes = featureRows.stream() - .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) + .map(row -> KeyValue.from(new byte[1], Optional.of(row.toByteArray()))) .collect(Collectors.toList()); featureRowBytes.add(null); @@ -200,9 +201,9 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List> expected = - ImmutableList.of( - Lists.newArrayList( + List> expected = + Lists.newArrayList( + Optional.of( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .setFeatureSet("project/featureSet") @@ -210,17 +211,10 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").build(), - Field.newBuilder().setName("feature2").build())) - .build())); - - List> actual = - redisOnlineRetriever.getOnlineFeatures(entityRows, ImmutableList.of(featureSetRequest)); + .build()), + Optional.empty()); + List> actual = + redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 1d38cc924b4..4ecdff98fb7 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -16,6 +16,7 @@ from feast.client import Client from feast.feature_set import FeatureSet, FeatureSetRef from feast.type_map import ValueType +from feast.wait import wait_retry_backoff from feast.constants import FEAST_DEFAULT_OPTIONS, CONFIG_PROJECT_KEY from google.protobuf.duration_pb2 import Duration from datetime import datetime @@ -32,6 +33,56 @@ PROJECT_NAME = 'basic_' + uuid.uuid4().hex.upper()[0:6] DIR_PATH = os.path.dirname(os.path.realpath(__file__)) +def basic_dataframe(entities, features, ingest_time, n_size, null_features=[]): + """ + Generate a basic feast-ingestable dataframe for testing. + Entity value incrementally increase from 1 to n_size + Features values are randomlly generated floats. + entities - names of entities + features - names of the features + ingest_time - ingestion timestamp + n_size - no. of rows in the generated dataframe. + null_features - names of features that contain null values + Returns the generated dataframe + """ + offset = random.randint(1000, 100000) # ensure a unique key space is used + df_dict = { + "datetime": [ingest_time.replace(tzinfo=pytz.utc) for _ in + range(n_size)], + } + for entity_name in entities: + df_dict[entity_name] = list(range(1, n_size + 1)) + for feature_name in features: + df_dict[feature_name] = [np.random.rand() for _ in range(n_size)] + for null_feature_name in null_features: + df_dict[null_feature_name] = [None for _ in range(n_size)] + return pd.DataFrame(df_dict) + +def check_online_response(feature_ref, ingest_df, response): + """ + Check the feature value and status in the given online serving response. + feature_refs - string feature ref used to access feature in response + ingest_df - dataframe of ingested values + response - response to extract retrieved feature value and metadata + Returns True if given response has expected feature value and metadata, otherwise False. + """ + feature_ref_splits = feature_ref.split(":") + if len(feature_ref_splits) == 1: + feature_name = feature_ref + else: + _, feature_name = feature_ref_splits + + returned_status = response.field_values[0].statuses[feature_ref] + if ingest_df.loc[0, feature_name] is None: + return returned_status == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE + else: + sent_value = float(ingest_df.iloc[0][feature_name]) + returned_value = float(response.field_values[0].fields[feature_ref].float_val) + return ( + math.isclose(sent_value, returned_value, abs_tol=FLOAT_TOLERANCE) + and returned_status == GetOnlineFeaturesResponse.FieldStatus.PRESENT + ) + @pytest.fixture(scope='module') def core_url(pytestconfig): @@ -66,19 +117,6 @@ def client(core_url, serving_url, allow_dirty): return client -def basic_dataframe(entities, features, ingest_time, n_size): - offset = random.randint(1000, 100000) # ensure a unique key space is used - df_dict = { - "datetime": [ingest_time.replace(tzinfo=pytz.utc) for _ in - range(n_size)], - } - for entity_name in entities: - df_dict[entity_name] = list(range(1, n_size + 1)) - for feature_name in features: - df_dict[feature_name] = [np.random.rand() for _ in range(n_size)] - return pd.DataFrame(df_dict) - - @pytest.fixture(scope="module") def ingest_time(): return datetime.utcnow() @@ -88,10 +126,10 @@ def ingest_time(): def cust_trans_df(ingest_time): return basic_dataframe(entities=["customer_id"], features=["daily_transactions", "total_transactions"], + null_features=["null_values"], ingest_time=ingest_time, n_size=5) - @pytest.fixture(scope="module") def driver_df(ingest_time): return basic_dataframe(entities=["driver_id"], @@ -99,7 +137,6 @@ def driver_df(ingest_time): ingest_time=ingest_time, n_size=5) - def test_version_returns_results(client): version_info = client.version() assert not version_info['core'] is 'not configured' @@ -174,9 +211,13 @@ def test_basic_ingest_success(client, cust_trans_df, driver_df): @pytest.mark.timeout(90) @pytest.mark.run(order=12) def test_basic_retrieve_online_success(client, cust_trans_df): + feature_refs=[ + "daily_transactions", + "total_transactions", + "null_values" + ] # Poll serving for feature values until the correct values are returned - while True: - time.sleep(1) + def try_get_features(): response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -187,45 +228,28 @@ def test_basic_retrieve_online_success(client, cust_trans_df): } ) ], - # Test retrieve with different variations of the string feature refs - feature_refs=[ - "daily_transactions", - "total_transactions", - ] - ) # type: GetOnlineFeaturesResponse - - if response is None: - continue - - returned_daily_transactions = float( - response.field_values[0] - .fields["daily_transactions"] - .float_val - ) - sent_daily_transactions = float( - cust_trans_df.iloc[0]["daily_transactions"]) - - if math.isclose( - sent_daily_transactions, - returned_daily_transactions, - abs_tol=FLOAT_TOLERANCE, - ): - break + feature_refs=feature_refs) + # type: GetOnlineFeaturesResponse + is_ok = all([check_online_response(ref, cust_trans_df, response) for ref in feature_refs]) + return response, is_ok + wait_retry_backoff(retry_fn=try_get_features, + timeout_secs=90, + timeout_msg="Timed out trying to get online feature values") @pytest.mark.timeout(90) @pytest.mark.run(order=13) def test_basic_retrieve_online_multiple_featureset(client, cust_trans_df, driver_df): + # Test retrieve with different variations of the string feature refs + # ie feature set inference for feature refs without specified feature set + feature_ref_df_mapping = [ + ("customer_transactions:daily_transactions", cust_trans_df), + ("driver:rating", driver_df), + ("total_transactions", cust_trans_df), + ] # Poll serving for feature values until the correct values are returned - while True: - time.sleep(1) - # Test retrieve with different variations of the string feature refs - # ie feature set inference for feature refs without specified feature set - feature_ref_df_mapping = [ - ("customer_transactions:daily_transactions", cust_trans_df), - ("driver:rating", driver_df), - ("total_transactions", cust_trans_df), - ] + def try_get_features(): + feature_refs = [mapping[0] for mapping in feature_ref_df_mapping] response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -239,35 +263,15 @@ def test_basic_retrieve_online_multiple_featureset(client, cust_trans_df, driver } ) ], - feature_refs=[mapping[0] for mapping in feature_ref_df_mapping], + feature_refs=feature_refs, ) # type: GetOnlineFeaturesResponse + is_ok = all([check_online_response(ref, df, response) + for ref, df in feature_ref_df_mapping]) + return response, is_ok - if response is None: - continue - - def check_response(ingest_df, response, feature_ref): - returned_value = float( - response.field_values[0] - .fields[feature_ref] - .float_val - ) - feature_ref_splits = feature_ref.split(":") - if len(feature_ref_splits) == 1: - feature_name = feature_ref - else: - _, feature_name = feature_ref_splits - - sent_value = float( - ingest_df.iloc[0][feature_name]) - - return math.isclose( - sent_value, - returned_value, - abs_tol=FLOAT_TOLERANCE, - ) - - if all([check_response(df, response, ref) for ref, df in feature_ref_df_mapping]): - break + wait_retry_backoff(retry_fn=try_get_features, + timeout_secs=90, + timeout_msg="Timed out trying to get online feature values") @pytest.mark.timeout(300) @@ -406,10 +410,23 @@ def test_all_types_ingest_success(client, all_types_dataframe): @pytest.mark.timeout(90) @pytest.mark.run(order=22) def test_all_types_retrieve_online_success(client, all_types_dataframe): - # Poll serving for feature values until the correct values are returned - while True: - time.sleep(1) - + # Poll serving for feature values until the correct values are returned_float_list + feature_refs = [ + "float_feature", + "int64_feature", + "int32_feature", + "double_feature", + "string_feature", + "bool_feature", + "bytes_feature", + "float_list_feature", + "int64_list_feature", + "int32_list_feature", + "string_list_feature", + "bytes_list_feature", + "double_list_feature", + ] + def try_get_features(): response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -417,39 +434,22 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): int64_val=all_types_dataframe.iloc[0]["user_id"])} ) ], - feature_refs=[ - "float_feature", - "int64_feature", - "int32_feature", - "string_feature", - "bytes_feature", - "bool_feature", - "double_feature", - "float_list_feature", - "int64_list_feature", - "int32_list_feature", - "string_list_feature", - "bytes_list_feature", - "double_list_feature", - ], + feature_refs=feature_refs, ) # type: GetOnlineFeaturesResponse + is_ok = check_online_response("float_feature", all_types_dataframe, response) + return response, is_ok - if response is None: - continue - - returned_float_list = ( - response.field_values[0] - .fields["float_list_feature"] - .float_list_val.val - ) - - sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] - - if math.isclose( - returned_float_list[0], sent_float_list[0], abs_tol=FLOAT_TOLERANCE - ): - break + response = wait_retry_backoff(retry_fn=try_get_features, + timeout_secs=90, + timeout_msg="Timed out trying to get online feature values") + # check returned values + returned_float_list = response.field_values[0].fields["float_list_feature"].float_list_val.val + sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] + assert math.isclose(returned_float_list[0], sent_float_list[0], abs_tol=FLOAT_TOLERANCE) + # check returned metadata + assert (response.field_values[0].statuses["float_list_feature"] + == GetOnlineFeaturesResponse.FieldStatus.PRESENT) @pytest.mark.timeout(300) @pytest.mark.run(order=29) @@ -532,9 +532,11 @@ def test_large_volume_ingest_success(client, large_volume_dataframe): @pytest.mark.run(order=32) def test_large_volume_retrieve_online_success(client, large_volume_dataframe): # Poll serving for feature values until the correct values are returned + feature_refs=[ + "daily_transactions_large", + "total_transactions_large", + ] while True: - time.sleep(1) - response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -546,30 +548,14 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): } ) ], - feature_refs=[ - "daily_transactions_large", - "total_transactions_large", - ], + feature_refs=feature_refs, ) # type: GetOnlineFeaturesResponse + is_ok = all([check_online_response(ref, large_volume_dataframe, response) for ref in feature_refs]) + return None, is_ok - if response is None: - continue - - returned_daily_transactions = float( - response.field_values[0] - .fields["daily_transactions_large"] - .float_val - ) - sent_daily_transactions = float( - large_volume_dataframe.iloc[0]["daily_transactions_large"]) - - if math.isclose( - sent_daily_transactions, - returned_daily_transactions, - abs_tol=FLOAT_TOLERANCE, - ): - break - + wait_retry_backoff(retry_fn=try_get_features, + timeout_secs=90, + timeout_msg="Timed out trying to get online feature values") @pytest.fixture(scope='module') def all_types_parquet_file(): diff --git a/tests/e2e/redis/basic/cust_trans_fs.yaml b/tests/e2e/redis/basic/cust_trans_fs.yaml index 14d46794a6d..941037670d3 100644 --- a/tests/e2e/redis/basic/cust_trans_fs.yaml +++ b/tests/e2e/redis/basic/cust_trans_fs.yaml @@ -9,4 +9,6 @@ spec: valueType: FLOAT - name: total_transactions valueType: FLOAT + - name: null_values + valueType: FLOAT maxAge: 3600s