From 377cb53f09af5b178cb1db9fd0a0d078cc552d2b Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 17 Apr 2020 10:24:10 +0800 Subject: [PATCH 01/65] Update ServingService protobuf with additions to support online metadata retrieval. Additions are outlined in detail in this RFC: https://docs.google.com/document/d/1VQngwBcx-yWgGpAbsFVdth9GnjL8q-ZgUNBGv57R0Fk/ --- protos/feast/serving/ServingService.proto | 59 ++++++++++++++++++----- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto index cd7d51bd59c..5ade63cd09c 100644 --- a/protos/feast/serving/ServingService.proto +++ b/protos/feast/serving/ServingService.proto @@ -90,9 +90,13 @@ message GetOnlineFeaturesRequest { // values will be returned. bool omit_entities_in_response = 3; + // Option to include feature metadata in the response. + // If true, response will include both feature values and metadata. + bool include_metadata_in_response = 5; + message EntityRow { - // Request timestamp of this row. This value will be used, together with maxAge, - // to determine feature staleness. + // Request timestamp of this row. This value will be used, + // together with maxAge, to determine feature staleness. google.protobuf.Timestamp entity_timestamp = 1; // Map containing mapping of entity name to entity value. @@ -100,6 +104,46 @@ message GetOnlineFeaturesRequest { } } +message GetOnlineFeaturesResponse { + // Data records retrieved from the online feast store. + // Each record represents the data retrieved for an entity row in the request. + repeated Record records = 1; + + message Record { + // Map of field name to data fields stored in this data record. + // Each field represents an individual feature in the data record. + map fields = 1; + } + + message Field { + // Value of this field. + feast.types.Value value = 1; + // Status of this field. + FieldStatus status = 2; + } + + enum FieldStatus { + // Status is unset for this field. + INVALID = 0; + + // Field value is present for this field and within maximum allowable range + PRESENT = 1; + + // Values could be found for entity key within the maximum allowable range, but + // this field value is assigned a null value on ingestion into feast. + NULL_VALUE = 2; + + // Entity key did not return any values as they do not exist in Feast. + // This could suggest that the feature values have not yet been ingested + // into feast or the ingestion failed. + NOT_FOUND = 3; + + // Values could be found for entity key, but field values are outside the maximum + // allowable range. + OUTSIDE_MAX_AGE = 4; + } +} + message GetBatchFeaturesRequest { // List of features that are being retrieved repeated FeatureReference features = 3; @@ -109,17 +153,6 @@ message GetBatchFeaturesRequest { DatasetSource dataset_source = 2; } -message GetOnlineFeaturesResponse { - // Feature values retrieved from feast. - repeated FieldValues field_values = 1; - - message FieldValues { - // Map of feature or entity name to feature/entity values. - // Timestamps are not returned in this response. - map fields = 1; - } -} - message GetBatchFeaturesResponse { Job job = 1; } From 9170e229251711d142c76fc252f81bb40b9575a7 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 17 Apr 2020 22:45:01 +0800 Subject: [PATCH 02/65] Change error thrown by sendMultiGet() to Unknown as it describes the fault more accurately --- .../redis/retriever/RedisOnlineRetriever.java | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index 0db4837c069..b248200cf4c 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -63,14 +63,17 @@ public static OnlineRetriever create(StatefulRedisConnection con } /** - * Gets online features from redis. This method returns a list of {@link FeatureRow}s - * corresponding to each feature set spec. Each feature row in the list then corresponds to an - * {@link EntityRow} provided by the user. + * Gets online features from redis store for the given entity rows using data retrieved from the + * feature/featureset specified in feature set requests. * - * @param entityRows list of entity rows in the feature request - * @param featureSetRequests Map of {@link feast.proto.core.FeatureSetProto.FeatureSetSpec} to - * feature references in the request tied to that feature set. - * @return List of List of {@link FeatureRow} + *

This method returns a list of {@link FeatureRow}s corresponding to each feature set spec. + * Each feature row in the list then corresponds to an {@link EntityRow} provided by the user. If + * retrieval fails for a given entity row, will return null in place of the {@link FeatureRow}. + * + * @param featureSetRequests List of {@link FeatureSetRequest} specifying the features/feature set + * to retrieve data from. + * @return list of lists of {@link FeatureRow}s corresponding to data retrieved for each feature + * set request and entity row. */ @Override public List> getOnlineFeatures( @@ -93,6 +96,7 @@ public List> getOnlineFeatures( .asRuntimeException(); } } + return featureRows; } @@ -181,10 +185,11 @@ private List sendAndProcessMultiGet( } /** - * Send a list of get request as an mget + * Pull the data stored in redis at the given keys as bytes using the mget command. If no data is + * stored at a given key in redis, will subsitute the data with null. * - * @param keys list of {@link RedisKey} - * @return list of {@link FeatureRow} in primitive byte representation for each {@link RedisKey} + * @param keys list of {@link RedisKey} to pull from redis. + * @return list of data bytes or null pulled from redis for each given key. */ private List sendMultiGet(List keys) { try { @@ -203,14 +208,14 @@ private List sendMultiGet(List keys) { }) .collect(Collectors.toList()); } catch (Exception e) { - throw Status.NOT_FOUND - .withDescription("Unable to retrieve feature from Redis") + throw Status.UNKNOWN + .withDescription("Unexpected error when pulling data from from Redis.") .withCause(e) .asRuntimeException(); } } - // TODO: Refactor this out to common package? + // TODO: Refactor this out to common package? move to Ref utils private static String generateFeatureSetStringRef(FeatureSetSpec featureSetSpec) { String ref = String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName()); return ref; From 2db28c19de77076c5069e4022af021256c41b179 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Sat, 18 Apr 2020 13:30:04 +0800 Subject: [PATCH 03/65] Return boolean primitive instead of Boolean object type in FeatureRowDecoder This removes the unneeded possiblity of a null being returning from the the decoder. --- .../storage/connectors/redis/retriever/FeatureRowDecoder.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java index fd9556841e4..aad3147f710 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/FeatureRowDecoder.java @@ -43,7 +43,7 @@ public FeatureRowDecoder(String featureSetRef, FeatureSetSpec spec) { * @param featureRow Feature row * @return boolean */ - public Boolean isEncoded(FeatureRow featureRow) { + public boolean isEncoded(FeatureRow featureRow) { return featureRow.getFeatureSet().isEmpty() && featureRow.getFieldsList().stream().allMatch(field -> field.getName().isEmpty()); } @@ -54,7 +54,7 @@ public Boolean isEncoded(FeatureRow featureRow) { * @param featureRow Feature row * @return boolean */ - public Boolean isEncodingValid(FeatureRow featureRow) { + public boolean isEncodingValid(FeatureRow featureRow) { return featureRow.getFieldsList().size() == spec.getFeaturesList().size(); } From 2b13bb58006f64f7b342399ed6a6279b8fd1dfa1 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Sun, 19 Apr 2020 10:46:07 +0800 Subject: [PATCH 04/65] Config sendAndProcessMultiGet() to throw an exception on feature row decoding failure. Instead of the current behaviour to silently fail with and return an empty feature row. --- .../redis/retriever/RedisOnlineRetriever.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index b248200cf4c..2b455c41a15 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -150,7 +150,7 @@ private List sendAndProcessMultiGet( List featureReferences) throws InvalidProtocolBufferException, ExecutionException { - List values = sendMultiGet(redisKeys); + List featureRowsBytes = sendMultiGet(redisKeys); List featureRows = new ArrayList<>(); FeatureRow.Builder nullFeatureRowBuilder = @@ -159,23 +159,26 @@ private List sendAndProcessMultiGet( nullFeatureRowBuilder.addFields(Field.newBuilder().setName(featureReference.getName())); } - for (int i = 0; i < values.size(); i++) { + for (int i = 0; i < featureRowsBytes.size(); i++) { - byte[] value = values.get(i); - if (value == null) { + byte[] featureRowBytes = featureRowsBytes.get(i); + if (featureRowBytes == null) { featureRows.add(nullFeatureRowBuilder.build()); continue; } - FeatureRow featureRow = FeatureRow.parseFrom(value); + FeatureRow featureRow = FeatureRow.parseFrom(featureRowBytes); String featureSetRef = redisKeys.get(i).getFeatureSet(); FeatureRowDecoder decoder = new FeatureRowDecoder(featureSetRef, featureSetSpec); if (decoder.isEncoded(featureRow)) { if (decoder.isEncodingValid(featureRow)) { featureRow = decoder.decode(featureRow); } else { - featureRows.add(nullFeatureRowBuilder.build()); - continue; + // decoding feature row failed: data corruption could have occurred + throw Status.DATA_LOSS + .withDescription("Failed to decode FeatureRow from bytes retrieved from redis" + + ": Possible data corruption") + .asRuntimeException(); } } From 4ba034bbfeae6270e430eedf7a6401a993a45d12 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Sun, 19 Apr 2020 11:15:09 +0800 Subject: [PATCH 05/65] Refactor sendMultiGetAndProcess() to getFeaturesForFeatureSet(). Rename as the original name described the implementation instead what the goal of the method is. Construct and pass the decoder outside method in getFeaturesForFeatureSet() instead of inside method in sendMultiGetAndProcess() as constructing it inside added uncessary arguments and code (ie getting the string ref out a redis key seemed out of place.) Changed getFeaturesForFeatureSet() to return null instead of a nullFeatureRow. when retrieving data from redis using sendMultiGet() using a key missing in Redis. This makes the missing key condition easier to detect. Removed unncessary arguments passed to sendMultiGetAndProcess() - featureReference used to contruct the nullFeatureRow - featureSetSpec used to contruct the decoder. --- .../redis/retriever/RedisOnlineRetriever.java | 56 +++++++++---------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index 2b455c41a15..4620186e702 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -81,13 +81,13 @@ public List> getOnlineFeatures( List> featureRows = new ArrayList<>(); for (FeatureSetRequest featureSetRequest : featureSetRequests) { - List redisKeys = buildRedisKeys(entityRows, featureSetRequest.getSpec()); + // get features for this features/featureset in featureset request + FeatureSetSpec featureSetSpec = featureSetRequest.getSpec(); + List redisKeys = buildRedisKeys(entityRows, featureSetSpec); + FeatureRowDecoder decoder = + new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); try { - List featureRowsForFeatureSet = - sendAndProcessMultiGet( - redisKeys, - featureSetRequest.getSpec(), - featureSetRequest.getFeatureReferences().asList()); + List featureRowsForFeatureSet = getFeaturesForFeatureSet(redisKeys, decoder); featureRows.add(featureRowsForFeatureSet); } catch (InvalidProtocolBufferException | ExecutionException e) { throw Status.INTERNAL @@ -144,52 +144,48 @@ private RedisKey makeRedisKey( return builder.build(); } - private List sendAndProcessMultiGet( - List redisKeys, - FeatureSetSpec featureSetSpec, - List featureReferences) + /** + * Get features from data pulled from the Redis for a specific featureset. + * + * @param redisKeys keys used to retrieve data from Redis for a specific featureset. + * @param decoder used to decode the data retrieved from Redis for a specific featureset. + * @return List of {@link FeatureRow}s + */ + private List getFeaturesForFeatureSet( + List redisKeys, FeatureRowDecoder decoder) throws InvalidProtocolBufferException, ExecutionException { - + // pull feature row data bytes from redis using given redis keys List featureRowsBytes = sendMultiGet(redisKeys); List featureRows = new ArrayList<>(); - FeatureRow.Builder nullFeatureRowBuilder = - FeatureRow.newBuilder().setFeatureSet(generateFeatureSetStringRef(featureSetSpec)); - for (FeatureReference featureReference : featureReferences) { - nullFeatureRowBuilder.addFields(Field.newBuilder().setName(featureReference.getName())); - } - - for (int i = 0; i < featureRowsBytes.size(); i++) { - - byte[] featureRowBytes = featureRowsBytes.get(i); + for (byte[] featureRowBytes : featureRowsBytes) { if (featureRowBytes == null) { - featureRows.add(nullFeatureRowBuilder.build()); + featureRows.add(null); continue; } + // decode feature rows from data bytes using decoder. FeatureRow featureRow = FeatureRow.parseFrom(featureRowBytes); - String featureSetRef = redisKeys.get(i).getFeatureSet(); - FeatureRowDecoder decoder = new FeatureRowDecoder(featureSetRef, featureSetSpec); if (decoder.isEncoded(featureRow)) { if (decoder.isEncodingValid(featureRow)) { featureRow = decoder.decode(featureRow); } else { // decoding feature row failed: data corruption could have occurred throw Status.DATA_LOSS - .withDescription("Failed to decode FeatureRow from bytes retrieved from redis" - + ": Possible data corruption") - .asRuntimeException(); + .withDescription( + "Failed to decode FeatureRow from bytes retrieved from redis" + + ": Possible data corruption") + .asRuntimeException(); } } - featureRows.add(featureRow); } return featureRows; } /** - * Pull the data stored in redis at the given keys as bytes using the mget command. If no data is - * stored at a given key in redis, will subsitute the data with null. + * Pull the data stored in Redis at the given keys as bytes using the mget command. If no data is + * stored at a given key in Redis, will subsitute the data with null. * * @param keys list of {@link RedisKey} to pull from redis. * @return list of data bytes or null pulled from redis for each given key. @@ -218,7 +214,7 @@ private List sendMultiGet(List keys) { } } - // TODO: Refactor this out to common package? move to Ref utils + // TODO: Refactor this out to common package private static String generateFeatureSetStringRef(FeatureSetSpec featureSetSpec) { String ref = String.format("%s/%s", featureSetSpec.getProject(), featureSetSpec.getName()); return ref; From 607b681453273bff0017be02ea87fc9515202415 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 20 Apr 2020 15:38:36 +0800 Subject: [PATCH 06/65] Update ResponseJSONMapper to use new GetOnlineFeatures protobuf --- .../serving/util/mappers/ResponseJSONMapper.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java b/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java index 6aba17ac431..cca40dc9643 100644 --- a/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java +++ b/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java @@ -16,9 +16,15 @@ */ package feast.serving.util.mappers; +<<<<<<< HEAD import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.ValueProto.Value; +======= +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; +import feast.types.ValueProto.Value; +>>>>>>> Update ResponseJSONMapper to use new GetOnlineFeatures protobuf import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -28,14 +34,14 @@ public class ResponseJSONMapper { public static List> mapGetOnlineFeaturesResponse( GetOnlineFeaturesResponse response) { - return response.getFieldValuesList().stream() - .map(fieldValue -> convertFieldValuesToMap(fieldValue)) + return response.getRecordsList().stream() + .map(fieldValue -> convertToMap(fieldValue)) .collect(Collectors.toList()); } - private static Map convertFieldValuesToMap(FieldValues fieldValues) { - return fieldValues.getFieldsMap().entrySet().stream() - .collect(Collectors.toMap(es -> es.getKey(), es -> extractValue(es.getValue()))); + private static Map convertToMap(Record record) { + return record.getFieldsMap().entrySet().stream() + .collect(Collectors.toMap(es -> es.getKey(), es -> extractValue(es.getValue().getValue()))); } private static Object extractValue(Value value) { From 2f4c710ee262a03b4a7fbd4d1da52ce9497aa9ab Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 20 Apr 2020 15:39:16 +0800 Subject: [PATCH 07/65] Remove references to FieldValues in OnlineServingService & Tests --- .../serving/service/OnlineServingService.java | 21 +- .../feast/serving/service/ServingService.java | 4 +- .../service/OnlineServingServiceTest.java | 195 +++++++++++++----- 3 files changed, 159 insertions(+), 61 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index bb73e34f51c..414211cbb30 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -21,8 +21,7 @@ import com.google.protobuf.Duration; import feast.proto.serving.ServingAPIProto.*; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; -import feast.proto.types.FeatureRowProto.FeatureRow; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; import feast.proto.types.ValueProto.Value; import feast.serving.specs.CachedSpecService; import feast.serving.util.Metrics; @@ -66,20 +65,24 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ try (Scope scope = tracer.buildSpan("getOnlineFeatures").startActive(true)) { GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder = GetOnlineFeaturesResponse.newBuilder(); + // Build featureset requests used to pass to the retriever to retrieve feature data. List featureSetRequests = specService.getFeatureSets(request.getFeaturesList()); List entityRows = request.getEntityRowsList(); + Map> featureValuesMap = entityRows.stream() .collect(Collectors.toMap(row -> row, row -> Maps.newHashMap(row.getFieldsMap()))); - // Get all feature rows from the retriever. Each feature row list corresponds to a single - // feature set request. + + // Pull feature rows for each feature set request from the retriever. + // Each feature row list corresponds to a single feature set request. List> featureRows = retriever.getOnlineFeatures(entityRows, featureSetRequests); if (scope != null) { scope.span().log(ImmutableMap.of("event", "featureRows", "value", featureRows)); } + // Match entity rows request with the features defined in the feature spec. // For each feature set request, read the feature rows returned by the retriever, and // populate the featureValuesMap with the feature values corresponding to that entity row. for (var fsIdx = 0; fsIdx < featureRows.size(); fsIdx++) { @@ -99,6 +102,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ EntityRow entityRow = entityRows.get(entityRowIdx); // If the row is stale, put an empty value into the featureValuesMap. + // TODO: online-feature-metadata: detect stale features here. if (isStale(featureSetRequest, entityRow, featureRow)) { featureSetRequest .getFeatureReferences() @@ -127,14 +131,13 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ } } - List fieldValues = - featureValuesMap.values().stream() - .map(valueMap -> FieldValues.newBuilder().putAllFields(valueMap).build()) - .collect(Collectors.toList()); - return getOnlineFeaturesResponseBuilder.addAllFieldValues(fieldValues).build(); + // TODO: update this to return actual records + Record record = Record.newBuilder().build(); + return getOnlineFeaturesResponseBuilder.addRecords(record).build(); } } + // TODO: call this based on what is returned in records. private void populateStaleKeyCountMetrics(String project, FeatureReference ref) { Metrics.staleKeyCount.labels(project, ref.getName()).inc(); } diff --git a/serving/src/main/java/feast/serving/service/ServingService.java b/serving/src/main/java/feast/serving/service/ServingService.java index 1fe9840d594..a9155019bb6 100644 --- a/serving/src/main/java/feast/serving/service/ServingService.java +++ b/serving/src/main/java/feast/serving/service/ServingService.java @@ -56,8 +56,8 @@ GetFeastServingInfoResponse getFeastServingInfo( * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow}s to join the * retrieved values to. * @return {@link GetOnlineFeaturesResponse} with list of {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues} for each {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow} supplied. + * feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record} for each {@link + * feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow} supplied. */ GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getFeaturesRequest); diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 6358460a070..6275f138ca6 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -30,7 +30,7 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; import feast.proto.types.FeatureRowProto.FeatureRow; import feast.proto.types.FieldProto.Field; import feast.proto.types.ValueProto.Value; @@ -119,18 +119,98 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() + .addRecords( + Record.newBuilder() + .putFields("entity1", intField(1)) + .putFields("entity2", strField("a")) + .putFields("project/feature1:1", intField(1)) + .putFields("project/feature2:1", intField(1))) + .addRecords( + Record.newBuilder() + .putFields("entity1", intField(2)) + .putFields("entity2", strField("b")) + .putFields("project/feature1:1", intField(2)) + .putFields("project/feature2:1", intField(2))) + .build(); + GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); + assertThat( + responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); + } + + @Test + public void shouldReturnKeysWithoutVersionIfNotProvided() { + GetOnlineFeaturesRequest request = + GetOnlineFeaturesRequest.newBuilder() + .addFeatures( + FeatureReference.newBuilder() + .setName("feature1") + .setVersion(1) + .setProject("project") + .build()) + .addFeatures( + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) + .addEntityRows( + EntityRow.newBuilder() + .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .putFields("feature1", intValue(1)) - .putFields("feature2", intValue(1))) - .addFieldValues( - FieldValues.newBuilder() + .putFields("entity2", strValue("a"))) + .addEntityRows( + EntityRow.newBuilder() + .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .putFields("feature1", intValue(2)) - .putFields("feature2", intValue(2))) + .putFields("entity2", strValue("b"))) + .build(); + + List featureRows = + Lists.newArrayList( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), + Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .setFeatureSet("featureSet:1") + .build(), + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), + Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), + Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) + .setFeatureSet("featureSet:1") + .build()); + + FeatureSetRequest featureSetRequest = + FeatureSetRequest.newBuilder() + .addAllFeatureReferences(request.getFeaturesList()) + .setSpec(getFeatureSetSpec()) + .build(); + + when(specService.getFeatureSets(request.getFeaturesList())) + .thenReturn(Collections.singletonList(featureSetRequest)); + when(retriever.getOnlineFeatures( + request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) + .thenReturn(Collections.singletonList(featureRows)); + when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); + + GetOnlineFeaturesResponse expected = + GetOnlineFeaturesResponse.newBuilder() + .addRecords( + Record.newBuilder() + .putFields("entity1", intField(1)) + .putFields("entity2", strField("a")) + .putFields("project/feature1:1", intField(1)) + .putFields("project/feature2", intField(1))) + .addRecords( + Record.newBuilder() + .putFields("entity1", intField(2)) + .putFields("entity2", strField("b")) + .putFields("project/feature1:1", intField(2)) + .putFields("project/feature2", intField(2))) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( @@ -189,18 +269,18 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .putFields("feature1", intValue(1)) - .putFields("feature2", intValue(1))) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .putFields("feature1", Value.newBuilder().build()) - .putFields("feature2", Value.newBuilder().build())) + .addRecords( + Record.newBuilder() + .putFields("entity1", intField(1)) + .putFields("entity2", strField("a")) + .putFields("project/feature1:1", intField(1)) + .putFields("project/feature2:1", intField(1))) + .addRecords( + Record.newBuilder() + .putFields("entity1", intField(2)) + .putFields("entity2", strField("b")) + .putFields("project/feature1:1", GetOnlineFeaturesResponse.Field.newBuilder().build()) + .putFields("project/feature2:1", GetOnlineFeaturesResponse.Field.newBuilder().build())) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( @@ -267,18 +347,18 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .putFields("feature1", intValue(1)) - .putFields("feature2", intValue(1))) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .putFields("feature1", Value.newBuilder().build()) - .putFields("feature2", Value.newBuilder().build())) + .addRecords( + Record.newBuilder() + .putFields("entity1", intField(1)) + .putFields("entity2", strField("a")) + .putFields("project/feature1:1", intField(1)) + .putFields("project/feature2:1", intField(1))) + .addRecords( + Record.newBuilder() + .putFields("entity1", intField(2)) + .putFields("entity2", strField("b")) + .putFields("project/feature1:1", GetOnlineFeaturesResponse.Field.newBuilder().build()) + .putFields("project/feature2:1", GetOnlineFeaturesResponse.Field.newBuilder().build())) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( @@ -339,16 +419,16 @@ public void shouldFilterOutUndesiredRows() { GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a")) - .putFields("feature1", intValue(1))) - .addFieldValues( - FieldValues.newBuilder() - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b")) - .putFields("feature1", intValue(2))) + .addRecords( + Record.newBuilder() + .putFields("entity1", intField(1)) + .putFields("entity2", strField("a")) + .putFields("project/feature1:1", intField(1))) + .addRecords( + Record.newBuilder() + .putFields("entity1", intField(2)) + .putFields("entity2", strField("b")) + .putFields("project/feature1:1", intField(2))) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( @@ -356,19 +436,34 @@ public void shouldFilterOutUndesiredRows() { } private List> responseToMapList(GetOnlineFeaturesResponse response) { - return response.getFieldValuesList().stream() - .map(FieldValues::getFieldsMap) + // TODO:: update with metadata + return response.getRecordsList().stream() + .map(Record::getFieldsMap) + .map(fieldsMap -> fieldsMap.entrySet().stream().collect( + Collectors.toMap(es -> es.getKey(), es -> { + GetOnlineFeaturesResponse.Field field = es.getValue(); + return field.getValue(); + }))) .collect(Collectors.toList()); } - + private Value intValue(int val) { - return Value.newBuilder().setInt64Val(val).build(); + return Value.newBuilder().setInt32Val(val).build(); } - + private Value strValue(String val) { return Value.newBuilder().setStringVal(val).build(); } + private GetOnlineFeaturesResponse.Field intField(int val) { + return GetOnlineFeaturesResponse.Field.newBuilder().setValue(intValue(val)).build(); + } + + private GetOnlineFeaturesResponse.Field strField(String val) { + return GetOnlineFeaturesResponse.Field.newBuilder().setValue(strValue(val)).build(); + } + + private FeatureSetSpec getFeatureSetSpec() { return FeatureSetSpec.newBuilder() .setName("featureSet") From 59692f2ac586da4faa9986a33c25583cc5b08dee Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 20 Apr 2020 16:07:39 +0800 Subject: [PATCH 08/65] Update java sdk's FieldClient to use the new GetOnlineFeatures protobuf --- .../java/com/gojek/feast/FeastClient.java | 19 +++++-------------- .../util/mappers/ResponseJSONMapper.java | 14 +++++--------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index a81fdab21a2..ab4f86008dd 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -141,22 +141,13 @@ public List getOnlineFeatures( .setOmitEntitiesInResponse(omitEntitiesInResponse) .build()); - return response.getFieldValuesList().stream() + return response.getRecordsList().stream() .map( - field -> { + record -> { Row row = Row.create(); - field - .getFieldsMap() - .forEach( - (String name, Value value) -> { - // Strip project from string Feature References from returned from serving - if (!entityRefs.contains(name)) { - FeatureReference featureRef = - RequestUtil.parseFeatureRef(name, true).build(); - name = RequestUtil.renderFeatureRef(featureRef); - } - row.set(name, value); - }); + record.getFieldsMap().forEach((name, field) -> { + row.set(name, field.getValue()); + }); return row; }) .collect(Collectors.toList()); diff --git a/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java b/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java index cca40dc9643..dbf06c12187 100644 --- a/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java +++ b/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java @@ -16,15 +16,10 @@ */ package feast.serving.util.mappers; -<<<<<<< HEAD import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Field; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; import feast.proto.types.ValueProto.Value; -======= -import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; -import feast.types.ValueProto.Value; ->>>>>>> Update ResponseJSONMapper to use new GetOnlineFeatures protobuf import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -41,10 +36,11 @@ public static List> mapGetOnlineFeaturesResponse( private static Map convertToMap(Record record) { return record.getFieldsMap().entrySet().stream() - .collect(Collectors.toMap(es -> es.getKey(), es -> extractValue(es.getValue().getValue()))); + .collect(Collectors.toMap(es -> es.getKey(), es -> extractField(es.getValue()))); } - private static Object extractValue(Value value) { + private static Object extractField(Field field) { + Value value = field.getValue(); switch (value.getValCase().getNumber()) { case 1: return value.getBytesVal(); From 7e89555738b85ea5184cbcfee569794df1dff1c0 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 20 Apr 2020 18:52:33 +0800 Subject: [PATCH 09/65] Update OnlineRetriever to return null on missing retrieval (such as missing key for redis). --- .../java/com/gojek/feast/FeastClient.java | 9 ++++-- .../service/OnlineServingServiceTest.java | 32 ++++++++++++------- .../api/retriever/OnlineRetriever.java | 22 +++++++------ .../redis/retriever/RedisOnlineRetriever.java | 2 +- .../retriever/RedisOnlineRetrieverTest.java | 11 ++----- 5 files changed, 41 insertions(+), 35 deletions(-) diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index ab4f86008dd..b9f2ea6bdb8 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -145,9 +145,12 @@ public List getOnlineFeatures( .map( record -> { Row row = Row.create(); - record.getFieldsMap().forEach((name, field) -> { - row.set(name, field.getValue()); - }); + record + .getFieldsMap() + .forEach( + (name, field) -> { + row.set(name, field.getValue()); + }); return row; }) .collect(Collectors.toList()); diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 6275f138ca6..cee34d04016 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -279,8 +279,10 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { Record.newBuilder() .putFields("entity1", intField(2)) .putFields("entity2", strField("b")) - .putFields("project/feature1:1", GetOnlineFeaturesResponse.Field.newBuilder().build()) - .putFields("project/feature2:1", GetOnlineFeaturesResponse.Field.newBuilder().build())) + .putFields( + "project/feature1:1", GetOnlineFeaturesResponse.Field.newBuilder().build()) + .putFields( + "project/feature2:1", GetOnlineFeaturesResponse.Field.newBuilder().build())) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( @@ -357,8 +359,10 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { Record.newBuilder() .putFields("entity1", intField(2)) .putFields("entity2", strField("b")) - .putFields("project/feature1:1", GetOnlineFeaturesResponse.Field.newBuilder().build()) - .putFields("project/feature2:1", GetOnlineFeaturesResponse.Field.newBuilder().build())) + .putFields( + "project/feature1:1", GetOnlineFeaturesResponse.Field.newBuilder().build()) + .putFields( + "project/feature2:1", GetOnlineFeaturesResponse.Field.newBuilder().build())) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat( @@ -439,18 +443,23 @@ private List> responseToMapList(GetOnlineFeaturesResponse res // TODO:: update with metadata return response.getRecordsList().stream() .map(Record::getFieldsMap) - .map(fieldsMap -> fieldsMap.entrySet().stream().collect( - Collectors.toMap(es -> es.getKey(), es -> { - GetOnlineFeaturesResponse.Field field = es.getValue(); - return field.getValue(); - }))) + .map( + fieldsMap -> + fieldsMap.entrySet().stream() + .collect( + Collectors.toMap( + es -> es.getKey(), + es -> { + GetOnlineFeaturesResponse.Field field = es.getValue(); + return field.getValue(); + }))) .collect(Collectors.toList()); } - + private Value intValue(int val) { return Value.newBuilder().setInt32Val(val).build(); } - + private Value strValue(String val) { return Value.newBuilder().setStringVal(val).build(); } @@ -463,7 +472,6 @@ private GetOnlineFeaturesResponse.Field strField(String val) { return GetOnlineFeaturesResponse.Field.newBuilder().setValue(strValue(val)).build(); } - private FeatureSetSpec getFeatureSetSpec() { return FeatureSetSpec.newBuilder() .setName("featureSet") diff --git a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java index 45d6b352632..71f2790fcd2 100644 --- a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java +++ b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java @@ -20,20 +20,22 @@ import feast.proto.types.FeatureRowProto.FeatureRow; import java.util.List; -/** - * An online retriever is a feature retriever that retrieves the latest feature data corresponding - * to provided entities. - */ +/** An online retriever is a feature retriever that retrieves the latest feature data. */ public interface OnlineRetriever { /** - * Get all values corresponding to the request. + * Get online features for the given entity rows using data retrieved from the feature/featureset + * specified in feature set requests. + * + *

This method returns a list of {@link FeatureRow}s corresponding to each feature set spec. + * Each feature row in the list then corresponds to an {@link EntityRow} provided by the user. If + * feature for a given entity row is not found, will return null in place of the {@link FeatureRow}. * - * @param entityRows list of entity rows in the feature request - * @param featureSetRequests List of {@link FeatureSetRequest} to feature references in the - * request tied to that feature set. - * @return list of lists of {@link FeatureRow}s corresponding to each feature set request and - * entity row. + * @param entityRows list of entity rows to request. + * @param featureSetRequests List of {@link FeatureSetRequest} specifying the features/feature set + * to retrieve data from. + * @return list of lists of {@link FeatureRow}s corresponding to data retrieved for each feature + * set request and entity row. */ List> getOnlineFeatures( List entityRows, List featureSetRequests); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index 4620186e702..fca4538c544 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -68,7 +68,7 @@ public static OnlineRetriever create(StatefulRedisConnection con * *

This method returns a list of {@link FeatureRow}s corresponding to each feature set spec. * Each feature row in the list then corresponds to an {@link EntityRow} provided by the user. If - * retrieval fails for a given entity row, will return null in place of the {@link FeatureRow}. + * feature for a given entity row is not found, will return null in place of the {@link FeatureRow}. * * @param featureSetRequests List of {@link FeatureSetRequest} specifying the features/feature set * to retrieve data from. diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java index adacacb941e..0da6d45069e 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java @@ -158,7 +158,7 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { } @Test - public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { + public void shouldReturnNullIfKeysNotPresent() { FeatureSetRequest featureSetRequest = FeatureSetRequest.newBuilder() .setSpec(getFeatureSetSpec()) @@ -211,14 +211,7 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) .build(), - FeatureRow.newBuilder() - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").build(), - Field.newBuilder().setName("feature2").build())) - .build())); - + null)); List> actual = redisOnlineRetriever.getOnlineFeatures(entityRows, ImmutableList.of(featureSetRequest)); assertThat(actual, equalTo(expected)); From 38815bbb1b5bbd7cdf1bfaef02857d4cc3a84b0e Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 21 Apr 2020 11:43:10 +0800 Subject: [PATCH 10/65] Refactor out unnecessary nesting of returned list in OnlineRetriever's getOnlineFeatures() This refactor is done to improve code readablity as it is difficult to conceptualise what each nesting of the list corresponds to: - developers have to keep in mind that the outer nest corresponds to faetures requests and the inner nesting corresponds to entity rows. The nesting is unnecessary as: - inside RedisOnlineRetriever we have a loop over feature set requests. - outside in OnlineServingService where this is used, we also loop over feature set requests. --- .../serving/service/OnlineServingService.java | 18 ++--- .../service/OnlineServingServiceTest.java | 20 +++--- .../api/retriever/OnlineRetriever.java | 22 +++---- .../redis/retriever/RedisOnlineRetriever.java | 48 +++++--------- .../retriever/RedisOnlineRetrieverTest.java | 66 +++++++++---------- 5 files changed, 73 insertions(+), 101 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index 414211cbb30..78302379e85 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -74,21 +74,13 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ entityRows.stream() .collect(Collectors.toMap(row -> row, row -> Maps.newHashMap(row.getFieldsMap()))); - // Pull feature rows for each feature set request from the retriever. - // Each feature row list corresponds to a single feature set request. - List> featureRows = - retriever.getOnlineFeatures(entityRows, featureSetRequests); - if (scope != null) { - scope.span().log(ImmutableMap.of("event", "featureRows", "value", featureRows)); - } - // Match entity rows request with the features defined in the feature spec. // For each feature set request, read the feature rows returned by the retriever, and // populate the featureValuesMap with the feature values corresponding to that entity row. - for (var fsIdx = 0; fsIdx < featureRows.size(); fsIdx++) { - List featureRowsForFs = featureRows.get(fsIdx); - FeatureSetRequest featureSetRequest = featureSetRequests.get(fsIdx); - + for (FeatureSetRequest featureSetRequest : featureSetRequests) { + // Pull feature rows for each feature set request from the retriever. + List featureRows = retriever.getOnlineFeatures(entityRows, featureSetRequest); + String project = featureSetRequest.getSpec().getProject(); // In order to return values containing the same feature references provided by the user, @@ -98,7 +90,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ // Each feature row returned (per feature set request) corresponds to a given entity row. // For each feature row, update the featureValuesMap. for (var entityRowIdx = 0; entityRowIdx < entityRows.size(); entityRowIdx++) { - FeatureRow featureRow = featureRowsForFs.get(entityRowIdx); + FeatureRow featureRow = featureRows.get(entityRowIdx); EntityRow entityRow = entityRows.get(entityRowIdx); // If the row is stale, put an empty value into the featureValuesMap. diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index cee34d04016..29cd18aecff 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -113,8 +113,8 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); when(retriever.getOnlineFeatures( - request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) - .thenReturn(Collections.singletonList(featureRows)); + request.getEntityRowsList(), featureSetRequest)) + .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -193,8 +193,8 @@ public void shouldReturnKeysWithoutVersionIfNotProvided() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); when(retriever.getOnlineFeatures( - request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) - .thenReturn(Collections.singletonList(featureRows)); + request.getEntityRowsList(), featureSetRequest)) + .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -263,8 +263,8 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); when(retriever.getOnlineFeatures( - request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) - .thenReturn(Collections.singletonList(featureRows)); + request.getEntityRowsList(), featureSetRequest)) + .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -343,8 +343,8 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); when(retriever.getOnlineFeatures( - request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) - .thenReturn(Collections.singletonList(featureRows)); + request.getEntityRowsList(), featureSetRequest)) + .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = @@ -417,8 +417,8 @@ public void shouldFilterOutUndesiredRows() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); when(retriever.getOnlineFeatures( - request.getEntityRowsList(), Collections.singletonList(featureSetRequest))) - .thenReturn(Collections.singletonList(featureRows)); + request.getEntityRowsList(), featureSetRequest)) + .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); GetOnlineFeaturesResponse expected = diff --git a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java index 71f2790fcd2..246025cc656 100644 --- a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java +++ b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java @@ -25,18 +25,18 @@ public interface OnlineRetriever { /** * Get online features for the given entity rows using data retrieved from the feature/featureset - * specified in feature set requests. + * specified in feature set request. * - *

This method returns a list of {@link FeatureRow}s corresponding to each feature set spec. - * Each feature row in the list then corresponds to an {@link EntityRow} provided by the user. If - * feature for a given entity row is not found, will return null in place of the {@link FeatureRow}. + *

Each {@link FeatureRow} in the returned list then corresponds to an {@link EntityRow} + * provided by the user. If feature for a given entity row is not found, will return null in place + * of the {@link FeatureRow}. The no. of {@link FeatureRow} returned should match the no. of given + * {@link EntityRow}s * - * @param entityRows list of entity rows to request. - * @param featureSetRequests List of {@link FeatureSetRequest} specifying the features/feature set - * to retrieve data from. - * @return list of lists of {@link FeatureRow}s corresponding to data retrieved for each feature - * set request and entity row. + * @param entityRows list of entity rows to request features for. + * @param featureSetRequest specifies the features/feature set to retrieve data from + * @return list of {@link FeatureRow}s corresponding to data retrieved for each entity row from + * feature/featureset specified in featureset request. */ - List> getOnlineFeatures( - List entityRows, List featureSetRequests); + List getOnlineFeatures( + List entityRows, FeatureSetRequest featureSetRequest); } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index fca4538c544..0948cd8974e 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -62,39 +62,23 @@ public static OnlineRetriever create(StatefulRedisConnection con return new RedisOnlineRetriever(connection); } - /** - * Gets online features from redis store for the given entity rows using data retrieved from the - * feature/featureset specified in feature set requests. - * - *

This method returns a list of {@link FeatureRow}s corresponding to each feature set spec. - * Each feature row in the list then corresponds to an {@link EntityRow} provided by the user. If - * feature for a given entity row is not found, will return null in place of the {@link FeatureRow}. - * - * @param featureSetRequests List of {@link FeatureSetRequest} specifying the features/feature set - * to retrieve data from. - * @return list of lists of {@link FeatureRow}s corresponding to data retrieved for each feature - * set request and entity row. - */ @Override - public List> getOnlineFeatures( - List entityRows, List featureSetRequests) { - - List> featureRows = new ArrayList<>(); - for (FeatureSetRequest featureSetRequest : featureSetRequests) { - // get features for this features/featureset in featureset request - FeatureSetSpec featureSetSpec = featureSetRequest.getSpec(); - List redisKeys = buildRedisKeys(entityRows, featureSetSpec); - FeatureRowDecoder decoder = - new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); - try { - List featureRowsForFeatureSet = getFeaturesForFeatureSet(redisKeys, decoder); - featureRows.add(featureRowsForFeatureSet); - } catch (InvalidProtocolBufferException | ExecutionException e) { - throw Status.INTERNAL - .withDescription("Unable to parse protobuf while retrieving feature") - .withCause(e) - .asRuntimeException(); - } + public List getOnlineFeatures( + List entityRows, FeatureSetRequest featureSetRequest) { + + // get features for this features/featureset in featureset request + FeatureSetSpec featureSetSpec = featureSetRequest.getSpec(); + List redisKeys = buildRedisKeys(entityRows, featureSetSpec); + FeatureRowDecoder decoder = + new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); + List featureRows = new ArrayList<>(); + try { + featureRows = getFeaturesForFeatureSet(redisKeys, decoder); + } catch (InvalidProtocolBufferException | ExecutionException e) { + throw Status.INTERNAL + .withDescription("Unable to parse protobuf while retrieving feature") + .withCause(e) + .asRuntimeException(); } return featureRows; diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java index 0da6d45069e..a029968373e 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java @@ -132,28 +132,26 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List> expected = - ImmutableList.of( - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build())); + List expected = + Lists.newArrayList( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet:1") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build(), + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet:1") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) + .build()); - List> actual = - redisOnlineRetriever.getOnlineFeatures(entityRows, ImmutableList.of(featureSetRequest)); + List actual = redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } @@ -200,20 +198,18 @@ public void shouldReturnNullIfKeysNotPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List> expected = - ImmutableList.of( - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - null)); - List> actual = - redisOnlineRetriever.getOnlineFeatures(entityRows, ImmutableList.of(featureSetRequest)); + List expected = + Lists.newArrayList( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet:1") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build(), + null); + List actual = redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } From 437da4509b940e93fd52882c3aff13ec87e8f934 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Thu, 23 Apr 2020 15:25:32 +0800 Subject: [PATCH 11/65] Add online feature metadata to OnlineServingService's getOnlineFeatures() Updated getOnlineFeatures() to return metadata on each response Field. - Added sanity check with exception to ensure that the feature rows retrieved from the OnlineRetriever matches the no. of entity rows pass to it. Refactor code out of getOnlineFeatures() to smaller methods to reduce method size and improve readabilty: - added unpackFields(entityRow) to unpack EntityRows into response Fields - added unpackFields(featureRow) to unpack FeatureRows into response Fields - added attachMetadata(field) to add metadata to response Fields. - moved loop over feature references into populateStaleKeyCountMetrics() --- .../serving/service/OnlineServingService.java | 261 +++++++++++++----- 1 file changed, 188 insertions(+), 73 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index 78302379e85..09196cb472d 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -21,6 +21,8 @@ import com.google.protobuf.Duration; import feast.proto.serving.ServingAPIProto.*; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Field; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; import feast.proto.types.ValueProto.Value; import feast.serving.specs.CachedSpecService; @@ -31,8 +33,7 @@ import io.grpc.Status; import io.opentracing.Scope; import io.opentracing.Tracer; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -63,95 +64,174 @@ public GetFeastServingInfoResponse getFeastServingInfo( @Override public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { try (Scope scope = tracer.buildSpan("getOnlineFeatures").startActive(true)) { - GetOnlineFeaturesResponse.Builder getOnlineFeaturesResponseBuilder = - GetOnlineFeaturesResponse.newBuilder(); - // Build featureset requests used to pass to the retriever to retrieve feature data. - List featureSetRequests = - specService.getFeatureSets(request.getFeaturesList()); List entityRows = request.getEntityRowsList(); + // Collect the response fields corresponding by entity row in entityFieldsMap. + Map> entityFieldsMap = + entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); - Map> featureValuesMap = - entityRows.stream() - .collect(Collectors.toMap(row -> row, row -> Maps.newHashMap(row.getFieldsMap()))); + if (request.getOmitEntitiesInResponse() == false) { + // Add entity row's fields as response fields + entityRows.forEach( + entityRow -> { + entityFieldsMap.get(entityRow).putAll(this.unpackFields(entityRow)); + }); + } - // Match entity rows request with the features defined in the feature spec. - // For each feature set request, read the feature rows returned by the retriever, and - // populate the featureValuesMap with the feature values corresponding to that entity row. + List featureSetRequests = + specService.getFeatureSets(request.getFeaturesList()); for (FeatureSetRequest featureSetRequest : featureSetRequests) { - // Pull feature rows for each feature set request from the retriever. + // Pull feature rows for given entity rows from the feature/featureset specified n feature + // set request. List featureRows = retriever.getOnlineFeatures(entityRows, featureSetRequest); - - String project = featureSetRequest.getSpec().getProject(); - - // In order to return values containing the same feature references provided by the user, - // we reuse the feature references in the request as the keys in the featureValuesMap - Map refsByName = featureSetRequest.getFeatureRefsByName(); - - // Each feature row returned (per feature set request) corresponds to a given entity row. - // For each feature row, update the featureValuesMap. - for (var entityRowIdx = 0; entityRowIdx < entityRows.size(); entityRowIdx++) { - FeatureRow featureRow = featureRows.get(entityRowIdx); - EntityRow entityRow = entityRows.get(entityRowIdx); - - // If the row is stale, put an empty value into the featureValuesMap. - // TODO: online-feature-metadata: detect stale features here. - if (isStale(featureSetRequest, entityRow, featureRow)) { - featureSetRequest - .getFeatureReferences() - .forEach( - ref -> { - populateStaleKeyCountMetrics(project, ref); - featureValuesMap - .get(entityRow) - .put(RefUtil.generateFeatureStringRef(ref), Value.newBuilder().build()); - }); - - } else { - populateRequestCountMetrics(featureSetRequest); - - // Else populate the featureValueMap at this entityRow with the values in the feature - // row. - featureRow.getFieldsList().stream() - .filter(field -> refsByName.containsKey(field.getName())) - .forEach( - field -> { - FeatureReference ref = refsByName.get(field.getName()); - String id = RefUtil.generateFeatureStringRef(ref); - featureValuesMap.get(entityRow).put(id, field.getValue()); - }); - } + // Check that feature row returned corresponds to a given entity row. + if (featureRows.size() != entityRows.size()) { + throw Status.INTERNAL + .withDescription( + "The no. of FeatureRow obtained from OnlineRetriever" + + "does not match no. of entityRow passed.") + .asRuntimeException(); + } + + for (var i = 0; i < entityRows.size(); i++) { + FeatureRow featureRow = featureRows.get(i); + EntityRow entityRow = entityRows.get(i); + // Unpack feature response fields from feature row + Map fields = + this.unpackFields( + featureRow, entityRow, featureSetRequest, request.getIncludeMetadataInResponse()); + // Merge feature response fields into entityFieldsMap + entityFieldsMap + .get(entityRow) + .putAll( + fields.entrySet().stream() + .collect( + Collectors.toMap( + es -> RefUtil.generateFeatureStringRef(es.getKey()), + es -> es.getValue()))); + + this.populateStaleKeyCountMetrics(fields, featureSetRequest); } + this.populateRequestCountMetrics(featureSetRequest); } - // TODO: update this to return actual records - Record record = Record.newBuilder().build(); - return getOnlineFeaturesResponseBuilder.addRecords(record).build(); + // Build response records from entityFieldsMap + List records = + entityFieldsMap.values().stream() + .map(fields -> Record.newBuilder().putAllFields(fields).build()) + .collect(Collectors.toList()); + return GetOnlineFeaturesResponse.newBuilder().addAllRecords(records).build(); } } - // TODO: call this based on what is returned in records. - private void populateStaleKeyCountMetrics(String project, FeatureReference ref) { - Metrics.staleKeyCount.labels(project, ref.getName()).inc(); + /** + * Unpack response fields from the given entity row's fields. + * + * @param entityRow to unpack for response fields + * @return Map mapping of name of field to response field. + */ + private Map unpackFields(EntityRow entityRow) { + return entityRow.getFieldsMap().entrySet().stream() + .collect( + Collectors.toMap( + es -> es.getKey(), + es -> { + return Field.newBuilder() + .setStatus(FieldStatus.PRESENT) + .setValue(es.getValue()) + .build(); + })); } - private void populateRequestCountMetrics(FeatureSetRequest featureSetRequest) { - String project = featureSetRequest.getSpec().getProject(); - featureSetRequest - .getFeatureReferences() - .parallelStream() - .forEach(ref -> Metrics.requestCount.labels(project, ref.getName()).inc()); - } + /** + * Unpack response fields using data from the given feature row for features specified in the + * given feature set request. + * + * @param featureRow to unpack for response fields. + * @param entityRow for which the feature row was retrieved for. + * @param featureSetRequest for which the feature row was retrieved. + * @param includeMetadata whether metadata should be included in the response fields + * @return Map mapping of feature ref to response field + */ + private Map unpackFields( + FeatureRow featureRow, + EntityRow entityRow, + FeatureSetRequest featureSetRequest, + boolean includeMetadata) { + // In order to return values containing the same feature references provided by the user, + // we reuse the feature references in the request as the keys in field builder map + Map refsByName = featureSetRequest.getFeatureRefsByName(); + Map fields = new HashMap<>(); - @Override - public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { - throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + // populate field builder map with feature row's field's data values + boolean hasStaleValues = this.isStale(featureSetRequest, entityRow, featureRow); + if (featureRow != null) { + Map featureFields = + featureRow.getFieldsList().stream() + .filter(featureRowField -> refsByName.containsKey(featureRowField.getName())) + .collect( + Collectors.toMap( + featureRowField -> refsByName.get(featureRowField.getName()), + featureRowField -> { + // omit stale feature values. + if (hasStaleValues) { + return Field.newBuilder().setValue(Value.newBuilder().build()); + } else { + return Field.newBuilder().setValue(featureRowField.getValue()); + } + })); + fields.putAll(featureFields); + } + + // create empty response fields for features specified in request but not present in feature + // row. + Set missingFeatures = new HashSet<>(refsByName.values()); + missingFeatures.removeAll(fields.keySet()); + missingFeatures.forEach(ref -> fields.put(ref, Field.newBuilder())); + + // attach metadata to the feature response fields & build response field + return fields.entrySet().stream() + .collect( + Collectors.toMap( + es -> es.getKey(), + es -> { + Field.Builder field = es.getValue(); + if (includeMetadata) { + field = this.attachMetadata(field, featureRow, hasStaleValues); + } + return field.build(); + })); } - @Override - public GetJobResponse getJob(GetJobRequest getJobRequest) { - throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + /** + * Attach metadata to the given response field. Attaches field status to the response providing + * metadata for the field. + * + * @param featureRow where the field was unpacked from. + * @param hasStaleValue whether the field contains a stale value + */ + private Field.Builder attachMetadata( + Field.Builder field, FeatureRow featureRow, boolean hasStaleValue) { + FieldStatus fieldStatus = FieldStatus.PRESENT; + if (featureRow == null) { + fieldStatus = FieldStatus.NOT_FOUND; + } else if (hasStaleValue) { + fieldStatus = FieldStatus.OUTSIDE_MAX_AGE; + } else if (field.getValue().getValCase() == Value.ValCase.VAL_NOT_SET) { + fieldStatus = FieldStatus.NULL_VALUE; + } + + return field.setStatus(fieldStatus); } + /** + * Determine if the feature data in the given feature row is considered stale. Data is considered + * to be stale when difference ingestion time set in feature row and the retrieval time set in + * entity row exceeds featureset max age. + * + * @param featureSetRequest contains the spec where feature's max age is extracted. + * @param entityRow contains the retrieval timing of when features are pulled. + * @param featureRow contains the ingestion timing and feature data. + */ private boolean isStale( FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { Duration maxAge = featureSetRequest.getSpec().getMaxAge(); @@ -165,4 +245,39 @@ private boolean isStale( long timeDifference = givenTimestamp - featureRow.getEventTimestamp().getSeconds(); return timeDifference > maxAge.getSeconds(); } + + private void populateStaleKeyCountMetrics( + Map fields, FeatureSetRequest featureSetRequest) { + String project = featureSetRequest.getSpec().getProject(); + fields + .entrySet() + .forEach( + es -> { + FeatureReference ref = es.getKey(); + Field field = es.getValue(); + if (field.getStatus() == FieldStatus.OUTSIDE_MAX_AGE) { + Metrics.staleKeyCount + .labels(project, RefUtil.generateFeatureStringRefWithoutProject(ref)) + .inc(); + } + }); + } + + private void populateRequestCountMetrics(FeatureSetRequest featureSetRequest) { + String project = featureSetRequest.getSpec().getProject(); + featureSetRequest + .getFeatureReferences() + .parallelStream() + .forEach(ref -> Metrics.requestCount.labels(project, ref.getName()).inc()); + } + + @Override + public GetBatchFeaturesResponse getBatchFeatures(GetBatchFeaturesRequest getFeaturesRequest) { + throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + } + + @Override + public GetJobResponse getJob(GetJobRequest getJobRequest) { + throw Status.UNIMPLEMENTED.withDescription("Method not implemented").asRuntimeException(); + } } From 4a781890c956f6ea6d4d3cd6622298d63935c60f Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Thu, 23 Apr 2020 15:59:05 +0800 Subject: [PATCH 12/65] Update OnlineServingServiceTest to check that online metadata is set correctly. --- .../service/OnlineServingServiceTest.java | 341 +++++++++++++----- 1 file changed, 257 insertions(+), 84 deletions(-) diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 29cd18aecff..8b6fbb1cbba 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -17,19 +17,22 @@ package feast.serving.service; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; import com.google.common.collect.Lists; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; + import feast.proto.core.FeatureSetProto.EntitySpec; import feast.proto.core.FeatureSetProto.FeatureSetSpec; import feast.proto.serving.ServingAPIProto.FeatureReference; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Field; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; import feast.proto.types.FeatureRowProto.FeatureRow; import feast.proto.types.FieldProto.Field; @@ -41,8 +44,6 @@ import io.opentracing.Tracer.SpanBuilder; import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatchers; @@ -66,11 +67,23 @@ public void setUp() { } @Test - public void shouldReturnResponseWithValuesIfKeysPresent() { + public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() - .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) - .addFeatures(FeatureReference.newBuilder().setName("feature2").build()) + .setOmitEntitiesInResponse(false) + .setIncludeMetadataInResponse(true) + .addFeatures( + FeatureReference.newBuilder() + .setName("feature1") + .setVersion(1) + .setProject("project") + .build()) + .addFeatures( + FeatureReference.newBuilder() + .setName("feature2") + .setVersion(1) + .setProject("project") + .build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -89,19 +102,45 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .addAllFields( Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .setFeatureSet("featureSet:1") .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .addAllFields( Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(2)) + .build())) + .setFeatureSet("featureSet:1") .build()); FeatureSetRequest featureSetRequest = @@ -112,8 +151,7 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures( - request.getEntityRowsList(), featureSetRequest)) + when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); @@ -133,14 +171,15 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { .putFields("project/feature2:1", intField(2))) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat( - responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); + assertThat(actual, equalTo(expected)); } @Test - public void shouldReturnKeysWithoutVersionIfNotProvided() { + public void shouldReturnFieldKeysWithoutVersionIfNotProvided() { GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() + .setIncludeMetadataInResponse(true) + .setOmitEntitiesInResponse(false) .addFeatures( FeatureReference.newBuilder() .setName("feature1") @@ -167,20 +206,44 @@ public void shouldReturnKeysWithoutVersionIfNotProvided() { .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .addAllFields( Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) .setFeatureSet("featureSet:1") .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .addAllFields( Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(2)) + .build())) .setFeatureSet("featureSet:1") .build()); @@ -192,8 +255,7 @@ public void shouldReturnKeysWithoutVersionIfNotProvided() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures( - request.getEntityRowsList(), featureSetRequest)) + when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); @@ -213,17 +275,33 @@ public void shouldReturnKeysWithoutVersionIfNotProvided() { .putFields("project/feature2", intField(2))) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat( - responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); + assertThat(actual, equalTo(expected)); } @Test - public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { + public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { // some keys not present, should have empty values GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() +<<<<<<< HEAD .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) .addFeatures(FeatureReference.newBuilder().setName("feature2").build()) +======= + .setOmitEntitiesInResponse(false) + .setIncludeMetadataInResponse(true) + .addFeatures( + FeatureReference.newBuilder() + .setName("feature1") + .setVersion(1) + .setProject("project") + .build()) + .addFeatures( + FeatureReference.newBuilder() + .setName("feature2") + .setVersion(1) + .setProject("project") + .build()) +>>>>>>> Update OnlineServingServiceTest to check that online metadata is set correctly. .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -249,21 +327,31 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) .build(), FeatureRow.newBuilder() +<<<<<<< HEAD .setFeatureSet("project/featureSet") +======= + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet:1") +>>>>>>> Update OnlineServingServiceTest to check that online metadata is set correctly. .addAllFields( Lists.newArrayList( - Field.newBuilder().setName("feature1").build(), - Field.newBuilder().setName("feature2").build())) + FieldProto.Field.newBuilder().setName("feature1").build(), + FieldProto.Field.newBuilder().setName("feature2").build())) .build()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures( - request.getEntityRowsList(), featureSetRequest)) + when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); @@ -280,22 +368,46 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { .putFields("entity1", intField(2)) .putFields("entity2", strField("b")) .putFields( - "project/feature1:1", GetOnlineFeaturesResponse.Field.newBuilder().build()) + "project/feature1:1", + Field.newBuilder() + .setValue(Value.newBuilder().build()) + .setStatus(FieldStatus.NULL_VALUE) + .build()) .putFields( - "project/feature2:1", GetOnlineFeaturesResponse.Field.newBuilder().build())) + "project/feature2:1", + Field.newBuilder() + .setValue(Value.newBuilder().build()) + .setStatus(FieldStatus.NULL_VALUE) + .build())) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat( - responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); + assertThat(actual, equalTo(expected)); } @Test - public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { - // keys present, but too stale comp. to maxAge + public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { + // keys present, but considered stale when compared to maxAge GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() +<<<<<<< HEAD .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) .addFeatures(FeatureReference.newBuilder().setName("feature2").build()) +======= + .setOmitEntitiesInResponse(false) + .setIncludeMetadataInResponse(true) + .addFeatures( + FeatureReference.newBuilder() + .setName("feature1") + .setVersion(1) + .setProject("project") + .build()) + .addFeatures( + FeatureReference.newBuilder() + .setName("feature2") + .setVersion(1) + .setProject("project") + .build()) +>>>>>>> Update OnlineServingServiceTest to check that online metadata is set correctly. .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -314,22 +426,62 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .addAllFields( Lists.newArrayList( +<<<<<<< HEAD Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) .setFeatureSet("project/featureSet") +======= + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .setFeatureSet("featureSet:1") +>>>>>>> Update OnlineServingServiceTest to check that online metadata is set correctly. .build(), FeatureRow.newBuilder() .setEventTimestamp( Timestamp.newBuilder().setSeconds(50)) // this value should be nulled .addAllFields( Lists.newArrayList( +<<<<<<< HEAD Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) .setFeatureSet("project/featureSet") +======= + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(2)) + .build())) + .setFeatureSet("featureSet:1") +>>>>>>> Update OnlineServingServiceTest to check that online metadata is set correctly. .build()); FeatureSetSpec spec = @@ -342,8 +494,7 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures( - request.getEntityRowsList(), featureSetRequest)) + when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); @@ -360,13 +511,21 @@ public void shouldReturnResponseWithUnsetValuesIfMaxAgeIsExceeded() { .putFields("entity1", intField(2)) .putFields("entity2", strField("b")) .putFields( - "project/feature1:1", GetOnlineFeaturesResponse.Field.newBuilder().build()) + "project/feature1:1", + Field.newBuilder() + .setValue(Value.newBuilder().build()) + .setStatus(FieldStatus.OUTSIDE_MAX_AGE) + .build()) .putFields( - "project/feature2:1", GetOnlineFeaturesResponse.Field.newBuilder().build())) + "project/feature2:1", + Field.newBuilder() + .setValue(Value.newBuilder().build()) + .setStatus(FieldStatus.OUTSIDE_MAX_AGE) + .build()) + .build()) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat( - responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); + assertThat(actual, equalTo(expected)); } @Test @@ -374,7 +533,14 @@ public void shouldFilterOutUndesiredRows() { // requested rows less than the rows available in the featureset GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() - .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) + .setIncludeMetadataInResponse(true) + .setOmitEntitiesInResponse(false) + .addFeatures( + FeatureReference.newBuilder() + .setName("feature1") + .setVersion(1) + .setProject("project") + .build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -393,19 +559,45 @@ public void shouldFilterOutUndesiredRows() { .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .addAllFields( Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .setFeatureSet("featureSet:1") .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .addAllFields( Lists.newArrayList( - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(2)) + .build())) + .setFeatureSet("featureSet:1") .build()); FeatureSetRequest featureSetRequest = @@ -416,8 +608,7 @@ public void shouldFilterOutUndesiredRows() { when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures( - request.getEntityRowsList(), featureSetRequest)) + when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) .thenReturn(featureRows); when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); @@ -435,25 +626,7 @@ public void shouldFilterOutUndesiredRows() { .putFields("project/feature1:1", intField(2))) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat( - responseToMapList(actual), containsInAnyOrder(responseToMapList(expected).toArray())); - } - - private List> responseToMapList(GetOnlineFeaturesResponse response) { - // TODO:: update with metadata - return response.getRecordsList().stream() - .map(Record::getFieldsMap) - .map( - fieldsMap -> - fieldsMap.entrySet().stream() - .collect( - Collectors.toMap( - es -> es.getKey(), - es -> { - GetOnlineFeaturesResponse.Field field = es.getValue(); - return field.getValue(); - }))) - .collect(Collectors.toList()); + assertThat(actual, equalTo(expected)); } private Value intValue(int val) { @@ -464,12 +637,12 @@ private Value strValue(String val) { return Value.newBuilder().setStringVal(val).build(); } - private GetOnlineFeaturesResponse.Field intField(int val) { - return GetOnlineFeaturesResponse.Field.newBuilder().setValue(intValue(val)).build(); + private Field intField(int val) { + return Field.newBuilder().setValue(intValue(val)).setStatus(FieldStatus.PRESENT).build(); } - private GetOnlineFeaturesResponse.Field strField(String val) { - return GetOnlineFeaturesResponse.Field.newBuilder().setValue(strValue(val)).build(); + private Field strField(String val) { + return Field.newBuilder().setValue(strValue(val)).setStatus(FieldStatus.PRESENT).build(); } private FeatureSetSpec getFeatureSetSpec() { From 4d6437ec79d638443c1fe7e933b323ec925caacc Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Thu, 23 Apr 2020 17:09:20 +0800 Subject: [PATCH 13/65] Added online metadata support java sdk's FeastClient. --- .../main/java/com/gojek/feast/FeastClient.java | 12 +++++++++--- sdk/java/src/main/java/com/gojek/feast/Row.java | 17 +++++++++++++++-- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index b9f2ea6bdb8..cbb59af7862 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -88,7 +88,7 @@ public List getOnlineFeatures(List featureRefs, List rows) { * @return list of {@link Row} containing retrieved data fields. */ public List getOnlineFeatures(List featureRefs, List rows, String project) { - return getOnlineFeatures(featureRefs, rows, project, false); + return getOnlineFeatures(featureRefs, rows, project, false, false); } /** @@ -114,10 +114,15 @@ public List getOnlineFeatures(List featureRefs, List rows, Str * Feature requested belong to. * @param omitEntitiesInResponse if true, the returned {@link Row} will not contain field and * value for the entity + * @param includeMetadataInResponse if true, will include field status metadata in {@link Row}. * @return list of {@link Row} containing retrieved data fields. */ public List getOnlineFeatures( - List featureRefs, List rows, String project, boolean omitEntitiesInResponse) { + List featureRefs, + List rows, String + project, + boolean omitEntitiesInResponse, + boolean includeMetadataInResponse) { List features = RequestUtil.createFeatureRefs(featureRefs, project); // build entity rows and collect entity references HashSet entityRefs = new HashSet<>(); @@ -139,6 +144,7 @@ public List getOnlineFeatures( .addAllFeatures(features) .addAllEntityRows(entityRows) .setOmitEntitiesInResponse(omitEntitiesInResponse) + .setIncludeMetadataInResponse(includeMetadataInResponse) .build()); return response.getRecordsList().stream() @@ -149,7 +155,7 @@ record -> { .getFieldsMap() .forEach( (name, field) -> { - row.set(name, field.getValue()); + row.set(name, field.getValue(), field.getStatus()); }); return row; }) diff --git a/sdk/java/src/main/java/com/gojek/feast/Row.java b/sdk/java/src/main/java/com/gojek/feast/Row.java index 4a3035f8f19..1e37b2d647a 100644 --- a/sdk/java/src/main/java/com/gojek/feast/Row.java +++ b/sdk/java/src/main/java/com/gojek/feast/Row.java @@ -21,6 +21,7 @@ import com.google.protobuf.util.Timestamps; import feast.proto.types.ValueProto.Value; import feast.proto.types.ValueProto.Value.ValCase; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; @@ -32,11 +33,13 @@ public class Row { private Timestamp entity_timestamp; private HashMap fields; + private HashMap fieldStatuses; public static Row create() { Row row = new Row(); row.entity_timestamp = Timestamps.fromMillis(System.currentTimeMillis()); row.fields = new HashMap<>(); + row.fieldStatuses = new HashMap<>(); return row; } @@ -54,7 +57,7 @@ public Row setEntityTimestamp(String dateTime) { return this; } - public Row set(String fieldName, Object value) { + public Row set(String fieldName, Object value, FieldStatus status) { String valueType = value.getClass().getCanonicalName(); switch (valueType) { case "java.lang.Integer": @@ -85,13 +88,15 @@ public Row set(String fieldName, Object value) { "Type '%s' is unsupported in Feast. Please use one of these value types: Integer, Long, Float, Double, String, byte[].", valueType)); } + + fieldStatuses.put(fieldName, status); return this; } public Map getFields() { return fields; } - + public Integer getInt(String fieldName) { return getValue(fieldName).map(Value::getInt32Val).orElse(null); } @@ -116,6 +121,14 @@ public byte[] getByte(String fieldName) { return getValue(fieldName).map(Value::getBytesVal).map(ByteString::toByteArray).orElse(null); } + public Map getFieldStatuses() { + return fieldStatuses; + } + + public FieldStatus getFieldStatus(String fieldName) { + return fieldStatuses.get(fieldName); + } + @Override public String toString() { List parts = new ArrayList<>(); From 9ca034c15cf4a691af377c1b3035cba0ab5fa605 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 24 Apr 2020 11:49:21 +0800 Subject: [PATCH 14/65] Changed go sdk's Row to store a map of fields instead of just Values --- go.sum | 2 ++ sdk/go/types.go | 15 +++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go.sum b/go.sum index 90a517da7e4..48887dc5cd3 100644 --- a/go.sum +++ b/go.sum @@ -162,6 +162,7 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= @@ -491,6 +492,7 @@ google.golang.org/genproto v0.0.0-20200319113533-08878b785e9c h1:5aI3/f/3eCZps9x google.golang.org/genproto v0.0.0-20200319113533-08878b785e9c/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587 h1:1Ym+vvUpq1ZHvxzn34gENJX8U4aKO+vhy2P/2+Xl6qQ= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/sdk/go/types.go b/sdk/go/types.go index dd6553163bd..b06fb3f8879 100644 --- a/sdk/go/types.go +++ b/sdk/go/types.go @@ -1,21 +1,20 @@ package feast import ( + "github.com/feast-dev/feast/sdk/go/protos/feast/serving" "github.com/feast-dev/feast/sdk/go/protos/feast/types" "github.com/golang/protobuf/proto" ) -// Row map of entity values -type Row map[string]*types.Value +// Row is a map of fields +type Row map[string]*serving.GetOnlineFeaturesResponse_Field func (r Row) equalTo(other Row) bool { - for k, v := range r { - if otherV, ok := other[k]; !ok { + for k, field := range r { + if otherField, ok := other[k]; !ok { + return false + } else if !proto.Equal(field, otherField) { return false - } else { - if !proto.Equal(v, otherV) { - return false - } } } return true From f91c3fe1a682d8c124b5aaeda1265453d5cd78d9 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 24 Apr 2020 11:50:14 +0800 Subject: [PATCH 15/65] Update response.go to use updated Row and take into account FieldStatus --- sdk/go/response.go | 22 +++++++++++----------- sdk/go/response_test.go | 39 ++++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/sdk/go/response.go b/sdk/go/response.go index 1bc80af9fbb..5ee87537e8b 100644 --- a/sdk/go/response.go +++ b/sdk/go/response.go @@ -24,9 +24,9 @@ type OnlineFeaturesResponse struct { // Rows retrieves the result of the request as a list of Rows. func (r OnlineFeaturesResponse) Rows() []Row { - rows := make([]Row, len(r.RawResponse.FieldValues)) - for i, val := range r.RawResponse.FieldValues { - rows[i] = val.Fields + rows := make([]Row, len(r.RawResponse.Records)) + for i, records := range r.RawResponse.Records { + rows[i] = records.Fields } return rows } @@ -34,18 +34,18 @@ func (r OnlineFeaturesResponse) Rows() []Row { // Int64Arrays retrieves the result of the request as a list of int64 slices. Any missing values will be filled // with the missing values provided. func (r OnlineFeaturesResponse) Int64Arrays(order []string, fillNa []int64) ([][]int64, error) { - rows := make([][]int64, len(r.RawResponse.FieldValues)) + rows := make([][]int64, len(r.RawResponse.Records)) if len(fillNa) != len(order) { return nil, fmt.Errorf(ErrLengthMismatch, len(fillNa), len(order)) } - for i, val := range r.RawResponse.FieldValues { + for i, record := range r.RawResponse.Records { rows[i] = make([]int64, len(order)) for j, fname := range order { - fValue, exists := val.Fields[fname] + field, exists := record.Fields[fname] if !exists { return nil, fmt.Errorf(ErrFeatureNotFound, fname) } - val := fValue.GetVal() + val := field.Value.GetVal() if val == nil { rows[i][j] = fillNa[j] } else if int64Val, ok := val.(*types.Value_Int64Val); ok { @@ -61,18 +61,18 @@ func (r OnlineFeaturesResponse) Int64Arrays(order []string, fillNa []int64) ([][ // Float64Arrays retrieves the result of the request as a list of float64 slices. Any missing values will be filled // with the missing values provided. func (r OnlineFeaturesResponse) Float64Arrays(order []string, fillNa []float64) ([][]float64, error) { - rows := make([][]float64, len(r.RawResponse.FieldValues)) + rows := make([][]float64, len(r.RawResponse.Records)) if len(fillNa) != len(order) { return nil, fmt.Errorf(ErrLengthMismatch, len(fillNa), len(order)) } - for i, val := range r.RawResponse.FieldValues { + for i, records := range r.RawResponse.Records { rows[i] = make([]float64, len(order)) for j, fname := range order { - fValue, exists := val.Fields[fname] + field, exists := records.Fields[fname] if !exists { return nil, fmt.Errorf(ErrFeatureNotFound, fname) } - val := fValue.GetVal() + val := field.Value.GetVal() if val == nil { rows[i][j] = fillNa[j] } else if doubleVal, ok := val.(*types.Value_DoubleVal); ok { diff --git a/sdk/go/response_test.go b/sdk/go/response_test.go index 6a7c4211187..0b80e924390 100644 --- a/sdk/go/response_test.go +++ b/sdk/go/response_test.go @@ -8,19 +8,30 @@ import ( "testing" ) +// constructs a response field from the given value +func toField(value *types.Value) *serving.GetOnlineFeaturesResponse_Field { + return &serving.GetOnlineFeaturesResponse_Field{ + Value: value, + Status: serving.GetOnlineFeaturesResponse_PRESENT, + } +} + var response = OnlineFeaturesResponse{ RawResponse: &serving.GetOnlineFeaturesResponse{ - FieldValues: []*serving.GetOnlineFeaturesResponse_FieldValues{ + Records: []*serving.GetOnlineFeaturesResponse_Record{ { - Fields: map[string]*types.Value{ - "project1/feature1": Int64Val(1), - "project1/feature2": {}, + Fields: map[string]*serving.GetOnlineFeaturesResponse_Field{ + "project1/feature1": toField(Int64Val(1)), + "project1/feature2": &serving.GetOnlineFeaturesResponse_Field{ + Value: &types.Value{}, + Status: serving.GetOnlineFeaturesResponse_NULL_VALUE, + }, }, }, { - Fields: map[string]*types.Value{ - "project1/feature1": Int64Val(2), - "project1/feature2": Int64Val(2), + Fields: map[string]*serving.GetOnlineFeaturesResponse_Field{ + "project1/feature1": toField(Int64Val(2)), + "project1/feature2": toField(Int64Val(2)), }, }, }, @@ -30,8 +41,17 @@ var response = OnlineFeaturesResponse{ func TestOnlineFeaturesResponseToRow(t *testing.T) { actual := response.Rows() expected := []Row{ - {"project1/feature1": Int64Val(1), "project1/feature2": &types.Value{}}, - {"project1/feature1": Int64Val(2), "project1/feature2": Int64Val(2)}, + { + "project1/feature1": toField(Int64Val(1)), + "project1/feature2": &serving.GetOnlineFeaturesResponse_Field{ + Value: &types.Value{}, + Status: serving.GetOnlineFeaturesResponse_NULL_VALUE, + }, + }, + { + "project1/feature1": toField(Int64Val(2)), + "project1/feature2": toField(Int64Val(2)), + }, } if len(expected) != len(actual) { t.Errorf("expected: %v, got: %v", expected, actual) @@ -64,6 +84,7 @@ func TestOnlineFeaturesResponseToInt64Array(t *testing.T) { want: [][]int64{{-1, 1}, {2, 2}}, wantErr: false, }, + { name: "length mismatch", args: args{ From 4df2d3437012beda66000a27299c5a84195eb60a Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 24 Apr 2020 12:51:07 +0800 Subject: [PATCH 16/65] Added Field() to go sdk's types.go to construct fields from a Value. --- sdk/go/request_test.go | 6 +++--- sdk/go/response_test.go | 20 ++++++-------------- sdk/go/types.go | 8 ++++++++ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sdk/go/request_test.go b/sdk/go/request_test.go index 70e1f0d8f5e..306ce2879b8 100644 --- a/sdk/go/request_test.go +++ b/sdk/go/request_test.go @@ -25,9 +25,9 @@ func TestGetOnlineFeaturesRequest(t *testing.T) { "driver_id", }, Entities: []Row{ - {"entity1": Int64Val(1), "entity2": StrVal("bob")}, - {"entity1": Int64Val(1), "entity2": StrVal("annie")}, - {"entity1": Int64Val(1), "entity2": StrVal("jane")}, + {"entity1": Field(Int64Val(1)), "entity2": Field(StrVal("bob"))}, + {"entity1": Field(Int64Val(1)), "entity2": Field(StrVal("annie"))}, + {"entity1": Field(Int64Val(1)), "entity2": Field(StrVal("jane"))}, }, Project: "driver_project", }, diff --git a/sdk/go/response_test.go b/sdk/go/response_test.go index 0b80e924390..a0ba85584e1 100644 --- a/sdk/go/response_test.go +++ b/sdk/go/response_test.go @@ -8,20 +8,12 @@ import ( "testing" ) -// constructs a response field from the given value -func toField(value *types.Value) *serving.GetOnlineFeaturesResponse_Field { - return &serving.GetOnlineFeaturesResponse_Field{ - Value: value, - Status: serving.GetOnlineFeaturesResponse_PRESENT, - } -} - var response = OnlineFeaturesResponse{ RawResponse: &serving.GetOnlineFeaturesResponse{ Records: []*serving.GetOnlineFeaturesResponse_Record{ { Fields: map[string]*serving.GetOnlineFeaturesResponse_Field{ - "project1/feature1": toField(Int64Val(1)), + "project1/feature1": Field(Int64Val(1)), "project1/feature2": &serving.GetOnlineFeaturesResponse_Field{ Value: &types.Value{}, Status: serving.GetOnlineFeaturesResponse_NULL_VALUE, @@ -30,8 +22,8 @@ var response = OnlineFeaturesResponse{ }, { Fields: map[string]*serving.GetOnlineFeaturesResponse_Field{ - "project1/feature1": toField(Int64Val(2)), - "project1/feature2": toField(Int64Val(2)), + "project1/feature1": Field(Int64Val(2)), + "project1/feature2": Field(Int64Val(2)), }, }, }, @@ -42,15 +34,15 @@ func TestOnlineFeaturesResponseToRow(t *testing.T) { actual := response.Rows() expected := []Row{ { - "project1/feature1": toField(Int64Val(1)), + "project1/feature1": Field(Int64Val(1)), "project1/feature2": &serving.GetOnlineFeaturesResponse_Field{ Value: &types.Value{}, Status: serving.GetOnlineFeaturesResponse_NULL_VALUE, }, }, { - "project1/feature1": toField(Int64Val(2)), - "project1/feature2": toField(Int64Val(2)), + "project1/feature1": Field(Int64Val(2)), + "project1/feature2": Field(Int64Val(2)), }, } if len(expected) != len(actual) { diff --git a/sdk/go/types.go b/sdk/go/types.go index b06fb3f8879..0117a9ecf1f 100644 --- a/sdk/go/types.go +++ b/sdk/go/types.go @@ -54,3 +54,11 @@ func BoolVal(val bool) *types.Value { func BytesVal(val []byte) *types.Value { return &types.Value{Val: &types.Value_BytesVal{BytesVal: val}} } + +// constructs a response field from the given value +func Field(value *types.Value) *serving.GetOnlineFeaturesResponse_Field { + return &serving.GetOnlineFeaturesResponse_Field{ + Value: value, + Status: serving.GetOnlineFeaturesResponse_PRESENT, + } +} From e896f17ec2b230a6f79b51677e46353c73fd4b3b Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 24 Apr 2020 12:55:26 +0800 Subject: [PATCH 17/65] Update go sdk's request.go to use updated Row and Add GetOnlineFeatures flags Added missing boolean flags for GetOnlineFeatures's IncludeMetadataInResponse and OmitEntitiesInResponse flags. --- sdk/go/request.go | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/sdk/go/request.go b/sdk/go/request.go index 683f17c16ed..0c75141ce8f 100644 --- a/sdk/go/request.go +++ b/sdk/go/request.go @@ -4,6 +4,9 @@ import ( "fmt" "github.com/feast-dev/feast/sdk/go/protos/feast/serving" "strings" + + "github.com/gojek/feast/sdk/go/protos/feast/serving" + "github.com/gojek/feast/sdk/go/protos/feast/types" ) var ( @@ -25,6 +28,12 @@ type OnlineFeaturesRequest struct { // Project specifies the project would contain the feature sets where the requested features belong to. Project string + + // whether to omit the entities fields in the response. + OmitEntities bool + + // whether to include field status metadata in the response. + IncludeMeta bool } // Builds the feast-specified request payload from the wrapper. @@ -34,16 +43,23 @@ func (r OnlineFeaturesRequest) buildRequest() (*serving.GetOnlineFeaturesRequest return nil, err } + // build request entity rows from native entities entityRows := make([]*serving.GetOnlineFeaturesRequest_EntityRow, len(r.Entities)) - - for i := range r.Entities { + for i, entity := range r.Entities { + fieldMap := make(map[string]*types.Value) + for key, field := range entity { + fieldMap[key] = field.Value + } entityRows[i] = &serving.GetOnlineFeaturesRequest_EntityRow{ - Fields: r.Entities[i], + Fields: fieldMap, } } + return &serving.GetOnlineFeaturesRequest{ - Features: featureRefs, - EntityRows: entityRows, + Features: features, + EntityRows: entityRows, + OmitEntitiesInResponse: r.OmitEntities, + IncludeMetadataInResponse: r.IncludeMeta, }, nil } From 5b23b55ce8f03d107ac810650cb3b8f744f31875 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 24 Apr 2020 13:07:44 +0800 Subject: [PATCH 18/65] Update go protobuf generated code for updated protobuf defintion --- go.mod | 8 + .../protos/feast/serving/ServingService.pb.go | 346 +++++++++++++----- .../tensorflow_metadata/proto/v0/path.pb.go | 11 + .../tensorflow_metadata/proto/v0/schema.pb.go | 10 + 4 files changed, 276 insertions(+), 99 deletions(-) diff --git a/go.mod b/go.mod index d5917cb33e3..0cc5494d067 100644 --- a/go.mod +++ b/go.mod @@ -23,12 +23,20 @@ require ( github.com/woop/protoc-gen-doc v1.3.0 // indirect go.opencensus.io v0.22.3 // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect +<<<<<<< HEAD golang.org/x/net v0.0.0-20200513185701-a91f0712d120 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc // indirect google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587 // indirect google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.24.0 // indirect +======= + golang.org/x/net v0.0.0-20200320220750-118fecf932d8 + golang.org/x/sys v0.0.0-20200321134203-328b4cd54aae // indirect + golang.org/x/tools v0.0.0-20200423205358-59e73619c742 // indirect + google.golang.org/genproto v0.0.0-20200319113533-08878b785e9c // indirect + google.golang.org/grpc v1.28.0 +>>>>>>> Update go protobuf generated code for updated protobuf defintion gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.2.4 istio.io/gogo-genproto v0.0.0-20191212213402-78a529a42cd8 // indirect diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index fc2748d5d84..f246f8169dd 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -243,6 +243,70 @@ func (DataFormat) EnumDescriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{3} } +type GetOnlineFeaturesResponse_FieldStatus int32 + +const ( + // Status is unset for this field. + GetOnlineFeaturesResponse_INVALID GetOnlineFeaturesResponse_FieldStatus = 0 + // Field value is present for this field and within maximum allowable range + GetOnlineFeaturesResponse_PRESENT GetOnlineFeaturesResponse_FieldStatus = 1 + // Values could be found for entity key within the maximum allowable range, but + // this field value is assigned a null value on ingestion into feast. + GetOnlineFeaturesResponse_NULL_VALUE GetOnlineFeaturesResponse_FieldStatus = 2 + // Entity key did not return any values as they do not exist in Feast. + // This could suggest that the feature values have not yet been ingested + // into feast or the ingestion failed. + GetOnlineFeaturesResponse_NOT_FOUND GetOnlineFeaturesResponse_FieldStatus = 3 + // Values could be found for entity key, but field values are outside the maximum + // allowable range. + GetOnlineFeaturesResponse_OUTSIDE_MAX_AGE GetOnlineFeaturesResponse_FieldStatus = 4 +) + +// Enum value maps for GetOnlineFeaturesResponse_FieldStatus. +var ( + GetOnlineFeaturesResponse_FieldStatus_name = map[int32]string{ + 0: "INVALID", + 1: "PRESENT", + 2: "NULL_VALUE", + 3: "NOT_FOUND", + 4: "OUTSIDE_MAX_AGE", + } + GetOnlineFeaturesResponse_FieldStatus_value = map[string]int32{ + "INVALID": 0, + "PRESENT": 1, + "NULL_VALUE": 2, + "NOT_FOUND": 3, + "OUTSIDE_MAX_AGE": 4, + } +) + +func (x GetOnlineFeaturesResponse_FieldStatus) Enum() *GetOnlineFeaturesResponse_FieldStatus { + p := new(GetOnlineFeaturesResponse_FieldStatus) + *p = x + return p +} + +func (x GetOnlineFeaturesResponse_FieldStatus) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GetOnlineFeaturesResponse_FieldStatus) Descriptor() protoreflect.EnumDescriptor { + return file_feast_serving_ServingService_proto_enumTypes[4].Descriptor() +} + +func (GetOnlineFeaturesResponse_FieldStatus) Type() protoreflect.EnumType { + return &file_feast_serving_ServingService_proto_enumTypes[4] +} + +func (x GetOnlineFeaturesResponse_FieldStatus) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GetOnlineFeaturesResponse_FieldStatus.Descriptor instead. +func (GetOnlineFeaturesResponse_FieldStatus) EnumDescriptor() ([]byte, []int) { + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 0} +} + type GetFeastServingInfoRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -431,6 +495,9 @@ type GetOnlineFeaturesRequest struct { // Option to omit entities from the response. If true, only feature // values will be returned. OmitEntitiesInResponse bool `protobuf:"varint,3,opt,name=omit_entities_in_response,json=omitEntitiesInResponse,proto3" json:"omit_entities_in_response,omitempty"` + // Option to include feature metadata in the response. + // If true, response will include both feature values and metadata. + IncludeMetadataInResponse bool `protobuf:"varint,5,opt,name=include_metadata_in_response,json=includeMetadataInResponse,proto3" json:"include_metadata_in_response,omitempty"` } func (x *GetOnlineFeaturesRequest) Reset() { @@ -486,20 +553,25 @@ func (x *GetOnlineFeaturesRequest) GetOmitEntitiesInResponse() bool { return false } -type GetBatchFeaturesRequest struct { +func (x *GetOnlineFeaturesRequest) GetIncludeMetadataInResponse() bool { + if x != nil { + return x.IncludeMetadataInResponse + } + return false +} + +type GetOnlineFeaturesResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // List of features that are being retrieved - Features []*FeatureReference `protobuf:"bytes,3,rep,name=features,proto3" json:"features,omitempty"` - // Source of the entity dataset containing the timestamps and entity keys to retrieve - // features for. - DatasetSource *DatasetSource `protobuf:"bytes,2,opt,name=dataset_source,json=datasetSource,proto3" json:"dataset_source,omitempty"` + // Data records retrieved from the online feast store. + // Each record represents the data retrieved for an entity row in the request. + Records []*GetOnlineFeaturesResponse_Record `protobuf:"bytes,1,rep,name=records,proto3" json:"records,omitempty"` } -func (x *GetBatchFeaturesRequest) Reset() { - *x = GetBatchFeaturesRequest{} +func (x *GetOnlineFeaturesResponse) Reset() { + *x = GetOnlineFeaturesResponse{} if protoimpl.UnsafeEnabled { mi := &file_feast_serving_ServingService_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -507,13 +579,13 @@ func (x *GetBatchFeaturesRequest) Reset() { } } -func (x *GetBatchFeaturesRequest) String() string { +func (x *GetOnlineFeaturesResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetBatchFeaturesRequest) ProtoMessage() {} +func (*GetOnlineFeaturesResponse) ProtoMessage() {} -func (x *GetBatchFeaturesRequest) ProtoReflect() protoreflect.Message { +func (x *GetOnlineFeaturesResponse) ProtoReflect() protoreflect.Message { mi := &file_feast_serving_ServingService_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -525,36 +597,32 @@ func (x *GetBatchFeaturesRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetBatchFeaturesRequest.ProtoReflect.Descriptor instead. -func (*GetBatchFeaturesRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use GetOnlineFeaturesResponse.ProtoReflect.Descriptor instead. +func (*GetOnlineFeaturesResponse) Descriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4} } -func (x *GetBatchFeaturesRequest) GetFeatures() []*FeatureReference { +func (x *GetOnlineFeaturesResponse) GetRecords() []*GetOnlineFeaturesResponse_Record { if x != nil { - return x.Features + return x.Records } return nil } -func (x *GetBatchFeaturesRequest) GetDatasetSource() *DatasetSource { - if x != nil { - return x.DatasetSource - } - return nil -} - -type GetOnlineFeaturesResponse struct { +type GetBatchFeaturesRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Feature values retrieved from feast. - FieldValues []*GetOnlineFeaturesResponse_FieldValues `protobuf:"bytes,1,rep,name=field_values,json=fieldValues,proto3" json:"field_values,omitempty"` + // List of features that are being retrieved + Features []*FeatureReference `protobuf:"bytes,3,rep,name=features,proto3" json:"features,omitempty"` + // Source of the entity dataset containing the timestamps and entity keys to retrieve + // features for. + DatasetSource *DatasetSource `protobuf:"bytes,2,opt,name=dataset_source,json=datasetSource,proto3" json:"dataset_source,omitempty"` } -func (x *GetOnlineFeaturesResponse) Reset() { - *x = GetOnlineFeaturesResponse{} +func (x *GetBatchFeaturesRequest) Reset() { + *x = GetBatchFeaturesRequest{} if protoimpl.UnsafeEnabled { mi := &file_feast_serving_ServingService_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -562,13 +630,13 @@ func (x *GetOnlineFeaturesResponse) Reset() { } } -func (x *GetOnlineFeaturesResponse) String() string { +func (x *GetBatchFeaturesRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetOnlineFeaturesResponse) ProtoMessage() {} +func (*GetBatchFeaturesRequest) ProtoMessage() {} -func (x *GetOnlineFeaturesResponse) ProtoReflect() protoreflect.Message { +func (x *GetBatchFeaturesRequest) ProtoReflect() protoreflect.Message { mi := &file_feast_serving_ServingService_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -580,14 +648,21 @@ func (x *GetOnlineFeaturesResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetOnlineFeaturesResponse.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesResponse) Descriptor() ([]byte, []int) { +// Deprecated: Use GetBatchFeaturesRequest.ProtoReflect.Descriptor instead. +func (*GetBatchFeaturesRequest) Descriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{5} } -func (x *GetOnlineFeaturesResponse) GetFieldValues() []*GetOnlineFeaturesResponse_FieldValues { +func (x *GetBatchFeaturesRequest) GetFeatures() []*FeatureReference { if x != nil { - return x.FieldValues + return x.Features + } + return nil +} + +func (x *GetBatchFeaturesRequest) GetDatasetSource() *DatasetSource { + if x != nil { + return x.DatasetSource } return nil } @@ -899,8 +974,8 @@ type GetOnlineFeaturesRequest_EntityRow struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Request timestamp of this row. This value will be used, together with maxAge, - // to determine feature staleness. + // Request timestamp of this row. This value will be used, + // together with maxAge, to determine feature staleness. EntityTimestamp *timestamp.Timestamp `protobuf:"bytes,1,opt,name=entity_timestamp,json=entityTimestamp,proto3" json:"entity_timestamp,omitempty"` // Map containing mapping of entity name to entity value. Fields map[string]*types.Value `protobuf:"bytes,2,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -952,18 +1027,18 @@ func (x *GetOnlineFeaturesRequest_EntityRow) GetFields() map[string]*types.Value return nil } -type GetOnlineFeaturesResponse_FieldValues struct { +type GetOnlineFeaturesResponse_Record struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Map of feature or entity name to feature/entity values. - // Timestamps are not returned in this response. - Fields map[string]*types.Value `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Map of field name to data fields stored in this data record. + // Each field represents an individual feature in the data record. + Fields map[string]*GetOnlineFeaturesResponse_Field `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } -func (x *GetOnlineFeaturesResponse_FieldValues) Reset() { - *x = GetOnlineFeaturesResponse_FieldValues{} +func (x *GetOnlineFeaturesResponse_Record) Reset() { + *x = GetOnlineFeaturesResponse_Record{} if protoimpl.UnsafeEnabled { mi := &file_feast_serving_ServingService_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -971,13 +1046,13 @@ func (x *GetOnlineFeaturesResponse_FieldValues) Reset() { } } -func (x *GetOnlineFeaturesResponse_FieldValues) String() string { +func (x *GetOnlineFeaturesResponse_Record) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetOnlineFeaturesResponse_FieldValues) ProtoMessage() {} +func (*GetOnlineFeaturesResponse_Record) ProtoMessage() {} -func (x *GetOnlineFeaturesResponse_FieldValues) ProtoReflect() protoreflect.Message { +func (x *GetOnlineFeaturesResponse_Record) ProtoReflect() protoreflect.Message { mi := &file_feast_serving_ServingService_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -989,18 +1064,75 @@ func (x *GetOnlineFeaturesResponse_FieldValues) ProtoReflect() protoreflect.Mess return mi.MessageOf(x) } -// Deprecated: Use GetOnlineFeaturesResponse_FieldValues.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesResponse_FieldValues) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{5, 0} +// Deprecated: Use GetOnlineFeaturesResponse_Record.ProtoReflect.Descriptor instead. +func (*GetOnlineFeaturesResponse_Record) Descriptor() ([]byte, []int) { + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 0} } -func (x *GetOnlineFeaturesResponse_FieldValues) GetFields() map[string]*types.Value { +func (x *GetOnlineFeaturesResponse_Record) GetFields() map[string]*GetOnlineFeaturesResponse_Field { if x != nil { return x.Fields } return nil } +type GetOnlineFeaturesResponse_Field struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Value of this field. + Value *types.Value `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` + // Status of this field. + Status GetOnlineFeaturesResponse_FieldStatus `protobuf:"varint,2,opt,name=status,proto3,enum=feast.serving.GetOnlineFeaturesResponse_FieldStatus" json:"status,omitempty"` +} + +func (x *GetOnlineFeaturesResponse_Field) Reset() { + *x = GetOnlineFeaturesResponse_Field{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_serving_ServingService_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetOnlineFeaturesResponse_Field) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetOnlineFeaturesResponse_Field) ProtoMessage() {} + +func (x *GetOnlineFeaturesResponse_Field) ProtoReflect() protoreflect.Message { + mi := &file_feast_serving_ServingService_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetOnlineFeaturesResponse_Field.ProtoReflect.Descriptor instead. +func (*GetOnlineFeaturesResponse_Field) Descriptor() ([]byte, []int) { + return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 1} +} + +func (x *GetOnlineFeaturesResponse_Field) GetValue() *types.Value { + if x != nil { + return x.Value + } + return nil +} + +func (x *GetOnlineFeaturesResponse_Field) GetStatus() GetOnlineFeaturesResponse_FieldStatus { + if x != nil { + return x.Status + } + return GetOnlineFeaturesResponse_INVALID +} + type DatasetSource_FileSource struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1017,7 +1149,7 @@ type DatasetSource_FileSource struct { func (x *DatasetSource_FileSource) Reset() { *x = DatasetSource_FileSource{} if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[15] + mi := &file_feast_serving_ServingService_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1030,7 +1162,7 @@ func (x *DatasetSource_FileSource) String() string { func (*DatasetSource_FileSource) ProtoMessage() {} func (x *DatasetSource_FileSource) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[15] + mi := &file_feast_serving_ServingService_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1254,19 +1386,20 @@ func file_feast_serving_ServingService_proto_rawDescGZIP() []byte { return file_feast_serving_ServingService_proto_rawDescData } -var file_feast_serving_ServingService_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_feast_serving_ServingService_proto_msgTypes = make([]protoimpl.MessageInfo, 16) +var file_feast_serving_ServingService_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_feast_serving_ServingService_proto_msgTypes = make([]protoimpl.MessageInfo, 17) var file_feast_serving_ServingService_proto_goTypes = []interface{}{ - (FeastServingType)(0), // 0: feast.serving.FeastServingType - (JobType)(0), // 1: feast.serving.JobType - (JobStatus)(0), // 2: feast.serving.JobStatus - (DataFormat)(0), // 3: feast.serving.DataFormat - (*GetFeastServingInfoRequest)(nil), // 4: feast.serving.GetFeastServingInfoRequest - (*GetFeastServingInfoResponse)(nil), // 5: feast.serving.GetFeastServingInfoResponse - (*FeatureReference)(nil), // 6: feast.serving.FeatureReference - (*GetOnlineFeaturesRequest)(nil), // 7: feast.serving.GetOnlineFeaturesRequest - (*GetBatchFeaturesRequest)(nil), // 8: feast.serving.GetBatchFeaturesRequest + (FeastServingType)(0), // 0: feast.serving.FeastServingType + (JobType)(0), // 1: feast.serving.JobType + (JobStatus)(0), // 2: feast.serving.JobStatus + (DataFormat)(0), // 3: feast.serving.DataFormat + (GetOnlineFeaturesResponse_FieldStatus)(0), // 4: feast.serving.GetOnlineFeaturesResponse.FieldStatus + (*GetFeastServingInfoRequest)(nil), // 5: feast.serving.GetFeastServingInfoRequest + (*GetFeastServingInfoResponse)(nil), // 6: feast.serving.GetFeastServingInfoResponse + (*FeatureReference)(nil), // 7: feast.serving.FeatureReference + (*GetOnlineFeaturesRequest)(nil), // 8: feast.serving.GetOnlineFeaturesRequest (*GetOnlineFeaturesResponse)(nil), // 9: feast.serving.GetOnlineFeaturesResponse +<<<<<<< HEAD (*GetBatchFeaturesResponse)(nil), // 10: feast.serving.GetBatchFeaturesResponse (*GetJobRequest)(nil), // 11: feast.serving.GetJobRequest (*GetJobResponse)(nil), // 12: feast.serving.GetJobResponse @@ -1282,37 +1415,40 @@ var file_feast_serving_ServingService_proto_goTypes = []interface{}{ } var file_feast_serving_ServingService_proto_depIdxs = []int32{ 0, // 0: feast.serving.GetFeastServingInfoResponse.type:type_name -> feast.serving.FeastServingType - 6, // 1: feast.serving.GetOnlineFeaturesRequest.features:type_name -> feast.serving.FeatureReference - 15, // 2: feast.serving.GetOnlineFeaturesRequest.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow - 6, // 3: feast.serving.GetBatchFeaturesRequest.features:type_name -> feast.serving.FeatureReference - 14, // 4: feast.serving.GetBatchFeaturesRequest.dataset_source:type_name -> feast.serving.DatasetSource - 17, // 5: feast.serving.GetOnlineFeaturesResponse.field_values:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues - 13, // 6: feast.serving.GetBatchFeaturesResponse.job:type_name -> feast.serving.Job - 13, // 7: feast.serving.GetJobRequest.job:type_name -> feast.serving.Job - 13, // 8: feast.serving.GetJobResponse.job:type_name -> feast.serving.Job - 1, // 9: feast.serving.Job.type:type_name -> feast.serving.JobType - 2, // 10: feast.serving.Job.status:type_name -> feast.serving.JobStatus - 3, // 11: feast.serving.Job.data_format:type_name -> feast.serving.DataFormat - 19, // 12: feast.serving.DatasetSource.file_source:type_name -> feast.serving.DatasetSource.FileSource - 20, // 13: feast.serving.GetOnlineFeaturesRequest.EntityRow.entity_timestamp:type_name -> google.protobuf.Timestamp - 16, // 14: feast.serving.GetOnlineFeaturesRequest.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry - 21, // 15: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry.value:type_name -> feast.types.Value - 18, // 16: feast.serving.GetOnlineFeaturesResponse.FieldValues.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry - 21, // 17: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry.value:type_name -> feast.types.Value - 3, // 18: feast.serving.DatasetSource.FileSource.data_format:type_name -> feast.serving.DataFormat - 4, // 19: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest - 7, // 20: feast.serving.ServingService.GetOnlineFeatures:input_type -> feast.serving.GetOnlineFeaturesRequest - 8, // 21: feast.serving.ServingService.GetBatchFeatures:input_type -> feast.serving.GetBatchFeaturesRequest - 11, // 22: feast.serving.ServingService.GetJob:input_type -> feast.serving.GetJobRequest - 5, // 23: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse - 9, // 24: feast.serving.ServingService.GetOnlineFeatures:output_type -> feast.serving.GetOnlineFeaturesResponse - 10, // 25: feast.serving.ServingService.GetBatchFeatures:output_type -> feast.serving.GetBatchFeaturesResponse - 12, // 26: feast.serving.ServingService.GetJob:output_type -> feast.serving.GetJobResponse - 23, // [23:27] is the sub-list for method output_type - 19, // [19:23] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 22, // 1: feast.serving.FeatureReference.max_age:type_name -> google.protobuf.Duration + 7, // 2: feast.serving.GetOnlineFeaturesRequest.features:type_name -> feast.serving.FeatureReference + 16, // 3: feast.serving.GetOnlineFeaturesRequest.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow + 18, // 4: feast.serving.GetOnlineFeaturesResponse.records:type_name -> feast.serving.GetOnlineFeaturesResponse.Record + 7, // 5: feast.serving.GetBatchFeaturesRequest.features:type_name -> feast.serving.FeatureReference + 15, // 6: feast.serving.GetBatchFeaturesRequest.dataset_source:type_name -> feast.serving.DatasetSource + 14, // 7: feast.serving.GetBatchFeaturesResponse.job:type_name -> feast.serving.Job + 14, // 8: feast.serving.GetJobRequest.job:type_name -> feast.serving.Job + 14, // 9: feast.serving.GetJobResponse.job:type_name -> feast.serving.Job + 1, // 10: feast.serving.Job.type:type_name -> feast.serving.JobType + 2, // 11: feast.serving.Job.status:type_name -> feast.serving.JobStatus + 3, // 12: feast.serving.Job.data_format:type_name -> feast.serving.DataFormat + 21, // 13: feast.serving.DatasetSource.file_source:type_name -> feast.serving.DatasetSource.FileSource + 23, // 14: feast.serving.GetOnlineFeaturesRequest.EntityRow.entity_timestamp:type_name -> google.protobuf.Timestamp + 17, // 15: feast.serving.GetOnlineFeaturesRequest.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry + 24, // 16: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry.value:type_name -> feast.types.Value + 20, // 17: feast.serving.GetOnlineFeaturesResponse.Record.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.Record.FieldsEntry + 24, // 18: feast.serving.GetOnlineFeaturesResponse.Field.value:type_name -> feast.types.Value + 4, // 19: feast.serving.GetOnlineFeaturesResponse.Field.status:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldStatus + 19, // 20: feast.serving.GetOnlineFeaturesResponse.Record.FieldsEntry.value:type_name -> feast.serving.GetOnlineFeaturesResponse.Field + 3, // 21: feast.serving.DatasetSource.FileSource.data_format:type_name -> feast.serving.DataFormat + 5, // 22: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest + 8, // 23: feast.serving.ServingService.GetOnlineFeatures:input_type -> feast.serving.GetOnlineFeaturesRequest + 10, // 24: feast.serving.ServingService.GetBatchFeatures:input_type -> feast.serving.GetBatchFeaturesRequest + 12, // 25: feast.serving.ServingService.GetJob:input_type -> feast.serving.GetJobRequest + 6, // 26: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse + 9, // 27: feast.serving.ServingService.GetOnlineFeatures:output_type -> feast.serving.GetOnlineFeaturesResponse + 11, // 28: feast.serving.ServingService.GetBatchFeatures:output_type -> feast.serving.GetBatchFeaturesResponse + 13, // 29: feast.serving.ServingService.GetJob:output_type -> feast.serving.GetJobResponse + 26, // [26:30] is the sub-list for method output_type + 22, // [22:26] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name } func init() { file_feast_serving_ServingService_proto_init() } @@ -1370,7 +1506,7 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetBatchFeaturesRequest); i { + switch v := v.(*GetOnlineFeaturesResponse); i { case 0: return &v.state case 1: @@ -1382,7 +1518,7 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesResponse); i { + switch v := v.(*GetBatchFeaturesRequest); i { case 0: return &v.state case 1: @@ -1466,7 +1602,19 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesResponse_FieldValues); i { + switch v := v.(*GetOnlineFeaturesResponse_Record); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_serving_ServingService_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetOnlineFeaturesResponse_Field); i { case 0: return &v.state case 1: @@ -1477,7 +1625,7 @@ func file_feast_serving_ServingService_proto_init() { return nil } } - file_feast_serving_ServingService_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + file_feast_serving_ServingService_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*DatasetSource_FileSource); i { case 0: return &v.state @@ -1498,8 +1646,8 @@ func file_feast_serving_ServingService_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_serving_ServingService_proto_rawDesc, - NumEnums: 4, - NumMessages: 16, + NumEnums: 5, + NumMessages: 17, NumExtensions: 0, NumServices: 1, }, diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go index f58247299d9..3b644381ca1 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go @@ -112,6 +112,7 @@ var file_tensorflow_metadata_proto_v0_path_proto_rawDesc = []byte{ 0x61, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x22, 0x1a, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x74, 0x65, +<<<<<<< HEAD 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x73, 0x74, 0x65, 0x70, 0x42, 0x68, 0x0a, 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, 0x45, 0x67, @@ -120,6 +121,16 @@ var file_tensorflow_metadata_proto_v0_path_proto_rawDesc = []byte{ 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, 0xf8, 0x01, 0x01, +======= + 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x73, 0x74, 0x65, 0x70, 0x42, 0x64, 0x0a, + 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, 0x41, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, + 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, + 0xf8, 0x01, 0x01, +>>>>>>> Update go protobuf generated code for updated protobuf defintion } var ( diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go index a04f5bba8ca..4ebfa7d5dba 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go @@ -3476,6 +3476,7 @@ var file_tensorflow_metadata_proto_v0_schema_proto_rawDesc = []byte{ 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x55, 0x43, 0x54, 0x10, 0x04, 0x42, +<<<<<<< HEAD 0x68, 0x0a, 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, @@ -3483,6 +3484,15 @@ var file_tensorflow_metadata_proto_v0_schema_proto_rawDesc = []byte{ 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, 0xf8, 0x01, 0x01, +======= + 0x64, 0x0a, 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, + 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, + 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, + 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, + 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, + 0x76, 0x30, 0xf8, 0x01, 0x01, +>>>>>>> Update go protobuf generated code for updated protobuf defintion } var ( From 1ef7f4d91a654eaf79eaed0bb6c2325e64e06b0b Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 24 Apr 2020 15:31:16 +0800 Subject: [PATCH 19/65] Update python sdk's client to document new fieldstatus metadata, added flags Added missing boolean flags for GetOnlineFeaturesRequest's omit_entities_in_response & include_metadata_in_response flags. Update unit tests with changes to GetOnlineFeaturesResponse protobuff --- protos/feast/serving/ServingService.proto | 4 ++-- sdk/python/feast/client.py | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto index 5ade63cd09c..d839f5d442d 100644 --- a/protos/feast/serving/ServingService.proto +++ b/protos/feast/serving/ServingService.proto @@ -126,7 +126,7 @@ message GetOnlineFeaturesResponse { // Status is unset for this field. INVALID = 0; - // Field value is present for this field and within maximum allowable range + // Field value is present for this field and within maximum allowable range. PRESENT = 1; // Values could be found for entity key within the maximum allowable range, but @@ -139,7 +139,7 @@ message GetOnlineFeaturesResponse { NOT_FOUND = 3; // Values could be found for entity key, but field values are outside the maximum - // allowable range. + // allowable range: values are older than max age. OUTSIDE_MAX_AGE = 4; } } diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 8d85f07cd33..b736fe23ac0 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -671,6 +671,8 @@ def get_online_features( feature_refs: List[str], entity_rows: List[GetOnlineFeaturesRequest.EntityRow], project: Optional[str] = None, + omit_entities: bool = False, + include_meta: bool = False, ) -> GetOnlineFeaturesResponse: """ Retrieves the latest online feature data from Feast Serving @@ -688,14 +690,17 @@ def get_online_features( which the requested features belong to. Returns: - Returns a list of maps where each item in the list contains the - latest feature values for the provided entities + GetOnlineFeaturesResponse containing the feature data in records. + Each EntityRow provided will yield one record, which contains + data fields with data value and field status metadata (if included). """ self._connect_serving() try: response = self._serving_service_stub.GetOnlineFeatures( GetOnlineFeaturesRequest( + omit_entities_in_response=omit_entities, + include_metadata_in_response=include_meta, features=_build_feature_references( feature_ref_strs=feature_refs, project=project if project is not None else self.project, From 240dc11c723cf4c25aa967367bc240195c0dcc38 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 24 Apr 2020 16:30:25 +0800 Subject: [PATCH 20/65] Fixed issue where getOnlineFeatures() returned Record in a non deterministic order. Enforce that getOnlineFeatures() returns Records in the same order as the user provides the EntityRow --- sdk/java/src/main/java/com/gojek/feast/Row.java | 10 ++++++++-- .../feast/serving/service/OnlineServingService.java | 11 +++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/sdk/java/src/main/java/com/gojek/feast/Row.java b/sdk/java/src/main/java/com/gojek/feast/Row.java index 1e37b2d647a..f4fdc3eb553 100644 --- a/sdk/java/src/main/java/com/gojek/feast/Row.java +++ b/sdk/java/src/main/java/com/gojek/feast/Row.java @@ -19,9 +19,15 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; +<<<<<<< HEAD import feast.proto.types.ValueProto.Value; import feast.proto.types.ValueProto.Value.ValCase; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; +======= +import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; +import feast.types.ValueProto.Value; +import feast.types.ValueProto.Value.ValCase; +>>>>>>> Fixed issue where getOnlineFeatures() returned Record in a non deterministic order. import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; @@ -88,7 +94,7 @@ public Row set(String fieldName, Object value, FieldStatus status) { "Type '%s' is unsupported in Feast. Please use one of these value types: Integer, Long, Float, Double, String, byte[].", valueType)); } - + fieldStatuses.put(fieldName, status); return this; } @@ -96,7 +102,7 @@ public Row set(String fieldName, Object value, FieldStatus status) { public Map getFields() { return fields; } - + public Integer getInt(String fieldName) { return getValue(fieldName).map(Value::getInt32Val).orElse(null); } diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index 09196cb472d..66b91c38df0 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -80,7 +80,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ List featureSetRequests = specService.getFeatureSets(request.getFeaturesList()); for (FeatureSetRequest featureSetRequest : featureSetRequests) { - // Pull feature rows for given entity rows from the feature/featureset specified n feature + // Pull feature rows for given entity rows from the feature/featureset specified in feature // set request. List featureRows = retriever.getOnlineFeatures(entityRows, featureSetRequest); // Check that feature row returned corresponds to a given entity row. @@ -114,10 +114,13 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ this.populateRequestCountMetrics(featureSetRequest); } - // Build response records from entityFieldsMap + // Build response records from entityFieldsMap. + // Records should be in the same order as the entityRows provided by the user. List records = - entityFieldsMap.values().stream() - .map(fields -> Record.newBuilder().putAllFields(fields).build()) + entityRows.stream() + .map( + entityRow -> + Record.newBuilder().putAllFields(entityFieldsMap.get(entityRow)).build()) .collect(Collectors.toList()); return GetOnlineFeaturesResponse.newBuilder().addAllRecords(records).build(); } From fea6f4a243736c95bd12eba5bef16a82a4a02280 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 27 Apr 2020 11:14:43 +0800 Subject: [PATCH 21/65] Update e2e ingestion tests to support new get_online_features api and check for metadata --- .../src/main/java/com/gojek/feast/Row.java | 6 -- tests/e2e/redis/basic-ingest-redis-serving.py | 95 +++++++++++-------- tests/e2e/redis/basic/cust_trans_fs.yaml | 2 + 3 files changed, 59 insertions(+), 44 deletions(-) diff --git a/sdk/java/src/main/java/com/gojek/feast/Row.java b/sdk/java/src/main/java/com/gojek/feast/Row.java index f4fdc3eb553..ef03e53ac73 100644 --- a/sdk/java/src/main/java/com/gojek/feast/Row.java +++ b/sdk/java/src/main/java/com/gojek/feast/Row.java @@ -19,15 +19,9 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; -<<<<<<< HEAD import feast.proto.types.ValueProto.Value; import feast.proto.types.ValueProto.Value.ValCase; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; -======= -import feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; -import feast.types.ValueProto.Value; -import feast.types.ValueProto.Value.ValCase; ->>>>>>> Fixed issue where getOnlineFeatures() returned Record in a non deterministic order. import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 1d38cc924b4..a15fd21aafc 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -71,6 +71,7 @@ def basic_dataframe(entities, features, ingest_time, n_size): df_dict = { "datetime": [ingest_time.replace(tzinfo=pytz.utc) for _ in range(n_size)], + "null_values": [ Value() for _ in range(n_size) ], } for entity_name in entities: df_dict[entity_name] = list(range(1, n_size + 1)) @@ -83,7 +84,7 @@ def basic_dataframe(entities, features, ingest_time, n_size): def ingest_time(): return datetime.utcnow() - +None @pytest.fixture(scope="module") def cust_trans_df(ingest_time): return basic_dataframe(entities=["customer_id"], @@ -191,26 +192,32 @@ def test_basic_retrieve_online_success(client, cust_trans_df): feature_refs=[ "daily_transactions", "total_transactions", - ] + "null_values" + ], ) # type: GetOnlineFeaturesResponse - if response is None: - continue - - returned_daily_transactions = float( - response.field_values[0] - .fields["daily_transactions"] - .float_val - ) - sent_daily_transactions = float( - cust_trans_df.iloc[0]["daily_transactions"]) + # wait & unpack response + if response is None: continue + fields = response.records[0].fields + daily_transactions_field = fields[PROJECT_NAME + "/daily_transactions"] + null_value_field = fields[PROJECT_NAME + "/null_values"] - if math.isclose( + # check values returned are correct + sent_daily_transactions = float(basic_dataframe.iloc[0]["daily_transactions"]) + is_values_correct = ( + math.isclose( sent_daily_transactions, - returned_daily_transactions, + float(daily_transactions_field.value.float_val), abs_tol=FLOAT_TOLERANCE, - ): - break + ) and null_value_field.value.WhichOneof("val") is None + ) + # check field status metadata + is_meta_correct = ( + daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT + and null_value_field == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE + ) + if is_values_correct and is_meta_correct: + break # complete test @pytest.mark.timeout(90) @@ -434,21 +441,26 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): ], ) # type: GetOnlineFeaturesResponse + # wait for and unpack response if response is None: + print("test_all_types_retrieve_online_success(): polling for response.") continue + float_list_field = response.records[0].fields[PROJECT_NAME+"/float_list_feature"] - returned_float_list = ( - response.field_values[0] - .fields["float_list_feature"] - .float_list_val.val - ) - + returned_float_list = float_list_field.float_list_val.val sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] + is_values_correct = math.isclose(returned_float_list[0], + sent_float_list[0], + abs_tol=FLOAT_TOLERANCE) + # check returned metadata + is_meta_correct = (float_list_field.status == + GetOnlineFeaturesResponse.FieldStatus.PRESENT) - if math.isclose( - returned_float_list[0], sent_float_list[0], abs_tol=FLOAT_TOLERANCE - ): - break + if not (is_values_correct and is_meta_correct): + print(f"test_all_types_retrieve_online_success(): polling for" + + " correct values ({is_values_correct}) or correct metadata ({is_meta_correct})") + # complete test + break @pytest.mark.timeout(300) @@ -552,23 +564,30 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): ], ) # type: GetOnlineFeaturesResponse + # wait for and unpack response if response is None: + print("test_all_types_retrieve_online_success(): polling for response.") continue + daily_transactions_field = response.records[0].fields[ + PROJECT_NAME + "/daily_transactions_large"] - returned_daily_transactions = float( - response.field_values[0] - .fields["daily_transactions_large"] - .float_val - ) + # check returned values + returned_daily_transactions = float(daily_transactions_field.value.float_val) sent_daily_transactions = float( large_volume_dataframe.iloc[0]["daily_transactions_large"]) - - if math.isclose( - sent_daily_transactions, - returned_daily_transactions, - abs_tol=FLOAT_TOLERANCE, - ): - break + is_values_correct = math.isclose( + sent_daily_transactions, + returned_daily_transactions, + abs_tol=FLOAT_TOLERANCE, + ) + # check returned metadata + is_meta_correct = (daily_transactions_field.status == + GetOnlineFeaturesResponse.FieldStatus.PRESENT) + if not (is_values_correct and is_meta_correct): + print(f"test_all_types_retrieve_online_success(): polling for" + + " correct values ({is_values_correct}) or correct metadata ({is_meta_correct})") + # complete test + break @pytest.fixture(scope='module') diff --git a/tests/e2e/redis/basic/cust_trans_fs.yaml b/tests/e2e/redis/basic/cust_trans_fs.yaml index 14d46794a6d..941037670d3 100644 --- a/tests/e2e/redis/basic/cust_trans_fs.yaml +++ b/tests/e2e/redis/basic/cust_trans_fs.yaml @@ -9,4 +9,6 @@ spec: valueType: FLOAT - name: total_transactions valueType: FLOAT + - name: null_values + valueType: FLOAT maxAge: 3600s From 6c4968f46831aa3100d42a098bb48cc2e091bcc4 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 27 Apr 2020 12:42:25 +0800 Subject: [PATCH 22/65] Fixed NullPointerException when feature row is null. --- .../feast/serving/service/OnlineServingService.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index 66b91c38df0..cd39b2def58 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -160,6 +160,7 @@ private Map unpackFields( EntityRow entityRow, FeatureSetRequest featureSetRequest, boolean includeMetadata) { + System.out.println("feature row: " +featureRow); // In order to return values containing the same feature references provided by the user, // we reuse the feature references in the request as the keys in field builder map Map refsByName = featureSetRequest.getFeatureRefsByName(); @@ -189,7 +190,8 @@ private Map unpackFields( // row. Set missingFeatures = new HashSet<>(refsByName.values()); missingFeatures.removeAll(fields.keySet()); - missingFeatures.forEach(ref -> fields.put(ref, Field.newBuilder())); + missingFeatures.forEach(ref -> fields.put(ref, + Field.newBuilder().setValue(Value.newBuilder().build()))); // attach metadata to the feature response fields & build response field return fields.entrySet().stream() @@ -238,9 +240,15 @@ private Field.Builder attachMetadata( private boolean isStale( FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { Duration maxAge = featureSetRequest.getSpec().getMaxAge(); + if(featureRow == null) { + // no data to consider: not stale + return false; + } if (maxAge.equals(Duration.getDefaultInstance())) { + // max age is not set: stale detection disabled return false; } + long givenTimestamp = entityRow.getEntityTimestamp().getSeconds(); if (givenTimestamp == 0) { givenTimestamp = System.currentTimeMillis() / 1000; From a7e97d64dc296aa33e98e7e40a2f7c655f9d4b7d Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 27 Apr 2020 12:53:36 +0800 Subject: [PATCH 23/65] Fixed issue in which unpackFields(entityRow) was not responding to includeMetadata flag --- .../serving/service/OnlineServingService.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index cd39b2def58..c00092da351 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -65,6 +65,7 @@ public GetFeastServingInfoResponse getFeastServingInfo( public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { try (Scope scope = tracer.buildSpan("getOnlineFeatures").startActive(true)) { List entityRows = request.getEntityRowsList(); + boolean includeMetadata = request.getIncludeMetadataInResponse(); // Collect the response fields corresponding by entity row in entityFieldsMap. Map> entityFieldsMap = entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); @@ -73,7 +74,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ // Add entity row's fields as response fields entityRows.forEach( entityRow -> { - entityFieldsMap.get(entityRow).putAll(this.unpackFields(entityRow)); + entityFieldsMap.get(entityRow).putAll(this.unpackFields(entityRow, includeMetadata)); }); } @@ -97,8 +98,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ EntityRow entityRow = entityRows.get(i); // Unpack feature response fields from feature row Map fields = - this.unpackFields( - featureRow, entityRow, featureSetRequest, request.getIncludeMetadataInResponse()); + this.unpackFields(featureRow, entityRow, featureSetRequest, includeMetadata); // Merge feature response fields into entityFieldsMap entityFieldsMap .get(entityRow) @@ -130,18 +130,21 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ * Unpack response fields from the given entity row's fields. * * @param entityRow to unpack for response fields + * @param includeMetadata whether metadata should be included in the response fields * @return Map mapping of name of field to response field. */ - private Map unpackFields(EntityRow entityRow) { + private Map unpackFields(EntityRow entityRow, boolean includeMetadata) { return entityRow.getFieldsMap().entrySet().stream() .collect( Collectors.toMap( es -> es.getKey(), es -> { - return Field.newBuilder() - .setStatus(FieldStatus.PRESENT) - .setValue(es.getValue()) - .build(); + Field.Builder field = Field.newBuilder() + .setValue(es.getValue()); + if(includeMetadata) { + field.setStatus(FieldStatus.PRESENT); + } + return field.build(); })); } From 55e1ce466b1ad66350bf887513bd16fc41de9d16 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 27 Apr 2020 15:57:02 +0800 Subject: [PATCH 24/65] Check FieldStatus in end to end tests to wait for ingestion of values to complete. --- .../serving/service/OnlineServingService.java | 1 - tests/e2e/redis/basic-ingest-redis-serving.py | 25 +++++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index c00092da351..9431c70d34c 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -163,7 +163,6 @@ private Map unpackFields( EntityRow entityRow, FeatureSetRequest featureSetRequest, boolean includeMetadata) { - System.out.println("feature row: " +featureRow); // In order to return values containing the same feature references provided by the user, // we reuse the feature references in the request as the keys in field builder map Map refsByName = featureSetRequest.getFeatureRefsByName(); diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index a15fd21aafc..9e684d4a8ae 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -197,10 +197,12 @@ def test_basic_retrieve_online_success(client, cust_trans_df): ) # type: GetOnlineFeaturesResponse # wait & unpack response - if response is None: continue fields = response.records[0].fields daily_transactions_field = fields[PROJECT_NAME + "/daily_transactions"] null_value_field = fields[PROJECT_NAME + "/null_values"] + if daily_transactions.status == GetOnlineFeaturesRespons.FieldStatus.NOT_FOUND: + print("test_basic_retrieve_online_success(): waiting for ingested values.") + continue # check values returned are correct sent_daily_transactions = float(basic_dataframe.iloc[0]["daily_transactions"]) @@ -216,8 +218,10 @@ def test_basic_retrieve_online_success(client, cust_trans_df): daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT and null_value_field == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE ) - if is_values_correct and is_meta_correct: - break # complete test + if not (is_values_correct and is_meta_correct): + print(f"test_basic_retrieve_online_success(): polling for" + " correct values ({is_values_correct}) " + " or correct metadata ({is_meta_correct})") @pytest.mark.timeout(90) @@ -446,6 +450,9 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): print("test_all_types_retrieve_online_success(): polling for response.") continue float_list_field = response.records[0].fields[PROJECT_NAME+"/float_list_feature"] + if float_list_field.status == GetOnlineFeaturesRespons.FieldStatus.NOT_FOUND: + print("test_all_types_retrieve_online_success(): polling for ingested values.") + continue returned_float_list = float_list_field.float_list_val.val sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] @@ -457,8 +464,9 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): GetOnlineFeaturesResponse.FieldStatus.PRESENT) if not (is_values_correct and is_meta_correct): - print(f"test_all_types_retrieve_online_success(): polling for" + - " correct values ({is_values_correct}) or correct metadata ({is_meta_correct})") + print(f"test_all_types_retrieve_online_success(): polling for" + " correct values ({is_values_correct}) " + " or correct metadata ({is_meta_correct})") # complete test break @@ -569,7 +577,7 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): print("test_all_types_retrieve_online_success(): polling for response.") continue daily_transactions_field = response.records[0].fields[ - PROJECT_NAME + "/daily_transactions_large"] + PROJECT_NAME+"/daily_transactions_large"] # check returned values returned_daily_transactions = float(daily_transactions_field.value.float_val) @@ -584,8 +592,9 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): is_meta_correct = (daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT) if not (is_values_correct and is_meta_correct): - print(f"test_all_types_retrieve_online_success(): polling for" + - " correct values ({is_values_correct}) or correct metadata ({is_meta_correct})") + print(f"test_all_types_retrieve_online_success(): polling for" + " correct values ({is_values_correct})" + " or correct metadata ({is_meta_correct})") # complete test break From d90cf5adf912bd31f615b4636750dd345a10372b Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 27 Apr 2020 16:09:20 +0800 Subject: [PATCH 25/65] Moved test checks out of polling loop in e2e online ingestion tests --- tests/e2e/redis/basic-ingest-redis-serving.py | 63 ++++++++----------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 9e684d4a8ae..09d28c9ec4c 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -196,32 +196,28 @@ def test_basic_retrieve_online_success(client, cust_trans_df): ], ) # type: GetOnlineFeaturesResponse - # wait & unpack response + # unpack response & wait for ingested values fields = response.records[0].fields daily_transactions_field = fields[PROJECT_NAME + "/daily_transactions"] null_value_field = fields[PROJECT_NAME + "/null_values"] - if daily_transactions.status == GetOnlineFeaturesRespons.FieldStatus.NOT_FOUND: + if (daily_transactions.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND + or null_value_field == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND): print("test_basic_retrieve_online_success(): waiting for ingested values.") continue + else: + break - # check values returned are correct - sent_daily_transactions = float(basic_dataframe.iloc[0]["daily_transactions"]) - is_values_correct = ( - math.isclose( - sent_daily_transactions, - float(daily_transactions_field.value.float_val), - abs_tol=FLOAT_TOLERANCE, - ) and null_value_field.value.WhichOneof("val") is None - ) - # check field status metadata - is_meta_correct = ( - daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT - and null_value_field == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE - ) - if not (is_values_correct and is_meta_correct): - print(f"test_basic_retrieve_online_success(): polling for" - " correct values ({is_values_correct}) " - " or correct metadata ({is_meta_correct})") + # check values returned are correct + sent_daily_transactions = float(basic_dataframe.iloc[0]["daily_transactions"]) + assert math.isclose( + sent_daily_transactions, + float(daily_transactions_field.value.float_val), + abs_tol=FLOAT_TOLERANCE, + ) + assert null_value_field.value.WhichOneof("val") is None + # check field status metadata + assert daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT + assert null_value_field == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE @pytest.mark.timeout(90) @@ -453,23 +449,15 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): if float_list_field.status == GetOnlineFeaturesRespons.FieldStatus.NOT_FOUND: print("test_all_types_retrieve_online_success(): polling for ingested values.") continue + else: + break - returned_float_list = float_list_field.float_list_val.val - sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] - is_values_correct = math.isclose(returned_float_list[0], - sent_float_list[0], - abs_tol=FLOAT_TOLERANCE) - # check returned metadata - is_meta_correct = (float_list_field.status == - GetOnlineFeaturesResponse.FieldStatus.PRESENT) - - if not (is_values_correct and is_meta_correct): - print(f"test_all_types_retrieve_online_success(): polling for" - " correct values ({is_values_correct}) " - " or correct metadata ({is_meta_correct})") - # complete test - break - + # check returned values + returned_float_list = float_list_field.float_list_val.val + sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] + assert math.isclose(returned_float_list[0], sent_float_list[0], abs_tol=FLOAT_TOLERANCE) + # check returned metadata + assert float_list_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT @pytest.mark.timeout(300) @pytest.mark.run(order=29) @@ -578,6 +566,9 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): continue daily_transactions_field = response.records[0].fields[ PROJECT_NAME+"/daily_transactions_large"] + if float_list_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND: + print("test_all_types_retrieve_online_success(): polling for response.") + continue # check returned values returned_daily_transactions = float(daily_transactions_field.value.float_val) From cd5ea470a9043b27d1e049db68263e5cfa39da33 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 27 Apr 2020 17:11:05 +0800 Subject: [PATCH 26/65] Apply spotless java formatting --- .../feast/serving/service/OnlineServingService.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index 9431c70d34c..d1bf311c231 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -139,9 +139,8 @@ private Map unpackFields(EntityRow entityRow, boolean includeMeta Collectors.toMap( es -> es.getKey(), es -> { - Field.Builder field = Field.newBuilder() - .setValue(es.getValue()); - if(includeMetadata) { + Field.Builder field = Field.newBuilder().setValue(es.getValue()); + if (includeMetadata) { field.setStatus(FieldStatus.PRESENT); } return field.build(); @@ -192,8 +191,8 @@ private Map unpackFields( // row. Set missingFeatures = new HashSet<>(refsByName.values()); missingFeatures.removeAll(fields.keySet()); - missingFeatures.forEach(ref -> fields.put(ref, - Field.newBuilder().setValue(Value.newBuilder().build()))); + missingFeatures.forEach( + ref -> fields.put(ref, Field.newBuilder().setValue(Value.newBuilder().build()))); // attach metadata to the feature response fields & build response field return fields.entrySet().stream() @@ -242,7 +241,7 @@ private Field.Builder attachMetadata( private boolean isStale( FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { Duration maxAge = featureSetRequest.getSpec().getMaxAge(); - if(featureRow == null) { + if (featureRow == null) { // no data to consider: not stale return false; } From 34430782ef2ce7f1efc218b006dda9f03df5e657 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 27 Apr 2020 17:53:04 +0800 Subject: [PATCH 27/65] Fixed misplaced break in end to end tests --- tests/e2e/redis/basic-ingest-redis-serving.py | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 09d28c9ec4c..770bebd3645 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -566,29 +566,19 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): continue daily_transactions_field = response.records[0].fields[ PROJECT_NAME+"/daily_transactions_large"] - if float_list_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND: + if daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND: print("test_all_types_retrieve_online_success(): polling for response.") continue + else: + break - # check returned values - returned_daily_transactions = float(daily_transactions_field.value.float_val) - sent_daily_transactions = float( - large_volume_dataframe.iloc[0]["daily_transactions_large"]) - is_values_correct = math.isclose( - sent_daily_transactions, - returned_daily_transactions, - abs_tol=FLOAT_TOLERANCE, - ) - # check returned metadata - is_meta_correct = (daily_transactions_field.status == - GetOnlineFeaturesResponse.FieldStatus.PRESENT) - if not (is_values_correct and is_meta_correct): - print(f"test_all_types_retrieve_online_success(): polling for" - " correct values ({is_values_correct})" - " or correct metadata ({is_meta_correct})") - # complete test - break - + # check returned values + returned_daily_transactions = float(daily_transactions_field.value.float_val) + sent_daily_transactions = float( + large_volume_dataframe.iloc[0]["daily_transactions_large"]) + assert math.isclose(sent_daily_transactions, returned_daily_transactions, abs_tol=FLOAT_TOLERANCE) + # check returned metadata + assert daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT @pytest.fixture(scope='module') def all_types_parquet_file(): From 3cc3a8eadfac8431fed8d8ac6f826c7630f3b2e9 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Mon, 27 Apr 2020 18:00:35 +0800 Subject: [PATCH 28/65] Fixed typos in e2e ingestion tests. --- tests/e2e/redis/basic-ingest-redis-serving.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 770bebd3645..64b65aacc25 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -200,7 +200,7 @@ def test_basic_retrieve_online_success(client, cust_trans_df): fields = response.records[0].fields daily_transactions_field = fields[PROJECT_NAME + "/daily_transactions"] null_value_field = fields[PROJECT_NAME + "/null_values"] - if (daily_transactions.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND + if (daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND or null_value_field == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND): print("test_basic_retrieve_online_success(): waiting for ingested values.") continue @@ -446,7 +446,7 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): print("test_all_types_retrieve_online_success(): polling for response.") continue float_list_field = response.records[0].fields[PROJECT_NAME+"/float_list_feature"] - if float_list_field.status == GetOnlineFeaturesRespons.FieldStatus.NOT_FOUND: + if float_list_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND: print("test_all_types_retrieve_online_success(): polling for ingested values.") continue else: From 3dedf4f6ba4bd3c7850f3a415c7146233c139f6f Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 28 Apr 2020 11:32:13 +0800 Subject: [PATCH 29/65] Apply changes on RedisOnlineRetriever to RedisClusterOnlineRetriever Changes * Removed unnessary nesting of list in FeaturRows returned * Refactored sendMultiGetAndProcess() to getOnlineFeaturesForFeatureSet() Necessary due to code duplciation between RedisOnlineRetriever and RedisOnlineRetriever. --- .../RedisClusterOnlineRetriever.java | 108 ++++++++---------- .../redis/retriever/RedisOnlineRetriever.java | 1 + .../RedisClusterOnlineRetrieverTest.java | 74 ++++++------ 3 files changed, 83 insertions(+), 100 deletions(-) diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java index 18619252c27..390575400fd 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java @@ -41,6 +41,7 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; +/** Defines a storage retriever */ public class RedisClusterOnlineRetriever implements OnlineRetriever { private final RedisAdvancedClusterCommands syncCommands; @@ -69,40 +70,29 @@ public static OnlineRetriever create(StatefulRedisClusterConnection> getOnlineFeatures( - List entityRows, List featureSetRequests) { - - List> featureRows = new ArrayList<>(); - for (FeatureSetRequest featureSetRequest : featureSetRequests) { - List redisKeys = buildRedisKeys(entityRows, featureSetRequest.getSpec()); - try { - List featureRowsForFeatureSet = - sendAndProcessMultiGet( - redisKeys, - featureSetRequest.getSpec(), - featureSetRequest.getFeatureReferences().asList()); - featureRows.add(featureRowsForFeatureSet); - } catch (InvalidProtocolBufferException | ExecutionException e) { - throw Status.INTERNAL - .withDescription("Unable to parse protobuf while retrieving feature") - .withCause(e) - .asRuntimeException(); - } + public List getOnlineFeatures( + List entityRows, FeatureSetRequest featureSetRequest) { + + // get features for this features/featureset in featureset request + FeatureSetSpec featureSetSpec = featureSetRequest.getSpec(); + List redisKeys = buildRedisKeys(entityRows, featureSetSpec); + FeatureRowDecoder decoder = + new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); + List featureRows = new ArrayList<>(); + try { + featureRows = getFeaturesForFeatureSet(redisKeys, decoder); + } catch (InvalidProtocolBufferException | ExecutionException e) { + throw Status.INTERNAL + .withDescription("Unable to parse protobuf while retrieving feature") + .withCause(e) + .asRuntimeException(); } return featureRows; } + private List buildRedisKeys(List entityRows, FeatureSetSpec featureSetSpec) { String featureSetRef = generateFeatureSetStringRef(featureSetSpec); List featureSetEntityNames = @@ -147,49 +137,51 @@ private RedisKey makeRedisKey( return builder.build(); } - private List sendAndProcessMultiGet( - List redisKeys, - FeatureSetSpec featureSetSpec, - List featureReferences) + /** + * Get features from data pulled from the Redis for a specific featureset. + * + * @param redisKeys keys used to retrieve data from Redis for a specific featureset. + * @param decoder used to decode the data retrieved from Redis for a specific featureset. + * @return List of {@link FeatureRow}s + */ + private List getFeaturesForFeatureSet( + List redisKeys, FeatureRowDecoder decoder) throws InvalidProtocolBufferException, ExecutionException { - - List values = sendMultiGet(redisKeys); + // pull feature row data bytes from redis using given redis keys + List featureRowsBytes = sendMultiGet(redisKeys); List featureRows = new ArrayList<>(); - FeatureRow.Builder nullFeatureRowBuilder = - FeatureRow.newBuilder().setFeatureSet(generateFeatureSetStringRef(featureSetSpec)); - for (FeatureReference featureReference : featureReferences) { - nullFeatureRowBuilder.addFields(Field.newBuilder().setName(featureReference.getName())); - } - - for (int i = 0; i < values.size(); i++) { - - byte[] value = values.get(i); - if (value == null) { - featureRows.add(nullFeatureRowBuilder.build()); + for (byte[] featureRowBytes : featureRowsBytes) { + if (featureRowBytes == null) { + featureRows.add(null); continue; } - FeatureRow featureRow = FeatureRow.parseFrom(value); - String featureSetRef = redisKeys.get(i).getFeatureSet(); - FeatureRowDecoder decoder = new FeatureRowDecoder(featureSetRef, featureSetSpec); - if (decoder.isEncodingValid(featureRow)) { - featureRow = decoder.decode(featureRow); - } else { - featureRows.add(nullFeatureRowBuilder.build()); - continue; + // decode feature rows from data bytes using decoder. + FeatureRow featureRow = FeatureRow.parseFrom(featureRowBytes); + if (decoder.isEncoded(featureRow)) { + if (decoder.isEncodingValid(featureRow)) { + featureRow = decoder.decode(featureRow); + } else { + // decoding feature row failed: data corruption could have occurred + throw Status.DATA_LOSS + .withDescription( + "Failed to decode FeatureRow from bytes retrieved from redis" + + ": Possible data corruption") + .asRuntimeException(); + } } - featureRows.add(featureRow); } return featureRows; } /** - * Send a list of get request as an mget + * Pull the data stored in Redis at the given keys as bytes using the mget command. If no data is + * stored at a given key in Redis, will subsitute the data with null. * - * @param keys list of {@link RedisKey} - * @return list of {@link FeatureRow} in primitive byte representation for each {@link RedisKey} + * @param keys list of {@link RedisKey} to pull from redis. + * @return list of data bytes or null pulled from redis for each given key. */ private List sendMultiGet(List keys) { try { diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index 0948cd8974e..5ef42ef96b4 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -62,6 +62,7 @@ public static OnlineRetriever create(StatefulRedisConnection con return new RedisOnlineRetriever(connection); } + /** {@inheritDoc} */ @Override public List getOnlineFeatures( List entityRows, FeatureSetRequest featureSetRequest) { diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java index 45842cef6ee..e56323bee32 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java @@ -132,34 +132,32 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List> expected = - ImmutableList.of( + List expected = + Lists.newArrayList( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build())); + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build(), + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) + .build())); - List> actual = - redisClusterOnlineRetriever.getOnlineFeatures( - entityRows, ImmutableList.of(featureSetRequest)); + List actual = + redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } @Test - public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { + public void shouldReturnNullIfKeysNotPresent() { FeatureSetRequest featureSetRequest = FeatureSetRequest.newBuilder() .setSpec(getFeatureSetSpec()) @@ -201,28 +199,20 @@ public void shouldReturnResponseWithUnsetValuesIfKeysNotPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List> expected = - ImmutableList.of( + List expected = + Lists.newArrayList( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").build(), - Field.newBuilder().setName("feature2").build())) - .build())); + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build(), + null); - List> actual = - redisClusterOnlineRetriever.getOnlineFeatures( - entityRows, ImmutableList.of(featureSetRequest)); + List actual = + redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } From 933e20d0a010adcaffcd77106ba2c24a02f2dfd8 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 28 Apr 2020 11:37:09 +0800 Subject: [PATCH 30/65] Rename getFeaturesForFeatureSet() to getFeaturesFromRedis() for better sementics --- .../redis/retriever/RedisClusterOnlineRetriever.java | 4 ++-- .../connectors/redis/retriever/RedisOnlineRetriever.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java index 390575400fd..6ce037bd3df 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java @@ -82,7 +82,7 @@ public List getOnlineFeatures( new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); List featureRows = new ArrayList<>(); try { - featureRows = getFeaturesForFeatureSet(redisKeys, decoder); + featureRows = getFeaturesFromRedis(redisKeys, decoder); } catch (InvalidProtocolBufferException | ExecutionException e) { throw Status.INTERNAL .withDescription("Unable to parse protobuf while retrieving feature") @@ -144,7 +144,7 @@ private RedisKey makeRedisKey( * @param decoder used to decode the data retrieved from Redis for a specific featureset. * @return List of {@link FeatureRow}s */ - private List getFeaturesForFeatureSet( + private List getFeaturesFromRedis( List redisKeys, FeatureRowDecoder decoder) throws InvalidProtocolBufferException, ExecutionException { // pull feature row data bytes from redis using given redis keys diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index 5ef42ef96b4..041dbc71367 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -74,7 +74,7 @@ public List getOnlineFeatures( new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); List featureRows = new ArrayList<>(); try { - featureRows = getFeaturesForFeatureSet(redisKeys, decoder); + featureRows = getFeaturesFromRedis(redisKeys, decoder); } catch (InvalidProtocolBufferException | ExecutionException e) { throw Status.INTERNAL .withDescription("Unable to parse protobuf while retrieving feature") @@ -136,7 +136,7 @@ private RedisKey makeRedisKey( * @param decoder used to decode the data retrieved from Redis for a specific featureset. * @return List of {@link FeatureRow}s */ - private List getFeaturesForFeatureSet( + private List getFeaturesFromRedis( List redisKeys, FeatureRowDecoder decoder) throws InvalidProtocolBufferException, ExecutionException { // pull feature row data bytes from redis using given redis keys From cf99f9c95762093e494da390e6237ec3a4b6bf81 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 28 Apr 2020 11:53:24 +0800 Subject: [PATCH 31/65] Update Serving application.yml to include example on how to config RedisCluster --- .../serving/service/OnlineServingService.java | 7 +-- serving/src/main/resources/application.yml | 15 ++++-- .../RedisClusterOnlineRetriever.java | 8 ++- .../redis/retriever/RedisOnlineRetriever.java | 3 +- .../RedisClusterOnlineRetrieverTest.java | 54 +++++++++---------- 5 files changed, 46 insertions(+), 41 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index d1bf311c231..ea22a54957d 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -66,11 +66,11 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ try (Scope scope = tracer.buildSpan("getOnlineFeatures").startActive(true)) { List entityRows = request.getEntityRowsList(); boolean includeMetadata = request.getIncludeMetadataInResponse(); - // Collect the response fields corresponding by entity row in entityFieldsMap. + // Collect the response fields for each entity row in entityFieldsMap. Map> entityFieldsMap = entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); - if (request.getOmitEntitiesInResponse() == false) { + if (!request.getOmitEntitiesInResponse()) { // Add entity row's fields as response fields entityRows.forEach( entityRow -> { @@ -109,6 +109,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ es -> RefUtil.generateFeatureStringRef(es.getKey()), es -> es.getValue()))); + // Populate metrics this.populateStaleKeyCountMetrics(fields, featureSetRequest); } this.populateRequestCountMetrics(featureSetRequest); @@ -167,9 +168,9 @@ private Map unpackFields( Map refsByName = featureSetRequest.getFeatureRefsByName(); Map fields = new HashMap<>(); - // populate field builder map with feature row's field's data values boolean hasStaleValues = this.isStale(featureSetRequest, entityRow, featureRow); if (featureRow != null) { + // unpack feature row's field's values & populate field map Map featureFields = featureRow.getFieldsList().stream() .filter(featureRowField -> refsByName.containsKey(featureRowField.getName())) diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index b5eff65391b..e185983a872 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -10,10 +10,9 @@ feast: # List of store configurations stores: - # Below are two store configurations. One for Redis and one for BigQuery. # Please see https://api.docs.feast.dev/grpc/feast.core.pb.html#Store for configuration options - name: online # Name of the store (referenced by active_store) - type: REDIS # Type of the store. REDIS, BIGQUERY are available options + type: REDIS # Type of the store. REDIS, REDIS_CLUSTER, BIGQUERY are available options config: # Store specific configuration. See host: localhost port: 6379 @@ -22,7 +21,15 @@ feast: # Wildcards match all options. No filtering is done. - name: "*" project: "*" - + - name: online_cluster + type: REDIS_CLUSTER + config: # Store specific configuration. + # Connection string specifies the IP and ports of Redis instances in the redis cluster. + connection_string: "10.203.164.86:6379,10.203.164.3:6379,10.203.164.14:6379,10.203.164.30:6379,10.203.164.25:6379,10.203.164.67:6379" + subscriptions: + - name: "*" + project: "*" + version: "*" - name: historical type: BIGQUERY config: # Store specific configuration. @@ -77,4 +84,4 @@ server: # The port number on which the Tomcat webserver that serves REST API endpoints should listen # It is set by default to 8081 so it does not conflict with Tomcat webserver on Feast Core # if both Feast Core and Serving are running on the same machine - port: ${SERVER_PORT:8081} \ No newline at end of file + port: ${SERVER_PORT:8081} diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java index 6ce037bd3df..81fd8569442 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java @@ -41,7 +41,7 @@ import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; -/** Defines a storage retriever */ +/** Defines a storage retriever */ public class RedisClusterOnlineRetriever implements OnlineRetriever { private final RedisAdvancedClusterCommands syncCommands; @@ -74,7 +74,7 @@ public static OnlineRetriever create(StatefulRedisClusterConnection getOnlineFeatures( List entityRows, FeatureSetRequest featureSetRequest) { - + // get features for this features/featureset in featureset request FeatureSetSpec featureSetSpec = featureSetRequest.getSpec(); List redisKeys = buildRedisKeys(entityRows, featureSetSpec); @@ -92,7 +92,6 @@ public List getOnlineFeatures( return featureRows; } - private List buildRedisKeys(List entityRows, FeatureSetSpec featureSetSpec) { String featureSetRef = generateFeatureSetStringRef(featureSetSpec); List featureSetEntityNames = @@ -144,8 +143,7 @@ private RedisKey makeRedisKey( * @param decoder used to decode the data retrieved from Redis for a specific featureset. * @return List of {@link FeatureRow}s */ - private List getFeaturesFromRedis( - List redisKeys, FeatureRowDecoder decoder) + private List getFeaturesFromRedis(List redisKeys, FeatureRowDecoder decoder) throws InvalidProtocolBufferException, ExecutionException { // pull feature row data bytes from redis using given redis keys List featureRowsBytes = sendMultiGet(redisKeys); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index 041dbc71367..f5571840232 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -136,8 +136,7 @@ private RedisKey makeRedisKey( * @param decoder used to decode the data retrieved from Redis for a specific featureset. * @return List of {@link FeatureRow}s */ - private List getFeaturesFromRedis( - List redisKeys, FeatureRowDecoder decoder) + private List getFeaturesFromRedis(List redisKeys, FeatureRowDecoder decoder) throws InvalidProtocolBufferException, ExecutionException { // pull feature row data bytes from redis using given redis keys List featureRowsBytes = sendMultiGet(redisKeys); diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java index e56323bee32..d4f35435ce9 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java @@ -133,23 +133,23 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); List expected = - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build())); + Lists.newArrayList( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build(), + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) + .build()); List actual = redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); @@ -200,16 +200,16 @@ public void shouldReturnNullIfKeysNotPresent() { when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); List expected = - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - null); + Lists.newArrayList( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build(), + null); List actual = redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); From 3cd4e22565659e4fb46d14084d8f4ed9f354ab41 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 28 Apr 2020 16:26:35 +0800 Subject: [PATCH 32/65] Correct missing include_meta flag in e2e tests get_online_features() calls --- serving/src/main/resources/application.yml | 4 ++-- tests/e2e/redis/basic-ingest-redis-serving.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index e185983a872..1fe8a4db68f 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -24,8 +24,8 @@ feast: - name: online_cluster type: REDIS_CLUSTER config: # Store specific configuration. - # Connection string specifies the IP and ports of Redis instances in the redis cluster. - connection_string: "10.203.164.86:6379,10.203.164.3:6379,10.203.164.14:6379,10.203.164.30:6379,10.203.164.25:6379,10.203.164.67:6379" + # Connection string specifies the host;ports of Redis instances in the redis cluster. + connection_string: "localhost:7000,localhost:7001,localhost:7002,localhost:7003,localhost:7004,localhost:7005" subscriptions: - name: "*" project: "*" diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 64b65aacc25..b72a7bf7a70 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -439,6 +439,7 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): "bytes_list_feature", "double_list_feature", ], + include_meta=True, ) # type: GetOnlineFeaturesResponse # wait for and unpack response @@ -453,7 +454,7 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): break # check returned values - returned_float_list = float_list_field.float_list_val.val + returned_float_list = float_list_field.value.float_list_val.val sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] assert math.isclose(returned_float_list[0], sent_float_list[0], abs_tol=FLOAT_TOLERANCE) # check returned metadata @@ -558,6 +559,7 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): "daily_transactions_large", "total_transactions_large", ], + include_meta=True, ) # type: GetOnlineFeaturesResponse # wait for and unpack response From 044b27d00fbaf05cc192e8e1f146dd798ceaa6a7 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 28 Apr 2020 16:39:26 +0800 Subject: [PATCH 33/65] Correct minor comment typo in serving's application.yml --- serving/src/main/resources/application.yml | 2 +- tests/e2e/redis/basic-ingest-redis-serving.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index 1fe8a4db68f..2399d132ef9 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -24,7 +24,7 @@ feast: - name: online_cluster type: REDIS_CLUSTER config: # Store specific configuration. - # Connection string specifies the host;ports of Redis instances in the redis cluster. + # Connection string specifies the host:port of Redis instances in the redis cluster. connection_string: "localhost:7000,localhost:7001,localhost:7002,localhost:7003,localhost:7004,localhost:7005" subscriptions: - name: "*" diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index b72a7bf7a70..20cbe377969 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -194,6 +194,7 @@ def test_basic_retrieve_online_success(client, cust_trans_df): "total_transactions", "null_values" ], + include_meta=True, ) # type: GetOnlineFeaturesResponse # unpack response & wait for ingested values From 0a7c1fc19bb56faebfd0b711be289ee50725c2e0 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 28 Apr 2020 17:23:32 +0800 Subject: [PATCH 34/65] Correct another minor typo in e2e test: missing .status --- tests/e2e/redis/basic-ingest-redis-serving.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 20cbe377969..9dac8abd9d3 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -202,7 +202,7 @@ def test_basic_retrieve_online_success(client, cust_trans_df): daily_transactions_field = fields[PROJECT_NAME + "/daily_transactions"] null_value_field = fields[PROJECT_NAME + "/null_values"] if (daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND - or null_value_field == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND): + or null_value_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND): print("test_basic_retrieve_online_success(): waiting for ingested values.") continue else: @@ -218,7 +218,7 @@ def test_basic_retrieve_online_success(client, cust_trans_df): assert null_value_field.value.WhichOneof("val") is None # check field status metadata assert daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT - assert null_value_field == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE + assert null_value_field.status == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE @pytest.mark.timeout(90) From 2cdf558a613d0966c47d3b6b132d40387d23266d Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 3 Jun 2020 11:55:05 +0800 Subject: [PATCH 35/65] Fix Feast core and serving compilation/test failures after rebase --- .../java/com/gojek/feast/FeastClient.java | 11 +- .../src/main/java/com/gojek/feast/Row.java | 2 +- .../serving/service/OnlineServingService.java | 7 +- .../feast/serving/service/ServingService.java | 4 +- .../service/OnlineServingServiceTest.java | 233 +++--------------- .../RedisClusterOnlineRetriever.java | 1 - .../redis/retriever/RedisOnlineRetriever.java | 1 - .../retriever/RedisOnlineRetrieverTest.java | 6 +- 8 files changed, 42 insertions(+), 223 deletions(-) diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index cbb59af7862..0a94aa87f2d 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -24,7 +24,6 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; import feast.proto.serving.ServingServiceGrpc; import feast.proto.serving.ServingServiceGrpc.ServingServiceBlockingStub; -import feast.proto.types.ValueProto.Value; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import java.util.HashSet; @@ -62,7 +61,7 @@ public GetFeastServingInfoResponse getFeastServingInfo() { /** * Get online features from Feast from FeatureSets * - *

See {@link #getOnlineFeatures(List, List, String, boolean)} + *

See {@link #getOnlineFeatures(List, List, String, boolean, boolean)} * * @param featureRefs list of string feature references to retrieve in the following format * featureSet:feature, where 'featureSet' and 'feature' refer to the FeatureSet and Feature @@ -77,7 +76,7 @@ public List getOnlineFeatures(List featureRefs, List rows) { /** * Get online features from Feast. * - *

See {@link #getOnlineFeatures(List, List, String, boolean)} + *

See {@link #getOnlineFeatures(List, List, String, boolean, boolean)} * * @param featureRefs list of string feature references to retrieve in the following format * featureSet:feature, where 'featureSet' and 'feature' refer to the FeatureSet and Feature @@ -118,9 +117,9 @@ public List getOnlineFeatures(List featureRefs, List rows, Str * @return list of {@link Row} containing retrieved data fields. */ public List getOnlineFeatures( - List featureRefs, - List rows, String - project, + List featureRefs, + List rows, + String project, boolean omitEntitiesInResponse, boolean includeMetadataInResponse) { List features = RequestUtil.createFeatureRefs(featureRefs, project); diff --git a/sdk/java/src/main/java/com/gojek/feast/Row.java b/sdk/java/src/main/java/com/gojek/feast/Row.java index ef03e53ac73..3c0b7ab5a86 100644 --- a/sdk/java/src/main/java/com/gojek/feast/Row.java +++ b/sdk/java/src/main/java/com/gojek/feast/Row.java @@ -19,9 +19,9 @@ import com.google.protobuf.ByteString; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.types.ValueProto.Value; import feast.proto.types.ValueProto.Value.ValCase; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index ea22a54957d..afcad319bf8 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -16,14 +16,13 @@ */ package feast.serving.service; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; import com.google.protobuf.Duration; import feast.proto.serving.ServingAPIProto.*; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Field; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; +import feast.proto.types.FeatureRowProto.FeatureRow; import feast.proto.types.ValueProto.Value; import feast.serving.specs.CachedSpecService; import feast.serving.util.Metrics; @@ -269,9 +268,7 @@ private void populateStaleKeyCountMetrics( FeatureReference ref = es.getKey(); Field field = es.getValue(); if (field.getStatus() == FieldStatus.OUTSIDE_MAX_AGE) { - Metrics.staleKeyCount - .labels(project, RefUtil.generateFeatureStringRefWithoutProject(ref)) - .inc(); + Metrics.staleKeyCount.labels(project, RefUtil.generateFeatureStringRef(ref)).inc(); } }); } diff --git a/serving/src/main/java/feast/serving/service/ServingService.java b/serving/src/main/java/feast/serving/service/ServingService.java index a9155019bb6..97a772c56af 100644 --- a/serving/src/main/java/feast/serving/service/ServingService.java +++ b/serving/src/main/java/feast/serving/service/ServingService.java @@ -56,8 +56,8 @@ GetFeastServingInfoResponse getFeastServingInfo( * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow}s to join the * retrieved values to. * @return {@link GetOnlineFeaturesResponse} with list of {@link - * feast.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record} for each {@link - * feast.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow} supplied. + * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record} for each {@link + * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow} supplied. */ GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getFeaturesRequest); diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 8b6fbb1cbba..8fe0f4c0500 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -24,7 +24,6 @@ import com.google.common.collect.Lists; import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; - import feast.proto.core.FeatureSetProto.EntitySpec; import feast.proto.core.FeatureSetProto.FeatureSetSpec; import feast.proto.serving.ServingAPIProto.FeatureReference; @@ -35,7 +34,7 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; import feast.proto.types.FeatureRowProto.FeatureRow; -import feast.proto.types.FieldProto.Field; +import feast.proto.types.FieldProto; import feast.proto.types.ValueProto.Value; import feast.serving.specs.CachedSpecService; import feast.storage.api.retriever.FeatureSetRequest; @@ -72,120 +71,7 @@ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { GetOnlineFeaturesRequest.newBuilder() .setOmitEntitiesInResponse(false) .setIncludeMetadataInResponse(true) - .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) - .addFeatures( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) - .addEntityRows( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(1)) - .putFields("entity2", strValue("a"))) - .addEntityRows( - EntityRow.newBuilder() - .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) - .putFields("entity1", intValue(2)) - .putFields("entity2", strValue("b"))) - .build(); - - List featureRows = - Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("entity1") - .setValue(intValue(1)) - .build(), - FieldProto.Field.newBuilder() - .setName("entity2") - .setValue(strValue("a")) - .build(), - FieldProto.Field.newBuilder() - .setName("feature1") - .setValue(intValue(1)) - .build(), - FieldProto.Field.newBuilder() - .setName("feature2") - .setValue(intValue(1)) - .build())) - .setFeatureSet("featureSet:1") - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("entity1") - .setValue(intValue(2)) - .build(), - FieldProto.Field.newBuilder() - .setName("entity2") - .setValue(strValue("b")) - .build(), - FieldProto.Field.newBuilder() - .setName("feature1") - .setValue(intValue(2)) - .build(), - FieldProto.Field.newBuilder() - .setName("feature2") - .setValue(intValue(2)) - .build())) - .setFeatureSet("featureSet:1") - .build()); - - FeatureSetRequest featureSetRequest = - FeatureSetRequest.newBuilder() - .addAllFeatureReferences(request.getFeaturesList()) - .setSpec(getFeatureSetSpec()) - .build(); - - when(specService.getFeatureSets(request.getFeaturesList())) - .thenReturn(Collections.singletonList(featureSetRequest)); - when(retriever.getOnlineFeatures(request.getEntityRowsList(), featureSetRequest)) - .thenReturn(featureRows); - when(tracer.buildSpan(ArgumentMatchers.any())).thenReturn(Mockito.mock(SpanBuilder.class)); - - GetOnlineFeaturesResponse expected = - GetOnlineFeaturesResponse.newBuilder() - .addRecords( - Record.newBuilder() - .putFields("entity1", intField(1)) - .putFields("entity2", strField("a")) - .putFields("project/feature1:1", intField(1)) - .putFields("project/feature2:1", intField(1))) - .addRecords( - Record.newBuilder() - .putFields("entity1", intField(2)) - .putFields("entity2", strField("b")) - .putFields("project/feature1:1", intField(2)) - .putFields("project/feature2:1", intField(2))) - .build(); - GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); - assertThat(actual, equalTo(expected)); - } - - @Test - public void shouldReturnFieldKeysWithoutVersionIfNotProvided() { - GetOnlineFeaturesRequest request = - GetOnlineFeaturesRequest.newBuilder() - .setIncludeMetadataInResponse(true) - .setOmitEntitiesInResponse(false) - .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) .addFeatures( FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .addEntityRows( @@ -222,7 +108,7 @@ public void shouldReturnFieldKeysWithoutVersionIfNotProvided() { .setName("feature2") .setValue(intValue(1)) .build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -244,7 +130,7 @@ public void shouldReturnFieldKeysWithoutVersionIfNotProvided() { .setName("feature2") .setValue(intValue(2)) .build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build()); FeatureSetRequest featureSetRequest = @@ -265,13 +151,13 @@ public void shouldReturnFieldKeysWithoutVersionIfNotProvided() { Record.newBuilder() .putFields("entity1", intField(1)) .putFields("entity2", strField("a")) - .putFields("project/feature1:1", intField(1)) + .putFields("feature1", intField(1)) .putFields("project/feature2", intField(1))) .addRecords( Record.newBuilder() .putFields("entity1", intField(2)) .putFields("entity2", strField("b")) - .putFields("project/feature1:1", intField(2)) + .putFields("feature1", intField(2)) .putFields("project/feature2", intField(2))) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); @@ -283,25 +169,10 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { // some keys not present, should have empty values GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() -<<<<<<< HEAD - .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) - .addFeatures(FeatureReference.newBuilder().setName("feature2").build()) -======= - .setOmitEntitiesInResponse(false) .setIncludeMetadataInResponse(true) + .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) - .addFeatures( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) ->>>>>>> Update OnlineServingServiceTest to check that online metadata is set correctly. + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -336,18 +207,7 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { .setValue(intValue(1)) .build())) .build(), - FeatureRow.newBuilder() -<<<<<<< HEAD - .setFeatureSet("project/featureSet") -======= - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") ->>>>>>> Update OnlineServingServiceTest to check that online metadata is set correctly. - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder().setName("feature1").build(), - FieldProto.Field.newBuilder().setName("feature2").build())) - .build()); + null); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); @@ -361,23 +221,23 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { Record.newBuilder() .putFields("entity1", intField(1)) .putFields("entity2", strField("a")) - .putFields("project/feature1:1", intField(1)) - .putFields("project/feature2:1", intField(1))) + .putFields("feature1", intField(1)) + .putFields("project/feature2", intField(1))) .addRecords( Record.newBuilder() .putFields("entity1", intField(2)) .putFields("entity2", strField("b")) .putFields( - "project/feature1:1", + "feature1", Field.newBuilder() .setValue(Value.newBuilder().build()) - .setStatus(FieldStatus.NULL_VALUE) + .setStatus(FieldStatus.NOT_FOUND) .build()) .putFields( - "project/feature2:1", + "project/feature2", Field.newBuilder() .setValue(Value.newBuilder().build()) - .setStatus(FieldStatus.NULL_VALUE) + .setStatus(FieldStatus.NOT_FOUND) .build())) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); @@ -389,25 +249,11 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { // keys present, but considered stale when compared to maxAge GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() -<<<<<<< HEAD .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) - .addFeatures(FeatureReference.newBuilder().setName("feature2").build()) -======= + .addFeatures( + FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .setOmitEntitiesInResponse(false) .setIncludeMetadataInResponse(true) - .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) - .addFeatures( - FeatureReference.newBuilder() - .setName("feature2") - .setVersion(1) - .setProject("project") - .build()) ->>>>>>> Update OnlineServingServiceTest to check that online metadata is set correctly. .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -426,13 +272,6 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) .addAllFields( Lists.newArrayList( -<<<<<<< HEAD - Field.newBuilder().setName("entity1").setValue(intValue(1)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("a")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .setFeatureSet("project/featureSet") -======= FieldProto.Field.newBuilder() .setName("entity1") .setValue(intValue(1)) @@ -449,21 +288,13 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { .setName("feature2") .setValue(intValue(1)) .build())) - .setFeatureSet("featureSet:1") ->>>>>>> Update OnlineServingServiceTest to check that online metadata is set correctly. + .setFeatureSet("project/featureSet") .build(), FeatureRow.newBuilder() .setEventTimestamp( Timestamp.newBuilder().setSeconds(50)) // this value should be nulled .addAllFields( Lists.newArrayList( -<<<<<<< HEAD - Field.newBuilder().setName("entity1").setValue(intValue(2)).build(), - Field.newBuilder().setName("entity2").setValue(strValue("b")).build(), - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .setFeatureSet("project/featureSet") -======= FieldProto.Field.newBuilder() .setName("entity1") .setValue(intValue(2)) @@ -477,11 +308,10 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { .setValue(intValue(2)) .build(), FieldProto.Field.newBuilder() - .setName("feature2") + .setName("project/feature2") .setValue(intValue(2)) .build())) - .setFeatureSet("featureSet:1") ->>>>>>> Update OnlineServingServiceTest to check that online metadata is set correctly. + .setFeatureSet("project/featureSet") .build()); FeatureSetSpec spec = @@ -504,20 +334,20 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { Record.newBuilder() .putFields("entity1", intField(1)) .putFields("entity2", strField("a")) - .putFields("project/feature1:1", intField(1)) - .putFields("project/feature2:1", intField(1))) + .putFields("feature1", intField(1)) + .putFields("project/feature2", intField(1))) .addRecords( Record.newBuilder() .putFields("entity1", intField(2)) .putFields("entity2", strField("b")) .putFields( - "project/feature1:1", + "feature1", Field.newBuilder() .setValue(Value.newBuilder().build()) .setStatus(FieldStatus.OUTSIDE_MAX_AGE) .build()) .putFields( - "project/feature2:1", + "project/feature2", Field.newBuilder() .setValue(Value.newBuilder().build()) .setStatus(FieldStatus.OUTSIDE_MAX_AGE) @@ -535,12 +365,7 @@ public void shouldFilterOutUndesiredRows() { GetOnlineFeaturesRequest.newBuilder() .setIncludeMetadataInResponse(true) .setOmitEntitiesInResponse(false) - .addFeatures( - FeatureReference.newBuilder() - .setName("feature1") - .setVersion(1) - .setProject("project") - .build()) + .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -575,7 +400,7 @@ public void shouldFilterOutUndesiredRows() { .setName("feature2") .setValue(intValue(1)) .build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -597,7 +422,7 @@ public void shouldFilterOutUndesiredRows() { .setName("feature2") .setValue(intValue(2)) .build())) - .setFeatureSet("featureSet:1") + .setFeatureSet("featureSet") .build()); FeatureSetRequest featureSetRequest = @@ -618,12 +443,12 @@ public void shouldFilterOutUndesiredRows() { Record.newBuilder() .putFields("entity1", intField(1)) .putFields("entity2", strField("a")) - .putFields("project/feature1:1", intField(1))) + .putFields("feature1", intField(1))) .addRecords( Record.newBuilder() .putFields("entity1", intField(2)) .putFields("entity2", strField("b")) - .putFields("project/feature1:1", intField(2))) + .putFields("feature1", intField(2))) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java index 81fd8569442..1ca8bdd18bc 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java @@ -20,7 +20,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import feast.proto.core.FeatureSetProto.EntitySpec; import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.storage.RedisProto.RedisKey; import feast.proto.types.FeatureRowProto.FeatureRow; diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index f5571840232..4b4875fa3db 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -20,7 +20,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import feast.proto.core.FeatureSetProto.EntitySpec; import feast.proto.core.FeatureSetProto.FeatureSetSpec; -import feast.proto.serving.ServingAPIProto.FeatureReference; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.storage.RedisProto.RedisKey; import feast.proto.types.FeatureRowProto.FeatureRow; diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java index a029968373e..7795722e647 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java @@ -136,7 +136,7 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { Lists.newArrayList( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), @@ -144,7 +144,7 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { .build(), FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), @@ -202,7 +202,7 @@ public void shouldReturnNullIfKeysNotPresent() { Lists.newArrayList( FeatureRow.newBuilder() .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet:1") + .setFeatureSet("project/featureSet") .addAllFields( Lists.newArrayList( Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), From 57babd5751b52a43f0d0b006536c0fd9ad020d3f Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 3 Jun 2020 14:47:27 +0800 Subject: [PATCH 36/65] Fix go SDK tests after rebase --- go.mod | 11 +- go.sum | 4 + sdk/go/protos/feast/core/Runner.pb.go | 50 ++- .../protos/feast/serving/ServingService.pb.go | 386 +++++++++--------- .../tensorflow_metadata/proto/v0/path.pb.go | 11 - .../tensorflow_metadata/proto/v0/schema.pb.go | 10 - sdk/go/request.go | 5 +- 7 files changed, 244 insertions(+), 233 deletions(-) diff --git a/go.mod b/go.mod index 0cc5494d067..38ba2cbf85e 100644 --- a/go.mod +++ b/go.mod @@ -23,20 +23,11 @@ require ( github.com/woop/protoc-gen-doc v1.3.0 // indirect go.opencensus.io v0.22.3 // indirect golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect -<<<<<<< HEAD golang.org/x/net v0.0.0-20200513185701-a91f0712d120 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect - golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc // indirect - google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587 // indirect + golang.org/x/tools v0.0.0-20200601175630-2caf76543d99 // indirect google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.24.0 // indirect -======= - golang.org/x/net v0.0.0-20200320220750-118fecf932d8 - golang.org/x/sys v0.0.0-20200321134203-328b4cd54aae // indirect - golang.org/x/tools v0.0.0-20200423205358-59e73619c742 // indirect - google.golang.org/genproto v0.0.0-20200319113533-08878b785e9c // indirect - google.golang.org/grpc v1.28.0 ->>>>>>> Update go protobuf generated code for updated protobuf defintion gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.2.4 istio.io/gogo-genproto v0.0.0-20191212213402-78a529a42cd8 // indirect diff --git a/go.sum b/go.sum index 48887dc5cd3..7c2d4d57528 100644 --- a/go.sum +++ b/go.sum @@ -320,6 +320,7 @@ github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -467,8 +468,10 @@ golang.org/x/tools v0.0.0-20200519205726-57a9e4404bf7 h1:nm4zDh9WvH4jiuUpMY5RUsv golang.org/x/tools v0.0.0-20200519205726-57a9e4404bf7/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc h1:6m2YO+AmBApbUOmhsghW+IfRyZOY4My4UYvQQrEpHfY= golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200601175630-2caf76543d99/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= @@ -542,6 +545,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/sdk/go/protos/feast/core/Runner.pb.go b/sdk/go/protos/feast/core/Runner.pb.go index b1255e52ba6..ac936df021a 100644 --- a/sdk/go/protos/feast/core/Runner.pb.go +++ b/sdk/go/protos/feast/core/Runner.pb.go @@ -131,6 +131,8 @@ type DataflowRunnerConfigOptions struct { MaxNumWorkers int32 `protobuf:"varint,11,opt,name=maxNumWorkers,proto3" json:"maxNumWorkers,omitempty"` // BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID DeadLetterTableSpec string `protobuf:"bytes,12,opt,name=deadLetterTableSpec,proto3" json:"deadLetterTableSpec,omitempty"` + // Labels to apply to the dataflow job + Labels map[string]string `protobuf:"bytes,13,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *DataflowRunnerConfigOptions) Reset() { @@ -249,6 +251,13 @@ func (x *DataflowRunnerConfigOptions) GetDeadLetterTableSpec() string { return "" } +func (x *DataflowRunnerConfigOptions) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + var File_feast_core_Runner_proto protoreflect.FileDescriptor var file_feast_core_Runner_proto_rawDesc = []byte{ @@ -262,7 +271,7 @@ var file_feast_core_Runner_proto_rawDesc = []byte{ 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, - 0x65, 0x63, 0x22, 0xc7, 0x03, 0x0a, 0x1b, 0x44, 0x61, 0x74, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x52, + 0x65, 0x63, 0x22, 0xcf, 0x04, 0x0a, 0x1b, 0x44, 0x61, 0x74, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, @@ -290,13 +299,22 @@ var file_feast_core_Runner_proto_rawDesc = []byte{ 0x4e, 0x75, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, - 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x42, 0x54, 0x0a, 0x10, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x42, 0x0b, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, - 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, - 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x4b, 0x0a, 0x06, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x66, 0x6c, + 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, + 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x42, 0x54, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0b, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -311,17 +329,19 @@ func file_feast_core_Runner_proto_rawDescGZIP() []byte { return file_feast_core_Runner_proto_rawDescData } -var file_feast_core_Runner_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_feast_core_Runner_proto_msgTypes = make([]protoimpl.MessageInfo, 3) var file_feast_core_Runner_proto_goTypes = []interface{}{ (*DirectRunnerConfigOptions)(nil), // 0: feast.core.DirectRunnerConfigOptions (*DataflowRunnerConfigOptions)(nil), // 1: feast.core.DataflowRunnerConfigOptions + nil, // 2: feast.core.DataflowRunnerConfigOptions.LabelsEntry } var file_feast_core_Runner_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 2, // 0: feast.core.DataflowRunnerConfigOptions.labels:type_name -> feast.core.DataflowRunnerConfigOptions.LabelsEntry + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_feast_core_Runner_proto_init() } @@ -361,7 +381,7 @@ func file_feast_core_Runner_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_Runner_proto_rawDesc, NumEnums: 0, - NumMessages: 2, + NumMessages: 3, NumExtensions: 0, NumServices: 0, }, diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index f246f8169dd..d89265e4740 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -248,7 +248,7 @@ type GetOnlineFeaturesResponse_FieldStatus int32 const ( // Status is unset for this field. GetOnlineFeaturesResponse_INVALID GetOnlineFeaturesResponse_FieldStatus = 0 - // Field value is present for this field and within maximum allowable range + // Field value is present for this field and within maximum allowable range. GetOnlineFeaturesResponse_PRESENT GetOnlineFeaturesResponse_FieldStatus = 1 // Values could be found for entity key within the maximum allowable range, but // this field value is assigned a null value on ingestion into feast. @@ -258,7 +258,7 @@ const ( // into feast or the ingestion failed. GetOnlineFeaturesResponse_NOT_FOUND GetOnlineFeaturesResponse_FieldStatus = 3 // Values could be found for entity key, but field values are outside the maximum - // allowable range. + // allowable range: values are older than max age. GetOnlineFeaturesResponse_OUTSIDE_MAX_AGE GetOnlineFeaturesResponse_FieldStatus = 4 ) @@ -1220,7 +1220,7 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x4a, - 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0xe1, 0x03, 0x0a, 0x18, + 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0xa2, 0x04, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, @@ -1235,143 +1235,161 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x74, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, 0x6f, 0x6d, 0x69, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x49, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0xf8, 0x01, 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, - 0x6f, 0x77, 0x12, 0x45, 0x0a, 0x10, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x55, 0x0a, 0x06, 0x66, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, - 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x2e, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, - 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, - 0x9b, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x08, 0x66, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, - 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0e, 0x64, 0x61, 0x74, 0x61, - 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x0d, - 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xad, 0x02, - 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0c, 0x66, - 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x73, 0x1a, 0xb6, 0x01, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x1c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x19, 0x69, 0x6e, 0x63, 0x6c, + 0x75, 0x64, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x49, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0xf8, 0x01, 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, + 0x52, 0x6f, 0x77, 0x12, 0x45, 0x0a, 0x10, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x55, 0x0a, 0x06, 0x66, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, + 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x2e, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x8f, 0x04, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, + 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x2f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, + 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, + 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x1a, 0xc8, 0x01, 0x0a, 0x06, 0x52, 0x65, + 0x63, 0x6f, 0x72, 0x64, 0x12, 0x53, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x4d, - 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, - 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, - 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x40, 0x0a, - 0x18, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, - 0x35, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, - 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x36, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0xe2, - 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, - 0x70, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, - 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, - 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, - 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, - 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, - 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x1a, 0x65, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, - 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, - 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, - 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x64, 0x61, 0x74, 0x61, - 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2a, 0x6f, 0x0a, 0x10, 0x46, 0x65, - 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, - 0x0a, 0x1a, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x1d, - 0x0a, 0x19, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1c, 0x0a, - 0x18, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x2a, 0x36, 0x0a, 0x07, 0x4a, - 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, - 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, - 0x44, 0x10, 0x01, 0x2a, 0x68, 0x0a, 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, - 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, - 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, - 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x4f, 0x42, 0x5f, - 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x2a, 0x3b, 0x0a, - 0x0a, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x17, 0x0a, 0x13, 0x44, - 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, - 0x49, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, - 0x4d, 0x41, 0x54, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, 0x01, 0x32, 0x92, 0x03, 0x0a, 0x0e, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, - 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, - 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, - 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x11, 0x47, - 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, - 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, + 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x69, 0x0a, 0x0b, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x44, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, - 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, - 0x6f, 0x62, 0x12, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, - 0x5e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x41, - 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x7f, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x28, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x4c, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, + 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x5b, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, + 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x53, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, + 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, + 0x0a, 0x09, 0x4e, 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, + 0x0f, 0x4f, 0x55, 0x54, 0x53, 0x49, 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, + 0x10, 0x04, 0x22, 0x9b, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, + 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0e, 0x64, + 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x22, 0x40, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, + 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, + 0x6f, 0x62, 0x22, 0x35, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x36, 0x0a, 0x0e, 0x47, 0x65, 0x74, + 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, + 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, + 0x62, 0x22, 0xe2, 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x52, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1b, 0x0a, + 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, 0x64, 0x61, + 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, + 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, + 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, + 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x73, + 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, + 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, + 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x1a, 0x65, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, + 0x3a, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, + 0x0a, 0x64, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x64, + 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2a, 0x6f, 0x0a, + 0x10, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, + 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, + 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, + 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, + 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, + 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x2a, 0x36, + 0x0a, 0x07, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x4a, 0x4f, 0x42, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, + 0x15, 0x0a, 0x11, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, + 0x4c, 0x4f, 0x41, 0x44, 0x10, 0x01, 0x2a, 0x68, 0x0a, 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4a, + 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, + 0x47, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, + 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x4a, + 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, + 0x2a, 0x3b, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x17, + 0x0a, 0x13, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x49, 0x4e, + 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x41, 0x54, 0x41, 0x5f, + 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, 0x01, 0x32, 0x92, 0x03, + 0x0a, 0x0e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x12, 0x6c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, + 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, + 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, + 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, + 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x06, 0x47, + 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x42, 0x5e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, + 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1399,56 +1417,56 @@ var file_feast_serving_ServingService_proto_goTypes = []interface{}{ (*FeatureReference)(nil), // 7: feast.serving.FeatureReference (*GetOnlineFeaturesRequest)(nil), // 8: feast.serving.GetOnlineFeaturesRequest (*GetOnlineFeaturesResponse)(nil), // 9: feast.serving.GetOnlineFeaturesResponse -<<<<<<< HEAD - (*GetBatchFeaturesResponse)(nil), // 10: feast.serving.GetBatchFeaturesResponse - (*GetJobRequest)(nil), // 11: feast.serving.GetJobRequest - (*GetJobResponse)(nil), // 12: feast.serving.GetJobResponse - (*Job)(nil), // 13: feast.serving.Job - (*DatasetSource)(nil), // 14: feast.serving.DatasetSource - (*GetOnlineFeaturesRequest_EntityRow)(nil), // 15: feast.serving.GetOnlineFeaturesRequest.EntityRow - nil, // 16: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry - (*GetOnlineFeaturesResponse_FieldValues)(nil), // 17: feast.serving.GetOnlineFeaturesResponse.FieldValues - nil, // 18: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry - (*DatasetSource_FileSource)(nil), // 19: feast.serving.DatasetSource.FileSource - (*timestamp.Timestamp)(nil), // 20: google.protobuf.Timestamp - (*types.Value)(nil), // 21: feast.types.Value + (*GetBatchFeaturesRequest)(nil), // 10: feast.serving.GetBatchFeaturesRequest + (*GetBatchFeaturesResponse)(nil), // 11: feast.serving.GetBatchFeaturesResponse + (*GetJobRequest)(nil), // 12: feast.serving.GetJobRequest + (*GetJobResponse)(nil), // 13: feast.serving.GetJobResponse + (*Job)(nil), // 14: feast.serving.Job + (*DatasetSource)(nil), // 15: feast.serving.DatasetSource + (*GetOnlineFeaturesRequest_EntityRow)(nil), // 16: feast.serving.GetOnlineFeaturesRequest.EntityRow + nil, // 17: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry + (*GetOnlineFeaturesResponse_Record)(nil), // 18: feast.serving.GetOnlineFeaturesResponse.Record + (*GetOnlineFeaturesResponse_Field)(nil), // 19: feast.serving.GetOnlineFeaturesResponse.Field + nil, // 20: feast.serving.GetOnlineFeaturesResponse.Record.FieldsEntry + (*DatasetSource_FileSource)(nil), // 21: feast.serving.DatasetSource.FileSource + (*timestamp.Timestamp)(nil), // 22: google.protobuf.Timestamp + (*types.Value)(nil), // 23: feast.types.Value } var file_feast_serving_ServingService_proto_depIdxs = []int32{ 0, // 0: feast.serving.GetFeastServingInfoResponse.type:type_name -> feast.serving.FeastServingType - 22, // 1: feast.serving.FeatureReference.max_age:type_name -> google.protobuf.Duration - 7, // 2: feast.serving.GetOnlineFeaturesRequest.features:type_name -> feast.serving.FeatureReference - 16, // 3: feast.serving.GetOnlineFeaturesRequest.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow - 18, // 4: feast.serving.GetOnlineFeaturesResponse.records:type_name -> feast.serving.GetOnlineFeaturesResponse.Record - 7, // 5: feast.serving.GetBatchFeaturesRequest.features:type_name -> feast.serving.FeatureReference - 15, // 6: feast.serving.GetBatchFeaturesRequest.dataset_source:type_name -> feast.serving.DatasetSource - 14, // 7: feast.serving.GetBatchFeaturesResponse.job:type_name -> feast.serving.Job - 14, // 8: feast.serving.GetJobRequest.job:type_name -> feast.serving.Job - 14, // 9: feast.serving.GetJobResponse.job:type_name -> feast.serving.Job - 1, // 10: feast.serving.Job.type:type_name -> feast.serving.JobType - 2, // 11: feast.serving.Job.status:type_name -> feast.serving.JobStatus - 3, // 12: feast.serving.Job.data_format:type_name -> feast.serving.DataFormat - 21, // 13: feast.serving.DatasetSource.file_source:type_name -> feast.serving.DatasetSource.FileSource - 23, // 14: feast.serving.GetOnlineFeaturesRequest.EntityRow.entity_timestamp:type_name -> google.protobuf.Timestamp - 17, // 15: feast.serving.GetOnlineFeaturesRequest.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry - 24, // 16: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry.value:type_name -> feast.types.Value - 20, // 17: feast.serving.GetOnlineFeaturesResponse.Record.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.Record.FieldsEntry - 24, // 18: feast.serving.GetOnlineFeaturesResponse.Field.value:type_name -> feast.types.Value - 4, // 19: feast.serving.GetOnlineFeaturesResponse.Field.status:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldStatus - 19, // 20: feast.serving.GetOnlineFeaturesResponse.Record.FieldsEntry.value:type_name -> feast.serving.GetOnlineFeaturesResponse.Field - 3, // 21: feast.serving.DatasetSource.FileSource.data_format:type_name -> feast.serving.DataFormat - 5, // 22: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest - 8, // 23: feast.serving.ServingService.GetOnlineFeatures:input_type -> feast.serving.GetOnlineFeaturesRequest - 10, // 24: feast.serving.ServingService.GetBatchFeatures:input_type -> feast.serving.GetBatchFeaturesRequest - 12, // 25: feast.serving.ServingService.GetJob:input_type -> feast.serving.GetJobRequest - 6, // 26: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse - 9, // 27: feast.serving.ServingService.GetOnlineFeatures:output_type -> feast.serving.GetOnlineFeaturesResponse - 11, // 28: feast.serving.ServingService.GetBatchFeatures:output_type -> feast.serving.GetBatchFeaturesResponse - 13, // 29: feast.serving.ServingService.GetJob:output_type -> feast.serving.GetJobResponse - 26, // [26:30] is the sub-list for method output_type - 22, // [22:26] is the sub-list for method input_type - 22, // [22:22] is the sub-list for extension type_name - 22, // [22:22] is the sub-list for extension extendee - 0, // [0:22] is the sub-list for field type_name + 7, // 1: feast.serving.GetOnlineFeaturesRequest.features:type_name -> feast.serving.FeatureReference + 16, // 2: feast.serving.GetOnlineFeaturesRequest.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow + 18, // 3: feast.serving.GetOnlineFeaturesResponse.records:type_name -> feast.serving.GetOnlineFeaturesResponse.Record + 7, // 4: feast.serving.GetBatchFeaturesRequest.features:type_name -> feast.serving.FeatureReference + 15, // 5: feast.serving.GetBatchFeaturesRequest.dataset_source:type_name -> feast.serving.DatasetSource + 14, // 6: feast.serving.GetBatchFeaturesResponse.job:type_name -> feast.serving.Job + 14, // 7: feast.serving.GetJobRequest.job:type_name -> feast.serving.Job + 14, // 8: feast.serving.GetJobResponse.job:type_name -> feast.serving.Job + 1, // 9: feast.serving.Job.type:type_name -> feast.serving.JobType + 2, // 10: feast.serving.Job.status:type_name -> feast.serving.JobStatus + 3, // 11: feast.serving.Job.data_format:type_name -> feast.serving.DataFormat + 21, // 12: feast.serving.DatasetSource.file_source:type_name -> feast.serving.DatasetSource.FileSource + 22, // 13: feast.serving.GetOnlineFeaturesRequest.EntityRow.entity_timestamp:type_name -> google.protobuf.Timestamp + 17, // 14: feast.serving.GetOnlineFeaturesRequest.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry + 23, // 15: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry.value:type_name -> feast.types.Value + 20, // 16: feast.serving.GetOnlineFeaturesResponse.Record.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.Record.FieldsEntry + 23, // 17: feast.serving.GetOnlineFeaturesResponse.Field.value:type_name -> feast.types.Value + 4, // 18: feast.serving.GetOnlineFeaturesResponse.Field.status:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldStatus + 19, // 19: feast.serving.GetOnlineFeaturesResponse.Record.FieldsEntry.value:type_name -> feast.serving.GetOnlineFeaturesResponse.Field + 3, // 20: feast.serving.DatasetSource.FileSource.data_format:type_name -> feast.serving.DataFormat + 5, // 21: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest + 8, // 22: feast.serving.ServingService.GetOnlineFeatures:input_type -> feast.serving.GetOnlineFeaturesRequest + 10, // 23: feast.serving.ServingService.GetBatchFeatures:input_type -> feast.serving.GetBatchFeaturesRequest + 12, // 24: feast.serving.ServingService.GetJob:input_type -> feast.serving.GetJobRequest + 6, // 25: feast.serving.ServingService.GetFeastServingInfo:output_type -> feast.serving.GetFeastServingInfoResponse + 9, // 26: feast.serving.ServingService.GetOnlineFeatures:output_type -> feast.serving.GetOnlineFeaturesResponse + 11, // 27: feast.serving.ServingService.GetBatchFeatures:output_type -> feast.serving.GetBatchFeaturesResponse + 13, // 28: feast.serving.ServingService.GetJob:output_type -> feast.serving.GetJobResponse + 25, // [25:29] is the sub-list for method output_type + 21, // [21:25] is the sub-list for method input_type + 21, // [21:21] is the sub-list for extension type_name + 21, // [21:21] is the sub-list for extension extendee + 0, // [0:21] is the sub-list for field type_name } func init() { file_feast_serving_ServingService_proto_init() } diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go index 3b644381ca1..f58247299d9 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/path.pb.go @@ -112,7 +112,6 @@ var file_tensorflow_metadata_proto_v0_path_proto_rawDesc = []byte{ 0x61, 0x74, 0x68, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x16, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x22, 0x1a, 0x0a, 0x04, 0x50, 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x74, 0x65, -<<<<<<< HEAD 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x73, 0x74, 0x65, 0x70, 0x42, 0x68, 0x0a, 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, 0x45, 0x67, @@ -121,16 +120,6 @@ var file_tensorflow_metadata_proto_v0_path_proto_rawDesc = []byte{ 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, 0xf8, 0x01, 0x01, -======= - 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x73, 0x74, 0x65, 0x70, 0x42, 0x64, 0x0a, - 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, 0x41, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, 0x6b, 0x2f, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, - 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, - 0xf8, 0x01, 0x01, ->>>>>>> Update go protobuf generated code for updated protobuf defintion } var ( diff --git a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go index 4ebfa7d5dba..a04f5bba8ca 100644 --- a/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go +++ b/sdk/go/protos/tensorflow_metadata/proto/v0/schema.pb.go @@ -3476,7 +3476,6 @@ var file_tensorflow_metadata_proto_v0_schema_proto_rawDesc = []byte{ 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x54, 0x52, 0x55, 0x43, 0x54, 0x10, 0x04, 0x42, -<<<<<<< HEAD 0x68, 0x0a, 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, @@ -3484,15 +3483,6 @@ var file_tensorflow_metadata_proto_v0_schema_proto_rawDesc = []byte{ 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, 0xf8, 0x01, 0x01, -======= - 0x64, 0x0a, 0x1a, 0x6f, 0x72, 0x67, 0x2e, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, - 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x76, 0x30, 0x50, 0x01, 0x5a, - 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6a, 0x65, - 0x6b, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x74, 0x65, 0x6e, 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, - 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, - 0x76, 0x30, 0xf8, 0x01, 0x01, ->>>>>>> Update go protobuf generated code for updated protobuf defintion } var ( diff --git a/sdk/go/request.go b/sdk/go/request.go index 0c75141ce8f..3fd9aa2920f 100644 --- a/sdk/go/request.go +++ b/sdk/go/request.go @@ -5,8 +5,7 @@ import ( "github.com/feast-dev/feast/sdk/go/protos/feast/serving" "strings" - "github.com/gojek/feast/sdk/go/protos/feast/serving" - "github.com/gojek/feast/sdk/go/protos/feast/types" + "github.com/feast-dev/feast/sdk/go/protos/feast/types" ) var ( @@ -56,7 +55,7 @@ func (r OnlineFeaturesRequest) buildRequest() (*serving.GetOnlineFeaturesRequest } return &serving.GetOnlineFeaturesRequest{ - Features: features, + Features: featureRefs, EntityRows: entityRows, OmitEntitiesInResponse: r.OmitEntities, IncludeMetadataInResponse: r.IncludeMeta, From 6e247db0008ec0a5bb3ef4778ab4a195dc21f3c5 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Thu, 4 Jun 2020 11:40:03 +0800 Subject: [PATCH 37/65] Update ServingService protobuf to include metadata without breaking changes. --- go.mod | 2 +- go.sum | 2 + protos/feast/serving/ServingService.proto | 32 +- .../protos/feast/serving/ServingService.pb.go | 398 ++++++++---------- 4 files changed, 187 insertions(+), 247 deletions(-) diff --git a/go.mod b/go.mod index 38ba2cbf85e..f116fffa831 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect golang.org/x/net v0.0.0-20200513185701-a91f0712d120 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9 // indirect - golang.org/x/tools v0.0.0-20200601175630-2caf76543d99 // indirect + golang.org/x/tools v0.0.0-20200604042327-9b20fe4cabe8 // indirect google.golang.org/grpc v1.29.1 google.golang.org/protobuf v1.24.0 // indirect gopkg.in/russross/blackfriday.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 7c2d4d57528..4be66277bcf 100644 --- a/go.sum +++ b/go.sum @@ -469,6 +469,8 @@ golang.org/x/tools v0.0.0-20200519205726-57a9e4404bf7/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc h1:6m2YO+AmBApbUOmhsghW+IfRyZOY4My4UYvQQrEpHfY= golang.org/x/tools v0.0.0-20200521211927-2b542361a4fc/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200601175630-2caf76543d99/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200604042327-9b20fe4cabe8 h1:8Xr1qwxn90MXYKftwNxIO2g4J+26naghxFS5rYiTZww= +golang.org/x/tools v0.0.0-20200604042327-9b20fe4cabe8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto index d839f5d442d..0a89c97015b 100644 --- a/protos/feast/serving/ServingService.proto +++ b/protos/feast/serving/ServingService.proto @@ -105,32 +105,26 @@ message GetOnlineFeaturesRequest { } message GetOnlineFeaturesResponse { - // Data records retrieved from the online feast store. - // Each record represents the data retrieved for an entity row in the request. - repeated Record records = 1; - - message Record { - // Map of field name to data fields stored in this data record. - // Each field represents an individual feature in the data record. - map fields = 1; + // Feature values retrieved from feast. + repeated FieldValues field_values = 1; + + message FieldValues { + // Map of feature or entity name to feature/entity values. + // Timestamps are not returned in this response. + map fields = 1; + // Map of feature or entity name to feature/entity statuses/metadata. + map statuses = 2; } - - message Field { - // Value of this field. - feast.types.Value value = 1; - // Status of this field. - FieldStatus status = 2; - } - + enum FieldStatus { // Status is unset for this field. INVALID = 0; - // Field value is present for this field and within maximum allowable range. + // Field value is present for this field and within maximum allowable range PRESENT = 1; // Values could be found for entity key within the maximum allowable range, but - // this field value is assigned a null value on ingestion into feast. + // this field value is assigned a value on ingestion into feast. NULL_VALUE = 2; // Entity key did not return any values as they do not exist in Feast. @@ -139,7 +133,7 @@ message GetOnlineFeaturesResponse { NOT_FOUND = 3; // Values could be found for entity key, but field values are outside the maximum - // allowable range: values are older than max age. + // allowable range. OUTSIDE_MAX_AGE = 4; } } diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index d89265e4740..6946233c28f 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -248,17 +248,17 @@ type GetOnlineFeaturesResponse_FieldStatus int32 const ( // Status is unset for this field. GetOnlineFeaturesResponse_INVALID GetOnlineFeaturesResponse_FieldStatus = 0 - // Field value is present for this field and within maximum allowable range. + // Field value is present for this field and within maximum allowable range GetOnlineFeaturesResponse_PRESENT GetOnlineFeaturesResponse_FieldStatus = 1 // Values could be found for entity key within the maximum allowable range, but - // this field value is assigned a null value on ingestion into feast. + // this field value is assigned a value on ingestion into feast. GetOnlineFeaturesResponse_NULL_VALUE GetOnlineFeaturesResponse_FieldStatus = 2 // Entity key did not return any values as they do not exist in Feast. // This could suggest that the feature values have not yet been ingested // into feast or the ingestion failed. GetOnlineFeaturesResponse_NOT_FOUND GetOnlineFeaturesResponse_FieldStatus = 3 // Values could be found for entity key, but field values are outside the maximum - // allowable range: values are older than max age. + // allowable range. GetOnlineFeaturesResponse_OUTSIDE_MAX_AGE GetOnlineFeaturesResponse_FieldStatus = 4 ) @@ -565,9 +565,8 @@ type GetOnlineFeaturesResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Data records retrieved from the online feast store. - // Each record represents the data retrieved for an entity row in the request. - Records []*GetOnlineFeaturesResponse_Record `protobuf:"bytes,1,rep,name=records,proto3" json:"records,omitempty"` + // Feature values retrieved from feast. + FieldValues []*GetOnlineFeaturesResponse_FieldValues `protobuf:"bytes,1,rep,name=field_values,json=fieldValues,proto3" json:"field_values,omitempty"` } func (x *GetOnlineFeaturesResponse) Reset() { @@ -602,9 +601,9 @@ func (*GetOnlineFeaturesResponse) Descriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4} } -func (x *GetOnlineFeaturesResponse) GetRecords() []*GetOnlineFeaturesResponse_Record { +func (x *GetOnlineFeaturesResponse) GetFieldValues() []*GetOnlineFeaturesResponse_FieldValues { if x != nil { - return x.Records + return x.FieldValues } return nil } @@ -1027,18 +1026,20 @@ func (x *GetOnlineFeaturesRequest_EntityRow) GetFields() map[string]*types.Value return nil } -type GetOnlineFeaturesResponse_Record struct { +type GetOnlineFeaturesResponse_FieldValues struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Map of field name to data fields stored in this data record. - // Each field represents an individual feature in the data record. - Fields map[string]*GetOnlineFeaturesResponse_Field `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Map of feature or entity name to feature/entity values. + // Timestamps are not returned in this response. + Fields map[string]*types.Value `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Map of feature or entity name to feature/entity statuses/metadata. + Statuses map[string]GetOnlineFeaturesResponse_FieldStatus `protobuf:"bytes,2,rep,name=statuses,proto3" json:"statuses,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3,enum=feast.serving.GetOnlineFeaturesResponse_FieldStatus"` } -func (x *GetOnlineFeaturesResponse_Record) Reset() { - *x = GetOnlineFeaturesResponse_Record{} +func (x *GetOnlineFeaturesResponse_FieldValues) Reset() { + *x = GetOnlineFeaturesResponse_FieldValues{} if protoimpl.UnsafeEnabled { mi := &file_feast_serving_ServingService_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -1046,13 +1047,13 @@ func (x *GetOnlineFeaturesResponse_Record) Reset() { } } -func (x *GetOnlineFeaturesResponse_Record) String() string { +func (x *GetOnlineFeaturesResponse_FieldValues) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetOnlineFeaturesResponse_Record) ProtoMessage() {} +func (*GetOnlineFeaturesResponse_FieldValues) ProtoMessage() {} -func (x *GetOnlineFeaturesResponse_Record) ProtoReflect() protoreflect.Message { +func (x *GetOnlineFeaturesResponse_FieldValues) ProtoReflect() protoreflect.Message { mi := &file_feast_serving_ServingService_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -1064,75 +1065,25 @@ func (x *GetOnlineFeaturesResponse_Record) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetOnlineFeaturesResponse_Record.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesResponse_Record) Descriptor() ([]byte, []int) { +// Deprecated: Use GetOnlineFeaturesResponse_FieldValues.ProtoReflect.Descriptor instead. +func (*GetOnlineFeaturesResponse_FieldValues) Descriptor() ([]byte, []int) { return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 0} } -func (x *GetOnlineFeaturesResponse_Record) GetFields() map[string]*GetOnlineFeaturesResponse_Field { +func (x *GetOnlineFeaturesResponse_FieldValues) GetFields() map[string]*types.Value { if x != nil { return x.Fields } return nil } -type GetOnlineFeaturesResponse_Field struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Value of this field. - Value *types.Value `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` - // Status of this field. - Status GetOnlineFeaturesResponse_FieldStatus `protobuf:"varint,2,opt,name=status,proto3,enum=feast.serving.GetOnlineFeaturesResponse_FieldStatus" json:"status,omitempty"` -} - -func (x *GetOnlineFeaturesResponse_Field) Reset() { - *x = GetOnlineFeaturesResponse_Field{} - if protoimpl.UnsafeEnabled { - mi := &file_feast_serving_ServingService_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *GetOnlineFeaturesResponse_Field) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetOnlineFeaturesResponse_Field) ProtoMessage() {} - -func (x *GetOnlineFeaturesResponse_Field) ProtoReflect() protoreflect.Message { - mi := &file_feast_serving_ServingService_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetOnlineFeaturesResponse_Field.ProtoReflect.Descriptor instead. -func (*GetOnlineFeaturesResponse_Field) Descriptor() ([]byte, []int) { - return file_feast_serving_ServingService_proto_rawDescGZIP(), []int{4, 1} -} - -func (x *GetOnlineFeaturesResponse_Field) GetValue() *types.Value { +func (x *GetOnlineFeaturesResponse_FieldValues) GetStatuses() map[string]GetOnlineFeaturesResponse_FieldStatus { if x != nil { - return x.Value + return x.Statuses } return nil } -func (x *GetOnlineFeaturesResponse_Field) GetStatus() GetOnlineFeaturesResponse_FieldStatus { - if x != nil { - return x.Status - } - return GetOnlineFeaturesResponse_INVALID -} - type DatasetSource_FileSource struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1255,141 +1206,146 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x8f, 0x04, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x49, - 0x0a, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x2f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, - 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, - 0x52, 0x07, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x1a, 0xc8, 0x01, 0x0a, 0x06, 0x52, 0x65, - 0x63, 0x6f, 0x72, 0x64, 0x12, 0x53, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, + 0x22, 0xdd, 0x04, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, + 0x0a, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, - 0x65, 0x63, 0x6f, 0x72, 0x64, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x1a, 0x69, 0x0a, 0x0b, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x44, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, - 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x7f, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x28, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x4c, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x89, 0x03, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x5b, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, - 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x53, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, - 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, - 0x0a, 0x09, 0x4e, 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, - 0x0f, 0x4f, 0x55, 0x54, 0x53, 0x49, 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, - 0x10, 0x04, 0x22, 0x9b, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, - 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x2e, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, - 0x65, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0e, 0x64, - 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x52, 0x0d, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x22, 0x40, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, - 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, - 0x6f, 0x62, 0x22, 0x35, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x36, 0x0a, 0x0e, 0x47, 0x65, 0x74, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, - 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, - 0x62, 0x22, 0xe2, 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x52, - 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, - 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1b, 0x0a, - 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, - 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, 0x64, 0x61, - 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, - 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, - 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x73, - 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, - 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, + 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x73, 0x12, 0x5e, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, + 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x1a, 0x71, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x4a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0x5b, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, + 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x53, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, + 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, + 0x4e, 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x4f, + 0x55, 0x54, 0x53, 0x49, 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, 0x10, 0x04, + 0x22, 0x9b, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x08, + 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, + 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0e, 0x64, 0x61, 0x74, + 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, + 0x0d, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x40, + 0x0a, 0x18, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, + 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, + 0x22, 0x35, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, + 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x36, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x4a, 0x6f, + 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, + 0xe2, 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x66, + 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, + 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, + 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, - 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, - 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x1a, 0x65, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, - 0x3a, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, - 0x0a, 0x64, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x64, - 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2a, 0x6f, 0x0a, - 0x10, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, - 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, - 0x00, 0x12, 0x1d, 0x0a, 0x19, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, - 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, - 0x12, 0x1c, 0x0a, 0x18, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, - 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x2a, 0x36, - 0x0a, 0x07, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x4a, 0x4f, 0x42, + 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x46, 0x6f, + 0x72, 0x6d, 0x61, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, + 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, + 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x1a, 0x65, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, + 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, + 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x64, 0x61, 0x74, + 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2a, 0x6f, 0x0a, 0x10, 0x46, + 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x1e, 0x0a, 0x1a, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, - 0x15, 0x0a, 0x11, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, - 0x4c, 0x4f, 0x41, 0x44, 0x10, 0x01, 0x2a, 0x68, 0x0a, 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, - 0x53, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4a, - 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, - 0x47, 0x10, 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, - 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x4a, - 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, - 0x2a, 0x3b, 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x17, - 0x0a, 0x13, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x49, 0x4e, - 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x41, 0x54, 0x41, 0x5f, - 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, 0x01, 0x32, 0x92, 0x03, - 0x0a, 0x0e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x6c, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, - 0x0a, 0x11, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, - 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, - 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, - 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x06, 0x47, - 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x12, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x42, 0x5e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, - 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, - 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x1d, 0x0a, 0x19, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1c, + 0x0a, 0x18, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x2a, 0x36, 0x0a, 0x07, + 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x4a, 0x4f, 0x42, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, + 0x11, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, + 0x41, 0x44, 0x10, 0x01, 0x2a, 0x68, 0x0a, 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, + 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, + 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, + 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x4f, 0x42, + 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x2a, 0x3b, + 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x17, 0x0a, 0x13, + 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x49, 0x4e, 0x56, 0x41, + 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, + 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, 0x01, 0x32, 0x92, 0x03, 0x0a, 0x0e, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, + 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x11, + 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x73, 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, + 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, + 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x06, 0x47, 0x65, 0x74, + 0x4a, 0x6f, 0x62, 0x12, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x42, 0x5e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1424,19 +1380,19 @@ var file_feast_serving_ServingService_proto_goTypes = []interface{}{ (*Job)(nil), // 14: feast.serving.Job (*DatasetSource)(nil), // 15: feast.serving.DatasetSource (*GetOnlineFeaturesRequest_EntityRow)(nil), // 16: feast.serving.GetOnlineFeaturesRequest.EntityRow - nil, // 17: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry - (*GetOnlineFeaturesResponse_Record)(nil), // 18: feast.serving.GetOnlineFeaturesResponse.Record - (*GetOnlineFeaturesResponse_Field)(nil), // 19: feast.serving.GetOnlineFeaturesResponse.Field - nil, // 20: feast.serving.GetOnlineFeaturesResponse.Record.FieldsEntry - (*DatasetSource_FileSource)(nil), // 21: feast.serving.DatasetSource.FileSource - (*timestamp.Timestamp)(nil), // 22: google.protobuf.Timestamp - (*types.Value)(nil), // 23: feast.types.Value + nil, // 17: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry + (*GetOnlineFeaturesResponse_FieldValues)(nil), // 18: feast.serving.GetOnlineFeaturesResponse.FieldValues + nil, // 19: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry + nil, // 20: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry + (*DatasetSource_FileSource)(nil), // 21: feast.serving.DatasetSource.FileSource + (*timestamp.Timestamp)(nil), // 22: google.protobuf.Timestamp + (*types.Value)(nil), // 23: feast.types.Value } var file_feast_serving_ServingService_proto_depIdxs = []int32{ 0, // 0: feast.serving.GetFeastServingInfoResponse.type:type_name -> feast.serving.FeastServingType 7, // 1: feast.serving.GetOnlineFeaturesRequest.features:type_name -> feast.serving.FeatureReference 16, // 2: feast.serving.GetOnlineFeaturesRequest.entity_rows:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow - 18, // 3: feast.serving.GetOnlineFeaturesResponse.records:type_name -> feast.serving.GetOnlineFeaturesResponse.Record + 18, // 3: feast.serving.GetOnlineFeaturesResponse.field_values:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues 7, // 4: feast.serving.GetBatchFeaturesRequest.features:type_name -> feast.serving.FeatureReference 15, // 5: feast.serving.GetBatchFeaturesRequest.dataset_source:type_name -> feast.serving.DatasetSource 14, // 6: feast.serving.GetBatchFeaturesResponse.job:type_name -> feast.serving.Job @@ -1449,10 +1405,10 @@ var file_feast_serving_ServingService_proto_depIdxs = []int32{ 22, // 13: feast.serving.GetOnlineFeaturesRequest.EntityRow.entity_timestamp:type_name -> google.protobuf.Timestamp 17, // 14: feast.serving.GetOnlineFeaturesRequest.EntityRow.fields:type_name -> feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry 23, // 15: feast.serving.GetOnlineFeaturesRequest.EntityRow.FieldsEntry.value:type_name -> feast.types.Value - 20, // 16: feast.serving.GetOnlineFeaturesResponse.Record.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.Record.FieldsEntry - 23, // 17: feast.serving.GetOnlineFeaturesResponse.Field.value:type_name -> feast.types.Value - 4, // 18: feast.serving.GetOnlineFeaturesResponse.Field.status:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldStatus - 19, // 19: feast.serving.GetOnlineFeaturesResponse.Record.FieldsEntry.value:type_name -> feast.serving.GetOnlineFeaturesResponse.Field + 19, // 16: feast.serving.GetOnlineFeaturesResponse.FieldValues.fields:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry + 20, // 17: feast.serving.GetOnlineFeaturesResponse.FieldValues.statuses:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry + 23, // 18: feast.serving.GetOnlineFeaturesResponse.FieldValues.FieldsEntry.value:type_name -> feast.types.Value + 4, // 19: feast.serving.GetOnlineFeaturesResponse.FieldValues.StatusesEntry.value:type_name -> feast.serving.GetOnlineFeaturesResponse.FieldStatus 3, // 20: feast.serving.DatasetSource.FileSource.data_format:type_name -> feast.serving.DataFormat 5, // 21: feast.serving.ServingService.GetFeastServingInfo:input_type -> feast.serving.GetFeastServingInfoRequest 8, // 22: feast.serving.ServingService.GetOnlineFeatures:input_type -> feast.serving.GetOnlineFeaturesRequest @@ -1620,19 +1576,7 @@ func file_feast_serving_ServingService_proto_init() { } } file_feast_serving_ServingService_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesResponse_Record); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_feast_serving_ServingService_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetOnlineFeaturesResponse_Field); i { + switch v := v.(*GetOnlineFeaturesResponse_FieldValues); i { case 0: return &v.state case 1: From d012ee70502e04e34385eef8247d4d71587167b1 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Thu, 4 Jun 2020 18:23:43 +0800 Subject: [PATCH 38/65] Revert Feast Serving's ServingService & ResponseJSONMapper to FieldValues --- .../feast/serving/service/ServingService.java | 2 +- .../serving/util/mappers/ResponseJSONMapper.java | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/ServingService.java b/serving/src/main/java/feast/serving/service/ServingService.java index 97a772c56af..1fe9840d594 100644 --- a/serving/src/main/java/feast/serving/service/ServingService.java +++ b/serving/src/main/java/feast/serving/service/ServingService.java @@ -56,7 +56,7 @@ GetFeastServingInfoResponse getFeastServingInfo( * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow}s to join the * retrieved values to. * @return {@link GetOnlineFeaturesResponse} with list of {@link - * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record} for each {@link + * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues} for each {@link * feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow} supplied. */ GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest getFeaturesRequest); diff --git a/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java b/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java index dbf06c12187..238df549513 100644 --- a/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java +++ b/serving/src/main/java/feast/serving/util/mappers/ResponseJSONMapper.java @@ -17,8 +17,7 @@ package feast.serving.util.mappers; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Field; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.ValueProto.Value; import java.util.List; import java.util.Map; @@ -29,18 +28,17 @@ public class ResponseJSONMapper { public static List> mapGetOnlineFeaturesResponse( GetOnlineFeaturesResponse response) { - return response.getRecordsList().stream() - .map(fieldValue -> convertToMap(fieldValue)) + return response.getFieldValuesList().stream() + .map(fieldValues -> convertFieldValuesToMap(fieldValues)) .collect(Collectors.toList()); } - private static Map convertToMap(Record record) { - return record.getFieldsMap().entrySet().stream() - .collect(Collectors.toMap(es -> es.getKey(), es -> extractField(es.getValue()))); + private static Map convertFieldValuesToMap(FieldValues fieldValues) { + return fieldValues.getFieldsMap().entrySet().stream() + .collect(Collectors.toMap(es -> es.getKey(), es -> extractValue(es.getValue()))); } - private static Object extractField(Field field) { - Value value = field.getValue(); + private static Object extractValue(Value value) { switch (value.getValCase().getNumber()) { case 1: return value.getBytesVal(); From 149a455cb982a0c5d54a7d104a4bfc62c0a70c05 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 5 Jun 2020 11:12:35 +0800 Subject: [PATCH 39/65] Revert serving's OnlineServingService to backwards compatible FieldValues API --- .../serving/service/OnlineServingService.java | 197 ++++++++---------- .../service/OnlineServingServiceTest.java | 161 +++++++------- 2 files changed, 172 insertions(+), 186 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index afcad319bf8..de4b48f77ea 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -19,9 +19,8 @@ import com.google.protobuf.Duration; import feast.proto.serving.ServingAPIProto.*; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Field; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.FeatureRowProto.FeatureRow; import feast.proto.types.ValueProto.Value; import feast.serving.specs.CachedSpecService; @@ -64,16 +63,20 @@ public GetFeastServingInfoResponse getFeastServingInfo( public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest request) { try (Scope scope = tracer.buildSpan("getOnlineFeatures").startActive(true)) { List entityRows = request.getEntityRowsList(); - boolean includeMetadata = request.getIncludeMetadataInResponse(); - // Collect the response fields for each entity row in entityFieldsMap. - Map> entityFieldsMap = + // Collect the feature/entity value for each entity row in entityValueMap + Map> entityValuesMap = + entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); + // Collect the feature/entity status metadata for each entity row in entityValueMap + Map> entityStatusesMap = entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); if (!request.getOmitEntitiesInResponse()) { // Add entity row's fields as response fields entityRows.forEach( entityRow -> { - entityFieldsMap.get(entityRow).putAll(this.unpackFields(entityRow, includeMetadata)); + Map valueMap = entityRow.getFieldsMap(); + entityValuesMap.get(entityRow).putAll(valueMap); + entityStatusesMap.get(entityRow).putAll(this.getMetadataMap(valueMap, false, false)); }); } @@ -82,6 +85,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ for (FeatureSetRequest featureSetRequest : featureSetRequests) { // Pull feature rows for given entity rows from the feature/featureset specified in feature // set request. + // from the configured online List featureRows = retriever.getOnlineFeatures(entityRows, featureSetRequest); // Check that feature row returned corresponds to a given entity row. if (featureRows.size() != entityRows.size()) { @@ -95,140 +99,115 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ for (var i = 0; i < entityRows.size(); i++) { FeatureRow featureRow = featureRows.get(i); EntityRow entityRow = entityRows.get(i); - // Unpack feature response fields from feature row - Map fields = - this.unpackFields(featureRow, entityRow, featureSetRequest, includeMetadata); - // Merge feature response fields into entityFieldsMap - entityFieldsMap - .get(entityRow) - .putAll( - fields.entrySet().stream() - .collect( - Collectors.toMap( - es -> RefUtil.generateFeatureStringRef(es.getKey()), - es -> es.getValue()))); + // Unpack feature field values and merge into entityValueMap + boolean isStaleValues = this.isStale(featureSetRequest, entityRow, featureRow); + Map valueMap = + this.unpackValueMap(featureRow, featureSetRequest, isStaleValues); + entityValuesMap.get(entityRow).putAll(valueMap); + + // Generate metadata for feature values and merge into entityFieldsMap + boolean isNotFound = featureRow == null; + Map statusMap = + this.getMetadataMap(valueMap, isNotFound, isStaleValues); + entityStatusesMap.get(entityRow).putAll(statusMap); // Populate metrics - this.populateStaleKeyCountMetrics(fields, featureSetRequest); + this.populateStaleKeyCountMetrics(statusMap, featureSetRequest); } this.populateRequestCountMetrics(featureSetRequest); } - // Build response records from entityFieldsMap. - // Records should be in the same order as the entityRows provided by the user. - List records = + // Build response field values from entityValuesMap and entityStatusesMap + // Reponse field values should be in the same order as the entityRows provided by the user. + List fieldValuesList = entityRows.stream() .map( - entityRow -> - Record.newBuilder().putAllFields(entityFieldsMap.get(entityRow)).build()) + entityRow -> { + return FieldValues.newBuilder() + .putAllFields(entityValuesMap.get(entityRow)) + .putAllStatuses(entityStatusesMap.get(entityRow)) + .build(); + }) .collect(Collectors.toList()); - return GetOnlineFeaturesResponse.newBuilder().addAllRecords(records).build(); + return GetOnlineFeaturesResponse.newBuilder().addAllFieldValues(fieldValuesList).build(); } } /** - * Unpack response fields from the given entity row's fields. - * - * @param entityRow to unpack for response fields - * @param includeMetadata whether metadata should be included in the response fields - * @return Map mapping of name of field to response field. - */ - private Map unpackFields(EntityRow entityRow, boolean includeMetadata) { - return entityRow.getFieldsMap().entrySet().stream() - .collect( - Collectors.toMap( - es -> es.getKey(), - es -> { - Field.Builder field = Field.newBuilder().setValue(es.getValue()); - if (includeMetadata) { - field.setStatus(FieldStatus.PRESENT); - } - return field.build(); - })); - } - - /** - * Unpack response fields using data from the given feature row for features specified in the - * given feature set request. + * Unpack feature values using data from the given feature row for features specified in the given + * feature set request. * - * @param featureRow to unpack for response fields. - * @param entityRow for which the feature row was retrieved for. + * @param featureRow to unpack for feature values. * @param featureSetRequest for which the feature row was retrieved. - * @param includeMetadata whether metadata should be included in the response fields - * @return Map mapping of feature ref to response field + * @param isStaleValues whether which the feature row contains stale values. + * @return valueMap mapping string feature reference to feature valuyesaq */ - private Map unpackFields( - FeatureRow featureRow, - EntityRow entityRow, - FeatureSetRequest featureSetRequest, - boolean includeMetadata) { + private Map unpackValueMap( + FeatureRow featureRow, FeatureSetRequest featureSetRequest, boolean isStaleValues) { + Map valueMap = new HashMap<>(); // In order to return values containing the same feature references provided by the user, // we reuse the feature references in the request as the keys in field builder map - Map refsByName = featureSetRequest.getFeatureRefsByName(); - Map fields = new HashMap<>(); - - boolean hasStaleValues = this.isStale(featureSetRequest, entityRow, featureRow); + Map nameRefMap = featureSetRequest.getFeatureRefsByName(); if (featureRow != null) { - // unpack feature row's field's values & populate field map - Map featureFields = + // unpack feature row's feature values and populate value map + Map featureValueMap = featureRow.getFieldsList().stream() - .filter(featureRowField -> refsByName.containsKey(featureRowField.getName())) + .filter(featureRowField -> nameRefMap.containsKey(featureRowField.getName())) .collect( Collectors.toMap( - featureRowField -> refsByName.get(featureRowField.getName()), featureRowField -> { - // omit stale feature values. - if (hasStaleValues) { - return Field.newBuilder().setValue(Value.newBuilder().build()); + FeatureReference featureRef = nameRefMap.get(featureRowField.getName()); + return RefUtil.generateFeatureStringRef(featureRef); + }, + featureRowField -> { + // drop stale feature values. + if (isStaleValues) { + return Value.newBuilder().build(); } else { - return Field.newBuilder().setValue(featureRowField.getValue()); + return featureRowField.getValue(); } })); - fields.putAll(featureFields); + valueMap.putAll(featureValueMap); } - // create empty response fields for features specified in request but not present in feature - // row. - Set missingFeatures = new HashSet<>(refsByName.values()); - missingFeatures.removeAll(fields.keySet()); - missingFeatures.forEach( - ref -> fields.put(ref, Field.newBuilder().setValue(Value.newBuilder().build()))); + // create empty values for features specified in request but not present in feature row. + Set missingFeatures = + nameRefMap.values().stream() + .map(ref -> RefUtil.generateFeatureStringRef(ref)) + .collect(Collectors.toSet()); + missingFeatures.removeAll(valueMap.keySet()); + missingFeatures.forEach(refString -> valueMap.put(refString, Value.newBuilder().build())); + + return valueMap; + } - // attach metadata to the feature response fields & build response field - return fields.entrySet().stream() + /** + * Generate Field level Status metadata for the given valueMap. + * + * @param isNotFound whether the given valueMap represents values that were not found in the + * online retriever. + * @param isStaleValues whether the given valueMap contains stale values. + * @return a 1:1 map containing field status metadata instead of values in the valueMap. + */ + private Map getMetadataMap( + Map valueMap, boolean isNotFound, boolean isStaleValues) { + return valueMap.entrySet().stream() .collect( Collectors.toMap( es -> es.getKey(), es -> { - Field.Builder field = es.getValue(); - if (includeMetadata) { - field = this.attachMetadata(field, featureRow, hasStaleValues); + Value fieldValue = es.getValue(); + if (isNotFound) { + return FieldStatus.NOT_FOUND; + } else if (isStaleValues) { + return FieldStatus.OUTSIDE_MAX_AGE; + } else if (fieldValue.getValCase().equals(Value.ValCase.VAL_NOT_SET)) { + return FieldStatus.NULL_VALUE; } - return field.build(); + return FieldStatus.PRESENT; })); } - /** - * Attach metadata to the given response field. Attaches field status to the response providing - * metadata for the field. - * - * @param featureRow where the field was unpacked from. - * @param hasStaleValue whether the field contains a stale value - */ - private Field.Builder attachMetadata( - Field.Builder field, FeatureRow featureRow, boolean hasStaleValue) { - FieldStatus fieldStatus = FieldStatus.PRESENT; - if (featureRow == null) { - fieldStatus = FieldStatus.NOT_FOUND; - } else if (hasStaleValue) { - fieldStatus = FieldStatus.OUTSIDE_MAX_AGE; - } else if (field.getValue().getValCase() == Value.ValCase.VAL_NOT_SET) { - fieldStatus = FieldStatus.NULL_VALUE; - } - - return field.setStatus(fieldStatus); - } - /** * Determine if the feature data in the given feature row is considered stale. Data is considered * to be stale when difference ingestion time set in feature row and the retrieval time set in @@ -259,16 +238,16 @@ private boolean isStale( } private void populateStaleKeyCountMetrics( - Map fields, FeatureSetRequest featureSetRequest) { + Map statusMap, FeatureSetRequest featureSetRequest) { String project = featureSetRequest.getSpec().getProject(); - fields + statusMap .entrySet() .forEach( es -> { - FeatureReference ref = es.getKey(); - Field field = es.getValue(); - if (field.getStatus() == FieldStatus.OUTSIDE_MAX_AGE) { - Metrics.staleKeyCount.labels(project, RefUtil.generateFeatureStringRef(ref)).inc(); + String featureRefString = es.getKey(); + FieldStatus status = es.getValue(); + if (status == FieldStatus.OUTSIDE_MAX_AGE) { + Metrics.staleKeyCount.labels(project, featureRefString).inc(); } }); } diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 8fe0f4c0500..19691e5e7f0 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -30,9 +30,8 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Field; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; -import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.Record; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.FeatureRowProto.FeatureRow; import feast.proto.types.FieldProto; import feast.proto.types.ValueProto.Value; @@ -70,7 +69,6 @@ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() .setOmitEntitiesInResponse(false) - .setIncludeMetadataInResponse(true) .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) .addFeatures( FeatureReference.newBuilder().setName("feature2").setProject("project").build()) @@ -147,18 +145,28 @@ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() - .addRecords( - Record.newBuilder() - .putFields("entity1", intField(1)) - .putFields("entity2", strField("a")) - .putFields("feature1", intField(1)) - .putFields("project/feature2", intField(1))) - .addRecords( - Record.newBuilder() - .putFields("entity1", intField(2)) - .putFields("entity2", strField("b")) - .putFields("feature1", intField(2)) - .putFields("project/feature2", intField(2))) + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(1)) + .putStatuses("entity1", FieldStatus.PRESENT) + .putFields("entity2", strValue("a")) + .putStatuses("entity2", FieldStatus.PRESENT) + .putFields("feature1", intValue(1)) + .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("project/feature2", intValue(1)) + .putStatuses("project/feature2", FieldStatus.PRESENT) + .build()) + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(2)) + .putStatuses("entity1", FieldStatus.PRESENT) + .putFields("entity2", strValue("b")) + .putStatuses("entity2", FieldStatus.PRESENT) + .putFields("feature1", intValue(2)) + .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("project/feature2", intValue(2)) + .putStatuses("project/feature2", FieldStatus.PRESENT) + .build()) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); @@ -217,28 +225,28 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() - .addRecords( - Record.newBuilder() - .putFields("entity1", intField(1)) - .putFields("entity2", strField("a")) - .putFields("feature1", intField(1)) - .putFields("project/feature2", intField(1))) - .addRecords( - Record.newBuilder() - .putFields("entity1", intField(2)) - .putFields("entity2", strField("b")) - .putFields( - "feature1", - Field.newBuilder() - .setValue(Value.newBuilder().build()) - .setStatus(FieldStatus.NOT_FOUND) - .build()) - .putFields( - "project/feature2", - Field.newBuilder() - .setValue(Value.newBuilder().build()) - .setStatus(FieldStatus.NOT_FOUND) - .build())) + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(1)) + .putStatuses("entity1", FieldStatus.PRESENT) + .putFields("entity2", strValue("a")) + .putStatuses("entity2", FieldStatus.PRESENT) + .putFields("feature1", intValue(1)) + .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("project/feature2", intValue(1)) + .putStatuses("project/feature2", FieldStatus.PRESENT) + .build()) + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(2)) + .putStatuses("entity1", FieldStatus.PRESENT) + .putFields("entity2", strValue("b")) + .putStatuses("entity2", FieldStatus.PRESENT) + .putFields("feature1", Value.newBuilder().build()) + .putStatuses("feature1", FieldStatus.NOT_FOUND) + .putFields("project/feature2", Value.newBuilder().build()) + .putStatuses("project/feature2", FieldStatus.NOT_FOUND) + .build()) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); @@ -330,28 +338,27 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() - .addRecords( - Record.newBuilder() - .putFields("entity1", intField(1)) - .putFields("entity2", strField("a")) - .putFields("feature1", intField(1)) - .putFields("project/feature2", intField(1))) - .addRecords( - Record.newBuilder() - .putFields("entity1", intField(2)) - .putFields("entity2", strField("b")) - .putFields( - "feature1", - Field.newBuilder() - .setValue(Value.newBuilder().build()) - .setStatus(FieldStatus.OUTSIDE_MAX_AGE) - .build()) - .putFields( - "project/feature2", - Field.newBuilder() - .setValue(Value.newBuilder().build()) - .setStatus(FieldStatus.OUTSIDE_MAX_AGE) - .build()) + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(1)) + .putStatuses("entity1", FieldStatus.PRESENT) + .putFields("entity2", strValue("a")) + .putStatuses("entity2", FieldStatus.PRESENT) + .putFields("feature1", intValue(1)) + .putStatuses("feature1", FieldStatus.PRESENT) + .putFields("project/feature2", intValue(1)) + .putStatuses("project/feature2", FieldStatus.PRESENT) + .build()) + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(2)) + .putStatuses("entity1", FieldStatus.PRESENT) + .putFields("entity2", strValue("b")) + .putStatuses("entity2", FieldStatus.PRESENT) + .putFields("feature1", Value.newBuilder().build()) + .putStatuses("feature1", FieldStatus.OUTSIDE_MAX_AGE) + .putFields("project/feature2", Value.newBuilder().build()) + .putStatuses("project/feature2", FieldStatus.OUTSIDE_MAX_AGE) .build()) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); @@ -439,16 +446,24 @@ public void shouldFilterOutUndesiredRows() { GetOnlineFeaturesResponse expected = GetOnlineFeaturesResponse.newBuilder() - .addRecords( - Record.newBuilder() - .putFields("entity1", intField(1)) - .putFields("entity2", strField("a")) - .putFields("feature1", intField(1))) - .addRecords( - Record.newBuilder() - .putFields("entity1", intField(2)) - .putFields("entity2", strField("b")) - .putFields("feature1", intField(2))) + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(1)) + .putStatuses("entity1", FieldStatus.PRESENT) + .putFields("entity2", strValue("a")) + .putStatuses("entity2", FieldStatus.PRESENT) + .putFields("feature1", intValue(1)) + .putStatuses("feature1", FieldStatus.PRESENT) + .build()) + .addFieldValues( + FieldValues.newBuilder() + .putFields("entity1", intValue(2)) + .putStatuses("entity1", FieldStatus.PRESENT) + .putFields("entity2", strValue("b")) + .putStatuses("entity2", FieldStatus.PRESENT) + .putFields("feature1", intValue(2)) + .putStatuses("feature1", FieldStatus.PRESENT) + .build()) .build(); GetOnlineFeaturesResponse actual = onlineServingService.getOnlineFeatures(request); assertThat(actual, equalTo(expected)); @@ -462,14 +477,6 @@ private Value strValue(String val) { return Value.newBuilder().setStringVal(val).build(); } - private Field intField(int val) { - return Field.newBuilder().setValue(intValue(val)).setStatus(FieldStatus.PRESENT).build(); - } - - private Field strField(String val) { - return Field.newBuilder().setValue(strValue(val)).setStatus(FieldStatus.PRESENT).build(); - } - private FeatureSetSpec getFeatureSetSpec() { return FeatureSetSpec.newBuilder() .setName("featureSet") From 253ca2834fec2a12ad1feb90a53d597646028d4f Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 5 Jun 2020 11:30:31 +0800 Subject: [PATCH 40/65] Revert Java SDK's FeastClient to backwards compatible FieldStatus API. --- .../main/java/com/gojek/feast/FeastClient.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index 0a94aa87f2d..ae7d9ca8944 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -146,16 +146,16 @@ public List getOnlineFeatures( .setIncludeMetadataInResponse(includeMetadataInResponse) .build()); - return response.getRecordsList().stream() + return response.getFieldValuesList().stream() .map( - record -> { + fieldValues -> { Row row = Row.create(); - record - .getFieldsMap() - .forEach( - (name, field) -> { - row.set(name, field.getValue(), field.getStatus()); - }); + for (String fieldName : fieldValues.getFieldsMap().keySet()) { + row.set( + fieldName, + fieldValues.getFieldsMap().get(fieldName), + fieldValues.getStatusesMap().get(fieldName)); + } return row; }) .collect(Collectors.toList()); From 53ad9a04140515d956de2033f66077fb5633438b Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 5 Jun 2020 11:48:05 +0800 Subject: [PATCH 41/65] Revert Go SDK's client.go to backwards compatible FieldStatus API. --- sdk/go/client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index 30b3dc4bf09..63d39aaec0b 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -60,9 +60,9 @@ func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeatures } // strip projects from to projects - for _, fieldValue := range resp.GetFieldValues() { - stripFields := make(map[string]*types.Value, len(fieldValue.Fields)) - for refStr, value := range fieldValue.Fields { + for _, fieldValues := range resp.GetFieldValues() { + stripFields := make(map[string]*types.Value, len(fieldValues.Fields)) + for refStr, value := range fieldValues.Fields { _, isEntity := entityRefs[refStr] if !isEntity { // is feature ref featureRef, err := parseFeatureRef(refStr, true) @@ -73,7 +73,7 @@ func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeatures } stripFields[refStr] = value } - fieldValue.Fields = stripFields + fieldValues.Fields = stripFields } return &OnlineFeaturesResponse{RawResponse: resp}, err From 06bcca745a8124c8330a3c6097a5c45140ce4dac Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 5 Jun 2020 12:51:48 +0800 Subject: [PATCH 42/65] Fix Java SDK unit tests --- .../src/main/java/com/gojek/feast/Row.java | 8 ++++-- .../java/com/gojek/feast/FeastClientTest.java | 28 +++++++++++++------ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/sdk/java/src/main/java/com/gojek/feast/Row.java b/sdk/java/src/main/java/com/gojek/feast/Row.java index 3c0b7ab5a86..51f820e320a 100644 --- a/sdk/java/src/main/java/com/gojek/feast/Row.java +++ b/sdk/java/src/main/java/com/gojek/feast/Row.java @@ -57,6 +57,10 @@ public Row setEntityTimestamp(String dateTime) { return this; } + public Row set(String fieldName, Object value) { + return this.set(fieldName, value, FieldStatus.PRESENT); + } + public Row set(String fieldName, Object value, FieldStatus status) { String valueType = value.getClass().getCanonicalName(); switch (valueType) { @@ -121,11 +125,11 @@ public byte[] getByte(String fieldName) { return getValue(fieldName).map(Value::getBytesVal).map(ByteString::toByteArray).orElse(null); } - public Map getFieldStatuses() { + public Map getStatuses() { return fieldStatuses; } - public FieldStatus getFieldStatus(String fieldName) { + public FieldStatus getStatus(String fieldName) { return fieldStatuses.get(fieldName); } diff --git a/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java b/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java index 717792244ee..b86d2ff31fe 100644 --- a/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java +++ b/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java @@ -25,6 +25,7 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse; +import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.serving.ServingServiceGrpc.ServingServiceImplBase; import feast.proto.types.ValueProto.Value; @@ -98,6 +99,17 @@ public void shouldGetOnlineFeatures() { put("driver_id", intValue(1)); put("driver:name", strValue("david")); put("rating", intValue(3)); + put("null_value", Value.newBuilder().build()); + } + }); + assertEquals( + rows.get(0).getStatuses(), + new HashMap() { + { + put("driver_id", FieldStatus.PRESENT); + put("driver:name", FieldStatus.PRESENT); + put("rating", FieldStatus.PRESENT); + put("null_value", FieldStatus.NULL_VALUE); } }); } @@ -124,14 +136,14 @@ private static GetOnlineFeaturesResponse getFakeResponse() { return GetOnlineFeaturesResponse.newBuilder() .addFieldValues( FieldValues.newBuilder() - .putAllFields( - new HashMap() { - { - put("driver_id", intValue(1)); - put("driver:name", strValue("david")); - put("rating", intValue(3)); - } - }) + .putFields("driver_id", intValue(1)) + .putStatuses("driver_id", FieldStatus.PRESENT) + .putFields("driver:name", strValue("david")) + .putStatuses("driver:name", FieldStatus.PRESENT) + .putFields("rating", intValue(3)) + .putStatuses("rating", FieldStatus.PRESENT) + .putFields("null_value", Value.newBuilder().build()) + .putStatuses("null_value", FieldStatus.NULL_VALUE) .build()) .build(); } From d9332adaeafc6f2c554c24a89f82a38358df34d8 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 5 Jun 2020 13:37:45 +0800 Subject: [PATCH 43/65] Added .Statuses() method to GO Sdk's OnlineFeaturesResponse to get metadata --- sdk/go/request.go | 8 +------- sdk/go/response.go | 40 +++++++++++++++++++++++++--------------- sdk/go/types.go | 11 +---------- 3 files changed, 27 insertions(+), 32 deletions(-) diff --git a/sdk/go/request.go b/sdk/go/request.go index 3fd9aa2920f..52e044b2656 100644 --- a/sdk/go/request.go +++ b/sdk/go/request.go @@ -4,8 +4,6 @@ import ( "fmt" "github.com/feast-dev/feast/sdk/go/protos/feast/serving" "strings" - - "github.com/feast-dev/feast/sdk/go/protos/feast/types" ) var ( @@ -45,12 +43,8 @@ func (r OnlineFeaturesRequest) buildRequest() (*serving.GetOnlineFeaturesRequest // build request entity rows from native entities entityRows := make([]*serving.GetOnlineFeaturesRequest_EntityRow, len(r.Entities)) for i, entity := range r.Entities { - fieldMap := make(map[string]*types.Value) - for key, field := range entity { - fieldMap[key] = field.Value - } entityRows[i] = &serving.GetOnlineFeaturesRequest_EntityRow{ - Fields: fieldMap, + Fields: entity, } } diff --git a/sdk/go/response.go b/sdk/go/response.go index 5ee87537e8b..7fa50761b69 100644 --- a/sdk/go/response.go +++ b/sdk/go/response.go @@ -24,31 +24,41 @@ type OnlineFeaturesResponse struct { // Rows retrieves the result of the request as a list of Rows. func (r OnlineFeaturesResponse) Rows() []Row { - rows := make([]Row, len(r.RawResponse.Records)) - for i, records := range r.RawResponse.Records { - rows[i] = records.Fields + rows := make([]Row, len(r.RawResponse.FieldValues)) + for i, fieldValues := range r.RawResponse.FieldValues { + rows[i] = fieldValues.Fields } return rows } +// Statuses retrieves field level status metadata for each row in Rows(). +// Each status map returned maps status 1:1 to each returned row from Rows() +func (r OnlineFeaturesResponse) Statuses() []map[string]serving.GetOnlineFeaturesResponse_FieldStatus { + statuses := make([]map[string]serving.GetOnlineFeaturesResponse_FieldStatus, len(r.RawResponse.FieldValues)) + for i, fieldValues := range r.RawResponse.FieldValues { + statuses[i] = fieldValues.Statuses + } + return statuses +} + // Int64Arrays retrieves the result of the request as a list of int64 slices. Any missing values will be filled // with the missing values provided. func (r OnlineFeaturesResponse) Int64Arrays(order []string, fillNa []int64) ([][]int64, error) { - rows := make([][]int64, len(r.RawResponse.Records)) + rows := make([][]int64, len(r.RawResponse.FieldValues)) if len(fillNa) != len(order) { return nil, fmt.Errorf(ErrLengthMismatch, len(fillNa), len(order)) } - for i, record := range r.RawResponse.Records { + for i, fieldValues := range r.RawResponse.FieldValues { rows[i] = make([]int64, len(order)) for j, fname := range order { - field, exists := record.Fields[fname] + value, exists := fieldValues.Fields[fname] if !exists { return nil, fmt.Errorf(ErrFeatureNotFound, fname) } - val := field.Value.GetVal() - if val == nil { + valType := value.GetVal() + if valType == nil { rows[i][j] = fillNa[j] - } else if int64Val, ok := val.(*types.Value_Int64Val); ok { + } else if int64Val, ok := valType.(*types.Value_Int64Val); ok { rows[i][j] = int64Val.Int64Val } else { return nil, fmt.Errorf(ErrTypeMismatch, "int64") @@ -61,21 +71,21 @@ func (r OnlineFeaturesResponse) Int64Arrays(order []string, fillNa []int64) ([][ // Float64Arrays retrieves the result of the request as a list of float64 slices. Any missing values will be filled // with the missing values provided. func (r OnlineFeaturesResponse) Float64Arrays(order []string, fillNa []float64) ([][]float64, error) { - rows := make([][]float64, len(r.RawResponse.Records)) + rows := make([][]float64, len(r.RawResponse.FieldValues)) if len(fillNa) != len(order) { return nil, fmt.Errorf(ErrLengthMismatch, len(fillNa), len(order)) } - for i, records := range r.RawResponse.Records { + for i, records := range r.RawResponse.FieldValues { rows[i] = make([]float64, len(order)) for j, fname := range order { - field, exists := records.Fields[fname] + value, exists := records.Fields[fname] if !exists { return nil, fmt.Errorf(ErrFeatureNotFound, fname) } - val := field.Value.GetVal() - if val == nil { + valType := value.GetVal() + if valType == nil { rows[i][j] = fillNa[j] - } else if doubleVal, ok := val.(*types.Value_DoubleVal); ok { + } else if doubleVal, ok := valType.(*types.Value_DoubleVal); ok { rows[i][j] = doubleVal.DoubleVal } else { return nil, fmt.Errorf(ErrTypeMismatch, "float64") diff --git a/sdk/go/types.go b/sdk/go/types.go index 0117a9ecf1f..c2bc43e9c37 100644 --- a/sdk/go/types.go +++ b/sdk/go/types.go @@ -1,13 +1,12 @@ package feast import ( - "github.com/feast-dev/feast/sdk/go/protos/feast/serving" "github.com/feast-dev/feast/sdk/go/protos/feast/types" "github.com/golang/protobuf/proto" ) // Row is a map of fields -type Row map[string]*serving.GetOnlineFeaturesResponse_Field +type Row map[string]*types.Value func (r Row) equalTo(other Row) bool { for k, field := range r { @@ -54,11 +53,3 @@ func BoolVal(val bool) *types.Value { func BytesVal(val []byte) *types.Value { return &types.Value{Val: &types.Value_BytesVal{BytesVal: val}} } - -// constructs a response field from the given value -func Field(value *types.Value) *serving.GetOnlineFeaturesResponse_Field { - return &serving.GetOnlineFeaturesResponse_Field{ - Value: value, - Status: serving.GetOnlineFeaturesResponse_PRESENT, - } -} From 5141a1ed373fbb8fc1cc2be9ccd01cfd23290cd7 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 5 Jun 2020 15:34:40 +0800 Subject: [PATCH 44/65] Fixed Java SDK not stripping project from field values and statuses --- .../java/com/gojek/feast/FeastClient.java | 6 ++++- .../java/com/gojek/feast/FeastClientTest.java | 26 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index ae7d9ca8944..502094ce4c7 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -151,8 +151,12 @@ public List getOnlineFeatures( fieldValues -> { Row row = Row.create(); for (String fieldName : fieldValues.getFieldsMap().keySet()) { + if (entityRefs.contains(fieldName)) continue; + // Strip project from string Feature References from returned from serving + FeatureReference featureRef = RequestUtil.parseFeatureRef(fieldName, true).build(); + String stripFieldName = RequestUtil.renderFeatureRef(featureRef); row.set( - fieldName, + stripFieldName, fieldValues.getFieldsMap().get(fieldName), fieldValues.getStatusesMap().get(fieldName)); } diff --git a/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java b/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java index b86d2ff31fe..5cb756f8de6 100644 --- a/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java +++ b/sdk/java/src/test/java/com/gojek/feast/FeastClientTest.java @@ -54,8 +54,9 @@ public class FeastClientTest { public void getOnlineFeatures( GetOnlineFeaturesRequest request, StreamObserver responseObserver) { + if (!request.equals(FeastClientTest.getFakeRequest())) { - responseObserver.onError(Status.UNKNOWN.asRuntimeException()); + responseObserver.onError(Status.FAILED_PRECONDITION.asRuntimeException()); } responseObserver.onNext(FeastClientTest.getFakeResponse()); @@ -87,7 +88,7 @@ public void setup() throws Exception { public void shouldGetOnlineFeatures() { List rows = this.client.getOnlineFeatures( - Arrays.asList("driver:name", "rating"), + Arrays.asList("driver:name", "rating", "null_value"), Arrays.asList( Row.create().set("driver_id", 1).setEntityTimestamp(Instant.ofEpochSecond(100))), "driver_project"); @@ -125,6 +126,11 @@ private static GetOnlineFeaturesRequest getFakeRequest() { .build()) .addFeatures( FeatureReference.newBuilder().setProject("driver_project").setName("rating").build()) + .addFeatures( + FeatureReference.newBuilder() + .setProject("driver_project") + .setName("null_value") + .build()) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -136,14 +142,14 @@ private static GetOnlineFeaturesResponse getFakeResponse() { return GetOnlineFeaturesResponse.newBuilder() .addFieldValues( FieldValues.newBuilder() - .putFields("driver_id", intValue(1)) - .putStatuses("driver_id", FieldStatus.PRESENT) - .putFields("driver:name", strValue("david")) - .putStatuses("driver:name", FieldStatus.PRESENT) - .putFields("rating", intValue(3)) - .putStatuses("rating", FieldStatus.PRESENT) - .putFields("null_value", Value.newBuilder().build()) - .putStatuses("null_value", FieldStatus.NULL_VALUE) + .putFields("driver_project/driver_id", intValue(1)) + .putStatuses("driver_project/driver_id", FieldStatus.PRESENT) + .putFields("driver_project/driver:name", strValue("david")) + .putStatuses("driver_project/driver:name", FieldStatus.PRESENT) + .putFields("driver_project/rating", intValue(3)) + .putStatuses("driver_project/rating", FieldStatus.PRESENT) + .putFields("driver_project/null_value", Value.newBuilder().build()) + .putStatuses("driver_project/null_value", FieldStatus.NULL_VALUE) .build()) .build(); } From 059ea9fd35bf86964775421b0e9c039e384ae79a Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 5 Jun 2020 15:36:14 +0800 Subject: [PATCH 45/65] Fixed Go SDK's client.go not stripping project from field values and statuses --- sdk/go/client.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index 63d39aaec0b..7c44886a8a2 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -59,21 +59,26 @@ func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeatures } } - // strip projects from to projects + // strip projects from feature refs recieved for _, fieldValues := range resp.GetFieldValues() { stripFields := make(map[string]*types.Value, len(fieldValues.Fields)) - for refStr, value := range fieldValues.Fields { + stripStatuses := make(map[string]serving.GetOnlineFeaturesResponse_FieldStatus, len(fieldValues.Statuses)) + for refStr, _ := range fieldValues.Fields { _, isEntity := entityRefs[refStr] + stripRefStr := refStr if !isEntity { // is feature ref featureRef, err := parseFeatureRef(refStr, true) if err != nil { return nil, err } - refStr = toFeatureRefStr(featureRef) + stripRefStr = toFeatureRefStr(featureRef) } - stripFields[refStr] = value + + stripFields[stripRefStr] = fieldValues.Fields[refStr] + stripStatuses[stripRefStr] = fieldValues.Statuses[refStr] } fieldValues.Fields = stripFields + fieldValues.Statuses = stripStatuses } return &OnlineFeaturesResponse{RawResponse: resp}, err From 8d5d0311fa132786f7582beb2a110731e759adba Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 5 Jun 2020 15:36:35 +0800 Subject: [PATCH 46/65] Fixed Go SDK's Unit tests --- sdk/go/client.go | 3 +- sdk/go/client_test.go | 13 +++++ sdk/go/request_test.go | 6 +- sdk/go/response_test.go | 55 ++++++++++++------- .../java/com/gojek/feast/FeastClient.java | 18 +++--- 5 files changed, 63 insertions(+), 32 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index 7c44886a8a2..617c37a6f10 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -77,8 +77,7 @@ func (fc *GrpcClient) GetOnlineFeatures(ctx context.Context, req *OnlineFeatures stripFields[stripRefStr] = fieldValues.Fields[refStr] stripStatuses[stripRefStr] = fieldValues.Statuses[refStr] } - fieldValues.Fields = stripFields - fieldValues.Statuses = stripStatuses + fieldValues.Fields, fieldValues.Statuses = stripFields, stripStatuses } return &OnlineFeaturesResponse{RawResponse: resp}, err diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 1ad6629e9af..49fbe83266c 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -27,6 +27,7 @@ func TestGetOnlineFeatures(t *testing.T) { Features: []string{ "driver:rating", "rating", + "null_value", }, Entities: []Row{ {"driver_id": Int64Val(1)}, @@ -41,6 +42,12 @@ func TestGetOnlineFeatures(t *testing.T) { Fields: map[string]*types.Value{ "driver_project/driver:rating": Int64Val(1), "driver_project/rating": Int64Val(1), + "driver_project/null_value": {}, + }, + Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{ + "driver_project/driver:rating": serving.GetOnlineFeaturesResponse_PRESENT, + "driver_project/rating": serving.GetOnlineFeaturesResponse_PRESENT, + "driver_project/null_value": serving.GetOnlineFeaturesResponse_NULL_VALUE, }, }, }, @@ -53,6 +60,12 @@ func TestGetOnlineFeatures(t *testing.T) { Fields: map[string]*types.Value{ "driver:rating": Int64Val(1), "rating": Int64Val(1), + "null_value": {}, + }, + Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{ + "driver:rating": serving.GetOnlineFeaturesResponse_PRESENT, + "rating": serving.GetOnlineFeaturesResponse_PRESENT, + "null_value": serving.GetOnlineFeaturesResponse_NULL_VALUE, }, }, }, diff --git a/sdk/go/request_test.go b/sdk/go/request_test.go index 306ce2879b8..70e1f0d8f5e 100644 --- a/sdk/go/request_test.go +++ b/sdk/go/request_test.go @@ -25,9 +25,9 @@ func TestGetOnlineFeaturesRequest(t *testing.T) { "driver_id", }, Entities: []Row{ - {"entity1": Field(Int64Val(1)), "entity2": Field(StrVal("bob"))}, - {"entity1": Field(Int64Val(1)), "entity2": Field(StrVal("annie"))}, - {"entity1": Field(Int64Val(1)), "entity2": Field(StrVal("jane"))}, + {"entity1": Int64Val(1), "entity2": StrVal("bob")}, + {"entity1": Int64Val(1), "entity2": StrVal("annie")}, + {"entity1": Int64Val(1), "entity2": StrVal("jane")}, }, Project: "driver_project", }, diff --git a/sdk/go/response_test.go b/sdk/go/response_test.go index a0ba85584e1..0949f24d679 100644 --- a/sdk/go/response_test.go +++ b/sdk/go/response_test.go @@ -10,20 +10,25 @@ import ( var response = OnlineFeaturesResponse{ RawResponse: &serving.GetOnlineFeaturesResponse{ - Records: []*serving.GetOnlineFeaturesResponse_Record{ + FieldValues: []*serving.GetOnlineFeaturesResponse_FieldValues{ { - Fields: map[string]*serving.GetOnlineFeaturesResponse_Field{ - "project1/feature1": Field(Int64Val(1)), - "project1/feature2": &serving.GetOnlineFeaturesResponse_Field{ - Value: &types.Value{}, - Status: serving.GetOnlineFeaturesResponse_NULL_VALUE, - }, + Fields: map[string]*types.Value{ + "project1/feature1": Int64Val(1), + "project1/feature2": {}, + }, + Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{ + "project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT, + "project1/feature2": serving.GetOnlineFeaturesResponse_NULL_VALUE, }, }, { - Fields: map[string]*serving.GetOnlineFeaturesResponse_Field{ - "project1/feature1": Field(Int64Val(2)), - "project1/feature2": Field(Int64Val(2)), + Fields: map[string]*types.Value{ + "project1/feature1": Int64Val(2), + "project1/feature2": Int64Val(2), + }, + Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{ + "project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT, + "project1/feature2": serving.GetOnlineFeaturesResponse_PRESENT, }, }, }, @@ -33,23 +38,36 @@ var response = OnlineFeaturesResponse{ func TestOnlineFeaturesResponseToRow(t *testing.T) { actual := response.Rows() expected := []Row{ + {"project1/feature1": Int64Val(1), "project1/feature2": &types.Value{}}, + {"project1/feature1": Int64Val(2), "project1/feature2": Int64Val(2)}, + } + if len(expected) != len(actual) { + t.Errorf("expected: %v, got: %v", expected, actual) + } + for i := range expected { + if !expected[i].equalTo(actual[i]) { + t.Errorf("expected: %v, got: %v", expected, actual) + } + } +} + +func TestOnlineFeaturesResponseoToStatuses(t *testing.T) { + actual := response.Statuses() + expected := []map[string]serving.GetOnlineFeaturesResponse_FieldStatus{ { - "project1/feature1": Field(Int64Val(1)), - "project1/feature2": &serving.GetOnlineFeaturesResponse_Field{ - Value: &types.Value{}, - Status: serving.GetOnlineFeaturesResponse_NULL_VALUE, - }, + "project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT, + "project1/feature2": serving.GetOnlineFeaturesResponse_NULL_VALUE, }, { - "project1/feature1": Field(Int64Val(2)), - "project1/feature2": Field(Int64Val(2)), + "project1/feature1": serving.GetOnlineFeaturesResponse_PRESENT, + "project1/feature2": serving.GetOnlineFeaturesResponse_PRESENT, }, } if len(expected) != len(actual) { t.Errorf("expected: %v, got: %v", expected, actual) } for i := range expected { - if !expected[i].equalTo(actual[i]) { + if !cmp.Equal(expected[i], actual[i]) { t.Errorf("expected: %v, got: %v", expected, actual) } } @@ -76,7 +94,6 @@ func TestOnlineFeaturesResponseToInt64Array(t *testing.T) { want: [][]int64{{-1, 1}, {2, 2}}, wantErr: false, }, - { name: "length mismatch", args: args{ diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index 502094ce4c7..6e21d2b662b 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -151,14 +151,16 @@ public List getOnlineFeatures( fieldValues -> { Row row = Row.create(); for (String fieldName : fieldValues.getFieldsMap().keySet()) { - if (entityRefs.contains(fieldName)) continue; - // Strip project from string Feature References from returned from serving - FeatureReference featureRef = RequestUtil.parseFeatureRef(fieldName, true).build(); - String stripFieldName = RequestUtil.renderFeatureRef(featureRef); - row.set( - stripFieldName, - fieldValues.getFieldsMap().get(fieldName), - fieldValues.getStatusesMap().get(fieldName)); + String stripFieldName = fieldName; + if (!entityRefs.contains(fieldName)) { + // Strip project from string Feature References from returned from serving + FeatureReference featureRef = RequestUtil.parseFeatureRef(fieldName, true).build(); + stripFieldName = RequestUtil.renderFeatureRef(featureRef); + row.set( + stripFieldName, + fieldValues.getFieldsMap().get(fieldName), + fieldValues.getStatusesMap().get(fieldName)); + } } return row; }) From bf9c11bcd95c5cb0fe9b8c7b34f0ffdfc9a18a1d Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 5 Jun 2020 16:29:51 +0800 Subject: [PATCH 47/65] Revert Python SDK to backward compatible FieldValues API & fix tests --- sdk/python/feast/client.py | 16 ++++++++++------ sdk/python/tests/test_client.py | 20 ++++++++++++++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index b736fe23ac0..244fb93bdd4 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -716,15 +716,19 @@ def get_online_features( strip_field_values = [] for field_value in response.field_values: # strip the project part the string feature references returned from serving - strip_fields = {} - for ref_str, value in field_value.fields.items(): - if ref_str not in entity_refs: - ref_str = repr( + strip_fields, strip_statuses = {}, {} + for ref_str in field_value.fields.keys(): + strip_ref_str = ref_str + if not ref_str in entity_refs: + strip_ref_str = repr( FeatureRef.from_str(ref_str, ignore_project=True) ) - strip_fields[ref_str] = value + strip_fields[strip_ref_str] = field_value.fields[ref_str] + strip_statuses[strip_ref_str] = field_value.statuses[ref_str] strip_field_values.append( - GetOnlineFeaturesResponse.FieldValues(fields=strip_fields) + GetOnlineFeaturesResponse.FieldValues( + fields=strip_fields, statuses=strip_statuses, + ) ) del response.field_values[:] diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 04626f73cd5..e1b3f8680fc 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -220,6 +220,7 @@ def int_val(x): project="driver_project", feature_set="driver", name="age" ), FeatureRefProto(project="driver_project", name="rating"), + FeatureRefProto(project="driver_project", name="null_value"), ] ) recieve_response = GetOnlineFeaturesResponse() @@ -234,6 +235,13 @@ def int_val(x): "driver_id": int_val(row_number), "driver_project/driver:age": int_val(1), "driver_project/rating": int_val(9), + "driver_project/null_value": ValueProto.Value(), + }, + statuses={ + "driver_id": GetOnlineFeaturesResponse.FieldStatus.PRESENT, + "driver_project/driver:age": GetOnlineFeaturesResponse.FieldStatus.PRESENT, + "driver_project/rating": GetOnlineFeaturesResponse.FieldStatus.PRESENT, + "driver_project/null_value": GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE, } ) recieve_response.field_values.append(field_values) @@ -245,18 +253,22 @@ def int_val(x): ) got_response = mocked_client.get_online_features( entity_rows=request.entity_rows, - feature_refs=["driver:age", "rating"], + feature_refs=["driver:age", "rating", "null_value"], project="driver_project", ) # type: GetOnlineFeaturesResponse - mocked_client._serving_service_stub.GetOnlineFeatures.assert_called_with( - request - ) + mocked_client._serving_service_stub.GetOnlineFeatures.assert_called_with(request) got_fields = got_response.field_values[0].fields + got_statuses = got_response.field_values[0].statuses assert ( got_fields["driver_id"] == int_val(1) + and got_statuses["driver_id"] == GetOnlineFeaturesResponse.FieldStatus.PRESENT and got_fields["driver:age"] == int_val(1) + and got_statuses["driver:age"] == GetOnlineFeaturesResponse.FieldStatus.PRESENT and got_fields["rating"] == int_val(9) + and got_statuses["rating"] == GetOnlineFeaturesResponse.FieldStatus.PRESENT + and got_fields["null_value"] == ValueProto.Value() + and got_statuses["null_value"] == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE ) @pytest.mark.parametrize( From 87249c8d3383006634a7a6f6a94c29de39ae9052 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Sun, 7 Jun 2020 11:03:00 +0800 Subject: [PATCH 48/65] Move hardcoded constants from job.py to constants.py in Python SDK to consolidate constants --- sdk/python/feast/constants.py | 5 ++++ sdk/python/feast/job.py | 52 ++++++++++++----------------------- 2 files changed, 23 insertions(+), 34 deletions(-) diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index c4bde75404a..720682d144a 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -35,6 +35,9 @@ "batch_feature_request_wait_time_seconds" ) +CONFIG_TIMEOUT_KEY = "timeout" +CONFIG_MAX_WAIT_INTERVAL_KEY = "max_wait_interval" + # Configuration option default values FEAST_DEFAULT_OPTIONS = { CONFIG_PROJECT_KEY: "default", @@ -45,4 +48,6 @@ CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY: "3", CONFIG_GRPC_CONNECTION_TIMEOUT_APPLY_KEY: "600", CONFIG_BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS_KEY: "600", + CONFIG_MAX_WAIT_INTERVAL_KEY: "21600", + CONFIG_MAX_WAIT_INTERVAL_KEY: "60", } diff --git a/sdk/python/feast/job.py b/sdk/python/feast/job.py index 21b08224bad..ff179975f3f 100644 --- a/sdk/python/feast/job.py +++ b/sdk/python/feast/job.py @@ -23,14 +23,9 @@ from feast.serving.ServingService_pb2 import Job as JobProto from feast.serving.ServingService_pb2_grpc import ServingServiceStub from feast.source import Source - -# Maximum no of seconds to wait until the retrieval jobs status is DONE in Feast -# Currently set to the maximum query execution time limit in BigQuery -DEFAULT_TIMEOUT_SEC: int = 21600 - -# Maximum no of seconds to wait before reloading the job status in Feast -MAX_WAIT_INTERVAL_SEC: int = 60 - +from feast.wait import wait_retry_backoff +from feast.constants import FEAST_DEFAULT_OPTIONS as defaults +from feast.constants import CONFIG_TIMEOUT_KEY, CONFIG_MAX_WAIT_INTERVAL_KEY class RetrievalJob: """ @@ -71,7 +66,7 @@ def reload(self): """ self.job_proto = self.serving_stub.GetJob(GetJobRequest(job=self.job_proto)).job - def get_avro_files(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): + def get_avro_files(self, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY]): """ Wait until job is done to get the file uri to Avro result files on Google Cloud Storage. @@ -87,18 +82,15 @@ def get_avro_files(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): max_wait_datetime = datetime.now() + timedelta(seconds=timeout_sec) wait_duration_sec = 2 - while self.status != JOB_STATUS_DONE: - if datetime.now() > max_wait_datetime: - raise Exception( - "Timeout exceeded while waiting for result. Please retry " - "this method or use a longer timeout value." - ) - + def try_retrieve(): self.reload() - time.sleep(wait_duration_sec) + return self.status != JOB_STATUS_DONE - # Backoff the wait duration exponentially up till MAX_WAIT_INTERVAL_SEC - wait_duration_sec = min(wait_duration_sec * 2, MAX_WAIT_INTERVAL_SEC) + wait_retry_backoff(retry_fn=try_retrieve, + timeout_secs=timeout_sec, + timeout_msg="Timeout exceeded while waiting for result. Please retry " + "this method or use a longer timeout value.", + max_wait_secs=defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]) if self.job_proto.error: raise Exception(self.job_proto.error) @@ -111,7 +103,7 @@ def get_avro_files(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): return [urlparse(uri) for uri in self.job_proto.file_uris] - def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): + def result(self, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY]): """ Wait until job is done to get an iterable rows of result. The row can only represent an Avro row in Feast 0.3. @@ -142,7 +134,7 @@ def result(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC): for record in avro_reader: yield record - def to_dataframe(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC) -> pd.DataFrame: + def to_dataframe(self, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY]) -> pd.DataFrame: """ Wait until a job is done to get an iterable rows of result. This method will split the response into chunked DataFrame of a specified size to @@ -164,7 +156,7 @@ def to_dataframe(self, timeout_sec: int = DEFAULT_TIMEOUT_SEC) -> pd.DataFrame: return pd.DataFrame.from_records(records) def to_chunked_dataframe( - self, max_chunk_size: int = -1, timeout_sec: int = DEFAULT_TIMEOUT_SEC + self, max_chunk_size: int = -1, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY] ) -> pd.DataFrame: """ Wait until a job is done to get an iterable rows of result. This method @@ -281,18 +273,10 @@ def wait(self, status: IngestionJobStatus, timeout_secs: float = 300): timeout_secs: Maximum seconds to wait before timing out. """ # poll & wait for job status to transition - wait_begin = time.time() - wait_secs = 2 - elapsed_secs = 0 - while self.status != status and elapsed_secs <= timeout_secs: - time.sleep(wait_secs) - # back off wait duration exponentially, capped at MAX_WAIT_INTERVAL_SEC - wait_secs = min(wait_secs * 2, MAX_WAIT_INTERVAL_SEC) - elapsed_secs = time.time() - wait_begin - - # raise error if timeout - if elapsed_secs > timeout_secs: - raise TimeoutError("Wait for IngestJob's status to transition timed out") + wait_retry_backoff(retry_fn=(lambda: self.status != status), + timeout_secs=timeout_secs, + timeout_msg="Wait for IngestJob's status to transition timed out", + max_wait_secs=defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]) def __str__(self): # render the contents of ingest job as human readable string From ccd632d260f668d47760cc6ac4bad2bdc3ceb770 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Sun, 7 Jun 2020 11:03:49 +0800 Subject: [PATCH 49/65] Added wait backoff implementation in Python SDK: wait_retry_backoff() --- sdk/python/feast/constants.py | 2 +- sdk/python/feast/wait.py | 43 +++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 sdk/python/feast/wait.py diff --git a/sdk/python/feast/constants.py b/sdk/python/feast/constants.py index 720682d144a..4b22a8e975d 100644 --- a/sdk/python/feast/constants.py +++ b/sdk/python/feast/constants.py @@ -48,6 +48,6 @@ CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY: "3", CONFIG_GRPC_CONNECTION_TIMEOUT_APPLY_KEY: "600", CONFIG_BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS_KEY: "600", - CONFIG_MAX_WAIT_INTERVAL_KEY: "21600", + CONFIG_TIMEOUT_KEY: "21600", CONFIG_MAX_WAIT_INTERVAL_KEY: "60", } diff --git a/sdk/python/feast/wait.py b/sdk/python/feast/wait.py new file mode 100644 index 00000000000..c2f17f0ac7f --- /dev/null +++ b/sdk/python/feast/wait.py @@ -0,0 +1,43 @@ +# Copyright 2019 The Feast Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from datetime import datetime, timedelta +from typing import Callable, Optional + +def wait_retry_backoff(retry_fn: Callable[[], bool], + timeout_secs :Optional[int]=None, + timeout_msg :Optional[str]="Timeout while waiting for retry_fn() to return True", + max_wait_secs: Optional[int]=None): + """ + Repeated try calling given go_func until it returns True. + Waits with a exponentiall backoff between retries + Args: + retry_fn: Callable that returns a boolean to retry. + timeout_secs: timeout in seconds to give up retrying and throw TimeoutError, + or None to retry perpectually. + timeout_msg: Message to use when throwing TimeoutError. + max_wait_secs: max wait in seconds to wait between retries. + """ + wait_begin = time.time() + wait_secs = 2 + elapsed_secs = 0 + while not retry_fn() and elapsed_secs <= timeout_secs: + # back off wait duration exponentially, capped at MAX_WAIT_INTERVAL_SEC + time.sleep(wait_secs) + wait_secs = min(wait_secs * 2, max_wait_secs) + elapsed_secs = time.time() - wait_begin + + if elapsed_secs > timeout_secs: + raise TimeoutError(timeout_msg) From 4580e04ef21f2946e3675a8d33143edcf1bc442c Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Sun, 7 Jun 2020 11:10:25 +0800 Subject: [PATCH 50/65] Update Python SDK's job.py to use shared backoff wait implemetation --- sdk/python/feast/cli.py | 2 +- sdk/python/feast/client.py | 4 +- sdk/python/feast/feature_set.py | 6 +-- sdk/python/feast/job.py | 45 +++++++++++-------- sdk/python/feast/loaders/ingest.py | 2 +- sdk/python/feast/type_map.py | 4 +- sdk/python/feast/wait.py | 11 +++-- sdk/python/tests/test_client.py | 15 ++++--- tests/e2e/redis/basic-ingest-redis-serving.py | 1 - 9 files changed, 52 insertions(+), 38 deletions(-) diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 68fee30c9b1..9eb356034a8 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -18,8 +18,8 @@ import click import pkg_resources - import yaml + from feast.client import Client from feast.config import Config from feast.core.IngestionJob_pb2 import IngestionJobStatus diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 244fb93bdd4..6ddd04689fa 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -26,10 +26,10 @@ import grpc import pandas as pd -from google.protobuf.timestamp_pb2 import Timestamp - import pyarrow as pa import pyarrow.parquet as pq +from google.protobuf.timestamp_pb2 import Timestamp + from feast.config import Config from feast.constants import ( CONFIG_CORE_SECURE_KEY, diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py index f4397f6cbd5..ee06463bc6f 100644 --- a/sdk/python/feast/feature_set.py +++ b/sdk/python/feast/feature_set.py @@ -16,14 +16,15 @@ from typing import Dict, List, MutableMapping, Optional import pandas as pd +import pyarrow as pa +import yaml from google.protobuf import json_format from google.protobuf.duration_pb2 import Duration from google.protobuf.json_format import MessageToDict, MessageToJson from google.protobuf.message import Message from pandas.api.types import is_datetime64_ns_dtype +from pyarrow.lib import TimestampType -import pyarrow as pa -import yaml from feast.core.FeatureSet_pb2 import FeatureSet as FeatureSetProto from feast.core.FeatureSet_pb2 import FeatureSetMeta as FeatureSetMetaProto from feast.core.FeatureSet_pb2 import FeatureSetSpec as FeatureSetSpecProto @@ -39,7 +40,6 @@ pa_to_feast_value_type, python_type_to_feast_value_type, ) -from pyarrow.lib import TimestampType from tensorflow_metadata.proto.v0 import schema_pb2 diff --git a/sdk/python/feast/job.py b/sdk/python/feast/job.py index ff179975f3f..401b032eb20 100644 --- a/sdk/python/feast/job.py +++ b/sdk/python/feast/job.py @@ -9,6 +9,8 @@ from google.cloud import storage from google.protobuf.json_format import MessageToJson +from feast.constants import CONFIG_MAX_WAIT_INTERVAL_KEY, CONFIG_TIMEOUT_KEY +from feast.constants import FEAST_DEFAULT_OPTIONS as defaults from feast.core.CoreService_pb2 import ListIngestionJobsRequest from feast.core.CoreService_pb2_grpc import CoreServiceStub from feast.core.IngestionJob_pb2 import IngestionJob as IngestJobProto @@ -24,8 +26,7 @@ from feast.serving.ServingService_pb2_grpc import ServingServiceStub from feast.source import Source from feast.wait import wait_retry_backoff -from feast.constants import FEAST_DEFAULT_OPTIONS as defaults -from feast.constants import CONFIG_TIMEOUT_KEY, CONFIG_MAX_WAIT_INTERVAL_KEY + class RetrievalJob: """ @@ -66,7 +67,7 @@ def reload(self): """ self.job_proto = self.serving_stub.GetJob(GetJobRequest(job=self.job_proto)).job - def get_avro_files(self, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY]): + def get_avro_files(self, timeout_sec: int = int(defaults[CONFIG_TIMEOUT_KEY])): """ Wait until job is done to get the file uri to Avro result files on Google Cloud Storage. @@ -79,18 +80,18 @@ def get_avro_files(self, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY]): Returns: str: Google Cloud Storage file uris of the returned Avro files. """ - max_wait_datetime = datetime.now() + timedelta(seconds=timeout_sec) - wait_duration_sec = 2 def try_retrieve(): self.reload() - return self.status != JOB_STATUS_DONE - - wait_retry_backoff(retry_fn=try_retrieve, - timeout_secs=timeout_sec, - timeout_msg="Timeout exceeded while waiting for result. Please retry " - "this method or use a longer timeout value.", - max_wait_secs=defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]) + return self.status == JOB_STATUS_DONE + + wait_retry_backoff( + retry_fn=try_retrieve, + timeout_secs=timeout_sec, + timeout_msg="Timeout exceeded while waiting for result. Please retry " + "this method or use a longer timeout value.", + max_wait_secs=int(defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]), + ) if self.job_proto.error: raise Exception(self.job_proto.error) @@ -103,7 +104,7 @@ def try_retrieve(): return [urlparse(uri) for uri in self.job_proto.file_uris] - def result(self, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY]): + def result(self, timeout_sec: int = int(defaults[CONFIG_TIMEOUT_KEY])): """ Wait until job is done to get an iterable rows of result. The row can only represent an Avro row in Feast 0.3. @@ -134,7 +135,9 @@ def result(self, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY]): for record in avro_reader: yield record - def to_dataframe(self, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY]) -> pd.DataFrame: + def to_dataframe( + self, timeout_sec: int = int(defaults[CONFIG_TIMEOUT_KEY]) + ) -> pd.DataFrame: """ Wait until a job is done to get an iterable rows of result. This method will split the response into chunked DataFrame of a specified size to @@ -156,7 +159,9 @@ def to_dataframe(self, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY]) -> pd.Da return pd.DataFrame.from_records(records) def to_chunked_dataframe( - self, max_chunk_size: int = -1, timeout_sec: int = defaults[CONFIG_TIMEOUT_KEY] + self, + max_chunk_size: int = -1, + timeout_sec: int = int(defaults[CONFIG_TIMEOUT_KEY]), ) -> pd.DataFrame: """ Wait until a job is done to get an iterable rows of result. This method @@ -273,10 +278,12 @@ def wait(self, status: IngestionJobStatus, timeout_secs: float = 300): timeout_secs: Maximum seconds to wait before timing out. """ # poll & wait for job status to transition - wait_retry_backoff(retry_fn=(lambda: self.status != status), - timeout_secs=timeout_secs, - timeout_msg="Wait for IngestJob's status to transition timed out", - max_wait_secs=defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]) + wait_retry_backoff( + retry_fn=(lambda: self.status == status), + timeout_secs=timeout_secs, + timeout_msg="Wait for IngestJob's status to transition timed out", + max_wait_secs=int(defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]), + ) def __str__(self): # render the contents of ingest job as human readable string diff --git a/sdk/python/feast/loaders/ingest.py b/sdk/python/feast/loaders/ingest.py index 5e8e5da4d41..b439dbd3027 100644 --- a/sdk/python/feast/loaders/ingest.py +++ b/sdk/python/feast/loaders/ingest.py @@ -4,8 +4,8 @@ from typing import Iterable, List import pandas as pd - import pyarrow.parquet as pq + from feast.constants import DATETIME_COLUMN from feast.feature_set import FeatureSet from feast.type_map import ( diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 7a483a6c2a0..85def25fcb9 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -17,9 +17,10 @@ import numpy as np import pandas as pd +import pyarrow as pa from google.protobuf.timestamp_pb2 import Timestamp +from pyarrow.lib import TimestampType -import pyarrow as pa from feast.constants import DATETIME_COLUMN from feast.types import FeatureRow_pb2 as FeatureRowProto from feast.types import Field_pb2 as FieldProto @@ -35,7 +36,6 @@ from feast.types.Value_pb2 import Value as ProtoValue from feast.types.Value_pb2 import ValueType as ProtoValueType from feast.value_type import ValueType -from pyarrow.lib import TimestampType def python_type_to_feast_value_type( diff --git a/sdk/python/feast/wait.py b/sdk/python/feast/wait.py index c2f17f0ac7f..01fe1721773 100644 --- a/sdk/python/feast/wait.py +++ b/sdk/python/feast/wait.py @@ -16,10 +16,13 @@ from datetime import datetime, timedelta from typing import Callable, Optional -def wait_retry_backoff(retry_fn: Callable[[], bool], - timeout_secs :Optional[int]=None, - timeout_msg :Optional[str]="Timeout while waiting for retry_fn() to return True", - max_wait_secs: Optional[int]=None): + +def wait_retry_backoff( + retry_fn: Callable[[], bool], + timeout_secs: Optional[int] = None, + timeout_msg: Optional[str] = "Timeout while waiting for retry_fn() to return True", + max_wait_secs: Optional[int] = None, +): """ Repeated try calling given go_func until it returns True. Waits with a exponentiall backoff between retries diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index e1b3f8680fc..9437ef735d1 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -242,7 +242,7 @@ def int_val(x): "driver_project/driver:age": GetOnlineFeaturesResponse.FieldStatus.PRESENT, "driver_project/rating": GetOnlineFeaturesResponse.FieldStatus.PRESENT, "driver_project/null_value": GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE, - } + }, ) recieve_response.field_values.append(field_values) @@ -256,19 +256,24 @@ def int_val(x): feature_refs=["driver:age", "rating", "null_value"], project="driver_project", ) # type: GetOnlineFeaturesResponse - mocked_client._serving_service_stub.GetOnlineFeatures.assert_called_with(request) + mocked_client._serving_service_stub.GetOnlineFeatures.assert_called_with( + request + ) got_fields = got_response.field_values[0].fields got_statuses = got_response.field_values[0].statuses assert ( got_fields["driver_id"] == int_val(1) - and got_statuses["driver_id"] == GetOnlineFeaturesResponse.FieldStatus.PRESENT + and got_statuses["driver_id"] + == GetOnlineFeaturesResponse.FieldStatus.PRESENT and got_fields["driver:age"] == int_val(1) - and got_statuses["driver:age"] == GetOnlineFeaturesResponse.FieldStatus.PRESENT + and got_statuses["driver:age"] + == GetOnlineFeaturesResponse.FieldStatus.PRESENT and got_fields["rating"] == int_val(9) and got_statuses["rating"] == GetOnlineFeaturesResponse.FieldStatus.PRESENT and got_fields["null_value"] == ValueProto.Value() - and got_statuses["null_value"] == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE + and got_statuses["null_value"] + == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE ) @pytest.mark.parametrize( diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 9dac8abd9d3..2d7a73acc7f 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -84,7 +84,6 @@ def basic_dataframe(entities, features, ingest_time, n_size): def ingest_time(): return datetime.utcnow() -None @pytest.fixture(scope="module") def cust_trans_df(ingest_time): return basic_dataframe(entities=["customer_id"], From 95f5f6b6e2eb6670d65a1dea3fc58d7eb22c9652 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 12 Jun 2020 12:52:56 +0800 Subject: [PATCH 51/65] Clarify that maximum range in ServingService proto to max age --- protos/feast/serving/ServingService.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto index 0a89c97015b..40e3f8b2957 100644 --- a/protos/feast/serving/ServingService.proto +++ b/protos/feast/serving/ServingService.proto @@ -120,10 +120,10 @@ message GetOnlineFeaturesResponse { // Status is unset for this field. INVALID = 0; - // Field value is present for this field and within maximum allowable range + // Field value is present for this field and age is within max age. PRESENT = 1; - // Values could be found for entity key within the maximum allowable range, but + // Values could be found for entity key and age is within max age, but // this field value is assigned a value on ingestion into feast. NULL_VALUE = 2; From 3bf17f212c88fde642ed2127523305fe7eda765e Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 12 Jun 2020 17:33:17 +0800 Subject: [PATCH 52/65] Fixed online serving e2e tests --- .../java/com/gojek/feast/FeastClient.java | 3 +- sdk/python/feast/job.py | 6 +- sdk/python/feast/wait.py | 38 ++- tests/e2e/redis/basic-ingest-redis-serving.py | 256 ++++++++---------- 4 files changed, 145 insertions(+), 158 deletions(-) diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index 6e21d2b662b..ec03ee8e504 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -154,7 +154,8 @@ public List getOnlineFeatures( String stripFieldName = fieldName; if (!entityRefs.contains(fieldName)) { // Strip project from string Feature References from returned from serving - FeatureReference featureRef = RequestUtil.parseFeatureRef(fieldName, true).build(); + FeatureReference featureRef = + RequestUtil.parseFeatureRef(fieldName, true).build(); stripFieldName = RequestUtil.renderFeatureRef(featureRef); row.set( stripFieldName, diff --git a/sdk/python/feast/job.py b/sdk/python/feast/job.py index 401b032eb20..f0f9e5c80f0 100644 --- a/sdk/python/feast/job.py +++ b/sdk/python/feast/job.py @@ -83,14 +83,13 @@ def get_avro_files(self, timeout_sec: int = int(defaults[CONFIG_TIMEOUT_KEY])): def try_retrieve(): self.reload() - return self.status == JOB_STATUS_DONE + return None, self.status == JOB_STATUS_DONE wait_retry_backoff( retry_fn=try_retrieve, timeout_secs=timeout_sec, timeout_msg="Timeout exceeded while waiting for result. Please retry " "this method or use a longer timeout value.", - max_wait_secs=int(defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]), ) if self.job_proto.error: @@ -279,10 +278,9 @@ def wait(self, status: IngestionJobStatus, timeout_secs: float = 300): """ # poll & wait for job status to transition wait_retry_backoff( - retry_fn=(lambda: self.status == status), + retry_fn=(lambda: (None, self.status == status)), timeout_secs=timeout_secs, timeout_msg="Wait for IngestJob's status to transition timed out", - max_wait_secs=int(defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]), ) def __str__(self): diff --git a/sdk/python/feast/wait.py b/sdk/python/feast/wait.py index 01fe1721773..d20f229b477 100644 --- a/sdk/python/feast/wait.py +++ b/sdk/python/feast/wait.py @@ -13,34 +13,44 @@ # limitations under the License. import time -from datetime import datetime, timedelta -from typing import Callable, Optional +from typing import Any, Callable, Optional, Tuple + +from feast.constants import CONFIG_MAX_WAIT_INTERVAL_KEY +from feast.constants import FEAST_DEFAULT_OPTIONS as defaults def wait_retry_backoff( - retry_fn: Callable[[], bool], + retry_fn: Callable[[], Tuple[Any, bool]], timeout_secs: Optional[int] = None, timeout_msg: Optional[str] = "Timeout while waiting for retry_fn() to return True", - max_wait_secs: Optional[int] = None, -): + max_interval_secs: Optional[int]=int(defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]), +) -> Any: """ - Repeated try calling given go_func until it returns True. - Waits with a exponentiall backoff between retries + Repeatedly try calling given retry_fn until it returns a True boolean success flag. + Waits with a exponential backoff between retries until timeout when it throws TimeoutError. Args: - retry_fn: Callable that returns a boolean to retry. + retry_fn: Callable that returns a result and a boolean success flag. timeout_secs: timeout in seconds to give up retrying and throw TimeoutError, or None to retry perpectually. timeout_msg: Message to use when throwing TimeoutError. - max_wait_secs: max wait in seconds to wait between retries. + max_interval_secs: max wait in seconds to wait between retries. + Returns: + Returned Result from retry_fn() if success flag is True. """ + wait_secs, elapsed_secs = 1, 0 + result, is_success = retry_fn() wait_begin = time.time() - wait_secs = 2 - elapsed_secs = 0 - while not retry_fn() and elapsed_secs <= timeout_secs: + while not is_success and elapsed_secs <= timeout_secs: # back off wait duration exponentially, capped at MAX_WAIT_INTERVAL_SEC + elapsed_secs = time.time() - wait_begin + till_timeout_secs = timeout_secs - elapsed_secs + wait_secs = min(wait_secs * 2, max_interval_secs, till_timeout_secs) time.sleep(wait_secs) - wait_secs = min(wait_secs * 2, max_wait_secs) + # retry call + result, is_success = retry_fn() elapsed_secs = time.time() - wait_begin - if elapsed_secs > timeout_secs: + if not is_success and elapsed_secs > timeout_secs: raise TimeoutError(timeout_msg) + + return result diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 2d7a73acc7f..989be273b81 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -16,6 +16,7 @@ from feast.client import Client from feast.feature_set import FeatureSet, FeatureSetRef from feast.type_map import ValueType +from feast.wait import wait_retry_backoff from feast.constants import FEAST_DEFAULT_OPTIONS, CONFIG_PROJECT_KEY from google.protobuf.duration_pb2 import Duration from datetime import datetime @@ -32,6 +33,56 @@ PROJECT_NAME = 'basic_' + uuid.uuid4().hex.upper()[0:6] DIR_PATH = os.path.dirname(os.path.realpath(__file__)) +def basic_dataframe(entities, features, ingest_time, n_size, null_features=[]): + """ + Generate a basic feast-ingestable dataframe for testing. + Entity value incrementally increase from 1 to n_size + Features values are randomlly generated floats. + entities - names of entities + features - names of the features + ingest_time - ingestion timestamp + n_size - no. of rows in the generated dataframe. + null_features - names of features that contain null values + Returns the generated dataframe + """ + offset = random.randint(1000, 100000) # ensure a unique key space is used + df_dict = { + "datetime": [ingest_time.replace(tzinfo=pytz.utc) for _ in + range(n_size)], + } + for entity_name in entities: + df_dict[entity_name] = list(range(1, n_size + 1)) + for feature_name in features: + df_dict[feature_name] = [np.random.rand() for _ in range(n_size)] + for null_feature_name in null_features: + df_dict[null_feature_name] = [None for _ in range(n_size)] + return pd.DataFrame(df_dict) + +def check_online_response(feature_ref, ingest_df, response): + """ + Check the feature value and status in the given online serving response. + feature_refs - string feature ref used to access feature in response + ingest_df - dataframe of ingested values + response - response to extract retrieved feature value and metadata + Returns True if given response has expected feature value and metadata, otherwise False. + """ + feature_ref_splits = feature_ref.split(":") + if len(feature_ref_splits) == 1: + feature_name = feature_ref + else: + _, feature_name = feature_ref_splits + + returned_status = response.field_values[0].statuses[feature_ref] + if ingest_df.loc[0, feature_name] is None: + return returned_status == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE + else: + sent_value = float(ingest_df.iloc[0][feature_name]) + returned_value = float(response.field_values[0].fields[feature_ref].float_val) + return ( + math.isclose(sent_value, returned_value, abs_tol=FLOAT_TOLERANCE) + and returned_status == GetOnlineFeaturesResponse.FieldStatus.PRESENT + ) + @pytest.fixture(scope='module') def core_url(pytestconfig): @@ -66,32 +117,19 @@ def client(core_url, serving_url, allow_dirty): return client -def basic_dataframe(entities, features, ingest_time, n_size): - offset = random.randint(1000, 100000) # ensure a unique key space is used - df_dict = { - "datetime": [ingest_time.replace(tzinfo=pytz.utc) for _ in - range(n_size)], - "null_values": [ Value() for _ in range(n_size) ], - } - for entity_name in entities: - df_dict[entity_name] = list(range(1, n_size + 1)) - for feature_name in features: - df_dict[feature_name] = [np.random.rand() for _ in range(n_size)] - return pd.DataFrame(df_dict) - - @pytest.fixture(scope="module") def ingest_time(): return datetime.utcnow() + @pytest.fixture(scope="module") def cust_trans_df(ingest_time): return basic_dataframe(entities=["customer_id"], features=["daily_transactions", "total_transactions"], + null_features=["null_values"], ingest_time=ingest_time, n_size=5) - @pytest.fixture(scope="module") def driver_df(ingest_time): return basic_dataframe(entities=["driver_id"], @@ -99,7 +137,6 @@ def driver_df(ingest_time): ingest_time=ingest_time, n_size=5) - def test_version_returns_results(client): version_info = client.version() assert not version_info['core'] is 'not configured' @@ -174,9 +211,13 @@ def test_basic_ingest_success(client, cust_trans_df, driver_df): @pytest.mark.timeout(90) @pytest.mark.run(order=12) def test_basic_retrieve_online_success(client, cust_trans_df): + feature_refs=[ + "daily_transactions", + "total_transactions", + "null_values" + ] # Poll serving for feature values until the correct values are returned - while True: - time.sleep(1) + def try_get_features(): response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -187,52 +228,29 @@ def test_basic_retrieve_online_success(client, cust_trans_df): } ) ], - # Test retrieve with different variations of the string feature refs - feature_refs=[ - "daily_transactions", - "total_transactions", - "null_values" - ], + feature_refs=feature_refs, include_meta=True, ) # type: GetOnlineFeaturesResponse + is_ok = all([check_online_response(ref, cust_trans_df, response) for ref in feature_refs]) + return response, is_ok - # unpack response & wait for ingested values - fields = response.records[0].fields - daily_transactions_field = fields[PROJECT_NAME + "/daily_transactions"] - null_value_field = fields[PROJECT_NAME + "/null_values"] - if (daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND - or null_value_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND): - print("test_basic_retrieve_online_success(): waiting for ingested values.") - continue - else: - break - - # check values returned are correct - sent_daily_transactions = float(basic_dataframe.iloc[0]["daily_transactions"]) - assert math.isclose( - sent_daily_transactions, - float(daily_transactions_field.value.float_val), - abs_tol=FLOAT_TOLERANCE, - ) - assert null_value_field.value.WhichOneof("val") is None - # check field status metadata - assert daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT - assert null_value_field.status == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE - + wait_retry_backoff(retry_fn=try_get_features, + timeout_secs=90, + timeout_msg="Timed out trying to get online feature values") @pytest.mark.timeout(90) @pytest.mark.run(order=13) def test_basic_retrieve_online_multiple_featureset(client, cust_trans_df, driver_df): + # Test retrieve with different variations of the string feature refs + # ie feature set inference for feature refs without specified feature set + feature_ref_df_mapping = [ + ("customer_transactions:daily_transactions", cust_trans_df), + ("driver:rating", driver_df), + ("total_transactions", cust_trans_df), + ] # Poll serving for feature values until the correct values are returned - while True: - time.sleep(1) - # Test retrieve with different variations of the string feature refs - # ie feature set inference for feature refs without specified feature set - feature_ref_df_mapping = [ - ("customer_transactions:daily_transactions", cust_trans_df), - ("driver:rating", driver_df), - ("total_transactions", cust_trans_df), - ] + def try_get_features(): + feature_refs = [mapping[0] for mapping in feature_ref_df_mapping] response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -246,35 +264,15 @@ def test_basic_retrieve_online_multiple_featureset(client, cust_trans_df, driver } ) ], - feature_refs=[mapping[0] for mapping in feature_ref_df_mapping], + feature_refs=feature_refs, ) # type: GetOnlineFeaturesResponse + is_ok = all([check_online_response(ref, df, response) + for ref, df in feature_ref_df_mapping]) + return response, is_ok - if response is None: - continue - - def check_response(ingest_df, response, feature_ref): - returned_value = float( - response.field_values[0] - .fields[feature_ref] - .float_val - ) - feature_ref_splits = feature_ref.split(":") - if len(feature_ref_splits) == 1: - feature_name = feature_ref - else: - _, feature_name = feature_ref_splits - - sent_value = float( - ingest_df.iloc[0][feature_name]) - - return math.isclose( - sent_value, - returned_value, - abs_tol=FLOAT_TOLERANCE, - ) - - if all([check_response(df, response, ref) for ref, df in feature_ref_df_mapping]): - break + wait_retry_backoff(retry_fn=try_get_features, + timeout_secs=90, + timeout_msg="Timed out trying to get online feature values") @pytest.mark.timeout(300) @@ -413,10 +411,23 @@ def test_all_types_ingest_success(client, all_types_dataframe): @pytest.mark.timeout(90) @pytest.mark.run(order=22) def test_all_types_retrieve_online_success(client, all_types_dataframe): - # Poll serving for feature values until the correct values are returned - while True: - time.sleep(1) - + # Poll serving for feature values until the correct values are returned_float_list + feature_refs = [ + "float_feature", + "int64_feature", + "int32_feature", + "double_feature", + "string_feature", + "bool_feature", + "bytes_feature", + "float_list_feature", + "int64_list_feature", + "int32_list_feature", + "string_list_feature", + "bytes_list_feature", + "double_list_feature", + ] + def try_get_features(): response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -424,41 +435,23 @@ def test_all_types_retrieve_online_success(client, all_types_dataframe): int64_val=all_types_dataframe.iloc[0]["user_id"])} ) ], - feature_refs=[ - "float_feature", - "int64_feature", - "int32_feature", - "string_feature", - "bytes_feature", - "bool_feature", - "double_feature", - "float_list_feature", - "int64_list_feature", - "int32_list_feature", - "string_list_feature", - "bytes_list_feature", - "double_list_feature", - ], + feature_refs=feature_refs, include_meta=True, ) # type: GetOnlineFeaturesResponse + is_ok = check_online_response("float_feature", all_types_dataframe, response) + return response, is_ok - # wait for and unpack response - if response is None: - print("test_all_types_retrieve_online_success(): polling for response.") - continue - float_list_field = response.records[0].fields[PROJECT_NAME+"/float_list_feature"] - if float_list_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND: - print("test_all_types_retrieve_online_success(): polling for ingested values.") - continue - else: - break + response = wait_retry_backoff(retry_fn=try_get_features, + timeout_secs=90, + timeout_msg="Timed out trying to get online feature values") # check returned values - returned_float_list = float_list_field.value.float_list_val.val + returned_float_list = response.field_values[0].fields["float_list_feature"].float_list_val.val sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] assert math.isclose(returned_float_list[0], sent_float_list[0], abs_tol=FLOAT_TOLERANCE) # check returned metadata - assert float_list_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT + assert (response.field_values[0].statuses["float_list_feature"] + == GetOnlineFeaturesResponse.FieldStatus.PRESENT) @pytest.mark.timeout(300) @pytest.mark.run(order=29) @@ -541,9 +534,11 @@ def test_large_volume_ingest_success(client, large_volume_dataframe): @pytest.mark.run(order=32) def test_large_volume_retrieve_online_success(client, large_volume_dataframe): # Poll serving for feature values until the correct values are returned + feature_refs=[ + "daily_transactions_large", + "total_transactions_large", + ] while True: - time.sleep(1) - response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( @@ -555,32 +550,15 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): } ) ], - feature_refs=[ - "daily_transactions_large", - "total_transactions_large", - ], + feature_refs=feature_refs, include_meta=True, ) # type: GetOnlineFeaturesResponse + is_ok = all([check_online_response(ref, large_volume_dataframe, response) for ref in feature_refs]) + return None, is_ok - # wait for and unpack response - if response is None: - print("test_all_types_retrieve_online_success(): polling for response.") - continue - daily_transactions_field = response.records[0].fields[ - PROJECT_NAME+"/daily_transactions_large"] - if daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.NOT_FOUND: - print("test_all_types_retrieve_online_success(): polling for response.") - continue - else: - break - - # check returned values - returned_daily_transactions = float(daily_transactions_field.value.float_val) - sent_daily_transactions = float( - large_volume_dataframe.iloc[0]["daily_transactions_large"]) - assert math.isclose(sent_daily_transactions, returned_daily_transactions, abs_tol=FLOAT_TOLERANCE) - # check returned metadata - assert daily_transactions_field.status == GetOnlineFeaturesResponse.FieldStatus.PRESENT + wait_retry_backoff(retry_fn=try_get_features, + timeout_secs=90, + timeout_msg="Timed out trying to get online feature values") @pytest.fixture(scope='module') def all_types_parquet_file(): From bd54fc754b41e7f080f9e8b393ddf193b39894a6 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 12 Jun 2020 17:41:29 +0800 Subject: [PATCH 53/65] Fix python lint. --- sdk/python/feast/client.py | 2 +- sdk/python/feast/job.py | 4 +--- sdk/python/feast/wait.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 6ddd04689fa..85337a43407 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -719,7 +719,7 @@ def get_online_features( strip_fields, strip_statuses = {}, {} for ref_str in field_value.fields.keys(): strip_ref_str = ref_str - if not ref_str in entity_refs: + if ref_str not in entity_refs: strip_ref_str = repr( FeatureRef.from_str(ref_str, ignore_project=True) ) diff --git a/sdk/python/feast/job.py b/sdk/python/feast/job.py index f0f9e5c80f0..829b4f79319 100644 --- a/sdk/python/feast/job.py +++ b/sdk/python/feast/job.py @@ -1,6 +1,4 @@ import tempfile -import time -from datetime import datetime, timedelta from typing import List from urllib.parse import urlparse @@ -9,7 +7,7 @@ from google.cloud import storage from google.protobuf.json_format import MessageToJson -from feast.constants import CONFIG_MAX_WAIT_INTERVAL_KEY, CONFIG_TIMEOUT_KEY +from feast.constants import CONFIG_TIMEOUT_KEY from feast.constants import FEAST_DEFAULT_OPTIONS as defaults from feast.core.CoreService_pb2 import ListIngestionJobsRequest from feast.core.CoreService_pb2_grpc import CoreServiceStub diff --git a/sdk/python/feast/wait.py b/sdk/python/feast/wait.py index d20f229b477..f25036d76ea 100644 --- a/sdk/python/feast/wait.py +++ b/sdk/python/feast/wait.py @@ -23,7 +23,7 @@ def wait_retry_backoff( retry_fn: Callable[[], Tuple[Any, bool]], timeout_secs: Optional[int] = None, timeout_msg: Optional[str] = "Timeout while waiting for retry_fn() to return True", - max_interval_secs: Optional[int]=int(defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]), + max_interval_secs: Optional[int] = int(defaults[CONFIG_MAX_WAIT_INTERVAL_KEY]), ) -> Any: """ Repeatedly try calling given retry_fn until it returns a True boolean success flag. From 64f59ac64afc5e0939c74651e794ebe7aa55ba6a Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 12 Jun 2020 18:07:37 +0800 Subject: [PATCH 54/65] Collect entity refs in Python SDK as a single line. --- sdk/python/feast/client.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 85337a43407..43e930b699c 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -708,11 +708,8 @@ def get_online_features( entity_rows=entity_rows, ) ) - # collect entity row refs - entity_refs = set() - for entity_row in entity_rows: - entity_refs.update(entity_row.fields.keys()) + entity_refs = {key for entity_row in entity_rows for key in entity_row.fields.keys()} strip_field_values = [] for field_value in response.field_values: # strip the project part the string feature references returned from serving From f1e52fcf6e00d50e2fbed39e04a4515c271c53c0 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 12 Jun 2020 18:08:37 +0800 Subject: [PATCH 55/65] Use guard style if statements in Go Sdk's types.go --- sdk/go/types.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/go/types.go b/sdk/go/types.go index c2bc43e9c37..521db80efb3 100644 --- a/sdk/go/types.go +++ b/sdk/go/types.go @@ -12,7 +12,8 @@ func (r Row) equalTo(other Row) bool { for k, field := range r { if otherField, ok := other[k]; !ok { return false - } else if !proto.Equal(field, otherField) { + } + if !proto.Equal(field, otherField) { return false } } From 9958e7e9244ded7a578ffd59bb238ffe9942b8d1 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 12 Jun 2020 18:25:17 +0800 Subject: [PATCH 56/65] Make strip field values part of Python SDK's get_online_features() more functional --- sdk/python/feast/client.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 43e930b699c..c619a06fa51 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -709,25 +709,32 @@ def get_online_features( ) ) - entity_refs = {key for entity_row in entity_rows for key in entity_row.fields.keys()} + entity_refs = { + key for entity_row in entity_rows for key in entity_row.fields.keys() + } + # strip the project part the string feature references returned from serving + strip = ( + lambda ref: repr(FeatureRef.from_str(ref, ignore_project=True)) + if ref not in entity_refs + else ref + ) strip_field_values = [] for field_value in response.field_values: - # strip the project part the string feature references returned from serving - strip_fields, strip_statuses = {}, {} - for ref_str in field_value.fields.keys(): - strip_ref_str = ref_str - if ref_str not in entity_refs: - strip_ref_str = repr( - FeatureRef.from_str(ref_str, ignore_project=True) - ) - strip_fields[strip_ref_str] = field_value.fields[ref_str] - strip_statuses[strip_ref_str] = field_value.statuses[ref_str] + keys, fields, statuses = ( + response.fields.keys(), + response.fields, + response.statuses, + ) + fields_and_statuses = [ + (strip(key), fields[key], statuses[key]) for key in keys + ] + keys, fields, statuses = zip(*fields_and_statuses) strip_field_values.append( GetOnlineFeaturesResponse.FieldValues( - fields=strip_fields, statuses=strip_statuses, + fields=dict(zip(keys, fields)), + statuses=dict(zip(keys, statuses)), ) ) - del response.field_values[:] response.field_values.extend(strip_field_values) From 6de82677829e7c32be0695ffc492a0767f24e39f Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Fri, 12 Jun 2020 18:29:35 +0800 Subject: [PATCH 57/65] Fix go unit tests --- sdk/go/types.go | 11 ++++++----- sdk/python/feast/client.py | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sdk/go/types.go b/sdk/go/types.go index 521db80efb3..600af0b658a 100644 --- a/sdk/go/types.go +++ b/sdk/go/types.go @@ -9,12 +9,13 @@ import ( type Row map[string]*types.Value func (r Row) equalTo(other Row) bool { - for k, field := range r { - if otherField, ok := other[k]; !ok { - return false - } - if !proto.Equal(field, otherField) { + for k, v := range r { + if otherV, ok := other[k]; !ok { return false + } else { + if !proto.Equal(v, otherV) { + return false + } } } return true diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index c619a06fa51..ac22cd3e721 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -721,9 +721,9 @@ def get_online_features( strip_field_values = [] for field_value in response.field_values: keys, fields, statuses = ( - response.fields.keys(), - response.fields, - response.statuses, + field_value.fields.keys(), + field_value.fields, + field_value.statuses, ) fields_and_statuses = [ (strip(key), fields[key], statuses[key]) for key in keys From 7952d989037db3fe32a71fff0d60067faa8ee095 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 16 Jun 2020 15:17:24 +0800 Subject: [PATCH 58/65] Remove include_metadata_in_response flag from ServingService proto as its redundant. --- protos/feast/serving/ServingService.proto | 4 - sdk/go/protos/feast/core/CoreService.pb.go | 546 +++++++++++++----- sdk/go/protos/feast/core/Runner.pb.go | 114 ++-- sdk/go/protos/feast/core/Store.pb.go | 87 +-- .../protos/feast/serving/ServingService.pb.go | 320 +++++----- sdk/go/request.go | 4 - .../java/com/gojek/feast/FeastClient.java | 14 +- sdk/python/feast/client.py | 2 - .../service/OnlineServingServiceTest.java | 3 - tests/e2e/redis/basic-ingest-redis-serving.py | 5 +- 10 files changed, 684 insertions(+), 415 deletions(-) diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto index 40e3f8b2957..8e36d459d66 100644 --- a/protos/feast/serving/ServingService.proto +++ b/protos/feast/serving/ServingService.proto @@ -90,10 +90,6 @@ message GetOnlineFeaturesRequest { // values will be returned. bool omit_entities_in_response = 3; - // Option to include feature metadata in the response. - // If true, response will include both feature values and metadata. - bool include_metadata_in_response = 5; - message EntityRow { // Request timestamp of this row. This value will be used, // together with maxAge, to determine feature staleness. diff --git a/sdk/go/protos/feast/core/CoreService.pb.go b/sdk/go/protos/feast/core/CoreService.pb.go index 4f22ae4c373..a6d1dee6651 100644 --- a/sdk/go/protos/feast/core/CoreService.pb.go +++ b/sdk/go/protos/feast/core/CoreService.pb.go @@ -24,7 +24,9 @@ package core import ( context "context" + v0 "github.com/feast-dev/feast/sdk/go/protos/tensorflow_metadata/proto/v0" proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" @@ -1273,6 +1275,168 @@ func (*StopIngestionJobResponse) Descriptor() ([]byte, []int) { return file_feast_core_CoreService_proto_rawDescGZIP(), []int{23} } +type GetFeatureStatisticsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Feature set to retrieve the statistics for. A fully qualified feature set + // id in the format of project/feature_set must be provided. + FeatureSetId string `protobuf:"bytes,1,opt,name=feature_set_id,json=featureSetId,proto3" json:"feature_set_id,omitempty"` + // Optional filter which filters returned statistics by selected features. These + // features must be present in the data that is being processed. + Features []string `protobuf:"bytes,2,rep,name=features,proto3" json:"features,omitempty"` + // Optional filter to select store over which the statistics will retrieved. + // Only historical stores are allowed. + Store string `protobuf:"bytes,3,opt,name=store,proto3" json:"store,omitempty"` + // Optional start and end dates over which to filter statistical data + // Start date is inclusive, but end date is not. + // Only dates are supported, not times. + // Cannot be used with dataset_ids. + // If this period spans multiple days, unaggregatable statistics will be dropped. + StartDate *timestamp.Timestamp `protobuf:"bytes,4,opt,name=start_date,json=startDate,proto3" json:"start_date,omitempty"` + EndDate *timestamp.Timestamp `protobuf:"bytes,5,opt,name=end_date,json=endDate,proto3" json:"end_date,omitempty"` + // Optional list of ingestion Ids by which to filter data before + // retrieving statistics. + // Cannot be used with the date ranges + // If multiple dataset ids are provided, unaggregatable statistics will be dropped. + IngestionIds []string `protobuf:"bytes,6,rep,name=ingestion_ids,json=ingestionIds,proto3" json:"ingestion_ids,omitempty"` + // Setting this flag to true will force a recalculation of statistics and overwrite results currently in the + // cache, if any. + ForceRefresh bool `protobuf:"varint,7,opt,name=force_refresh,json=forceRefresh,proto3" json:"force_refresh,omitempty"` +} + +func (x *GetFeatureStatisticsRequest) Reset() { + *x = GetFeatureStatisticsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_CoreService_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetFeatureStatisticsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetFeatureStatisticsRequest) ProtoMessage() {} + +func (x *GetFeatureStatisticsRequest) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[24] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetFeatureStatisticsRequest.ProtoReflect.Descriptor instead. +func (*GetFeatureStatisticsRequest) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{24} +} + +func (x *GetFeatureStatisticsRequest) GetFeatureSetId() string { + if x != nil { + return x.FeatureSetId + } + return "" +} + +func (x *GetFeatureStatisticsRequest) GetFeatures() []string { + if x != nil { + return x.Features + } + return nil +} + +func (x *GetFeatureStatisticsRequest) GetStore() string { + if x != nil { + return x.Store + } + return "" +} + +func (x *GetFeatureStatisticsRequest) GetStartDate() *timestamp.Timestamp { + if x != nil { + return x.StartDate + } + return nil +} + +func (x *GetFeatureStatisticsRequest) GetEndDate() *timestamp.Timestamp { + if x != nil { + return x.EndDate + } + return nil +} + +func (x *GetFeatureStatisticsRequest) GetIngestionIds() []string { + if x != nil { + return x.IngestionIds + } + return nil +} + +func (x *GetFeatureStatisticsRequest) GetForceRefresh() bool { + if x != nil { + return x.ForceRefresh + } + return false +} + +type GetFeatureStatisticsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Contains statistics for the requested data. + // Due to the limitations of TFDV and Facets, only a single dataset can be returned in, + // despite the message being of list type. + DatasetFeatureStatisticsList *v0.DatasetFeatureStatisticsList `protobuf:"bytes,1,opt,name=dataset_feature_statistics_list,json=datasetFeatureStatisticsList,proto3" json:"dataset_feature_statistics_list,omitempty"` +} + +func (x *GetFeatureStatisticsResponse) Reset() { + *x = GetFeatureStatisticsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_feast_core_CoreService_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetFeatureStatisticsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetFeatureStatisticsResponse) ProtoMessage() {} + +func (x *GetFeatureStatisticsResponse) ProtoReflect() protoreflect.Message { + mi := &file_feast_core_CoreService_proto_msgTypes[25] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetFeatureStatisticsResponse.ProtoReflect.Descriptor instead. +func (*GetFeatureStatisticsResponse) Descriptor() ([]byte, []int) { + return file_feast_core_CoreService_proto_rawDescGZIP(), []int{25} +} + +func (x *GetFeatureStatisticsResponse) GetDatasetFeatureStatisticsList() *v0.DatasetFeatureStatisticsList { + if x != nil { + return x.DatasetFeatureStatisticsList + } + return nil +} + type ListFeatureSetsRequest_Filter struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1299,7 +1463,7 @@ type ListFeatureSetsRequest_Filter struct { func (x *ListFeatureSetsRequest_Filter) Reset() { *x = ListFeatureSetsRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[24] + mi := &file_feast_core_CoreService_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1312,7 +1476,7 @@ func (x *ListFeatureSetsRequest_Filter) String() string { func (*ListFeatureSetsRequest_Filter) ProtoMessage() {} func (x *ListFeatureSetsRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[24] + mi := &file_feast_core_CoreService_proto_msgTypes[26] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1354,7 +1518,7 @@ type ListStoresRequest_Filter struct { func (x *ListStoresRequest_Filter) Reset() { *x = ListStoresRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[25] + mi := &file_feast_core_CoreService_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1367,7 +1531,7 @@ func (x *ListStoresRequest_Filter) String() string { func (*ListStoresRequest_Filter) ProtoMessage() {} func (x *ListStoresRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[25] + mi := &file_feast_core_CoreService_proto_msgTypes[27] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1406,7 +1570,7 @@ type ListIngestionJobsRequest_Filter struct { func (x *ListIngestionJobsRequest_Filter) Reset() { *x = ListIngestionJobsRequest_Filter{} if protoimpl.UnsafeEnabled { - mi := &file_feast_core_CoreService_proto_msgTypes[26] + mi := &file_feast_core_CoreService_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1419,7 +1583,7 @@ func (x *ListIngestionJobsRequest_Filter) String() string { func (*ListIngestionJobsRequest_Filter) ProtoMessage() {} func (x *ListIngestionJobsRequest_Filter) ProtoReflect() protoreflect.Message { - mi := &file_feast_core_CoreService_proto_msgTypes[26] + mi := &file_feast_core_CoreService_proto_msgTypes[28] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1461,7 +1625,12 @@ var File_feast_core_CoreService_proto protoreflect.FileDescriptor var file_feast_core_CoreService_proto_rawDesc = []byte{ 0x0a, 0x1c, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x1a, 0x1b, 0x66, 0x65, 0x61, 0x73, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x2d, 0x74, 0x65, 0x6e, + 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x76, 0x30, 0x2f, 0x73, 0x74, 0x61, 0x74, 0x69, 0x73, + 0x74, 0x69, 0x63, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x16, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, @@ -1586,82 +1755,118 @@ var file_feast_core_CoreService_proto_rawDesc = []byte{ 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x1a, 0x0a, 0x18, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, - 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xcb, 0x08, 0x0a, - 0x0b, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x66, 0x0a, 0x13, - 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, - 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, - 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xb1, 0x02, 0x0a, + 0x1b, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, + 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x0e, + 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, + 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x14, + 0x0a, 0x05, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x64, 0x61, + 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x73, 0x74, 0x61, 0x72, 0x74, 0x44, 0x61, 0x74, 0x65, 0x12, + 0x35, 0x0a, 0x08, 0x65, 0x6e, 0x64, 0x5f, 0x64, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x65, + 0x6e, 0x64, 0x44, 0x61, 0x74, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x69, 0x6e, 0x67, 0x65, 0x73, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x69, + 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x66, + 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0c, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, + 0x22, 0x9b, 0x01, 0x0a, 0x1c, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, + 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x7b, 0x0a, 0x1f, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x5f, 0x66, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x5f, + 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x74, 0x65, 0x6e, + 0x73, 0x6f, 0x72, 0x66, 0x6c, 0x6f, 0x77, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x76, 0x30, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x4c, 0x69, 0x73, 0x74, + 0x52, 0x1c, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x4c, 0x69, 0x73, 0x74, 0x32, 0xb6, + 0x09, 0x0a, 0x0b, 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x66, + 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x27, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, + 0x61, 0x73, 0x74, 0x43, 0x6f, 0x72, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, - 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x4c, 0x69, - 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x12, 0x22, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, - 0x6f, 0x72, 0x65, 0x73, 0x12, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, - 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x53, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x4e, 0x0a, 0x0b, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1e, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x54, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x12, 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, - 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5a, 0x0a, 0x0f, + 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x12, + 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x69, 0x0a, 0x14, 0x47, 0x65, 0x74, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, + 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, + 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, + 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, + 0x73, 0x12, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1e, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x5a, 0x0a, 0x0f, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x53, 0x65, 0x74, 0x12, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x70, 0x70, 0x6c, 0x79, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x53, 0x65, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0b, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x1e, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, + 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x53, + 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0d, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x20, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x12, 0x21, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x41, 0x72, 0x63, 0x68, 0x69, 0x76, 0x65, 0x50, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, - 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x20, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x60, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, - 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x12, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, - 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, - 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, - 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x13, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, - 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, - 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, - 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, 0x53, - 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, - 0x23, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, - 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, - 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x59, 0x0a, 0x10, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x10, - 0x43, 0x6f, 0x72, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, - 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x4c, + 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x1f, 0x2e, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, + 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x60, + 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, + 0x6f, 0x62, 0x73, 0x12, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, + 0x62, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, + 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x66, 0x0a, 0x13, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, + 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, + 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x52, 0x65, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5d, 0x0a, 0x10, 0x53, 0x74, 0x6f, 0x70, + 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x12, 0x23, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x49, 0x6e, + 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x24, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, + 0x74, 0x6f, 0x70, 0x49, 0x6e, 0x67, 0x65, 0x73, 0x74, 0x69, 0x6f, 0x6e, 0x4a, 0x6f, 0x62, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x59, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x10, 0x43, 0x6f, 0x72, + 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, + 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, + 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1677,7 +1882,7 @@ func file_feast_core_CoreService_proto_rawDescGZIP() []byte { } var file_feast_core_CoreService_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_feast_core_CoreService_proto_msgTypes = make([]protoimpl.MessageInfo, 27) +var file_feast_core_CoreService_proto_msgTypes = make([]protoimpl.MessageInfo, 29) var file_feast_core_CoreService_proto_goTypes = []interface{}{ (ApplyFeatureSetResponse_Status)(0), // 0: feast.core.ApplyFeatureSetResponse.Status (UpdateStoreResponse_Status)(0), // 1: feast.core.UpdateStoreResponse.Status @@ -1705,58 +1910,67 @@ var file_feast_core_CoreService_proto_goTypes = []interface{}{ (*RestartIngestionJobResponse)(nil), // 23: feast.core.RestartIngestionJobResponse (*StopIngestionJobRequest)(nil), // 24: feast.core.StopIngestionJobRequest (*StopIngestionJobResponse)(nil), // 25: feast.core.StopIngestionJobResponse - (*ListFeatureSetsRequest_Filter)(nil), // 26: feast.core.ListFeatureSetsRequest.Filter - (*ListStoresRequest_Filter)(nil), // 27: feast.core.ListStoresRequest.Filter - (*ListIngestionJobsRequest_Filter)(nil), // 28: feast.core.ListIngestionJobsRequest.Filter - (*FeatureSet)(nil), // 29: feast.core.FeatureSet - (*Store)(nil), // 30: feast.core.Store - (*IngestionJob)(nil), // 31: feast.core.IngestionJob - (*FeatureSetReference)(nil), // 32: feast.core.FeatureSetReference + (*GetFeatureStatisticsRequest)(nil), // 26: feast.core.GetFeatureStatisticsRequest + (*GetFeatureStatisticsResponse)(nil), // 27: feast.core.GetFeatureStatisticsResponse + (*ListFeatureSetsRequest_Filter)(nil), // 28: feast.core.ListFeatureSetsRequest.Filter + (*ListStoresRequest_Filter)(nil), // 29: feast.core.ListStoresRequest.Filter + (*ListIngestionJobsRequest_Filter)(nil), // 30: feast.core.ListIngestionJobsRequest.Filter + (*FeatureSet)(nil), // 31: feast.core.FeatureSet + (*Store)(nil), // 32: feast.core.Store + (*IngestionJob)(nil), // 33: feast.core.IngestionJob + (*timestamp.Timestamp)(nil), // 34: google.protobuf.Timestamp + (*v0.DatasetFeatureStatisticsList)(nil), // 35: tensorflow.metadata.v0.DatasetFeatureStatisticsList + (*FeatureSetReference)(nil), // 36: feast.core.FeatureSetReference } var file_feast_core_CoreService_proto_depIdxs = []int32{ - 29, // 0: feast.core.GetFeatureSetResponse.feature_set:type_name -> feast.core.FeatureSet - 26, // 1: feast.core.ListFeatureSetsRequest.filter:type_name -> feast.core.ListFeatureSetsRequest.Filter - 29, // 2: feast.core.ListFeatureSetsResponse.feature_sets:type_name -> feast.core.FeatureSet - 27, // 3: feast.core.ListStoresRequest.filter:type_name -> feast.core.ListStoresRequest.Filter - 30, // 4: feast.core.ListStoresResponse.store:type_name -> feast.core.Store - 29, // 5: feast.core.ApplyFeatureSetRequest.feature_set:type_name -> feast.core.FeatureSet - 29, // 6: feast.core.ApplyFeatureSetResponse.feature_set:type_name -> feast.core.FeatureSet + 31, // 0: feast.core.GetFeatureSetResponse.feature_set:type_name -> feast.core.FeatureSet + 28, // 1: feast.core.ListFeatureSetsRequest.filter:type_name -> feast.core.ListFeatureSetsRequest.Filter + 31, // 2: feast.core.ListFeatureSetsResponse.feature_sets:type_name -> feast.core.FeatureSet + 29, // 3: feast.core.ListStoresRequest.filter:type_name -> feast.core.ListStoresRequest.Filter + 32, // 4: feast.core.ListStoresResponse.store:type_name -> feast.core.Store + 31, // 5: feast.core.ApplyFeatureSetRequest.feature_set:type_name -> feast.core.FeatureSet + 31, // 6: feast.core.ApplyFeatureSetResponse.feature_set:type_name -> feast.core.FeatureSet 0, // 7: feast.core.ApplyFeatureSetResponse.status:type_name -> feast.core.ApplyFeatureSetResponse.Status - 30, // 8: feast.core.UpdateStoreRequest.store:type_name -> feast.core.Store - 30, // 9: feast.core.UpdateStoreResponse.store:type_name -> feast.core.Store + 32, // 8: feast.core.UpdateStoreRequest.store:type_name -> feast.core.Store + 32, // 9: feast.core.UpdateStoreResponse.store:type_name -> feast.core.Store 1, // 10: feast.core.UpdateStoreResponse.status:type_name -> feast.core.UpdateStoreResponse.Status - 28, // 11: feast.core.ListIngestionJobsRequest.filter:type_name -> feast.core.ListIngestionJobsRequest.Filter - 31, // 12: feast.core.ListIngestionJobsResponse.jobs:type_name -> feast.core.IngestionJob - 32, // 13: feast.core.ListIngestionJobsRequest.Filter.feature_set_reference:type_name -> feast.core.FeatureSetReference - 10, // 14: feast.core.CoreService.GetFeastCoreVersion:input_type -> feast.core.GetFeastCoreVersionRequest - 2, // 15: feast.core.CoreService.GetFeatureSet:input_type -> feast.core.GetFeatureSetRequest - 4, // 16: feast.core.CoreService.ListFeatureSets:input_type -> feast.core.ListFeatureSetsRequest - 6, // 17: feast.core.CoreService.ListStores:input_type -> feast.core.ListStoresRequest - 8, // 18: feast.core.CoreService.ApplyFeatureSet:input_type -> feast.core.ApplyFeatureSetRequest - 12, // 19: feast.core.CoreService.UpdateStore:input_type -> feast.core.UpdateStoreRequest - 14, // 20: feast.core.CoreService.CreateProject:input_type -> feast.core.CreateProjectRequest - 16, // 21: feast.core.CoreService.ArchiveProject:input_type -> feast.core.ArchiveProjectRequest - 18, // 22: feast.core.CoreService.ListProjects:input_type -> feast.core.ListProjectsRequest - 20, // 23: feast.core.CoreService.ListIngestionJobs:input_type -> feast.core.ListIngestionJobsRequest - 22, // 24: feast.core.CoreService.RestartIngestionJob:input_type -> feast.core.RestartIngestionJobRequest - 24, // 25: feast.core.CoreService.StopIngestionJob:input_type -> feast.core.StopIngestionJobRequest - 11, // 26: feast.core.CoreService.GetFeastCoreVersion:output_type -> feast.core.GetFeastCoreVersionResponse - 3, // 27: feast.core.CoreService.GetFeatureSet:output_type -> feast.core.GetFeatureSetResponse - 5, // 28: feast.core.CoreService.ListFeatureSets:output_type -> feast.core.ListFeatureSetsResponse - 7, // 29: feast.core.CoreService.ListStores:output_type -> feast.core.ListStoresResponse - 9, // 30: feast.core.CoreService.ApplyFeatureSet:output_type -> feast.core.ApplyFeatureSetResponse - 13, // 31: feast.core.CoreService.UpdateStore:output_type -> feast.core.UpdateStoreResponse - 15, // 32: feast.core.CoreService.CreateProject:output_type -> feast.core.CreateProjectResponse - 17, // 33: feast.core.CoreService.ArchiveProject:output_type -> feast.core.ArchiveProjectResponse - 19, // 34: feast.core.CoreService.ListProjects:output_type -> feast.core.ListProjectsResponse - 21, // 35: feast.core.CoreService.ListIngestionJobs:output_type -> feast.core.ListIngestionJobsResponse - 23, // 36: feast.core.CoreService.RestartIngestionJob:output_type -> feast.core.RestartIngestionJobResponse - 25, // 37: feast.core.CoreService.StopIngestionJob:output_type -> feast.core.StopIngestionJobResponse - 26, // [26:38] is the sub-list for method output_type - 14, // [14:26] is the sub-list for method input_type - 14, // [14:14] is the sub-list for extension type_name - 14, // [14:14] is the sub-list for extension extendee - 0, // [0:14] is the sub-list for field type_name + 30, // 11: feast.core.ListIngestionJobsRequest.filter:type_name -> feast.core.ListIngestionJobsRequest.Filter + 33, // 12: feast.core.ListIngestionJobsResponse.jobs:type_name -> feast.core.IngestionJob + 34, // 13: feast.core.GetFeatureStatisticsRequest.start_date:type_name -> google.protobuf.Timestamp + 34, // 14: feast.core.GetFeatureStatisticsRequest.end_date:type_name -> google.protobuf.Timestamp + 35, // 15: feast.core.GetFeatureStatisticsResponse.dataset_feature_statistics_list:type_name -> tensorflow.metadata.v0.DatasetFeatureStatisticsList + 36, // 16: feast.core.ListIngestionJobsRequest.Filter.feature_set_reference:type_name -> feast.core.FeatureSetReference + 10, // 17: feast.core.CoreService.GetFeastCoreVersion:input_type -> feast.core.GetFeastCoreVersionRequest + 2, // 18: feast.core.CoreService.GetFeatureSet:input_type -> feast.core.GetFeatureSetRequest + 4, // 19: feast.core.CoreService.ListFeatureSets:input_type -> feast.core.ListFeatureSetsRequest + 26, // 20: feast.core.CoreService.GetFeatureStatistics:input_type -> feast.core.GetFeatureStatisticsRequest + 6, // 21: feast.core.CoreService.ListStores:input_type -> feast.core.ListStoresRequest + 8, // 22: feast.core.CoreService.ApplyFeatureSet:input_type -> feast.core.ApplyFeatureSetRequest + 12, // 23: feast.core.CoreService.UpdateStore:input_type -> feast.core.UpdateStoreRequest + 14, // 24: feast.core.CoreService.CreateProject:input_type -> feast.core.CreateProjectRequest + 16, // 25: feast.core.CoreService.ArchiveProject:input_type -> feast.core.ArchiveProjectRequest + 18, // 26: feast.core.CoreService.ListProjects:input_type -> feast.core.ListProjectsRequest + 20, // 27: feast.core.CoreService.ListIngestionJobs:input_type -> feast.core.ListIngestionJobsRequest + 22, // 28: feast.core.CoreService.RestartIngestionJob:input_type -> feast.core.RestartIngestionJobRequest + 24, // 29: feast.core.CoreService.StopIngestionJob:input_type -> feast.core.StopIngestionJobRequest + 11, // 30: feast.core.CoreService.GetFeastCoreVersion:output_type -> feast.core.GetFeastCoreVersionResponse + 3, // 31: feast.core.CoreService.GetFeatureSet:output_type -> feast.core.GetFeatureSetResponse + 5, // 32: feast.core.CoreService.ListFeatureSets:output_type -> feast.core.ListFeatureSetsResponse + 27, // 33: feast.core.CoreService.GetFeatureStatistics:output_type -> feast.core.GetFeatureStatisticsResponse + 7, // 34: feast.core.CoreService.ListStores:output_type -> feast.core.ListStoresResponse + 9, // 35: feast.core.CoreService.ApplyFeatureSet:output_type -> feast.core.ApplyFeatureSetResponse + 13, // 36: feast.core.CoreService.UpdateStore:output_type -> feast.core.UpdateStoreResponse + 15, // 37: feast.core.CoreService.CreateProject:output_type -> feast.core.CreateProjectResponse + 17, // 38: feast.core.CoreService.ArchiveProject:output_type -> feast.core.ArchiveProjectResponse + 19, // 39: feast.core.CoreService.ListProjects:output_type -> feast.core.ListProjectsResponse + 21, // 40: feast.core.CoreService.ListIngestionJobs:output_type -> feast.core.ListIngestionJobsResponse + 23, // 41: feast.core.CoreService.RestartIngestionJob:output_type -> feast.core.RestartIngestionJobResponse + 25, // 42: feast.core.CoreService.StopIngestionJob:output_type -> feast.core.StopIngestionJobResponse + 30, // [30:43] is the sub-list for method output_type + 17, // [17:30] is the sub-list for method input_type + 17, // [17:17] is the sub-list for extension type_name + 17, // [17:17] is the sub-list for extension extendee + 0, // [0:17] is the sub-list for field type_name } func init() { file_feast_core_CoreService_proto_init() } @@ -2058,7 +2272,7 @@ func file_feast_core_CoreService_proto_init() { } } file_feast_core_CoreService_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListFeatureSetsRequest_Filter); i { + switch v := v.(*GetFeatureStatisticsRequest); i { case 0: return &v.state case 1: @@ -2070,7 +2284,7 @@ func file_feast_core_CoreService_proto_init() { } } file_feast_core_CoreService_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListStoresRequest_Filter); i { + switch v := v.(*GetFeatureStatisticsResponse); i { case 0: return &v.state case 1: @@ -2082,6 +2296,30 @@ func file_feast_core_CoreService_proto_init() { } } file_feast_core_CoreService_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListFeatureSetsRequest_Filter); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_CoreService_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ListStoresRequest_Filter); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_feast_core_CoreService_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*ListIngestionJobsRequest_Filter); i { case 0: return &v.state @@ -2100,7 +2338,7 @@ func file_feast_core_CoreService_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_feast_core_CoreService_proto_rawDesc, NumEnums: 2, - NumMessages: 27, + NumMessages: 29, NumExtensions: 0, NumServices: 1, }, @@ -2138,6 +2376,10 @@ type CoreServiceClient interface { // If no filter is provided in the request, the response will contain all the feature // sets currently stored in the registry. ListFeatureSets(ctx context.Context, in *ListFeatureSetsRequest, opts ...grpc.CallOption) (*ListFeatureSetsResponse, error) + // Get feature statistics computed over the data in the batch stores. + // + // Returns a dataset containing TFDV statistics mapped to each valid historical store. + GetFeatureStatistics(ctx context.Context, in *GetFeatureStatisticsRequest, opts ...grpc.CallOption) (*GetFeatureStatisticsResponse, error) // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. @@ -2220,6 +2462,15 @@ func (c *coreServiceClient) ListFeatureSets(ctx context.Context, in *ListFeature return out, nil } +func (c *coreServiceClient) GetFeatureStatistics(ctx context.Context, in *GetFeatureStatisticsRequest, opts ...grpc.CallOption) (*GetFeatureStatisticsResponse, error) { + out := new(GetFeatureStatisticsResponse) + err := c.cc.Invoke(ctx, "/feast.core.CoreService/GetFeatureStatistics", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *coreServiceClient) ListStores(ctx context.Context, in *ListStoresRequest, opts ...grpc.CallOption) (*ListStoresResponse, error) { out := new(ListStoresResponse) err := c.cc.Invoke(ctx, "/feast.core.CoreService/ListStores", in, out, opts...) @@ -2314,6 +2565,10 @@ type CoreServiceServer interface { // If no filter is provided in the request, the response will contain all the feature // sets currently stored in the registry. ListFeatureSets(context.Context, *ListFeatureSetsRequest) (*ListFeatureSetsResponse, error) + // Get feature statistics computed over the data in the batch stores. + // + // Returns a dataset containing TFDV statistics mapped to each valid historical store. + GetFeatureStatistics(context.Context, *GetFeatureStatisticsRequest) (*GetFeatureStatisticsResponse, error) // Retrieve store details given a filter. // // Returns all stores matching that filter. If none are found, an empty list will be returned. @@ -2374,6 +2629,9 @@ func (*UnimplementedCoreServiceServer) GetFeatureSet(context.Context, *GetFeatur func (*UnimplementedCoreServiceServer) ListFeatureSets(context.Context, *ListFeatureSetsRequest) (*ListFeatureSetsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListFeatureSets not implemented") } +func (*UnimplementedCoreServiceServer) GetFeatureStatistics(context.Context, *GetFeatureStatisticsRequest) (*GetFeatureStatisticsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetFeatureStatistics not implemented") +} func (*UnimplementedCoreServiceServer) ListStores(context.Context, *ListStoresRequest) (*ListStoresResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListStores not implemented") } @@ -2460,6 +2718,24 @@ func _CoreService_ListFeatureSets_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _CoreService_GetFeatureStatistics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetFeatureStatisticsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(CoreServiceServer).GetFeatureStatistics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/feast.core.CoreService/GetFeatureStatistics", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(CoreServiceServer).GetFeatureStatistics(ctx, req.(*GetFeatureStatisticsRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _CoreService_ListStores_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ListStoresRequest) if err := dec(in); err != nil { @@ -2638,6 +2914,10 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ MethodName: "ListFeatureSets", Handler: _CoreService_ListFeatureSets_Handler, }, + { + MethodName: "GetFeatureStatistics", + Handler: _CoreService_GetFeatureStatistics_Handler, + }, { MethodName: "ListStores", Handler: _CoreService_ListStores_Handler, diff --git a/sdk/go/protos/feast/core/Runner.pb.go b/sdk/go/protos/feast/core/Runner.pb.go index ac936df021a..74dcf9fad77 100644 --- a/sdk/go/protos/feast/core/Runner.pb.go +++ b/sdk/go/protos/feast/core/Runner.pb.go @@ -53,6 +53,9 @@ type DirectRunnerConfigOptions struct { TargetParallelism int32 `protobuf:"varint,1,opt,name=targetParallelism,proto3" json:"targetParallelism,omitempty"` // BigQuery table specification, e.g. PROJECT_ID:DATASET_ID.PROJECT_ID DeadLetterTableSpec string `protobuf:"bytes,2,opt,name=deadLetterTableSpec,proto3" json:"deadLetterTableSpec,omitempty"` + // A pipeline level default location for storing temporary files. + // Support Google Cloud Storage locations or local path + TempLocation string `protobuf:"bytes,3,opt,name=tempLocation,proto3" json:"tempLocation,omitempty"` } func (x *DirectRunnerConfigOptions) Reset() { @@ -101,6 +104,13 @@ func (x *DirectRunnerConfigOptions) GetDeadLetterTableSpec() string { return "" } +func (x *DirectRunnerConfigOptions) GetTempLocation() string { + if x != nil { + return x.TempLocation + } + return "" +} + type DataflowRunnerConfigOptions struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -263,58 +273,60 @@ var File_feast_core_Runner_proto protoreflect.FileDescriptor var file_feast_core_Runner_proto_rawDesc = []byte{ 0x0a, 0x17, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x7b, 0x0a, 0x19, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x52, - 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x72, 0x61, - 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, 0x6d, - 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, - 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, - 0x65, 0x63, 0x22, 0xcf, 0x04, 0x0a, 0x1b, 0x44, 0x61, 0x74, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x52, - 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, - 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x72, 0x65, - 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x26, 0x0a, 0x0e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, - 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x1e, 0x0a, 0x0a, 0x73, 0x75, - 0x62, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x2c, 0x0a, 0x11, 0x77, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x32, 0x0a, 0x14, 0x61, 0x75, 0x74, 0x6f, - 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, - 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, - 0x69, 0x6e, 0x67, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x22, 0x0a, 0x0c, - 0x75, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x70, 0x73, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0c, 0x75, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x70, 0x73, - 0x12, 0x22, 0x0a, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x61, 0x78, 0x4e, 0x75, 0x6d, 0x57, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x6d, 0x61, 0x78, - 0x4e, 0x75, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x22, 0x9f, 0x01, 0x0a, 0x19, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x2c, 0x0a, 0x11, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x72, + 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, + 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x69, 0x73, + 0x6d, 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, + 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, + 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, + 0x70, 0x65, 0x63, 0x12, 0x22, 0x0a, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xcf, 0x04, 0x0a, 0x1b, 0x44, 0x61, 0x74, 0x61, + 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x12, 0x16, 0x0a, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x7a, 0x6f, 0x6e, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x7a, 0x6f, 0x6e, 0x65, 0x12, 0x26, 0x0a, + 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x41, 0x63, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, + 0x1e, 0x0a, 0x0a, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, 0x75, 0x62, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, + 0x2c, 0x0a, 0x11, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, + 0x54, 0x79, 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x77, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x32, 0x0a, + 0x14, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x41, 0x6c, 0x67, 0x6f, + 0x72, 0x69, 0x74, 0x68, 0x6d, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x61, 0x75, 0x74, + 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, + 0x6d, 0x12, 0x22, 0x0a, 0x0c, 0x75, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x49, 0x70, + 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x75, 0x73, 0x65, 0x50, 0x75, 0x62, 0x6c, + 0x69, 0x63, 0x49, 0x70, 0x73, 0x12, 0x22, 0x0a, 0x0c, 0x74, 0x65, 0x6d, 0x70, 0x4c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x74, 0x65, 0x6d, + 0x70, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x61, 0x78, + 0x4e, 0x75, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x0d, 0x6d, 0x61, 0x78, 0x4e, 0x75, 0x6d, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, + 0x30, 0x0a, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, + 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, - 0x63, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x65, 0x61, 0x64, 0x4c, 0x65, 0x74, - 0x74, 0x65, 0x72, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x70, 0x65, 0x63, 0x12, 0x4b, 0x0a, 0x06, - 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x33, 0x2e, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x66, 0x6c, - 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x42, 0x54, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0b, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, - 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x63, 0x12, 0x4b, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x33, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x44, + 0x61, 0x74, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x1a, 0x39, + 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x54, 0x0a, 0x10, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0b, 0x52, + 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, + 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, + 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sdk/go/protos/feast/core/Store.pb.go b/sdk/go/protos/feast/core/Store.pb.go index a36c6afc6ab..829e2e7c180 100644 --- a/sdk/go/protos/feast/core/Store.pb.go +++ b/sdk/go/protos/feast/core/Store.pb.go @@ -377,11 +377,12 @@ type Store_BigQueryConfig struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` - DatasetId string `protobuf:"bytes,2,opt,name=dataset_id,json=datasetId,proto3" json:"dataset_id,omitempty"` - StagingLocation string `protobuf:"bytes,3,opt,name=staging_location,json=stagingLocation,proto3" json:"staging_location,omitempty"` - InitialRetryDelaySeconds int32 `protobuf:"varint,4,opt,name=initial_retry_delay_seconds,json=initialRetryDelaySeconds,proto3" json:"initial_retry_delay_seconds,omitempty"` - TotalTimeoutSeconds int32 `protobuf:"varint,5,opt,name=total_timeout_seconds,json=totalTimeoutSeconds,proto3" json:"total_timeout_seconds,omitempty"` + ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` + DatasetId string `protobuf:"bytes,2,opt,name=dataset_id,json=datasetId,proto3" json:"dataset_id,omitempty"` + StagingLocation string `protobuf:"bytes,3,opt,name=staging_location,json=stagingLocation,proto3" json:"staging_location,omitempty"` + InitialRetryDelaySeconds int32 `protobuf:"varint,4,opt,name=initial_retry_delay_seconds,json=initialRetryDelaySeconds,proto3" json:"initial_retry_delay_seconds,omitempty"` + TotalTimeoutSeconds int32 `protobuf:"varint,5,opt,name=total_timeout_seconds,json=totalTimeoutSeconds,proto3" json:"total_timeout_seconds,omitempty"` + WriteTriggeringFrequencySeconds int32 `protobuf:"varint,6,opt,name=write_triggering_frequency_seconds,json=writeTriggeringFrequencySeconds,proto3" json:"write_triggering_frequency_seconds,omitempty"` } func (x *Store_BigQueryConfig) Reset() { @@ -451,6 +452,13 @@ func (x *Store_BigQueryConfig) GetTotalTimeoutSeconds() int32 { return 0 } +func (x *Store_BigQueryConfig) GetWriteTriggeringFrequencySeconds() int32 { + if x != nil { + return x.WriteTriggeringFrequencySeconds + } + return 0 +} + type Store_CassandraConfig struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -643,7 +651,7 @@ var File_feast_core_Store_proto protoreflect.FileDescriptor var file_feast_core_Store_proto_rawDesc = []byte{ 0x0a, 0x16, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x22, 0xb4, 0x09, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, + 0x63, 0x6f, 0x72, 0x65, 0x22, 0x81, 0x0a, 0x0a, 0x05, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x53, 0x74, @@ -680,7 +688,7 @@ var file_feast_core_Store_proto_rawDesc = []byte{ 0x6d, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0xec, 0x01, 0x0a, 0x0e, + 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0xb9, 0x02, 0x0a, 0x0e, 0x42, 0x69, 0x67, 0x51, 0x75, 0x65, 0x72, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x49, 0x64, 0x12, 0x1d, 0x0a, @@ -695,36 +703,41 @@ var file_feast_core_Store_proto_rawDesc = []byte{ 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x15, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x13, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x54, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x1a, 0x39, 0x0a, 0x0f, 0x43, 0x61, - 0x73, 0x73, 0x61, 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, - 0x04, 0x68, 0x6f, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, - 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x04, 0x70, 0x6f, 0x72, 0x74, 0x1a, 0x90, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x64, 0x69, 0x73, 0x43, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2b, 0x0a, 0x11, - 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x69, - 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, - 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, - 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6d, 0x61, - 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0x42, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, - 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0x53, 0x0a, 0x09, - 0x53, 0x74, 0x6f, 0x72, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, - 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x44, 0x49, 0x53, 0x10, - 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x42, 0x49, 0x47, 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, - 0x0d, 0x0a, 0x09, 0x43, 0x41, 0x53, 0x53, 0x41, 0x4e, 0x44, 0x52, 0x41, 0x10, 0x03, 0x12, 0x11, - 0x0a, 0x0d, 0x52, 0x45, 0x44, 0x49, 0x53, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x10, - 0x04, 0x42, 0x08, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x53, 0x0a, 0x10, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, - 0x0a, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, - 0x76, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x75, 0x74, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x12, 0x4b, 0x0a, 0x22, 0x77, 0x72, + 0x69, 0x74, 0x65, 0x5f, 0x74, 0x72, 0x69, 0x67, 0x67, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x66, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x1f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x54, 0x72, 0x69, + 0x67, 0x67, 0x65, 0x72, 0x69, 0x6e, 0x67, 0x46, 0x72, 0x65, 0x71, 0x75, 0x65, 0x6e, 0x63, 0x79, + 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x1a, 0x39, 0x0a, 0x0f, 0x43, 0x61, 0x73, 0x73, 0x61, + 0x6e, 0x64, 0x72, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x6f, + 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, 0x6f, 0x73, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x70, 0x6f, + 0x72, 0x74, 0x1a, 0x90, 0x01, 0x0a, 0x12, 0x52, 0x65, 0x64, 0x69, 0x73, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2b, 0x0a, 0x11, 0x63, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, + 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x12, 0x2c, 0x0a, 0x12, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, + 0x6c, 0x5f, 0x62, 0x61, 0x63, 0x6b, 0x6f, 0x66, 0x66, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x10, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x6f, + 0x66, 0x66, 0x4d, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, + 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, + 0x74, 0x72, 0x69, 0x65, 0x73, 0x1a, 0x42, 0x0a, 0x0c, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x4a, 0x04, 0x08, 0x02, 0x10, 0x03, 0x22, 0x53, 0x0a, 0x09, 0x53, 0x74, 0x6f, + 0x72, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, + 0x44, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x52, 0x45, 0x44, 0x49, 0x53, 0x10, 0x01, 0x12, 0x0c, + 0x0a, 0x08, 0x42, 0x49, 0x47, 0x51, 0x55, 0x45, 0x52, 0x59, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, + 0x43, 0x41, 0x53, 0x53, 0x41, 0x4e, 0x44, 0x52, 0x41, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x52, + 0x45, 0x44, 0x49, 0x53, 0x5f, 0x43, 0x4c, 0x55, 0x53, 0x54, 0x45, 0x52, 0x10, 0x04, 0x42, 0x08, + 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x42, 0x53, 0x0a, 0x10, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x42, 0x0a, 0x53, 0x74, + 0x6f, 0x72, 0x65, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sdk/go/protos/feast/serving/ServingService.pb.go b/sdk/go/protos/feast/serving/ServingService.pb.go index 6946233c28f..c8b09a352b5 100644 --- a/sdk/go/protos/feast/serving/ServingService.pb.go +++ b/sdk/go/protos/feast/serving/ServingService.pb.go @@ -248,9 +248,9 @@ type GetOnlineFeaturesResponse_FieldStatus int32 const ( // Status is unset for this field. GetOnlineFeaturesResponse_INVALID GetOnlineFeaturesResponse_FieldStatus = 0 - // Field value is present for this field and within maximum allowable range + // Field value is present for this field and age is within max age. GetOnlineFeaturesResponse_PRESENT GetOnlineFeaturesResponse_FieldStatus = 1 - // Values could be found for entity key within the maximum allowable range, but + // Values could be found for entity key and age is within max age, but // this field value is assigned a value on ingestion into feast. GetOnlineFeaturesResponse_NULL_VALUE GetOnlineFeaturesResponse_FieldStatus = 2 // Entity key did not return any values as they do not exist in Feast. @@ -495,9 +495,6 @@ type GetOnlineFeaturesRequest struct { // Option to omit entities from the response. If true, only feature // values will be returned. OmitEntitiesInResponse bool `protobuf:"varint,3,opt,name=omit_entities_in_response,json=omitEntitiesInResponse,proto3" json:"omit_entities_in_response,omitempty"` - // Option to include feature metadata in the response. - // If true, response will include both feature values and metadata. - IncludeMetadataInResponse bool `protobuf:"varint,5,opt,name=include_metadata_in_response,json=includeMetadataInResponse,proto3" json:"include_metadata_in_response,omitempty"` } func (x *GetOnlineFeaturesRequest) Reset() { @@ -553,13 +550,6 @@ func (x *GetOnlineFeaturesRequest) GetOmitEntitiesInResponse() bool { return false } -func (x *GetOnlineFeaturesRequest) GetIncludeMetadataInResponse() bool { - if x != nil { - return x.IncludeMetadataInResponse - } - return false -} - type GetOnlineFeaturesResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1171,7 +1161,7 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x53, 0x65, 0x74, 0x4a, - 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0xa2, 0x04, 0x0a, 0x18, + 0x04, 0x08, 0x03, 0x10, 0x04, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0xe1, 0x03, 0x0a, 0x18, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x65, 0x61, @@ -1186,166 +1176,162 @@ var file_feast_serving_ServingService_proto_rawDesc = []byte{ 0x74, 0x5f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x5f, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x16, 0x6f, 0x6d, 0x69, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x49, 0x6e, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x1c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x5f, - 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x19, 0x69, 0x6e, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x49, 0x6e, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0xf8, 0x01, 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, - 0x52, 0x6f, 0x77, 0x12, 0x45, 0x0a, 0x10, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x65, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x55, 0x0a, 0x06, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, - 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x2e, 0x46, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, - 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0xdd, 0x04, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, - 0x0a, 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, - 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, - 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x89, 0x03, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, - 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, - 0x73, 0x12, 0x5e, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0xf8, 0x01, 0x0a, 0x09, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, + 0x6f, 0x77, 0x12, 0x45, 0x0a, 0x10, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x74, 0x69, 0x6d, + 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x55, 0x0a, 0x06, 0x66, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, + 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x6f, 0x77, 0x2e, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, + 0xdd, 0x04, 0x0a, 0x19, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, + 0x0c, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, - 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, - 0x73, 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x1a, 0x71, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x4a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x52, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x1a, 0x89, 0x03, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x12, 0x5e, 0x0a, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x42, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, - 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x22, 0x5b, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, - 0x0b, 0x0a, 0x07, 0x50, 0x52, 0x45, 0x53, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, - 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, - 0x4e, 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x4f, - 0x55, 0x54, 0x53, 0x49, 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, 0x10, 0x04, - 0x22, 0x9b, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x08, - 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, - 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, - 0x08, 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0e, 0x64, 0x61, 0x74, - 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, - 0x0d, 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x40, - 0x0a, 0x18, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, - 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, - 0x22, 0x35, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, - 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, - 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x36, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x4a, 0x6f, - 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, + 0x6c, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, + 0x1a, 0x4d, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, + 0x71, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x4a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x34, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x5b, 0x0a, 0x0b, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x0b, + 0x0a, 0x07, 0x50, 0x52, 0x45, 0x53, 0x45, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, + 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55, 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x4e, + 0x4f, 0x54, 0x5f, 0x46, 0x4f, 0x55, 0x4e, 0x44, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x4f, 0x55, + 0x54, 0x53, 0x49, 0x44, 0x45, 0x5f, 0x4d, 0x41, 0x58, 0x5f, 0x41, 0x47, 0x45, 0x10, 0x04, 0x22, + 0x9b, 0x01, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x08, 0x66, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x65, + 0x61, 0x74, 0x75, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x08, + 0x66, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x43, 0x0a, 0x0e, 0x64, 0x61, 0x74, 0x61, + 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x52, 0x0d, + 0x64, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x40, 0x0a, + 0x18, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, - 0xe2, 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, - 0x79, 0x70, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x66, - 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, - 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, - 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, - 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, - 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x46, 0x6f, - 0x72, 0x6d, 0x61, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, - 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x66, 0x65, - 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, - 0x73, 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x1a, 0x65, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, - 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, - 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, - 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x64, 0x61, 0x74, - 0x61, 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2a, 0x6f, 0x0a, 0x10, 0x46, - 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x1e, 0x0a, 0x1a, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, - 0x1d, 0x0a, 0x19, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, - 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1c, - 0x0a, 0x18, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, - 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x2a, 0x36, 0x0a, 0x07, - 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x4a, 0x4f, 0x42, 0x5f, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, - 0x11, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, - 0x41, 0x44, 0x10, 0x01, 0x2a, 0x68, 0x0a, 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, - 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, - 0x01, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x4f, 0x42, - 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x2a, 0x3b, - 0x0a, 0x0a, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x17, 0x0a, 0x13, - 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x49, 0x4e, 0x56, 0x41, - 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, - 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, 0x01, 0x32, 0x92, 0x03, 0x0a, 0x0e, - 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, - 0x0a, 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x11, - 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x73, 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, - 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, - 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, - 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, - 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, - 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, - 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x06, 0x47, 0x65, 0x74, - 0x4a, 0x6f, 0x62, 0x12, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, - 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, - 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x42, 0x5e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x41, 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, - 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x35, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, + 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0x36, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x03, 0x6a, 0x6f, 0x62, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, + 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x52, 0x03, 0x6a, 0x6f, 0x62, 0x22, 0xe2, + 0x01, 0x0a, 0x03, 0x4a, 0x6f, 0x62, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, + 0x70, 0x65, 0x12, 0x30, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x2e, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x69, + 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x66, + 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x5f, + 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x66, + 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, + 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, + 0x6d, 0x61, 0x74, 0x22, 0xd4, 0x01, 0x0a, 0x0d, 0x44, 0x61, 0x74, 0x61, 0x73, 0x65, 0x74, 0x53, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x4a, 0x0a, 0x0b, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x66, 0x65, 0x61, + 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x73, + 0x65, 0x74, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x48, 0x00, 0x52, 0x0a, 0x66, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x1a, 0x65, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, + 0x1b, 0x0a, 0x09, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x69, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x55, 0x72, 0x69, 0x73, 0x12, 0x3a, 0x0a, 0x0b, + 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x19, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, + 0x67, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x52, 0x0a, 0x64, 0x61, + 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x42, 0x10, 0x0a, 0x0e, 0x64, 0x61, 0x74, 0x61, + 0x73, 0x65, 0x74, 0x5f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2a, 0x6f, 0x0a, 0x10, 0x46, 0x65, + 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1e, + 0x0a, 0x1a, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x1d, + 0x0a, 0x19, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x4c, 0x49, 0x4e, 0x45, 0x10, 0x01, 0x12, 0x1c, 0x0a, + 0x18, 0x46, 0x45, 0x41, 0x53, 0x54, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x4e, 0x47, 0x5f, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x42, 0x41, 0x54, 0x43, 0x48, 0x10, 0x02, 0x2a, 0x36, 0x0a, 0x07, 0x4a, + 0x6f, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, + 0x4a, 0x4f, 0x42, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x57, 0x4e, 0x4c, 0x4f, 0x41, + 0x44, 0x10, 0x01, 0x2a, 0x68, 0x0a, 0x09, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x49, + 0x4e, 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x01, + 0x12, 0x16, 0x0a, 0x12, 0x4a, 0x4f, 0x42, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, + 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x4a, 0x4f, 0x42, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x03, 0x2a, 0x3b, 0x0a, + 0x0a, 0x44, 0x61, 0x74, 0x61, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x17, 0x0a, 0x13, 0x44, + 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, 0x4d, 0x41, 0x54, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, + 0x49, 0x44, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x44, 0x41, 0x54, 0x41, 0x5f, 0x46, 0x4f, 0x52, + 0x4d, 0x41, 0x54, 0x5f, 0x41, 0x56, 0x52, 0x4f, 0x10, 0x01, 0x32, 0x92, 0x03, 0x0a, 0x0e, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x6c, 0x0a, + 0x13, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, + 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x6e, 0x67, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2a, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, + 0x47, 0x65, 0x74, 0x46, 0x65, 0x61, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x49, + 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x66, 0x0a, 0x11, 0x47, + 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, + 0x12, 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x66, 0x65, 0x61, 0x73, + 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x6c, + 0x69, 0x6e, 0x65, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, + 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x26, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, + 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x27, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x2e, + 0x47, 0x65, 0x74, 0x42, 0x61, 0x74, 0x63, 0x68, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4a, + 0x6f, 0x62, 0x12, 0x1c, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x6e, 0x67, 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1d, 0x2e, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, + 0x2e, 0x47, 0x65, 0x74, 0x4a, 0x6f, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, + 0x5e, 0x0a, 0x13, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x42, 0x0f, 0x53, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x41, + 0x50, 0x49, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2d, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x65, + 0x61, 0x73, 0x74, 0x2f, 0x73, 0x64, 0x6b, 0x2f, 0x67, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x73, 0x2f, 0x66, 0x65, 0x61, 0x73, 0x74, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x6e, 0x67, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sdk/go/request.go b/sdk/go/request.go index 52e044b2656..a11999ae890 100644 --- a/sdk/go/request.go +++ b/sdk/go/request.go @@ -28,9 +28,6 @@ type OnlineFeaturesRequest struct { // whether to omit the entities fields in the response. OmitEntities bool - - // whether to include field status metadata in the response. - IncludeMeta bool } // Builds the feast-specified request payload from the wrapper. @@ -52,7 +49,6 @@ func (r OnlineFeaturesRequest) buildRequest() (*serving.GetOnlineFeaturesRequest Features: featureRefs, EntityRows: entityRows, OmitEntitiesInResponse: r.OmitEntities, - IncludeMetadataInResponse: r.IncludeMeta, }, nil } diff --git a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java index ec03ee8e504..11b4f352f64 100644 --- a/sdk/java/src/main/java/com/gojek/feast/FeastClient.java +++ b/sdk/java/src/main/java/com/gojek/feast/FeastClient.java @@ -61,7 +61,7 @@ public GetFeastServingInfoResponse getFeastServingInfo() { /** * Get online features from Feast from FeatureSets * - *

See {@link #getOnlineFeatures(List, List, String, boolean, boolean)} + *

See {@link #getOnlineFeatures(List, List, String, boolean)} * * @param featureRefs list of string feature references to retrieve in the following format * featureSet:feature, where 'featureSet' and 'feature' refer to the FeatureSet and Feature @@ -76,7 +76,7 @@ public List getOnlineFeatures(List featureRefs, List rows) { /** * Get online features from Feast. * - *

See {@link #getOnlineFeatures(List, List, String, boolean, boolean)} + *

See {@link #getOnlineFeatures(List, List, String, boolean)} * * @param featureRefs list of string feature references to retrieve in the following format * featureSet:feature, where 'featureSet' and 'feature' refer to the FeatureSet and Feature @@ -87,7 +87,7 @@ public List getOnlineFeatures(List featureRefs, List rows) { * @return list of {@link Row} containing retrieved data fields. */ public List getOnlineFeatures(List featureRefs, List rows, String project) { - return getOnlineFeatures(featureRefs, rows, project, false, false); + return getOnlineFeatures(featureRefs, rows, project, false); } /** @@ -113,15 +113,10 @@ public List getOnlineFeatures(List featureRefs, List rows, Str * Feature requested belong to. * @param omitEntitiesInResponse if true, the returned {@link Row} will not contain field and * value for the entity - * @param includeMetadataInResponse if true, will include field status metadata in {@link Row}. * @return list of {@link Row} containing retrieved data fields. */ public List getOnlineFeatures( - List featureRefs, - List rows, - String project, - boolean omitEntitiesInResponse, - boolean includeMetadataInResponse) { + List featureRefs, List rows, String project, boolean omitEntitiesInResponse) { List features = RequestUtil.createFeatureRefs(featureRefs, project); // build entity rows and collect entity references HashSet entityRefs = new HashSet<>(); @@ -143,7 +138,6 @@ public List getOnlineFeatures( .addAllFeatures(features) .addAllEntityRows(entityRows) .setOmitEntitiesInResponse(omitEntitiesInResponse) - .setIncludeMetadataInResponse(includeMetadataInResponse) .build()); return response.getFieldValuesList().stream() diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index ac22cd3e721..130a8149017 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -672,7 +672,6 @@ def get_online_features( entity_rows: List[GetOnlineFeaturesRequest.EntityRow], project: Optional[str] = None, omit_entities: bool = False, - include_meta: bool = False, ) -> GetOnlineFeaturesResponse: """ Retrieves the latest online feature data from Feast Serving @@ -700,7 +699,6 @@ def get_online_features( response = self._serving_service_stub.GetOnlineFeatures( GetOnlineFeaturesRequest( omit_entities_in_response=omit_entities, - include_metadata_in_response=include_meta, features=_build_feature_references( feature_ref_strs=feature_refs, project=project if project is not None else self.project, diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 19691e5e7f0..13dca6b8ec0 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -177,7 +177,6 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { // some keys not present, should have empty values GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() - .setIncludeMetadataInResponse(true) .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) .addFeatures( FeatureReference.newBuilder().setName("feature2").setProject("project").build()) @@ -261,7 +260,6 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { .addFeatures( FeatureReference.newBuilder().setName("feature2").setProject("project").build()) .setOmitEntitiesInResponse(false) - .setIncludeMetadataInResponse(true) .addEntityRows( EntityRow.newBuilder() .setEntityTimestamp(Timestamp.newBuilder().setSeconds(100)) @@ -370,7 +368,6 @@ public void shouldFilterOutUndesiredRows() { // requested rows less than the rows available in the featureset GetOnlineFeaturesRequest request = GetOnlineFeaturesRequest.newBuilder() - .setIncludeMetadataInResponse(true) .setOmitEntitiesInResponse(false) .addFeatures(FeatureReference.newBuilder().setName("feature1").build()) .addEntityRows( diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 989be273b81..205420988f9 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -229,8 +229,7 @@ def try_get_features(): ) ], feature_refs=feature_refs, - include_meta=True, - ) # type: GetOnlineFeaturesResponse + # type: GetOnlineFeaturesResponse is_ok = all([check_online_response(ref, cust_trans_df, response) for ref in feature_refs]) return response, is_ok @@ -436,7 +435,6 @@ def try_get_features(): ) ], feature_refs=feature_refs, - include_meta=True, ) # type: GetOnlineFeaturesResponse is_ok = check_online_response("float_feature", all_types_dataframe, response) return response, is_ok @@ -551,7 +549,6 @@ def test_large_volume_retrieve_online_success(client, large_volume_dataframe): ) ], feature_refs=feature_refs, - include_meta=True, ) # type: GetOnlineFeaturesResponse is_ok = all([check_online_response(ref, large_volume_dataframe, response) for ref in feature_refs]) return None, is_ok From 32cccaa19459a0f6d040cd4709d6e453e978cf43 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Tue, 16 Jun 2020 18:35:37 +0800 Subject: [PATCH 59/65] Readd missing span log call to log retrieved feature rows. --- sdk/python/feast/client.py | 1 + .../serving/service/OnlineServingService.java | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/sdk/python/feast/client.py b/sdk/python/feast/client.py index 130a8149017..c30e4265df7 100644 --- a/sdk/python/feast/client.py +++ b/sdk/python/feast/client.py @@ -687,6 +687,7 @@ def get_online_features( retrieval. All entity types within a feature project: Specifies the project which contain the FeatureSets which the requested features belong to. + omit_entities: If true will omit entity values in the returned feature data. Returns: GetOnlineFeaturesResponse containing the feature data in records. diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index de4b48f77ea..f3df9725063 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -16,6 +16,7 @@ */ package feast.serving.service; +import com.google.common.collect.ImmutableMap; import com.google.protobuf.Duration; import feast.proto.serving.ServingAPIProto.*; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; @@ -69,7 +70,9 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ // Collect the feature/entity status metadata for each entity row in entityValueMap Map> entityStatusesMap = entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); - + // Collect featureRows retrieved for logging/tracing + List> logFeatureRows = new LinkedList<>(); + if (!request.getOmitEntitiesInResponse()) { // Add entity row's fields as response fields entityRows.forEach( @@ -100,7 +103,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ FeatureRow featureRow = featureRows.get(i); EntityRow entityRow = entityRows.get(i); // Unpack feature field values and merge into entityValueMap - boolean isStaleValues = this.isStale(featureSetRequest, entityRow, featureRow); + boolean isStaleValues = this.isOutsideMaxAge(featureSetRequest, entityRow, featureRow); Map valueMap = this.unpackValueMap(featureRow, featureSetRequest, isStaleValues); entityValuesMap.get(entityRow).putAll(valueMap); @@ -116,6 +119,11 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ } this.populateRequestCountMetrics(featureSetRequest); } + + + if (scope != null) { + scope.span().log(ImmutableMap.of("event", "featureRows", "value", logFeatureRows)); + } // Build response field values from entityValuesMap and entityStatusesMap // Reponse field values should be in the same order as the entityRows provided by the user. @@ -209,7 +217,7 @@ private Map getMetadataMap( } /** - * Determine if the feature data in the given feature row is considered stale. Data is considered + * Determine if the feature data in the given feature row is outside maxAge. Data is outside maxAge * to be stale when difference ingestion time set in feature row and the retrieval time set in * entity row exceeds featureset max age. * @@ -217,7 +225,7 @@ private Map getMetadataMap( * @param entityRow contains the retrieval timing of when features are pulled. * @param featureRow contains the ingestion timing and feature data. */ - private boolean isStale( + private boolean isOutsideMaxAge( FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { Duration maxAge = featureSetRequest.getSpec().getMaxAge(); if (featureRow == null) { From c05614d8c8e8376ea99b6250985040a79849deb0 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 17 Jun 2020 10:39:29 +0800 Subject: [PATCH 60/65] Use more modern Streams.zip() to iterate entity rows and feature rows in OnlineServingService --- .../serving/service/OnlineServingService.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index f3df9725063..538002abc1c 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -34,6 +34,9 @@ import io.opentracing.Tracer; import java.util.*; import java.util.stream.Collectors; + +import org.apache.beam.vendor.grpc.v1p21p0.com.google.common.collect.Streams; +import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; public class OnlineServingService implements ServingService { @@ -72,7 +75,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); // Collect featureRows retrieved for logging/tracing List> logFeatureRows = new LinkedList<>(); - + if (!request.getOmitEntitiesInResponse()) { // Add entity row's fields as response fields entityRows.forEach( @@ -99,9 +102,9 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ .asRuntimeException(); } - for (var i = 0; i < entityRows.size(); i++) { - FeatureRow featureRow = featureRows.get(i); - EntityRow entityRow = entityRows.get(i); + Streams.zip(entityRows.stream(), featureRows.stream(), Pair::of).forEach(entityFeaturePair -> { + EntityRow entityRow = entityFeaturePair.getLeft(); + FeatureRow featureRow = entityFeaturePair.getRight(); // Unpack feature field values and merge into entityValueMap boolean isStaleValues = this.isOutsideMaxAge(featureSetRequest, entityRow, featureRow); Map valueMap = @@ -116,11 +119,10 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ // Populate metrics this.populateStaleKeyCountMetrics(statusMap, featureSetRequest); - } + }); this.populateRequestCountMetrics(featureSetRequest); + logFeatureRows.add(featureRows); } - - if (scope != null) { scope.span().log(ImmutableMap.of("event", "featureRows", "value", logFeatureRows)); } @@ -148,7 +150,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ * @param featureRow to unpack for feature values. * @param featureSetRequest for which the feature row was retrieved. * @param isStaleValues whether which the feature row contains stale values. - * @return valueMap mapping string feature reference to feature valuyesaq + * @return valueMap mapping string feature name to feature value for the given feature set request. */ private Map unpackValueMap( FeatureRow featureRow, FeatureSetRequest featureSetRequest, boolean isStaleValues) { @@ -169,11 +171,7 @@ private Map unpackValueMap( }, featureRowField -> { // drop stale feature values. - if (isStaleValues) { - return Value.newBuilder().build(); - } else { - return featureRowField.getValue(); - } + return (isStaleValues) ? Value.newBuilder().build() : featureRowField.getValue(); })); valueMap.putAll(featureValueMap); } @@ -192,10 +190,11 @@ private Map unpackValueMap( /** * Generate Field level Status metadata for the given valueMap. * + * @param valueMap map of field name to value to generate metadata for. * @param isNotFound whether the given valueMap represents values that were not found in the * online retriever. * @param isStaleValues whether the given valueMap contains stale values. - * @return a 1:1 map containing field status metadata instead of values in the valueMap. + * @return a 1:1 map keyed by field name containing field status metadata instead of values in the given valueMap. */ private Map getMetadataMap( Map valueMap, boolean isNotFound, boolean isStaleValues) { @@ -217,9 +216,9 @@ private Map getMetadataMap( } /** - * Determine if the feature data in the given feature row is outside maxAge. Data is outside maxAge - * to be stale when difference ingestion time set in feature row and the retrieval time set in - * entity row exceeds featureset max age. + * Determine if the feature data in the given feature row is outside maxAge. Data is outside + * maxAge to be stale when difference ingestion time set in feature row and the retrieval time set + * in entity row exceeds featureset max age. * * @param featureSetRequest contains the spec where feature's max age is extracted. * @param entityRow contains the retrieval timing of when features are pulled. From 81422ccd392043250c2f00e6a6ff61471208f367 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 17 Jun 2020 10:58:29 +0800 Subject: [PATCH 61/65] Change utility methods in OnlineServingService to static. --- .../serving/service/OnlineServingService.java | 55 ++++++++++--------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index 538002abc1c..c57d0d18fdc 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -34,7 +34,6 @@ import io.opentracing.Tracer; import java.util.*; import java.util.stream.Collectors; - import org.apache.beam.vendor.grpc.v1p21p0.com.google.common.collect.Streams; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; @@ -82,7 +81,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ entityRow -> { Map valueMap = entityRow.getFieldsMap(); entityValuesMap.get(entityRow).putAll(valueMap); - entityStatusesMap.get(entityRow).putAll(this.getMetadataMap(valueMap, false, false)); + entityStatusesMap.get(entityRow).putAll(getMetadataMap(valueMap, false, false)); }); } @@ -102,25 +101,27 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ .asRuntimeException(); } - Streams.zip(entityRows.stream(), featureRows.stream(), Pair::of).forEach(entityFeaturePair -> { - EntityRow entityRow = entityFeaturePair.getLeft(); - FeatureRow featureRow = entityFeaturePair.getRight(); - // Unpack feature field values and merge into entityValueMap - boolean isStaleValues = this.isOutsideMaxAge(featureSetRequest, entityRow, featureRow); - Map valueMap = - this.unpackValueMap(featureRow, featureSetRequest, isStaleValues); - entityValuesMap.get(entityRow).putAll(valueMap); + Streams.zip(entityRows.stream(), featureRows.stream(), Pair::of) + .forEach( + entityFeaturePair -> { + EntityRow entityRow = entityFeaturePair.getLeft(); + FeatureRow featureRow = entityFeaturePair.getRight(); + // Unpack feature field values and merge into entityValueMap + boolean isStaleValues = isOutsideMaxAge(featureSetRequest, entityRow, featureRow); + Map valueMap = + this.unpackValueMap(featureRow, featureSetRequest, isStaleValues); + entityValuesMap.get(entityRow).putAll(valueMap); - // Generate metadata for feature values and merge into entityFieldsMap - boolean isNotFound = featureRow == null; - Map statusMap = - this.getMetadataMap(valueMap, isNotFound, isStaleValues); - entityStatusesMap.get(entityRow).putAll(statusMap); + // Generate metadata for feature values and merge into entityFieldsMap + boolean isNotFound = featureRow == null; + Map statusMap = + this.getMetadataMap(valueMap, isNotFound, isStaleValues); + entityStatusesMap.get(entityRow).putAll(statusMap); - // Populate metrics - this.populateStaleKeyCountMetrics(statusMap, featureSetRequest); - }); - this.populateRequestCountMetrics(featureSetRequest); + // Populate metrics + populateStaleKeyCountMetrics(statusMap, featureSetRequest); + }); + populateRequestCountMetrics(featureSetRequest); logFeatureRows.add(featureRows); } if (scope != null) { @@ -150,9 +151,10 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ * @param featureRow to unpack for feature values. * @param featureSetRequest for which the feature row was retrieved. * @param isStaleValues whether which the feature row contains stale values. - * @return valueMap mapping string feature name to feature value for the given feature set request. + * @return valueMap mapping string feature name to feature value for the given feature set + * request. */ - private Map unpackValueMap( + private static Map unpackValueMap( FeatureRow featureRow, FeatureSetRequest featureSetRequest, boolean isStaleValues) { Map valueMap = new HashMap<>(); // In order to return values containing the same feature references provided by the user, @@ -171,7 +173,9 @@ private Map unpackValueMap( }, featureRowField -> { // drop stale feature values. - return (isStaleValues) ? Value.newBuilder().build() : featureRowField.getValue(); + return (isStaleValues) + ? Value.newBuilder().build() + : featureRowField.getValue(); })); valueMap.putAll(featureValueMap); } @@ -194,9 +198,10 @@ private Map unpackValueMap( * @param isNotFound whether the given valueMap represents values that were not found in the * online retriever. * @param isStaleValues whether the given valueMap contains stale values. - * @return a 1:1 map keyed by field name containing field status metadata instead of values in the given valueMap. + * @return a 1:1 map keyed by field name containing field status metadata instead of values in the + * given valueMap. */ - private Map getMetadataMap( + private static Map getMetadataMap( Map valueMap, boolean isNotFound, boolean isStaleValues) { return valueMap.entrySet().stream() .collect( @@ -224,7 +229,7 @@ private Map getMetadataMap( * @param entityRow contains the retrieval timing of when features are pulled. * @param featureRow contains the ingestion timing and feature data. */ - private boolean isOutsideMaxAge( + private static boolean isOutsideMaxAge( FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { Duration maxAge = featureSetRequest.getSpec().getMaxAge(); if (featureRow == null) { From 8df3e57f9e9b7c74dd9dfef924c8300573c369b8 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 17 Jun 2020 11:14:23 +0800 Subject: [PATCH 62/65] Use "outside max age" instead "stale" term in OnlineServingService as "stale" is a overloaded term. --- .../serving/service/OnlineServingService.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index c57d0d18fdc..f108c6e87e1 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -107,18 +107,19 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ EntityRow entityRow = entityFeaturePair.getLeft(); FeatureRow featureRow = entityFeaturePair.getRight(); // Unpack feature field values and merge into entityValueMap - boolean isStaleValues = isOutsideMaxAge(featureSetRequest, entityRow, featureRow); + boolean isOutsideMaxAge = + checkOutsideMaxAge(featureSetRequest, entityRow, featureRow); Map valueMap = - this.unpackValueMap(featureRow, featureSetRequest, isStaleValues); + unpackValueMap(featureRow, featureSetRequest, isOutsideMaxAge); entityValuesMap.get(entityRow).putAll(valueMap); // Generate metadata for feature values and merge into entityFieldsMap boolean isNotFound = featureRow == null; Map statusMap = - this.getMetadataMap(valueMap, isNotFound, isStaleValues); + getMetadataMap(valueMap, isNotFound, isOutsideMaxAge); entityStatusesMap.get(entityRow).putAll(statusMap); - // Populate metrics + // Populate metrics/log request populateStaleKeyCountMetrics(statusMap, featureSetRequest); }); populateRequestCountMetrics(featureSetRequest); @@ -150,12 +151,12 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ * * @param featureRow to unpack for feature values. * @param featureSetRequest for which the feature row was retrieved. - * @param isStaleValues whether which the feature row contains stale values. + * @param isOutsideMaxAge whether which the feature row contains values that is outside max age. * @return valueMap mapping string feature name to feature value for the given feature set * request. */ private static Map unpackValueMap( - FeatureRow featureRow, FeatureSetRequest featureSetRequest, boolean isStaleValues) { + FeatureRow featureRow, FeatureSetRequest featureSetRequest, boolean isOutsideMaxAge) { Map valueMap = new HashMap<>(); // In order to return values containing the same feature references provided by the user, // we reuse the feature references in the request as the keys in field builder map @@ -172,8 +173,8 @@ private static Map unpackValueMap( return RefUtil.generateFeatureStringRef(featureRef); }, featureRowField -> { - // drop stale feature values. - return (isStaleValues) + // drop feature values with an age outside feature set's max age. + return (isOutsideMaxAge) ? Value.newBuilder().build() : featureRowField.getValue(); })); @@ -197,12 +198,13 @@ private static Map unpackValueMap( * @param valueMap map of field name to value to generate metadata for. * @param isNotFound whether the given valueMap represents values that were not found in the * online retriever. - * @param isStaleValues whether the given valueMap contains stale values. + * @param isOutsideMaxAge whether the given valueMap contains values with age outside feature + * set's max age. * @return a 1:1 map keyed by field name containing field status metadata instead of values in the * given valueMap. */ private static Map getMetadataMap( - Map valueMap, boolean isNotFound, boolean isStaleValues) { + Map valueMap, boolean isNotFound, boolean isOutsideMaxAge) { return valueMap.entrySet().stream() .collect( Collectors.toMap( @@ -211,7 +213,7 @@ private static Map getMetadataMap( Value fieldValue = es.getValue(); if (isNotFound) { return FieldStatus.NOT_FOUND; - } else if (isStaleValues) { + } else if (isOutsideMaxAge) { return FieldStatus.OUTSIDE_MAX_AGE; } else if (fieldValue.getValCase().equals(Value.ValCase.VAL_NOT_SET)) { return FieldStatus.NULL_VALUE; @@ -222,22 +224,20 @@ private static Map getMetadataMap( /** * Determine if the feature data in the given feature row is outside maxAge. Data is outside - * maxAge to be stale when difference ingestion time set in feature row and the retrieval time set + * maxAge to be when the difference ingestion time set in feature row and the retrieval time set * in entity row exceeds featureset max age. * * @param featureSetRequest contains the spec where feature's max age is extracted. * @param entityRow contains the retrieval timing of when features are pulled. * @param featureRow contains the ingestion timing and feature data. */ - private static boolean isOutsideMaxAge( + private static boolean checkOutsideMaxAge( FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { Duration maxAge = featureSetRequest.getSpec().getMaxAge(); - if (featureRow == null) { - // no data to consider: not stale + if (featureRow == null) { // no data to consider return false; } - if (maxAge.equals(Duration.getDefaultInstance())) { - // max age is not set: stale detection disabled + if (maxAge.equals(Duration.getDefaultInstance())) { // max age is not set return false; } From a03a1a93f2411ade839f4209c7230f0f4904ad32 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 17 Jun 2020 14:30:05 +0800 Subject: [PATCH 63/65] Refactor serving's OnlineRetriever to return a empty Optional instead of returning null. This allows the code to more semenatic and readable compared to dealing with nullable FeatureRow values --- .../serving/service/OnlineServingService.java | 64 +++- .../service/OnlineServingServiceTest.java | 312 +++++++++--------- .../api/retriever/OnlineRetriever.java | 9 +- .../RedisClusterOnlineRetriever.java | 16 +- .../redis/retriever/RedisOnlineRetriever.java | 16 +- .../RedisClusterOnlineRetrieverTest.java | 61 ++-- .../retriever/RedisOnlineRetrieverTest.java | 65 ++-- 7 files changed, 301 insertions(+), 242 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index f108c6e87e1..3d1d7817486 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -23,6 +23,7 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldStatus; import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesResponse.FieldValues; import feast.proto.types.FeatureRowProto.FeatureRow; +import feast.proto.types.FieldProto.Field; import feast.proto.types.ValueProto.Value; import feast.serving.specs.CachedSpecService; import feast.serving.util.Metrics; @@ -73,7 +74,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ Map> entityStatusesMap = entityRows.stream().collect(Collectors.toMap(row -> row, row -> new HashMap<>())); // Collect featureRows retrieved for logging/tracing - List> logFeatureRows = new LinkedList<>(); + List>> logFeatureRows = new LinkedList<>(); if (!request.getOmitEntitiesInResponse()) { // Add entity row's fields as response fields @@ -91,7 +92,8 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ // Pull feature rows for given entity rows from the feature/featureset specified in feature // set request. // from the configured online - List featureRows = retriever.getOnlineFeatures(entityRows, featureSetRequest); + List> featureRows = + retriever.getOnlineFeatures(entityRows, featureSetRequest); // Check that feature row returned corresponds to a given entity row. if (featureRows.size() != entityRows.size()) { throw Status.INTERNAL @@ -105,7 +107,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ .forEach( entityFeaturePair -> { EntityRow entityRow = entityFeaturePair.getLeft(); - FeatureRow featureRow = entityFeaturePair.getRight(); + Optional featureRow = entityFeaturePair.getRight(); // Unpack feature field values and merge into entityValueMap boolean isOutsideMaxAge = checkOutsideMaxAge(featureSetRequest, entityRow, featureRow); @@ -114,7 +116,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ entityValuesMap.get(entityRow).putAll(valueMap); // Generate metadata for feature values and merge into entityFieldsMap - boolean isNotFound = featureRow == null; + boolean isNotFound = featureRow.isEmpty(); Map statusMap = getMetadataMap(valueMap, isNotFound, isOutsideMaxAge); entityStatusesMap.get(entityRow).putAll(statusMap); @@ -126,7 +128,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ logFeatureRows.add(featureRows); } if (scope != null) { - scope.span().log(ImmutableMap.of("event", "featureRows", "value", logFeatureRows)); + logFeatureRowTrace(scope, logFeatureRows, featureSetRequests); } // Build response field values from entityValuesMap and entityStatusesMap @@ -149,22 +151,24 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ * Unpack feature values using data from the given feature row for features specified in the given * feature set request. * - * @param featureRow to unpack for feature values. + * @param featureRow optional to unpack for feature values. * @param featureSetRequest for which the feature row was retrieved. * @param isOutsideMaxAge whether which the feature row contains values that is outside max age. * @return valueMap mapping string feature name to feature value for the given feature set * request. */ private static Map unpackValueMap( - FeatureRow featureRow, FeatureSetRequest featureSetRequest, boolean isOutsideMaxAge) { + Optional featureRow, + FeatureSetRequest featureSetRequest, + boolean isOutsideMaxAge) { Map valueMap = new HashMap<>(); // In order to return values containing the same feature references provided by the user, // we reuse the feature references in the request as the keys in field builder map Map nameRefMap = featureSetRequest.getFeatureRefsByName(); - if (featureRow != null) { + if (featureRow.isPresent()) { // unpack feature row's feature values and populate value map Map featureValueMap = - featureRow.getFieldsList().stream() + featureRow.get().getFieldsList().stream() .filter(featureRowField -> nameRefMap.containsKey(featureRowField.getName())) .collect( Collectors.toMap( @@ -180,7 +184,6 @@ private static Map unpackValueMap( })); valueMap.putAll(featureValueMap); } - // create empty values for features specified in request but not present in feature row. Set missingFeatures = nameRefMap.values().stream() @@ -232,9 +235,9 @@ private static Map getMetadataMap( * @param featureRow contains the ingestion timing and feature data. */ private static boolean checkOutsideMaxAge( - FeatureSetRequest featureSetRequest, EntityRow entityRow, FeatureRow featureRow) { + FeatureSetRequest featureSetRequest, EntityRow entityRow, Optional featureRow) { Duration maxAge = featureSetRequest.getSpec().getMaxAge(); - if (featureRow == null) { // no data to consider + if (featureRow.isEmpty()) { // no data to consider return false; } if (maxAge.equals(Duration.getDefaultInstance())) { // max age is not set @@ -245,10 +248,45 @@ private static boolean checkOutsideMaxAge( if (givenTimestamp == 0) { givenTimestamp = System.currentTimeMillis() / 1000; } - long timeDifference = givenTimestamp - featureRow.getEventTimestamp().getSeconds(); + long timeDifference = givenTimestamp - featureRow.get().getEventTimestamp().getSeconds(); return timeDifference > maxAge.getSeconds(); } + /** TODO: docs */ + private void logFeatureRowTrace( + Scope scope, + List>> logFeatureRows, + List featureSetRequests) { + List> loggableFeatureRows = + Streams.zip( + logFeatureRows.stream(), + featureSetRequests.stream(), + (featureRows, featureSetRequest) -> { + // log null feature row when feature row is missing + FeatureRow.Builder nullFeatureRowBuilder = + FeatureRow.newBuilder() + .setFeatureSet( + RefUtil.generateFeatureSetStringRef(featureSetRequest.getSpec())); + for (FeatureReference featureReference : + featureSetRequest.getFeatureReferences()) { + nullFeatureRowBuilder.addFields( + Field.newBuilder().setName(featureReference.getName())); + } + + return featureRows.stream() + .map( + featureRow -> { + return (featureRow.isPresent()) + ? nullFeatureRowBuilder.build() + : featureRow.get(); + }) + .collect(Collectors.toList()); + }) + .collect(Collectors.toList()); + + scope.span().log(ImmutableMap.of("event", "featureRows", "value", loggableFeatureRows)); + } + private void populateStaleKeyCountMetrics( Map statusMap, FeatureSetRequest featureSetRequest) { String project = featureSetRequest.getSpec().getProject(); diff --git a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java index 13dca6b8ec0..9ab892a0c44 100644 --- a/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java +++ b/serving/src/test/java/feast/serving/service/OnlineServingServiceTest.java @@ -42,6 +42,7 @@ import io.opentracing.Tracer.SpanBuilder; import java.util.Collections; import java.util.List; +import java.util.Optional; import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatchers; @@ -84,52 +85,54 @@ public void shouldReturnResponseWithValuesAndMetadataIfKeysPresent() { .putFields("entity2", strValue("b"))) .build(); - List featureRows = + List> featureRows = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("entity1") - .setValue(intValue(1)) - .build(), - FieldProto.Field.newBuilder() - .setName("entity2") - .setValue(strValue("a")) - .build(), - FieldProto.Field.newBuilder() - .setName("feature1") - .setValue(intValue(1)) - .build(), - FieldProto.Field.newBuilder() - .setName("feature2") - .setValue(intValue(1)) - .build())) - .setFeatureSet("featureSet") - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("entity1") - .setValue(intValue(2)) - .build(), - FieldProto.Field.newBuilder() - .setName("entity2") - .setValue(strValue("b")) - .build(), - FieldProto.Field.newBuilder() - .setName("feature1") - .setValue(intValue(2)) - .build(), - FieldProto.Field.newBuilder() - .setName("feature2") - .setValue(intValue(2)) - .build())) - .setFeatureSet("featureSet") - .build()); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .setFeatureSet("featureSet") + .build()), + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(2)) + .build())) + .setFeatureSet("featureSet") + .build())); FeatureSetRequest featureSetRequest = FeatureSetRequest.newBuilder() @@ -198,23 +201,24 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfKeysNotPresent() { .setSpec(getFeatureSetSpec()) .build(); - List featureRows = + List> featureRows = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("feature1") - .setValue(intValue(1)) - .build(), - FieldProto.Field.newBuilder() - .setName("feature2") - .setValue(intValue(1)) - .build())) - .build(), - null); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .build()), + Optional.empty()); when(specService.getFeatureSets(request.getFeaturesList())) .thenReturn(Collections.singletonList(featureSetRequest)); @@ -272,53 +276,55 @@ public void shouldReturnResponseWithUnsetValuesAndMetadataIfMaxAgeIsExceeded() { .putFields("entity2", strValue("b"))) .build(); - List featureRows = + List> featureRows = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("entity1") - .setValue(intValue(1)) - .build(), - FieldProto.Field.newBuilder() - .setName("entity2") - .setValue(strValue("a")) - .build(), - FieldProto.Field.newBuilder() - .setName("feature1") - .setValue(intValue(1)) - .build(), - FieldProto.Field.newBuilder() - .setName("feature2") - .setValue(intValue(1)) - .build())) - .setFeatureSet("project/featureSet") - .build(), - FeatureRow.newBuilder() - .setEventTimestamp( - Timestamp.newBuilder().setSeconds(50)) // this value should be nulled - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("entity1") - .setValue(intValue(2)) - .build(), - FieldProto.Field.newBuilder() - .setName("entity2") - .setValue(strValue("b")) - .build(), - FieldProto.Field.newBuilder() - .setName("feature1") - .setValue(intValue(2)) - .build(), - FieldProto.Field.newBuilder() - .setName("project/feature2") - .setValue(intValue(2)) - .build())) - .setFeatureSet("project/featureSet") - .build()); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .setFeatureSet("project/featureSet") + .build()), + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp( + Timestamp.newBuilder().setSeconds(50)) // this value should be nulled + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("project/feature2") + .setValue(intValue(2)) + .build())) + .setFeatureSet("project/featureSet") + .build())); FeatureSetSpec spec = getFeatureSetSpec().toBuilder().setMaxAge(Duration.newBuilder().setSeconds(1)).build(); @@ -382,52 +388,54 @@ public void shouldFilterOutUndesiredRows() { .putFields("entity2", strValue("b"))) .build(); - List featureRows = + List> featureRows = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("entity1") - .setValue(intValue(1)) - .build(), - FieldProto.Field.newBuilder() - .setName("entity2") - .setValue(strValue("a")) - .build(), - FieldProto.Field.newBuilder() - .setName("feature1") - .setValue(intValue(1)) - .build(), - FieldProto.Field.newBuilder() - .setName("feature2") - .setValue(intValue(1)) - .build())) - .setFeatureSet("featureSet") - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .addAllFields( - Lists.newArrayList( - FieldProto.Field.newBuilder() - .setName("entity1") - .setValue(intValue(2)) - .build(), - FieldProto.Field.newBuilder() - .setName("entity2") - .setValue(strValue("b")) - .build(), - FieldProto.Field.newBuilder() - .setName("feature1") - .setValue(intValue(2)) - .build(), - FieldProto.Field.newBuilder() - .setName("feature2") - .setValue(intValue(2)) - .build())) - .setFeatureSet("featureSet") - .build()); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("a")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(1)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(1)) + .build())) + .setFeatureSet("featureSet") + .build()), + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .addAllFields( + Lists.newArrayList( + FieldProto.Field.newBuilder() + .setName("entity1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("entity2") + .setValue(strValue("b")) + .build(), + FieldProto.Field.newBuilder() + .setName("feature1") + .setValue(intValue(2)) + .build(), + FieldProto.Field.newBuilder() + .setName("feature2") + .setValue(intValue(2)) + .build())) + .setFeatureSet("featureSet") + .build())); FeatureSetRequest featureSetRequest = FeatureSetRequest.newBuilder() diff --git a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java index 246025cc656..f56cf6ca022 100644 --- a/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java +++ b/storage/api/src/main/java/feast/storage/api/retriever/OnlineRetriever.java @@ -19,6 +19,7 @@ import feast.proto.serving.ServingAPIProto.GetOnlineFeaturesRequest.EntityRow; import feast.proto.types.FeatureRowProto.FeatureRow; import java.util.List; +import java.util.Optional; /** An online retriever is a feature retriever that retrieves the latest feature data. */ public interface OnlineRetriever { @@ -27,9 +28,9 @@ public interface OnlineRetriever { * Get online features for the given entity rows using data retrieved from the feature/featureset * specified in feature set request. * - *

Each {@link FeatureRow} in the returned list then corresponds to an {@link EntityRow} - * provided by the user. If feature for a given entity row is not found, will return null in place - * of the {@link FeatureRow}. The no. of {@link FeatureRow} returned should match the no. of given + *

Each {@link FeatureRow} optional in the returned list then corresponds to an {@link + * EntityRow} provided by the user. If feature for a given entity row is not found, will return an + * empty optional instead. The no. of {@link FeatureRow} returned should match the no. of given * {@link EntityRow}s * * @param entityRows list of entity rows to request features for. @@ -37,6 +38,6 @@ public interface OnlineRetriever { * @return list of {@link FeatureRow}s corresponding to data retrieved for each entity row from * feature/featureset specified in featureset request. */ - List getOnlineFeatures( + List> getOnlineFeatures( List entityRows, FeatureSetRequest featureSetRequest); } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java index 1ca8bdd18bc..c006149cd51 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetriever.java @@ -37,6 +37,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -71,7 +72,7 @@ public static OnlineRetriever create(StatefulRedisClusterConnection getOnlineFeatures( + public List> getOnlineFeatures( List entityRows, FeatureSetRequest featureSetRequest) { // get features for this features/featureset in featureset request @@ -79,7 +80,7 @@ public List getOnlineFeatures( List redisKeys = buildRedisKeys(entityRows, featureSetSpec); FeatureRowDecoder decoder = new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); - List featureRows = new ArrayList<>(); + List> featureRows = new ArrayList<>(); try { featureRows = getFeaturesFromRedis(redisKeys, decoder); } catch (InvalidProtocolBufferException | ExecutionException e) { @@ -140,17 +141,18 @@ private RedisKey makeRedisKey( * * @param redisKeys keys used to retrieve data from Redis for a specific featureset. * @param decoder used to decode the data retrieved from Redis for a specific featureset. - * @return List of {@link FeatureRow}s + * @return List of {@link FeatureRow} optionals */ - private List getFeaturesFromRedis(List redisKeys, FeatureRowDecoder decoder) + private List> getFeaturesFromRedis( + List redisKeys, FeatureRowDecoder decoder) throws InvalidProtocolBufferException, ExecutionException { // pull feature row data bytes from redis using given redis keys List featureRowsBytes = sendMultiGet(redisKeys); - List featureRows = new ArrayList<>(); + List> featureRows = new ArrayList<>(); for (byte[] featureRowBytes : featureRowsBytes) { if (featureRowBytes == null) { - featureRows.add(null); + featureRows.add(Optional.empty()); continue; } @@ -168,7 +170,7 @@ private List getFeaturesFromRedis(List redisKeys, FeatureR .asRuntimeException(); } } - featureRows.add(featureRow); + featureRows.add(Optional.of(featureRow)); } return featureRows; } diff --git a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java index 4b4875fa3db..2b09f7346bb 100644 --- a/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java +++ b/storage/connectors/redis/src/main/java/feast/storage/connectors/redis/retriever/RedisOnlineRetriever.java @@ -36,6 +36,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -63,7 +64,7 @@ public static OnlineRetriever create(StatefulRedisConnection con /** {@inheritDoc} */ @Override - public List getOnlineFeatures( + public List> getOnlineFeatures( List entityRows, FeatureSetRequest featureSetRequest) { // get features for this features/featureset in featureset request @@ -71,7 +72,7 @@ public List getOnlineFeatures( List redisKeys = buildRedisKeys(entityRows, featureSetSpec); FeatureRowDecoder decoder = new FeatureRowDecoder(generateFeatureSetStringRef(featureSetSpec), featureSetSpec); - List featureRows = new ArrayList<>(); + List> featureRows = new ArrayList<>(); try { featureRows = getFeaturesFromRedis(redisKeys, decoder); } catch (InvalidProtocolBufferException | ExecutionException e) { @@ -133,17 +134,18 @@ private RedisKey makeRedisKey( * * @param redisKeys keys used to retrieve data from Redis for a specific featureset. * @param decoder used to decode the data retrieved from Redis for a specific featureset. - * @return List of {@link FeatureRow}s + * @return List of {@link FeatureRow} optionals */ - private List getFeaturesFromRedis(List redisKeys, FeatureRowDecoder decoder) + private List> getFeaturesFromRedis( + List redisKeys, FeatureRowDecoder decoder) throws InvalidProtocolBufferException, ExecutionException { // pull feature row data bytes from redis using given redis keys List featureRowsBytes = sendMultiGet(redisKeys); - List featureRows = new ArrayList<>(); + List> featureRows = new ArrayList<>(); for (byte[] featureRowBytes : featureRowsBytes) { if (featureRowBytes == null) { - featureRows.add(null); + featureRows.add(Optional.empty()); continue; } @@ -161,7 +163,7 @@ private List getFeaturesFromRedis(List redisKeys, FeatureR .asRuntimeException(); } } - featureRows.add(featureRow); + featureRows.add(Optional.of(featureRow)); } return featureRows; } diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java index d4f35435ce9..76e22d07c54 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisClusterOnlineRetrieverTest.java @@ -132,26 +132,28 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List expected = + List> expected = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build()); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build()), + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) + .build())); - List actual = + List> actual = redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } @@ -199,19 +201,20 @@ public void shouldReturnNullIfKeysNotPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List expected = + List> expected = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - null); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build()), + Optional.empty()); - List actual = + List> actual = redisClusterOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } diff --git a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java index 7795722e647..1292f4ab0dc 100644 --- a/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java +++ b/storage/connectors/redis/src/test/java/feast/storage/connectors/redis/retriever/RedisOnlineRetrieverTest.java @@ -132,26 +132,29 @@ public void shouldReturnResponseWithValuesIfKeysPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List expected = + List> expected = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) - .build()); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build()), + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(2)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(2)).build())) + .build())); - List actual = redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); + List> actual = + redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } @@ -190,7 +193,7 @@ public void shouldReturnNullIfKeysNotPresent() { List> featureRowBytes = featureRows.stream() - .map(x -> KeyValue.from(new byte[1], Optional.of(x.toByteArray()))) + .map(row -> KeyValue.from(new byte[1], Optional.of(row.toByteArray()))) .collect(Collectors.toList()); featureRowBytes.add(null); @@ -198,18 +201,20 @@ public void shouldReturnNullIfKeysNotPresent() { when(connection.sync()).thenReturn(syncCommands); when(syncCommands.mget(redisKeyList)).thenReturn(featureRowBytes); - List expected = + List> expected = Lists.newArrayList( - FeatureRow.newBuilder() - .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) - .setFeatureSet("project/featureSet") - .addAllFields( - Lists.newArrayList( - Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), - Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) - .build(), - null); - List actual = redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); + Optional.of( + FeatureRow.newBuilder() + .setEventTimestamp(Timestamp.newBuilder().setSeconds(100)) + .setFeatureSet("project/featureSet") + .addAllFields( + Lists.newArrayList( + Field.newBuilder().setName("feature1").setValue(intValue(1)).build(), + Field.newBuilder().setName("feature2").setValue(intValue(1)).build())) + .build()), + Optional.empty()); + List> actual = + redisOnlineRetriever.getOnlineFeatures(entityRows, featureSetRequest); assertThat(actual, equalTo(expected)); } From 1e97264b6ea6a72bc9d33600f5604c95f5c07939 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 17 Jun 2020 15:58:35 +0800 Subject: [PATCH 64/65] Fixed typo in basic redis e2e tests --- tests/e2e/redis/basic-ingest-redis-serving.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/redis/basic-ingest-redis-serving.py b/tests/e2e/redis/basic-ingest-redis-serving.py index 205420988f9..4ecdff98fb7 100644 --- a/tests/e2e/redis/basic-ingest-redis-serving.py +++ b/tests/e2e/redis/basic-ingest-redis-serving.py @@ -228,7 +228,7 @@ def try_get_features(): } ) ], - feature_refs=feature_refs, + feature_refs=feature_refs) # type: GetOnlineFeaturesResponse is_ok = all([check_online_response(ref, cust_trans_df, response) for ref in feature_refs]) return response, is_ok From d4ef594bac0fb37bc38e3a8816e7cf97375d54a5 Mon Sep 17 00:00:00 2001 From: Zhu Zhanyan Date: Wed, 17 Jun 2020 23:37:41 +0800 Subject: [PATCH 65/65] Fixed typo OnlineServingService's logFeatureRowsTrace() --- .../java/feast/serving/service/OnlineServingService.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/serving/src/main/java/feast/serving/service/OnlineServingService.java b/serving/src/main/java/feast/serving/service/OnlineServingService.java index 3d1d7817486..6d581f6c710 100644 --- a/serving/src/main/java/feast/serving/service/OnlineServingService.java +++ b/serving/src/main/java/feast/serving/service/OnlineServingService.java @@ -128,7 +128,7 @@ public GetOnlineFeaturesResponse getOnlineFeatures(GetOnlineFeaturesRequest requ logFeatureRows.add(featureRows); } if (scope != null) { - logFeatureRowTrace(scope, logFeatureRows, featureSetRequests); + logFeatureRowsTrace(scope, logFeatureRows, featureSetRequests); } // Build response field values from entityValuesMap and entityStatusesMap @@ -252,8 +252,7 @@ private static boolean checkOutsideMaxAge( return timeDifference > maxAge.getSeconds(); } - /** TODO: docs */ - private void logFeatureRowTrace( + private void logFeatureRowsTrace( Scope scope, List>> logFeatureRows, List featureSetRequests) { @@ -262,7 +261,6 @@ private void logFeatureRowTrace( logFeatureRows.stream(), featureSetRequests.stream(), (featureRows, featureSetRequest) -> { - // log null feature row when feature row is missing FeatureRow.Builder nullFeatureRowBuilder = FeatureRow.newBuilder() .setFeatureSet( @@ -273,10 +271,11 @@ private void logFeatureRowTrace( Field.newBuilder().setName(featureReference.getName())); } + // log null feature row when feature row is empty return featureRows.stream() .map( featureRow -> { - return (featureRow.isPresent()) + return (featureRow.isEmpty()) ? nullFeatureRowBuilder.build() : featureRow.get(); })