From 56c61416c2c1645b00576307bc46db4e5f145950 Mon Sep 17 00:00:00 2001 From: Tim Sell Date: Mon, 28 Jan 2019 01:04:02 +0800 Subject: [PATCH 1/3] rename feature store classes to feature store factories --- .../feast/ingestion/boot/ImportJobModule.java | 24 ++++---- .../transform/ErrorsStoreTransform.java | 16 ++--- .../transform/ServingStoreTransform.java | 6 +- .../transform/SplitOutputByStore.java | 12 ++-- .../transform/WarehouseStoreTransform.java | 6 +- .../transform/fn/ValidateFeatureRowsDoFn.java | 12 ++-- ...re.java => FeatureErrorsStoreFactory.java} | 2 +- ...on.java => FeatureServingStoreClient.java} | 26 ++++++--- ...e.java => FeatureServingStoreFactory.java} | 2 +- ...ureStore.java => FeatureStoreFactory.java} | 2 +- ...java => FeatureWarehouseStoreFactory.java} | 2 +- .../main/java/feast/storage/StoreClient.java | 51 ---------------- ...ava => BigQueryWarehouseStoreFactory.java} | 6 +- .../bigtable/BigTableServingStoreFactory.java | 6 +- ...tores.java => JsonFileStoreFactories.java} | 20 +++---- ...tore.java => NoOpServingStoreFactory.java} | 7 +-- ...re.java => NoOpWarehouseStoreFactory.java} | 6 +- ...ore.java => RedisServingStoreFactory.java} | 6 +- ... => FeatureErrorsStoreFactoryService.java} | 23 ++++---- ...=> FeatureServingStoreFactoryService.java} | 23 ++++---- ... FeatureWarehouseStoreFactoryService.java} | 25 ++++---- ...a => StderrFeatureErrorsStoreFactory.java} | 6 +- ...a => StdoutFeatureErrorsStoreFactory.java} | 6 +- .../sdk/io/gcp/pubsub/FeastPubsubHelper.java | 58 ------------------- .../feast/ingestion/ImportJobCSVTest.java | 40 ++++++------- .../transform/ErrorsStoreTransformTest.java | 22 +++---- .../transform/SplitOutputByStoreTest.java | 10 ++-- ...ava => MockFeatureErrorsStoreFactory.java} | 7 ++- .../java/feast/storage/MockFeatureStore.java | 2 +- ...tore.java => MockServingStoreFactory.java} | 7 ++- ...re.java => MockWarehouseStoreFactory.java} | 7 ++- .../redis/FeatureRowRedisIOWriteTest.java | 6 +- 32 files changed, 182 insertions(+), 272 deletions(-) rename ingestion/src/main/java/feast/storage/{ErrorsStore.java => FeatureErrorsStoreFactory.java} (89%) rename ingestion/src/main/java/feast/storage/{FeatureStoreMigration.java => FeatureServingStoreClient.java} (56%) rename ingestion/src/main/java/feast/storage/{WarehouseStore.java => FeatureServingStoreFactory.java} (89%) rename ingestion/src/main/java/feast/storage/{FeatureStore.java => FeatureStoreFactory.java} (95%) rename ingestion/src/main/java/feast/storage/{ServingStore.java => FeatureWarehouseStoreFactory.java} (89%) delete mode 100644 ingestion/src/main/java/feast/storage/StoreClient.java rename ingestion/src/main/java/feast/storage/bigquery/{BigQueryWarehouseStore.java => BigQueryWarehouseStoreFactory.java} (88%) rename ingestion/src/main/java/feast/storage/file/json/{JsonFileStores.java => JsonFileStoreFactories.java} (78%) rename ingestion/src/main/java/feast/storage/noop/{NoOpWarehouseStore.java => NoOpServingStoreFactory.java} (85%) rename ingestion/src/main/java/feast/storage/noop/{NoOpServingStore.java => NoOpWarehouseStoreFactory.java} (84%) rename ingestion/src/main/java/feast/storage/redis/{RedisServingStore.java => RedisServingStoreFactory.java} (88%) rename ingestion/src/main/java/feast/storage/service/{ErrorsStoreService.java => FeatureErrorsStoreFactoryService.java} (60%) rename ingestion/src/main/java/feast/storage/service/{ServingStoreService.java => FeatureServingStoreFactoryService.java} (60%) rename ingestion/src/main/java/feast/storage/service/{WarehouseStoreService.java => FeatureWarehouseStoreFactoryService.java} (59%) rename ingestion/src/main/java/feast/storage/stderr/{StderrErrorsStore.java => StderrFeatureErrorsStoreFactory.java} (86%) rename ingestion/src/main/java/feast/storage/stderr/{StdoutErrorsStore.java => StdoutFeatureErrorsStoreFactory.java} (85%) delete mode 100644 ingestion/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/FeastPubsubHelper.java rename ingestion/src/test/java/feast/storage/{MockErrorsStore.java => MockFeatureErrorsStoreFactory.java} (79%) rename ingestion/src/test/java/feast/storage/{MockServingStore.java => MockServingStoreFactory.java} (80%) rename ingestion/src/test/java/feast/storage/{MockWarehouseStore.java => MockWarehouseStoreFactory.java} (80%) diff --git a/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java b/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java index 7e9e235b5b..de3d84b169 100644 --- a/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java +++ b/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java @@ -28,12 +28,12 @@ import feast.ingestion.service.SpecService.Builder; import feast.ingestion.service.SpecService.UnsupportedBuilder; import feast.specs.ImportSpecProto.ImportSpec; -import feast.storage.ErrorsStore; -import feast.storage.ServingStore; -import feast.storage.WarehouseStore; -import feast.storage.service.ErrorsStoreService; -import feast.storage.service.ServingStoreService; -import feast.storage.service.WarehouseStoreService; +import feast.storage.FeatureErrorsStoreFactory; +import feast.storage.FeatureServingStoreFactory; +import feast.storage.FeatureWarehouseStoreFactory; +import feast.storage.service.FeatureErrorsStoreFactoryService; +import feast.storage.service.FeatureServingStoreFactoryService; +import feast.storage.service.FeatureWarehouseStoreFactoryService; import java.util.List; import org.apache.beam.sdk.options.PipelineOptions; @@ -78,19 +78,19 @@ Specs provideSpecs(SpecService.Builder specService) { @Provides @Singleton - List provideWarehouseStores() { - return WarehouseStoreService.getAll(); + List provideWarehouseStores() { + return FeatureWarehouseStoreFactoryService.getAll(); } @Provides @Singleton - List provideServingStores() { - return ServingStoreService.getAll(); + List provideServingStores() { + return FeatureServingStoreFactoryService.getAll(); } @Provides @Singleton - List provideErrorsStores() { - return ErrorsStoreService.getAll(); + List provideErrorsStores() { + return FeatureErrorsStoreFactoryService.getAll(); } } diff --git a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java index 7b8e41030e..1ccdf1aa7e 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java @@ -25,7 +25,7 @@ import feast.ingestion.options.ImportJobPipelineOptions; import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ErrorsStore; +import feast.storage.FeatureErrorsStoreFactory; import feast.storage.noop.NoOpIO; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.List; @@ -40,13 +40,13 @@ public class ErrorsStoreTransform extends FeatureIO.Write { private String errorsStoreType; private StorageSpec errorsStoreSpec; private Specs specs; - private List errorsStores; + private List errorsStoreFactories; @Inject public ErrorsStoreTransform( - ImportJobPipelineOptions options, Specs specs, List errorsStores) { + ImportJobPipelineOptions options, Specs specs, List errorsStoreFactories) { this.specs = specs; - this.errorsStores = errorsStores; + this.errorsStoreFactories = errorsStoreFactories; this.errorsStoreType = options.getErrorsStoreType(); if (!Strings.isEmpty(errorsStoreType)) { @@ -70,11 +70,11 @@ public PDone expand(PCollection input) { return PDone.in(input.getPipeline()); } - ErrorsStore getErrorStore() { + FeatureErrorsStoreFactory getErrorStore() { checkArgument(!errorsStoreType.isEmpty(), "Errors store type not provided"); - for (ErrorsStore errorsStore : errorsStores) { - if (errorsStore.getType().equals(errorsStoreType)) { - return errorsStore; + for (FeatureErrorsStoreFactory errorsStoreFactory : errorsStoreFactories) { + if (errorsStoreFactory.getType().equals(errorsStoreType)) { + return errorsStoreFactory; } } throw new IllegalArgumentException("Errors store type not found"); diff --git a/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java index feb712da0d..d5f3e321ed 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java @@ -21,7 +21,7 @@ import feast.ingestion.metrics.FeastMetrics; import feast.ingestion.model.Specs; import feast.ingestion.values.PFeatureRows; -import feast.storage.ServingStore; +import feast.storage.FeatureServingStoreFactory; import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; @@ -31,11 +31,11 @@ @Slf4j public class ServingStoreTransform extends PTransform { - private List stores; + private List stores; private Specs specs; @Inject - public ServingStoreTransform(List stores, Specs specs) { + public ServingStoreTransform(List stores, Specs specs) { this.stores = stores; this.specs = specs; } diff --git a/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java b/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java index cb11490dd3..7c4ee2ee23 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java +++ b/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java @@ -25,7 +25,7 @@ import feast.ingestion.values.PFeatureRows; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureStore; +import feast.storage.FeatureStoreFactory; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.Collection; import java.util.HashMap; @@ -46,7 +46,7 @@ @Slf4j public class SplitOutputByStore extends PTransform { - private Collection stores; + private Collection stores; private SerializableFunction selector; private Specs specs; @@ -72,16 +72,16 @@ public PFeatureRows expand(PFeatureRows input) { input.getErrors()); } - private Map getStoresMap() { - Map storesMap = new HashMap<>(); - for (FeatureStore servingStore : stores) { + private Map getStoresMap() { + Map storesMap = new HashMap<>(); + for (FeatureStoreFactory servingStore : stores) { storesMap.put(servingStore.getType(), servingStore); } return storesMap; } private Map getFeatureStoreTransforms() { - Map storesMap = getStoresMap(); + Map storesMap = getStoresMap(); Map transforms = new HashMap<>(); Map storageSpecs = specs.getStorageSpecs(); for (String storeId : storageSpecs.keySet()) { diff --git a/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java index dcfb9c4433..6a5ee2f0b9 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java @@ -25,16 +25,16 @@ import feast.ingestion.metrics.FeastMetrics; import feast.ingestion.model.Specs; import feast.ingestion.values.PFeatureRows; -import feast.storage.WarehouseStore; +import feast.storage.FeatureWarehouseStoreFactory; @Slf4j public class WarehouseStoreTransform extends PTransform { - private List stores; + private List stores; private Specs specs; @Inject - public WarehouseStoreTransform(List stores, Specs specs) { + public WarehouseStoreTransform(List stores, Specs specs) { this.stores = stores; this.specs = specs; } diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java index 40fc6054fd..34b7cabfff 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java @@ -31,10 +31,10 @@ import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; -import feast.storage.WarehouseStore; -import feast.storage.service.ServingStoreService; -import feast.storage.service.WarehouseStoreService; +import feast.storage.FeatureServingStoreFactory; +import feast.storage.FeatureWarehouseStoreFactory; +import feast.storage.service.FeatureServingStoreFactoryService; +import feast.storage.service.FeatureWarehouseStoreFactoryService; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowProto.FeatureRow; import feast.types.GranularityProto.Granularity.Enum; @@ -66,10 +66,10 @@ public void setup() { featureIds.add(field.getFeatureId()); } } - for (ServingStore store : ServingStoreService.getAll()) { + for (FeatureServingStoreFactory store : FeatureServingStoreFactoryService.getAll()) { supportedServingTypes.add(store.getType()); } - for (WarehouseStore store : WarehouseStoreService.getAll()) { + for (FeatureWarehouseStoreFactory store : FeatureWarehouseStoreFactoryService.getAll()) { supportedWarehouseTypes.add(store.getType()); } } diff --git a/ingestion/src/main/java/feast/storage/ErrorsStore.java b/ingestion/src/main/java/feast/storage/FeatureErrorsStoreFactory.java similarity index 89% rename from ingestion/src/main/java/feast/storage/ErrorsStore.java rename to ingestion/src/main/java/feast/storage/FeatureErrorsStoreFactory.java index 4fde4ec00f..593b4b02e5 100644 --- a/ingestion/src/main/java/feast/storage/ErrorsStore.java +++ b/ingestion/src/main/java/feast/storage/FeatureErrorsStoreFactory.java @@ -17,4 +17,4 @@ package feast.storage; -public interface ErrorsStore extends FeatureStore {} +public interface FeatureErrorsStoreFactory extends FeatureStoreFactory {} diff --git a/ingestion/src/main/java/feast/storage/FeatureStoreMigration.java b/ingestion/src/main/java/feast/storage/FeatureServingStoreClient.java similarity index 56% rename from ingestion/src/main/java/feast/storage/FeatureStoreMigration.java rename to ingestion/src/main/java/feast/storage/FeatureServingStoreClient.java index a35e7db3fc..af62be1fec 100644 --- a/ingestion/src/main/java/feast/storage/FeatureStoreMigration.java +++ b/ingestion/src/main/java/feast/storage/FeatureServingStoreClient.java @@ -18,16 +18,24 @@ package feast.storage; import feast.specs.FeatureSpecProto.FeatureSpec; +import feast.types.FeatureRowProto.FeatureRow; +import java.util.Collection; import java.util.List; -import lombok.Value; /** - * This class is for notifying storage plugins about new or changed FeatureSpecs so they can update - * appropriate schemas. + * Abstraction of Client for Feast Serving Store */ -@Value -public class FeatureStoreMigration { - List create; - List drop; - List update; -} +public interface FeatureServingStoreClient { + + /** + * Get a the latest FeatureRow containing features for a single entity id + */ + FeatureRow get(String entityName, String entityId, List featureSpecs); + + /** + * Get a the latest FeatureRow for containing features for each entity id + */ + List get( + String entityName, Collection entityIds, List featureSpecs); + +} \ No newline at end of file diff --git a/ingestion/src/main/java/feast/storage/WarehouseStore.java b/ingestion/src/main/java/feast/storage/FeatureServingStoreFactory.java similarity index 89% rename from ingestion/src/main/java/feast/storage/WarehouseStore.java rename to ingestion/src/main/java/feast/storage/FeatureServingStoreFactory.java index b4bda0a33d..178e02bf0b 100644 --- a/ingestion/src/main/java/feast/storage/WarehouseStore.java +++ b/ingestion/src/main/java/feast/storage/FeatureServingStoreFactory.java @@ -17,4 +17,4 @@ package feast.storage; -public interface WarehouseStore extends FeatureStore {} +public interface FeatureServingStoreFactory extends FeatureStoreFactory {} diff --git a/ingestion/src/main/java/feast/storage/FeatureStore.java b/ingestion/src/main/java/feast/storage/FeatureStoreFactory.java similarity index 95% rename from ingestion/src/main/java/feast/storage/FeatureStore.java rename to ingestion/src/main/java/feast/storage/FeatureStoreFactory.java index a969c1476a..2989d45dcc 100644 --- a/ingestion/src/main/java/feast/storage/FeatureStore.java +++ b/ingestion/src/main/java/feast/storage/FeatureStoreFactory.java @@ -21,7 +21,7 @@ import feast.ingestion.transform.FeatureIO; import feast.specs.StorageSpecProto.StorageSpec; -public interface FeatureStore { +public interface FeatureStoreFactory { FeatureIO.Write create(StorageSpec storageSpec, Specs specs); String getType(); diff --git a/ingestion/src/main/java/feast/storage/ServingStore.java b/ingestion/src/main/java/feast/storage/FeatureWarehouseStoreFactory.java similarity index 89% rename from ingestion/src/main/java/feast/storage/ServingStore.java rename to ingestion/src/main/java/feast/storage/FeatureWarehouseStoreFactory.java index aa08181381..d88350bcae 100644 --- a/ingestion/src/main/java/feast/storage/ServingStore.java +++ b/ingestion/src/main/java/feast/storage/FeatureWarehouseStoreFactory.java @@ -17,4 +17,4 @@ package feast.storage; -public interface ServingStore extends FeatureStore {} +public interface FeatureWarehouseStoreFactory extends FeatureStoreFactory {} diff --git a/ingestion/src/main/java/feast/storage/StoreClient.java b/ingestion/src/main/java/feast/storage/StoreClient.java deleted file mode 100644 index 9f1302187b..0000000000 --- a/ingestion/src/main/java/feast/storage/StoreClient.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage; - -import java.util.Collection; -import java.util.List; -import java.util.Map; -import feast.serving.ServingAPIProto.TimestampRange; -import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.FeatureRowProto.FeatureRow; - -/** Abstraction of Client for Feast Storage. */ -public interface StoreClient { - - /** Get a the latest FeatureRow containing features for a single entity id */ - FeatureRow getLatest(String entityName, String entityId, List featureSpecs); - - /** Get a the latest FeatureRow for containing features for each entity id */ - List getLatest( - String entityName, Collection entityIds, List featureSpecs); - - /** - * Get a the N most recent FeatureRows for single entity id. Each featureRow will contain at least - * one of the features provided - */ - List getLatest( - String entityName, String entityId, List featureSpecs, int limit); - - /** - * Get up to N most recent FeatureRows for each entity id. Each featureRow will contain at least - * one of the features provided - * @return A map of feature ids to lists of FeatureRows - */ - Map> getLatestRange( - String entityName, Collection entityId, List featureSpecs, int limit, TimestampRange tsRange); -} diff --git a/ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStore.java b/ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStoreFactory.java similarity index 88% rename from ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStore.java rename to ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStoreFactory.java index a7682908bd..9d4f5285fb 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStore.java +++ b/ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStoreFactory.java @@ -23,10 +23,10 @@ import feast.ingestion.transform.FeatureIO.Write; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.WarehouseStore; +import feast.storage.FeatureWarehouseStoreFactory; -@AutoService(WarehouseStore.class) -public class BigQueryWarehouseStore implements WarehouseStore { +@AutoService(FeatureWarehouseStoreFactory.class) +public class BigQueryWarehouseStoreFactory implements FeatureWarehouseStoreFactory { public static String TYPE_BIGQUERY = "bigquery"; @Override diff --git a/ingestion/src/main/java/feast/storage/bigtable/BigTableServingStoreFactory.java b/ingestion/src/main/java/feast/storage/bigtable/BigTableServingStoreFactory.java index e6aa7f3232..842c5be5cf 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/BigTableServingStoreFactory.java +++ b/ingestion/src/main/java/feast/storage/bigtable/BigTableServingStoreFactory.java @@ -23,10 +23,10 @@ import feast.ingestion.transform.FeatureIO.Write; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; +import feast.storage.FeatureServingStoreFactory; -@AutoService(ServingStore.class) -public class BigTableServingStoreFactory implements ServingStore { +@AutoService(FeatureServingStoreFactory.class) +public class BigTableServingStoreFactory implements FeatureServingStoreFactory { public static String TYPE_BIGTABLE = "bigtable"; @Override diff --git a/ingestion/src/main/java/feast/storage/file/json/JsonFileStores.java b/ingestion/src/main/java/feast/storage/file/json/JsonFileStoreFactories.java similarity index 78% rename from ingestion/src/main/java/feast/storage/file/json/JsonFileStores.java rename to ingestion/src/main/java/feast/storage/file/json/JsonFileStoreFactories.java index 11dda1e3b7..2babb379d5 100644 --- a/ingestion/src/main/java/feast/storage/file/json/JsonFileStores.java +++ b/ingestion/src/main/java/feast/storage/file/json/JsonFileStoreFactories.java @@ -22,19 +22,19 @@ import feast.ingestion.transform.FeatureIO.Write; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ErrorsStore; -import feast.storage.ServingStore; -import feast.storage.WarehouseStore; +import feast.storage.FeatureErrorsStoreFactory; +import feast.storage.FeatureServingStoreFactory; +import feast.storage.FeatureWarehouseStoreFactory; import feast.storage.file.FileStoreOptions; import lombok.AllArgsConstructor; -public class JsonFileStores { +public class JsonFileStoreFactories { private static final String JSON_FILES_TYPE = "file.json"; - @AutoService(WarehouseStore.class) + @AutoService(FeatureWarehouseStoreFactory.class) @AllArgsConstructor - public static class JsonFileWarehouseStore implements WarehouseStore { + public static class JsonFileWarehouseStoreFactory implements FeatureWarehouseStoreFactory { @Override public Write create(StorageSpec storageSpec, Specs specs) { @@ -50,9 +50,9 @@ public String getType() { } } - @AutoService(ServingStore.class) + @AutoService(FeatureServingStoreFactory.class) @AllArgsConstructor - public static class JsonFileServingStore implements ServingStore { + public static class JsonFileServingStoreFactory implements FeatureServingStoreFactory { @Override public Write create(StorageSpec storageSpec, Specs specs) { @@ -68,9 +68,9 @@ public String getType() { } } - @AutoService(ErrorsStore.class) + @AutoService(FeatureErrorsStoreFactory.class) @AllArgsConstructor - public static class JsonFileErrorsStore implements ErrorsStore { + public static class JsonFileFeatureErrorsStoreFactory implements FeatureErrorsStoreFactory { @Override public Write create(StorageSpec storageSpec, Specs specs) { diff --git a/ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStore.java b/ingestion/src/main/java/feast/storage/noop/NoOpServingStoreFactory.java similarity index 85% rename from ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStore.java rename to ingestion/src/main/java/feast/storage/noop/NoOpServingStoreFactory.java index f69e7c9fbb..7aadb7f961 100644 --- a/ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStore.java +++ b/ingestion/src/main/java/feast/storage/noop/NoOpServingStoreFactory.java @@ -21,11 +21,10 @@ import feast.ingestion.model.Specs; import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; -import feast.storage.WarehouseStore; +import feast.storage.FeatureServingStoreFactory; -@AutoService(WarehouseStore.class) -public class NoOpWarehouseStore implements WarehouseStore { +@AutoService(FeatureServingStoreFactory.class) +public class NoOpServingStoreFactory implements FeatureServingStoreFactory { @Override public Write create(StorageSpec storageSpec, Specs specs) { diff --git a/ingestion/src/main/java/feast/storage/noop/NoOpServingStore.java b/ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStoreFactory.java similarity index 84% rename from ingestion/src/main/java/feast/storage/noop/NoOpServingStore.java rename to ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStoreFactory.java index 15de592144..dbee993767 100644 --- a/ingestion/src/main/java/feast/storage/noop/NoOpServingStore.java +++ b/ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStoreFactory.java @@ -21,10 +21,10 @@ import feast.ingestion.model.Specs; import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; +import feast.storage.FeatureWarehouseStoreFactory; -@AutoService(ServingStore.class) -public class NoOpServingStore implements ServingStore { +@AutoService(FeatureWarehouseStoreFactory.class) +public class NoOpWarehouseStoreFactory implements FeatureWarehouseStoreFactory { @Override public Write create(StorageSpec storageSpec, Specs specs) { diff --git a/ingestion/src/main/java/feast/storage/redis/RedisServingStore.java b/ingestion/src/main/java/feast/storage/redis/RedisServingStoreFactory.java similarity index 88% rename from ingestion/src/main/java/feast/storage/redis/RedisServingStore.java rename to ingestion/src/main/java/feast/storage/redis/RedisServingStoreFactory.java index 6b5fc23ce5..6d16e062ba 100644 --- a/ingestion/src/main/java/feast/storage/redis/RedisServingStore.java +++ b/ingestion/src/main/java/feast/storage/redis/RedisServingStoreFactory.java @@ -23,10 +23,10 @@ import feast.ingestion.transform.FeatureIO.Write; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ServingStore; +import feast.storage.FeatureServingStoreFactory; -@AutoService(ServingStore.class) -public class RedisServingStore implements ServingStore { +@AutoService(FeatureServingStoreFactory.class) +public class RedisServingStoreFactory implements FeatureServingStoreFactory { public static final String TYPE_REDIS = "redis"; @Override diff --git a/ingestion/src/main/java/feast/storage/service/ErrorsStoreService.java b/ingestion/src/main/java/feast/storage/service/FeatureErrorsStoreFactoryService.java similarity index 60% rename from ingestion/src/main/java/feast/storage/service/ErrorsStoreService.java rename to ingestion/src/main/java/feast/storage/service/FeatureErrorsStoreFactoryService.java index 0f2e8a30e0..bed7ed9631 100644 --- a/ingestion/src/main/java/feast/storage/service/ErrorsStoreService.java +++ b/ingestion/src/main/java/feast/storage/service/FeatureErrorsStoreFactoryService.java @@ -19,33 +19,36 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import feast.storage.ErrorsStore; +import feast.storage.FeatureErrorsStoreFactory; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; +/** + * Service class for fetching all the FeatureErrorsStoreFactory instances available + */ @Slf4j -public class ErrorsStoreService { +public class FeatureErrorsStoreFactoryService { - private static ServiceLoader serviceLoader = ServiceLoader.load(ErrorsStore.class); - private static List manuallyRegistered = new ArrayList<>(); + private static ServiceLoader serviceLoader = ServiceLoader.load(FeatureErrorsStoreFactory.class); + private static List manuallyRegistered = new ArrayList<>(); static { - for (ErrorsStore store : getAll()) { - log.info("ErrorsStore type found: " + store.getType()); + for (FeatureErrorsStoreFactory store : getAll()) { + log.info("FeatureErrorsStoreFactory type found: " + store.getType()); } } - public static List getAll() { + public static List getAll() { return Lists.newArrayList( Iterators.concat(manuallyRegistered.iterator(), serviceLoader.iterator())); } /** Get store of the given subclass. */ - public static T get(Class clazz) { - for (ErrorsStore store : getAll()) { + public static T get(Class clazz) { + for (FeatureErrorsStoreFactory store : getAll()) { if (clazz.isInstance(store)) { //noinspection unchecked return (T) store; @@ -54,7 +57,7 @@ public static T get(Class clazz) { return null; } - public static void register(ErrorsStore store) { + public static void register(FeatureErrorsStoreFactory store) { manuallyRegistered.add(store); } } diff --git a/ingestion/src/main/java/feast/storage/service/ServingStoreService.java b/ingestion/src/main/java/feast/storage/service/FeatureServingStoreFactoryService.java similarity index 60% rename from ingestion/src/main/java/feast/storage/service/ServingStoreService.java rename to ingestion/src/main/java/feast/storage/service/FeatureServingStoreFactoryService.java index a1ab609594..0028ff9dda 100644 --- a/ingestion/src/main/java/feast/storage/service/ServingStoreService.java +++ b/ingestion/src/main/java/feast/storage/service/FeatureServingStoreFactoryService.java @@ -19,31 +19,34 @@ import avro.shaded.com.google.common.collect.Lists; import com.google.common.collect.Iterators; +import feast.storage.FeatureServingStoreFactory; import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; import lombok.extern.slf4j.Slf4j; -import feast.storage.ServingStore; +/** + * Service class for fetching all the FeatureServingStoreFactory instances available + */ @Slf4j -public class ServingStoreService { - private static ServiceLoader serviceLoader = ServiceLoader.load(ServingStore.class); - private static List manuallyRegistered = new ArrayList<>(); +public class FeatureServingStoreFactoryService { + private static ServiceLoader serviceLoader = ServiceLoader.load(FeatureServingStoreFactory.class); + private static List manuallyRegistered = new ArrayList<>(); static { - for (ServingStore store : getAll()) { - log.info("ServingStore type found: " + store.getType()); + for (FeatureServingStoreFactory store : getAll()) { + log.info("FeatureServingStoreFactory type found: " + store.getType()); } } - public static List getAll() { + public static List getAll() { return Lists.newArrayList( Iterators.concat(manuallyRegistered.iterator(), serviceLoader.iterator())); } /** Get store of the given subclass. */ - public static T get(Class clazz) { - for (ServingStore store : getAll()) { + public static T get(Class clazz) { + for (FeatureServingStoreFactory store : getAll()) { if (clazz.isInstance(store)) { //noinspection unchecked return (T) store; @@ -52,7 +55,7 @@ public static T get(Class clazz) { return null; } - public static void register(ServingStore store) { + public static void register(FeatureServingStoreFactory store) { manuallyRegistered.add(store); } } diff --git a/ingestion/src/main/java/feast/storage/service/WarehouseStoreService.java b/ingestion/src/main/java/feast/storage/service/FeatureWarehouseStoreFactoryService.java similarity index 59% rename from ingestion/src/main/java/feast/storage/service/WarehouseStoreService.java rename to ingestion/src/main/java/feast/storage/service/FeatureWarehouseStoreFactoryService.java index 6236be0ab0..9f41ba558d 100644 --- a/ingestion/src/main/java/feast/storage/service/WarehouseStoreService.java +++ b/ingestion/src/main/java/feast/storage/service/FeatureWarehouseStoreFactoryService.java @@ -19,32 +19,35 @@ import com.google.common.collect.Iterators; import com.google.common.collect.Lists; +import feast.storage.FeatureWarehouseStoreFactory; import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; import lombok.extern.slf4j.Slf4j; -import feast.storage.WarehouseStore; +/** + * Service class for fetching all the FeatureWarehouseStoreFactory instances available + */ @Slf4j -public class WarehouseStoreService { - private static ServiceLoader serviceLoader = - ServiceLoader.load(WarehouseStore.class); - private static List manuallyRegistered = new ArrayList<>(); +public class FeatureWarehouseStoreFactoryService { + private static ServiceLoader serviceLoader = + ServiceLoader.load(FeatureWarehouseStoreFactory.class); + private static List manuallyRegistered = new ArrayList<>(); static { - for (WarehouseStore store : getAll()) { - log.info("WarehouseStore type found: " + store.getType()); + for (FeatureWarehouseStoreFactory store : getAll()) { + log.info("FeatureWarehouseStoreFactory type found: " + store.getType()); } } - public static List getAll() { + public static List getAll() { return Lists.newArrayList( Iterators.concat(manuallyRegistered.iterator(), serviceLoader.iterator())); } /** Get store of the given subclass. */ - public static T get(Class clazz) { - for (WarehouseStore store : getAll()) { + public static T get(Class clazz) { + for (FeatureWarehouseStoreFactory store : getAll()) { if (clazz.isInstance(store)) { //noinspection unchecked return (T) store; @@ -53,7 +56,7 @@ public static T get(Class clazz) { return null; } - public static void register(WarehouseStore store) { + public static void register(FeatureWarehouseStoreFactory store) { manuallyRegistered.add(store); } } diff --git a/ingestion/src/main/java/feast/storage/stderr/StderrErrorsStore.java b/ingestion/src/main/java/feast/storage/stderr/StderrFeatureErrorsStoreFactory.java similarity index 86% rename from ingestion/src/main/java/feast/storage/stderr/StderrErrorsStore.java rename to ingestion/src/main/java/feast/storage/stderr/StderrFeatureErrorsStoreFactory.java index 623b7383aa..43edef4e00 100644 --- a/ingestion/src/main/java/feast/storage/stderr/StderrErrorsStore.java +++ b/ingestion/src/main/java/feast/storage/stderr/StderrFeatureErrorsStoreFactory.java @@ -21,11 +21,11 @@ import feast.ingestion.model.Specs; import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ErrorsStore; +import feast.storage.FeatureErrorsStoreFactory; import org.slf4j.event.Level; -@AutoService(ErrorsStore.class) -public class StderrErrorsStore implements ErrorsStore { +@AutoService(FeatureErrorsStoreFactory.class) +public class StderrFeatureErrorsStoreFactory implements FeatureErrorsStoreFactory { public static final String TYPE_STDERR = "stderr"; diff --git a/ingestion/src/main/java/feast/storage/stderr/StdoutErrorsStore.java b/ingestion/src/main/java/feast/storage/stderr/StdoutFeatureErrorsStoreFactory.java similarity index 85% rename from ingestion/src/main/java/feast/storage/stderr/StdoutErrorsStore.java rename to ingestion/src/main/java/feast/storage/stderr/StdoutFeatureErrorsStoreFactory.java index 7c5cc46437..5a33ab88ce 100644 --- a/ingestion/src/main/java/feast/storage/stderr/StdoutErrorsStore.java +++ b/ingestion/src/main/java/feast/storage/stderr/StdoutFeatureErrorsStoreFactory.java @@ -21,11 +21,11 @@ import feast.ingestion.model.Specs; import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.ErrorsStore; +import feast.storage.FeatureErrorsStoreFactory; import org.slf4j.event.Level; -@AutoService(ErrorsStore.class) -public class StdoutErrorsStore implements ErrorsStore { +@AutoService(FeatureErrorsStoreFactory.class) +public class StdoutFeatureErrorsStoreFactory implements FeatureErrorsStoreFactory { public static final String TYPE_STDOUT = "stdout"; diff --git a/ingestion/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/FeastPubsubHelper.java b/ingestion/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/FeastPubsubHelper.java deleted file mode 100644 index 9f2853bc47..0000000000 --- a/ingestion/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/FeastPubsubHelper.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.beam.sdk.io.gcp.pubsub; - -import com.google.common.collect.Lists; -import com.google.protobuf.Message; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.OutgoingMessage; -import org.apache.beam.sdk.io.gcp.pubsub.PubsubClient.TopicPath; -import org.apache.beam.sdk.options.PipelineOptions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This PubSub helper exists because including the canonical client libraries can result in - * dependency problems. We place it in the beam package because we need protected methods. - */ -public class FeastPubsubHelper { - - private final PipelineOptions options; - private transient PubsubClient client; - - public FeastPubsubHelper(PipelineOptions options) { - this.options = options; - } - - private PubsubClient getClient() throws IOException { - if (client == null) { - client = PubsubJsonClient.FACTORY.newClient(null, null, options.as(PubsubOptions.class)); - } - return client; - } - - public int publish(TopicPath topicPath, Message message) throws IOException { - List messages = - Lists.newArrayList( - new OutgoingMessage( - message.toByteArray(), new HashMap<>(), System.currentTimeMillis(), null)); - return getClient().publish(topicPath, messages); - } -} diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java index bd7460e637..1f39c81563 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java @@ -19,7 +19,7 @@ import static feast.FeastMatchers.hasCount; import static feast.NormalizeFeatureRows.normalize; -import static feast.storage.MockErrorsStore.MOCK_ERRORS_STORE_TYPE; +import static feast.storage.MockFeatureErrorsStoreFactory.MOCK_ERRORS_STORE_TYPE; import static org.junit.Assert.assertEquals; import com.google.common.base.Charsets; @@ -38,12 +38,12 @@ import feast.ingestion.service.SpecRetrievalException; import feast.ingestion.util.ProtoUtil; import feast.specs.ImportSpecProto.ImportSpec; -import feast.storage.MockErrorsStore; -import feast.storage.MockServingStore; -import feast.storage.MockWarehouseStore; -import feast.storage.service.ErrorsStoreService; -import feast.storage.service.ServingStoreService; -import feast.storage.service.WarehouseStoreService; +import feast.storage.MockFeatureErrorsStoreFactory; +import feast.storage.MockServingStoreFactory; +import feast.storage.MockWarehouseStoreFactory; +import feast.storage.service.FeatureErrorsStoreFactoryService; +import feast.storage.service.FeatureServingStoreFactoryService; +import feast.storage.service.FeatureWarehouseStoreFactoryService; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; import feast.types.GranularityProto.Granularity; @@ -121,16 +121,16 @@ public void testImportCSV() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureServingStoreFactoryService.get(MockServingStoreFactory.class).getWrite().getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = PCollectionList.of( - WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs()) + FeatureWarehouseStoreFactoryService.get(MockWarehouseStoreFactory.class).getWrite().getInputs()) .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureErrorsStoreFactoryService.get(MockFeatureErrorsStoreFactory.class).getWrite().getInputs()) .apply("flatten errors input", Flatten.pCollections()); List expectedRows = @@ -212,16 +212,16 @@ public void testImportCSV_withSample1() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureServingStoreFactoryService.get(MockServingStoreFactory.class).getWrite().getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = PCollectionList.of( - WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs()) + FeatureWarehouseStoreFactoryService.get(MockWarehouseStoreFactory.class).getWrite().getInputs()) .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureErrorsStoreFactoryService.get(MockFeatureErrorsStoreFactory.class).getWrite().getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToServing).satisfies(hasCount(1)); @@ -271,16 +271,16 @@ public void testImportCSV_withCoalesceRows() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureServingStoreFactoryService.get(MockServingStoreFactory.class).getWrite().getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = PCollectionList.of( - WarehouseStoreService.get(MockWarehouseStore.class).getWrite().getInputs()) + FeatureWarehouseStoreFactoryService.get(MockWarehouseStoreFactory.class).getWrite().getInputs()) .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureErrorsStoreFactoryService.get(MockFeatureErrorsStoreFactory.class).getWrite().getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToErrors).satisfies(hasCount(0)); @@ -394,11 +394,11 @@ public void testImportWithErrors() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureServingStoreFactoryService.get(MockServingStoreFactory.class).getWrite().getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureErrorsStoreFactoryService.get(MockFeatureErrorsStoreFactory.class).getWrite().getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToErrors) @@ -459,11 +459,11 @@ public void testImportWithoutWarehouseStore() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(ServingStoreService.get(MockServingStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureServingStoreFactoryService.get(MockServingStoreFactory.class).getWrite().getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(ErrorsStoreService.get(MockErrorsStore.class).getWrite().getInputs()) + PCollectionList.of(FeatureErrorsStoreFactoryService.get(MockFeatureErrorsStoreFactory.class).getWrite().getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToErrors) diff --git a/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java b/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java index 83c0ef3e02..e5abaf28fa 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java @@ -18,14 +18,14 @@ package feast.ingestion.transform; import static feast.ingestion.model.Errors.toError; -import static feast.storage.MockErrorsStore.MOCK_ERRORS_STORE_TYPE; +import static feast.storage.MockFeatureErrorsStoreFactory.MOCK_ERRORS_STORE_TYPE; import feast.ingestion.model.Specs; import feast.ingestion.options.ImportJobPipelineOptions; -import feast.storage.MockErrorsStore; -import feast.storage.service.ErrorsStoreService; -import feast.storage.stderr.StderrErrorsStore; -import feast.storage.stderr.StdoutErrorsStore; +import feast.storage.MockFeatureErrorsStoreFactory; +import feast.storage.service.FeatureErrorsStoreFactoryService; +import feast.storage.stderr.StderrFeatureErrorsStoreFactory; +import feast.storage.stderr.StdoutFeatureErrorsStoreFactory; import feast.types.FeatureRowExtendedProto.Attempt; import feast.types.FeatureRowExtendedProto.Error; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; @@ -75,7 +75,7 @@ private FeatureRowExtended errorOf(String transform, Throwable cause) { @Test public void shouldWriteToGivenErrorsStore() { - MockErrorsStore mockStore = new MockErrorsStore(); + MockFeatureErrorsStoreFactory mockStore = new MockFeatureErrorsStoreFactory(); options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); ErrorsStoreTransform transform = new ErrorsStoreTransform(options, specs, Lists.newArrayList(mockStore)); @@ -91,9 +91,9 @@ public void shouldWriteToGivenErrorsStore() { @Test public void logErrorsToStdErr() { - options.setErrorsStoreType(StderrErrorsStore.TYPE_STDERR); + options.setErrorsStoreType(StderrFeatureErrorsStoreFactory.TYPE_STDERR); ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, ErrorsStoreService.getAll()); + new ErrorsStoreTransform(options, specs, FeatureErrorsStoreFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } @@ -101,9 +101,9 @@ public void logErrorsToStdErr() { @Test public void logErrorsToStdOut() { - options.setErrorsStoreType(StdoutErrorsStore.TYPE_STDOUT); + options.setErrorsStoreType(StdoutFeatureErrorsStoreFactory.TYPE_STDOUT); ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, ErrorsStoreService.getAll()); + new ErrorsStoreTransform(options, specs, FeatureErrorsStoreFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } @@ -112,7 +112,7 @@ public void logErrorsToStdOut() { public void logToNull() { //options.setErrorsStoreType(...); // no errors store type set ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, ErrorsStoreService.getAll()); + new ErrorsStoreTransform(options, specs, FeatureErrorsStoreFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } diff --git a/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java b/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java index 7758a77489..ceb9ca642b 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java @@ -37,7 +37,7 @@ import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureStore; +import feast.storage.FeatureStoreFactory; import feast.storage.MockFeatureStore; import feast.storage.MockTransforms; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; @@ -78,7 +78,7 @@ public void testSplit() { DataStores.newBuilder().setServing(DataStore.newBuilder().setId("store2"))).build()) .addStorage(StorageSpec.newBuilder().setId("store1").setType("type1").build()) .addStorage(StorageSpec.newBuilder().setId("store2").setType("type2").build()); - List stores = + List stores = Lists.newArrayList(new MockFeatureStore("type1"), new MockFeatureStore("type2")); Specs specs = Specs.of( @@ -186,7 +186,7 @@ public void testSplitWhereFeature2HasNoStoreId() { "store1", StorageSpec.newBuilder().setId("store1").setType("type1").build()); specService.storageSpecs.put( "store2", StorageSpec.newBuilder().setId("store2").setType("type2").build()); - List stores = + List stores = Lists.newArrayList(new MockFeatureStore("type1"), new MockFeatureStore("type2")); Specs specs = Specs.of( @@ -284,7 +284,7 @@ public void testSplitWhereNoStorageSpec() { .build(), specService); assertNull(specs.getError()); - List stores = Collections.emptyList(); + List stores = Collections.emptyList(); SplitOutputByStore split = new SplitOutputByStore(stores, selector, specs); PCollection input = @@ -346,7 +346,7 @@ public void testSplitWhereNoStorageSpecForAFeature() { .build(), specService); assertNull(specs.getError()); - List stores = Collections.emptyList(); + List stores = Collections.emptyList(); SplitOutputByStore split = new SplitOutputByStore(stores, selector, specs); PCollection input = diff --git a/ingestion/src/test/java/feast/storage/MockErrorsStore.java b/ingestion/src/test/java/feast/storage/MockFeatureErrorsStoreFactory.java similarity index 79% rename from ingestion/src/test/java/feast/storage/MockErrorsStore.java rename to ingestion/src/test/java/feast/storage/MockFeatureErrorsStoreFactory.java index e83ba5641e..eab730d768 100644 --- a/ingestion/src/test/java/feast/storage/MockErrorsStore.java +++ b/ingestion/src/test/java/feast/storage/MockFeatureErrorsStoreFactory.java @@ -19,11 +19,12 @@ import com.google.auto.service.AutoService; -@AutoService(ErrorsStore.class) -public class MockErrorsStore extends MockFeatureStore implements ErrorsStore { +@AutoService(FeatureErrorsStoreFactory.class) +public class MockFeatureErrorsStoreFactory extends MockFeatureStore implements + FeatureErrorsStoreFactory { public static final String MOCK_ERRORS_STORE_TYPE = "errors.mock"; - public MockErrorsStore() { + public MockFeatureErrorsStoreFactory() { super(MOCK_ERRORS_STORE_TYPE); } } diff --git a/ingestion/src/test/java/feast/storage/MockFeatureStore.java b/ingestion/src/test/java/feast/storage/MockFeatureStore.java index 2e957d4de2..f04d187ea2 100644 --- a/ingestion/src/test/java/feast/storage/MockFeatureStore.java +++ b/ingestion/src/test/java/feast/storage/MockFeatureStore.java @@ -21,7 +21,7 @@ import feast.specs.StorageSpecProto.StorageSpec; import lombok.Getter; -public class MockFeatureStore implements FeatureStore { +public class MockFeatureStore implements FeatureStoreFactory { private final String type; @Getter private MockTransforms.Write write; diff --git a/ingestion/src/test/java/feast/storage/MockServingStore.java b/ingestion/src/test/java/feast/storage/MockServingStoreFactory.java similarity index 80% rename from ingestion/src/test/java/feast/storage/MockServingStore.java rename to ingestion/src/test/java/feast/storage/MockServingStoreFactory.java index 2c60bcbf78..02ec97f382 100644 --- a/ingestion/src/test/java/feast/storage/MockServingStore.java +++ b/ingestion/src/test/java/feast/storage/MockServingStoreFactory.java @@ -19,11 +19,12 @@ import com.google.auto.service.AutoService; -@AutoService(ServingStore.class) -public class MockServingStore extends MockFeatureStore implements ServingStore { +@AutoService(FeatureServingStoreFactory.class) +public class MockServingStoreFactory extends MockFeatureStore implements + FeatureServingStoreFactory { public static final String MOCK_SERVING_STORE_TYPE = "serving.mock"; - public MockServingStore() { + public MockServingStoreFactory() { super(MOCK_SERVING_STORE_TYPE); } } diff --git a/ingestion/src/test/java/feast/storage/MockWarehouseStore.java b/ingestion/src/test/java/feast/storage/MockWarehouseStoreFactory.java similarity index 80% rename from ingestion/src/test/java/feast/storage/MockWarehouseStore.java rename to ingestion/src/test/java/feast/storage/MockWarehouseStoreFactory.java index dbe9ab8caf..8bebf5baa0 100644 --- a/ingestion/src/test/java/feast/storage/MockWarehouseStore.java +++ b/ingestion/src/test/java/feast/storage/MockWarehouseStoreFactory.java @@ -19,11 +19,12 @@ import com.google.auto.service.AutoService; -@AutoService(WarehouseStore.class) -public class MockWarehouseStore extends MockFeatureStore implements WarehouseStore { +@AutoService(FeatureWarehouseStoreFactory.class) +public class MockWarehouseStoreFactory extends MockFeatureStore implements + FeatureWarehouseStoreFactory { public static final String MOCK_WAREHOUSE_STORE_TYPE = "warehouse.mock"; - public MockWarehouseStore() { + public MockWarehouseStoreFactory() { super(MOCK_WAREHOUSE_STORE_TYPE); } } diff --git a/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java b/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java index 591736c1be..74959ccb6c 100644 --- a/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java +++ b/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java @@ -110,7 +110,7 @@ public void testWriteNoneGranularity() throws IOException { Specs specs = getSpecs(); specs.validate(); - new RedisServingStore().create(specs.getStorageSpec("REDIS1"), specs); + new RedisServingStoreFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowRedisIO.Write write = new FeatureRowRedisIO.Write( RedisStoreOptions.builder().host("localhost").port(REDIS_PORT).build(), specs); @@ -154,7 +154,7 @@ public void testWriteNoneGranularity() throws IOException { @Test public void testWriteNoneGranularityFromOptions() throws IOException { Specs specs = getSpecs(); - FeatureIO.Write write = new RedisServingStore().create(specs.getStorageSpec("REDIS1"), specs); + FeatureIO.Write write = new RedisServingStoreFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() @@ -195,7 +195,7 @@ public void testWriteNoneGranularityFromOptions() throws IOException { @Test public void testWriteHourGranularity() throws IOException { Specs specs = getSpecs(); - FeatureIO.Write write = new RedisServingStore().create(specs.getStorageSpec("REDIS1"), specs); + FeatureIO.Write write = new RedisServingStoreFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() From 7c217e2f79820d19aba991b834da38e966057632 Mon Sep 17 00:00:00 2001 From: Tim Sell Date: Mon, 28 Jan 2019 03:05:23 +0800 Subject: [PATCH 2/3] split stores up into serving and warehouse and rename some things --- .../feast/ingestion/boot/ImportJobModule.java | 24 ++--- .../transform/ErrorsStoreTransform.java | 13 +-- .../transform/ServingStoreTransform.java | 12 +-- .../transform/SplitOutputByStore.java | 2 +- .../transform/WarehouseStoreTransform.java | 18 ++-- .../transform/fn/ValidateFeatureRowsDoFn.java | 12 +-- .../bigquery/BigQueryToFeatureRowFn.java | 4 +- .../storage/file/json/JsonFileFeatureIO.java | 66 -------------- .../file/json/JsonFileStoreFactories.java | 88 ------------------- .../FeatureStoreFactory.java | 2 +- .../java/feast/store/FeatureStoreWrite.java | 27 ++++++ .../file => store}/FileStoreOptions.java | 2 +- .../feast/{storage/noop => store}/NoOpIO.java | 2 +- .../TextFileDynamicIO.java} | 39 ++++---- .../errors/FeatureErrorsFactory.java} | 6 +- .../errors/FeatureErrorsFactoryService.java} | 34 +++---- .../errors/json/JsonFileErrorsFactory.java} | 26 ++++-- .../errors/json/JsonFileErrorsWrite.java | 56 ++++++++++++ .../errors/logging}/LogIO.java | 6 +- .../logging/StderrFeatureErrorsFactory.java} | 14 +-- .../logging/StdoutFeatureErrorsFactory.java} | 12 +-- .../serving/FeatureServingFactory.java} | 6 +- .../FeatureServingFactoryService.java} | 29 +++--- .../serving}/FeatureServingStoreClient.java | 2 +- .../bigtable/BigTableFeatureOptions.java | 2 +- .../bigtable/BigTableServingStoreFactory.java | 9 +- .../bigtable/BigTableStoreOptions.java | 2 +- .../bigtable/FeatureRowBigTableIO.java | 2 +- .../FeatureRowToBigTableMutationDoFn.java | 2 +- .../serving}/redis/FeatureRowRedisIO.java | 5 +- .../redis/FeatureRowToRedisMutationDoFn.java | 6 +- .../serving}/redis/RedisCustomIO.java | 2 +- .../serving}/redis/RedisFeatureOptions.java | 2 +- .../serving/redis/RedisServingFactory.java} | 9 +- .../serving}/redis/RedisStoreOptions.java | 2 +- .../warehouse/FeatureWarehouseFactory.java} | 8 +- .../FeatureWarehouseFactoryService.java} | 32 ++++--- .../bigquery/BigQueryStoreOptions.java | 3 +- .../bigquery/BigQueryWarehouseFactory.java} | 8 +- .../bigquery/FeatureRowBigQueryIO.java | 2 +- .../FeatureRowToBigQueryTableRowDoFn.java | 4 +- .../bigquery/ValueBigQueryBuilder.java | 2 +- .../json/JsonFileWarehouseFactory.java} | 23 +++-- .../json/JsonFileWarehouseWrite.java | 56 ++++++++++++ .../feast/ingestion/ImportJobCSVTest.java | 63 ++++++++----- .../transform/ErrorsStoreTransformTest.java | 22 ++--- .../transform/SplitOutputByStoreTest.java | 6 +- .../file => store}/FileStoreOptionsTest.java | 7 +- .../MockFeatureErrorsFactory.java} | 15 ++-- .../{storage => store}/MockFeatureStore.java | 6 +- .../MockServingFactory.java} | 15 ++-- .../{storage => store}/MockTransforms.java | 6 +- .../MockWarehouseFactory.java} | 15 ++-- .../TextFileDynamicIOTest.java} | 51 ++++++----- .../bigtable/BigTableFeatureOptionsTest.java | 6 +- .../bigtable/BigTableStoreOptionsTest.java | 6 +- .../redis/FeatureRowRedisIOWriteTest.java | 16 ++-- .../FeatureRowToRedisMutationDoFnTest.java | 6 +- .../redis/RedisFeatureOptionsTest.java | 6 +- .../serving}/redis/RedisStoreOptionsTest.java | 6 +- .../bigquery/BigQueryStoreOptionsTest.java | 2 +- .../bigquery/ValueBigQueryBuilderTest.java | 2 +- .../json/JsonFileFeatureIOTest.java | 50 +++++++---- 63 files changed, 533 insertions(+), 456 deletions(-) delete mode 100644 ingestion/src/main/java/feast/storage/file/json/JsonFileFeatureIO.java delete mode 100644 ingestion/src/main/java/feast/storage/file/json/JsonFileStoreFactories.java rename ingestion/src/main/java/feast/{storage => store}/FeatureStoreFactory.java (97%) create mode 100644 ingestion/src/main/java/feast/store/FeatureStoreWrite.java rename ingestion/src/main/java/feast/{storage/file => store}/FileStoreOptions.java (97%) rename ingestion/src/main/java/feast/{storage/noop => store}/NoOpIO.java (97%) rename ingestion/src/main/java/feast/{storage/file/TextFileFeatureIO.java => store/TextFileDynamicIO.java} (65%) rename ingestion/src/main/java/feast/{storage/FeatureErrorsStoreFactory.java => store/errors/FeatureErrorsFactory.java} (81%) rename ingestion/src/main/java/feast/{storage/service/FeatureWarehouseStoreFactoryService.java => store/errors/FeatureErrorsFactoryService.java} (55%) rename ingestion/src/main/java/feast/{storage/noop/NoOpServingStoreFactory.java => store/errors/json/JsonFileErrorsFactory.java} (54%) create mode 100644 ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java rename ingestion/src/main/java/feast/{storage/stderr => store/errors/logging}/LogIO.java (91%) rename ingestion/src/main/java/feast/{storage/stderr/StderrFeatureErrorsStoreFactory.java => store/errors/logging/StderrFeatureErrorsFactory.java} (76%) rename ingestion/src/main/java/feast/{storage/stderr/StdoutFeatureErrorsStoreFactory.java => store/errors/logging/StdoutFeatureErrorsFactory.java} (76%) rename ingestion/src/main/java/feast/{storage/FeatureServingStoreFactory.java => store/serving/FeatureServingFactory.java} (81%) rename ingestion/src/main/java/feast/{storage/service/FeatureServingStoreFactoryService.java => store/serving/FeatureServingFactoryService.java} (57%) rename ingestion/src/main/java/feast/{storage => store/serving}/FeatureServingStoreClient.java (97%) rename ingestion/src/main/java/feast/{storage => store/serving}/bigtable/BigTableFeatureOptions.java (95%) rename ingestion/src/main/java/feast/{storage => store/serving}/bigtable/BigTableServingStoreFactory.java (90%) rename ingestion/src/main/java/feast/{storage => store/serving}/bigtable/BigTableStoreOptions.java (95%) rename ingestion/src/main/java/feast/{storage => store/serving}/bigtable/FeatureRowBigTableIO.java (98%) rename ingestion/src/main/java/feast/{storage => store/serving}/bigtable/FeatureRowToBigTableMutationDoFn.java (99%) rename ingestion/src/main/java/feast/{storage => store/serving}/redis/FeatureRowRedisIO.java (94%) rename ingestion/src/main/java/feast/{storage => store/serving}/redis/FeatureRowToRedisMutationDoFn.java (96%) rename ingestion/src/main/java/feast/{storage => store/serving}/redis/RedisCustomIO.java (99%) rename ingestion/src/main/java/feast/{storage => store/serving}/redis/RedisFeatureOptions.java (97%) rename ingestion/src/main/java/feast/{storage/redis/RedisServingStoreFactory.java => store/serving/redis/RedisServingFactory.java} (86%) rename ingestion/src/main/java/feast/{storage => store/serving}/redis/RedisStoreOptions.java (97%) rename ingestion/src/main/java/feast/{storage/FeatureWarehouseStoreFactory.java => store/warehouse/FeatureWarehouseFactory.java} (80%) rename ingestion/src/main/java/feast/{storage/service/FeatureErrorsStoreFactoryService.java => store/warehouse/FeatureWarehouseFactoryService.java} (59%) rename ingestion/src/main/java/feast/{storage => store/warehouse}/bigquery/BigQueryStoreOptions.java (93%) rename ingestion/src/main/java/feast/{storage/bigquery/BigQueryWarehouseStoreFactory.java => store/warehouse/bigquery/BigQueryWarehouseFactory.java} (86%) rename ingestion/src/main/java/feast/{storage => store/warehouse}/bigquery/FeatureRowBigQueryIO.java (99%) rename ingestion/src/main/java/feast/{storage => store/warehouse}/bigquery/FeatureRowToBigQueryTableRowDoFn.java (96%) rename ingestion/src/main/java/feast/{storage => store/warehouse}/bigquery/ValueBigQueryBuilder.java (99%) rename ingestion/src/main/java/feast/{storage/noop/NoOpWarehouseStoreFactory.java => store/warehouse/json/JsonFileWarehouseFactory.java} (56%) create mode 100644 ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java rename ingestion/src/test/java/feast/{storage/file => store}/FileStoreOptionsTest.java (93%) rename ingestion/src/test/java/feast/{storage/MockFeatureErrorsStoreFactory.java => store/MockFeatureErrorsFactory.java} (67%) rename ingestion/src/test/java/feast/{storage => store}/MockFeatureStore.java (90%) rename ingestion/src/test/java/feast/{storage/MockServingStoreFactory.java => store/MockServingFactory.java} (67%) rename ingestion/src/test/java/feast/{storage => store}/MockTransforms.java (93%) rename ingestion/src/test/java/feast/{storage/MockWarehouseStoreFactory.java => store/MockWarehouseFactory.java} (67%) rename ingestion/src/test/java/feast/{storage/file/TextFileFeatureIOTest.java => store/TextFileDynamicIOTest.java} (56%) rename ingestion/src/test/java/feast/{storage => store/serving}/bigtable/BigTableFeatureOptionsTest.java (91%) rename ingestion/src/test/java/feast/{storage => store/serving}/bigtable/BigTableStoreOptionsTest.java (95%) rename ingestion/src/test/java/feast/{storage => store/serving}/redis/FeatureRowRedisIOWriteTest.java (94%) rename ingestion/src/test/java/feast/{storage => store/serving}/redis/FeatureRowToRedisMutationDoFnTest.java (93%) rename ingestion/src/test/java/feast/{storage => store/serving}/redis/RedisFeatureOptionsTest.java (95%) rename ingestion/src/test/java/feast/{storage => store/serving}/redis/RedisStoreOptionsTest.java (92%) rename ingestion/src/test/java/feast/{storage => store/warehouse}/bigquery/BigQueryStoreOptionsTest.java (97%) rename ingestion/src/test/java/feast/{storage => store/warehouse}/bigquery/ValueBigQueryBuilderTest.java (98%) rename ingestion/src/test/java/feast/{storage/file => store/warehouse}/json/JsonFileFeatureIOTest.java (59%) diff --git a/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java b/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java index de3d84b169..81dadaf38e 100644 --- a/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java +++ b/ingestion/src/main/java/feast/ingestion/boot/ImportJobModule.java @@ -28,12 +28,12 @@ import feast.ingestion.service.SpecService.Builder; import feast.ingestion.service.SpecService.UnsupportedBuilder; import feast.specs.ImportSpecProto.ImportSpec; -import feast.storage.FeatureErrorsStoreFactory; -import feast.storage.FeatureServingStoreFactory; -import feast.storage.FeatureWarehouseStoreFactory; -import feast.storage.service.FeatureErrorsStoreFactoryService; -import feast.storage.service.FeatureServingStoreFactoryService; -import feast.storage.service.FeatureWarehouseStoreFactoryService; +import feast.store.errors.FeatureErrorsFactory; +import feast.store.errors.FeatureErrorsFactoryService; +import feast.store.serving.FeatureServingFactory; +import feast.store.serving.FeatureServingFactoryService; +import feast.store.warehouse.FeatureWarehouseFactory; +import feast.store.warehouse.FeatureWarehouseFactoryService; import java.util.List; import org.apache.beam.sdk.options.PipelineOptions; @@ -78,19 +78,19 @@ Specs provideSpecs(SpecService.Builder specService) { @Provides @Singleton - List provideWarehouseStores() { - return FeatureWarehouseStoreFactoryService.getAll(); + List provideWarehouseStores() { + return FeatureWarehouseFactoryService.getAll(); } @Provides @Singleton - List provideServingStores() { - return FeatureServingStoreFactoryService.getAll(); + List provideServingStores() { + return FeatureServingFactoryService.getAll(); } @Provides @Singleton - List provideErrorsStores() { - return FeatureErrorsStoreFactoryService.getAll(); + List provideErrorsStores() { + return FeatureErrorsFactoryService.getAll(); } } diff --git a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java index 1ccdf1aa7e..8ddd37422b 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java @@ -25,8 +25,8 @@ import feast.ingestion.options.ImportJobPipelineOptions; import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureErrorsStoreFactory; -import feast.storage.noop.NoOpIO; +import feast.store.NoOpIO; +import feast.store.errors.FeatureErrorsFactory; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.List; import lombok.extern.slf4j.Slf4j; @@ -40,11 +40,12 @@ public class ErrorsStoreTransform extends FeatureIO.Write { private String errorsStoreType; private StorageSpec errorsStoreSpec; private Specs specs; - private List errorsStoreFactories; + private List errorsStoreFactories; @Inject public ErrorsStoreTransform( - ImportJobPipelineOptions options, Specs specs, List errorsStoreFactories) { + ImportJobPipelineOptions options, Specs specs, + List errorsStoreFactories) { this.specs = specs; this.errorsStoreFactories = errorsStoreFactories; this.errorsStoreType = options.getErrorsStoreType(); @@ -70,9 +71,9 @@ public PDone expand(PCollection input) { return PDone.in(input.getPipeline()); } - FeatureErrorsStoreFactory getErrorStore() { + FeatureErrorsFactory getErrorStore() { checkArgument(!errorsStoreType.isEmpty(), "Errors store type not provided"); - for (FeatureErrorsStoreFactory errorsStoreFactory : errorsStoreFactories) { + for (FeatureErrorsFactory errorsStoreFactory : errorsStoreFactories) { if (errorsStoreFactory.getType().equals(errorsStoreType)) { return errorsStoreFactory; } diff --git a/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java index d5f3e321ed..024687bdfa 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ServingStoreTransform.java @@ -21,21 +21,20 @@ import feast.ingestion.metrics.FeastMetrics; import feast.ingestion.model.Specs; import feast.ingestion.values.PFeatureRows; -import feast.storage.FeatureServingStoreFactory; +import feast.store.serving.FeatureServingFactory; +import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; -import java.util.List; - @Slf4j public class ServingStoreTransform extends PTransform { - private List stores; + private List stores; private Specs specs; @Inject - public ServingStoreTransform(List stores, Specs specs) { + public ServingStoreTransform(List stores, Specs specs) { this.stores = stores; this.specs = specs; } @@ -50,7 +49,8 @@ public PFeatureRows expand(PFeatureRows input) { output.getMain().apply("metrics.store.lag", ParDo.of(FeastMetrics.lagUpdateDoFn())); output.getMain().apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("serving_stored"))); - output.getErrors().apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("serving_errors"))); + output.getErrors() + .apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("serving_errors"))); return output; } } diff --git a/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java b/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java index 7c4ee2ee23..d5eba40444 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java +++ b/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java @@ -25,7 +25,7 @@ import feast.ingestion.values.PFeatureRows; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureStoreFactory; +import feast.store.FeatureStoreFactory; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.Collection; import java.util.HashMap; diff --git a/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java index 6a5ee2f0b9..de70541961 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/WarehouseStoreTransform.java @@ -18,23 +18,23 @@ package feast.ingestion.transform; import com.google.inject.Inject; +import feast.ingestion.metrics.FeastMetrics; +import feast.ingestion.model.Specs; +import feast.ingestion.values.PFeatureRows; +import feast.store.warehouse.FeatureWarehouseFactory; import java.util.List; import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; -import feast.ingestion.metrics.FeastMetrics; -import feast.ingestion.model.Specs; -import feast.ingestion.values.PFeatureRows; -import feast.storage.FeatureWarehouseStoreFactory; @Slf4j public class WarehouseStoreTransform extends PTransform { - private List stores; + private List stores; private Specs specs; @Inject - public WarehouseStoreTransform(List stores, Specs specs) { + public WarehouseStoreTransform(List stores, Specs specs) { this.stores = stores; this.specs = specs; } @@ -48,8 +48,10 @@ public PFeatureRows expand(PFeatureRows input) { stores, (featureSpec) -> featureSpec.getDataStores().getWarehouse().getId(), specs)); - output.getMain().apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("warehouse_stored"))); - output.getErrors().apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("warehouse_errors"))); + output.getMain() + .apply("metrics.store.main", ParDo.of(FeastMetrics.incrDoFn("warehouse_stored"))); + output.getErrors() + .apply("metrics.store.errors", ParDo.of(FeastMetrics.incrDoFn("warehouse_errors"))); return output; } } diff --git a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java index 34b7cabfff..2704c5e766 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java +++ b/ingestion/src/main/java/feast/ingestion/transform/fn/ValidateFeatureRowsDoFn.java @@ -31,10 +31,10 @@ import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureServingStoreFactory; -import feast.storage.FeatureWarehouseStoreFactory; -import feast.storage.service.FeatureServingStoreFactoryService; -import feast.storage.service.FeatureWarehouseStoreFactoryService; +import feast.store.serving.FeatureServingFactory; +import feast.store.serving.FeatureServingFactoryService; +import feast.store.warehouse.FeatureWarehouseFactory; +import feast.store.warehouse.FeatureWarehouseFactoryService; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowProto.FeatureRow; import feast.types.GranularityProto.Granularity.Enum; @@ -66,10 +66,10 @@ public void setup() { featureIds.add(field.getFeatureId()); } } - for (FeatureServingStoreFactory store : FeatureServingStoreFactoryService.getAll()) { + for (FeatureServingFactory store : FeatureServingFactoryService.getAll()) { supportedServingTypes.add(store.getType()); } - for (FeatureWarehouseStoreFactory store : FeatureWarehouseStoreFactoryService.getAll()) { + for (FeatureWarehouseFactory store : FeatureWarehouseFactoryService.getAll()) { supportedWarehouseTypes.add(store.getType()); } } diff --git a/ingestion/src/main/java/feast/source/bigquery/BigQueryToFeatureRowFn.java b/ingestion/src/main/java/feast/source/bigquery/BigQueryToFeatureRowFn.java index 795735410d..dfe26c45b7 100644 --- a/ingestion/src/main/java/feast/source/bigquery/BigQueryToFeatureRowFn.java +++ b/ingestion/src/main/java/feast/source/bigquery/BigQueryToFeatureRowFn.java @@ -23,7 +23,7 @@ import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.common.collect.Maps; import com.google.protobuf.Timestamp; -import feast.storage.bigquery.ValueBigQueryBuilder; +import feast.store.warehouse.bigquery.ValueBigQueryBuilder; import java.util.Map; import org.apache.avro.generic.GenericRecord; import org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord; @@ -34,8 +34,6 @@ import feast.types.FeatureProto.Feature; import feast.types.FeatureRowProto.FeatureRow; import feast.types.ValueProto.Value; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * This is a serializable function used with the BigQueryIO for fetching feature rows directly from diff --git a/ingestion/src/main/java/feast/storage/file/json/JsonFileFeatureIO.java b/ingestion/src/main/java/feast/storage/file/json/JsonFileFeatureIO.java deleted file mode 100644 index 0ebbbcb540..0000000000 --- a/ingestion/src/main/java/feast/storage/file/json/JsonFileFeatureIO.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage.file.json; - -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; -import com.google.protobuf.util.JsonFormat; -import feast.ingestion.transform.FeatureIO; -import feast.storage.file.FileStoreOptions; -import feast.storage.file.TextFileFeatureIO; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import lombok.AllArgsConstructor; -import org.apache.beam.sdk.transforms.SerializableFunction; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; - -public class JsonFileFeatureIO { - - public static Write writeRowExtended(FileStoreOptions options) { - return new Write(options, (rowExtended) -> rowExtended); - } - - public static Write writeRow(FileStoreOptions options) { - return new Write(options, (rowExtended) -> rowExtended.getRow()); - } - - @AllArgsConstructor - public static class Write extends FeatureIO.Write { - - private FileStoreOptions options; - private SerializableFunction messageFunction; - - @Override - public PDone expand(PCollection input) { - return input.apply( - "Write Text Files", - new TextFileFeatureIO.Write( - options, - (rowExtended) -> { - try { - return JsonFormat.printer() - .omittingInsignificantWhitespace() - .print(messageFunction.apply(rowExtended)); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); - } - }, - ".json")); - } - } -} diff --git a/ingestion/src/main/java/feast/storage/file/json/JsonFileStoreFactories.java b/ingestion/src/main/java/feast/storage/file/json/JsonFileStoreFactories.java deleted file mode 100644 index 2babb379d5..0000000000 --- a/ingestion/src/main/java/feast/storage/file/json/JsonFileStoreFactories.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.storage.file.json; - -import com.google.auto.service.AutoService; -import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; -import feast.options.OptionsParser; -import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureErrorsStoreFactory; -import feast.storage.FeatureServingStoreFactory; -import feast.storage.FeatureWarehouseStoreFactory; -import feast.storage.file.FileStoreOptions; -import lombok.AllArgsConstructor; - -public class JsonFileStoreFactories { - - private static final String JSON_FILES_TYPE = "file.json"; - - @AutoService(FeatureWarehouseStoreFactory.class) - @AllArgsConstructor - public static class JsonFileWarehouseStoreFactory implements FeatureWarehouseStoreFactory { - - @Override - public Write create(StorageSpec storageSpec, Specs specs) { - FileStoreOptions options = - OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); - options.jobName = specs.getJobName(); - return JsonFileFeatureIO.writeRow(options); - } - - @Override - public String getType() { - return JSON_FILES_TYPE; - } - } - - @AutoService(FeatureServingStoreFactory.class) - @AllArgsConstructor - public static class JsonFileServingStoreFactory implements FeatureServingStoreFactory { - - @Override - public Write create(StorageSpec storageSpec, Specs specs) { - FileStoreOptions options = - OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); - options.jobName = specs.getJobName(); - return JsonFileFeatureIO.writeRow(options); - } - - @Override - public String getType() { - return JSON_FILES_TYPE; - } - } - - @AutoService(FeatureErrorsStoreFactory.class) - @AllArgsConstructor - public static class JsonFileFeatureErrorsStoreFactory implements FeatureErrorsStoreFactory { - - @Override - public Write create(StorageSpec storageSpec, Specs specs) { - FileStoreOptions options = - OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); - options.jobName = specs.getJobName(); - return JsonFileFeatureIO.writeRowExtended(options); - } - - @Override - public String getType() { - return JSON_FILES_TYPE; - } - } -} diff --git a/ingestion/src/main/java/feast/storage/FeatureStoreFactory.java b/ingestion/src/main/java/feast/store/FeatureStoreFactory.java similarity index 97% rename from ingestion/src/main/java/feast/storage/FeatureStoreFactory.java rename to ingestion/src/main/java/feast/store/FeatureStoreFactory.java index 2989d45dcc..5bb8dc6d6c 100644 --- a/ingestion/src/main/java/feast/storage/FeatureStoreFactory.java +++ b/ingestion/src/main/java/feast/store/FeatureStoreFactory.java @@ -15,7 +15,7 @@ * */ -package feast.storage; +package feast.store; import feast.ingestion.model.Specs; import feast.ingestion.transform.FeatureIO; diff --git a/ingestion/src/main/java/feast/store/FeatureStoreWrite.java b/ingestion/src/main/java/feast/store/FeatureStoreWrite.java new file mode 100644 index 0000000000..c039ea01ee --- /dev/null +++ b/ingestion/src/main/java/feast/store/FeatureStoreWrite.java @@ -0,0 +1,27 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.store; + +import feast.types.FeatureRowProto.FeatureRow; +import org.apache.beam.sdk.transforms.PTransform; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PDone; + +public abstract class FeatureStoreWrite extends PTransform, PDone> { + +} diff --git a/ingestion/src/main/java/feast/storage/file/FileStoreOptions.java b/ingestion/src/main/java/feast/store/FileStoreOptions.java similarity index 97% rename from ingestion/src/main/java/feast/storage/file/FileStoreOptions.java rename to ingestion/src/main/java/feast/store/FileStoreOptions.java index 84a5caa137..5df669f0e9 100644 --- a/ingestion/src/main/java/feast/storage/file/FileStoreOptions.java +++ b/ingestion/src/main/java/feast/store/FileStoreOptions.java @@ -15,7 +15,7 @@ * */ -package feast.storage.file; +package feast.store; import com.fasterxml.jackson.annotation.JsonIgnore; import javax.validation.constraints.NotEmpty; diff --git a/ingestion/src/main/java/feast/storage/noop/NoOpIO.java b/ingestion/src/main/java/feast/store/NoOpIO.java similarity index 97% rename from ingestion/src/main/java/feast/storage/noop/NoOpIO.java rename to ingestion/src/main/java/feast/store/NoOpIO.java index 45e06e5801..8893313c80 100644 --- a/ingestion/src/main/java/feast/storage/noop/NoOpIO.java +++ b/ingestion/src/main/java/feast/store/NoOpIO.java @@ -15,7 +15,7 @@ * */ -package feast.storage.noop; +package feast.store; import feast.ingestion.transform.FeatureIO; import feast.ingestion.transform.fn.Identity; diff --git a/ingestion/src/main/java/feast/storage/file/TextFileFeatureIO.java b/ingestion/src/main/java/feast/store/TextFileDynamicIO.java similarity index 65% rename from ingestion/src/main/java/feast/storage/file/TextFileFeatureIO.java rename to ingestion/src/main/java/feast/store/TextFileDynamicIO.java index 740a269212..ee7ed10751 100644 --- a/ingestion/src/main/java/feast/storage/file/TextFileFeatureIO.java +++ b/ingestion/src/main/java/feast/store/TextFileDynamicIO.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ * */ -package feast.storage.file; +package feast.store; import lombok.AllArgsConstructor; import org.apache.beam.sdk.coders.StringUtf8Coder; @@ -23,43 +23,46 @@ import org.apache.beam.sdk.io.TextIO; import org.apache.beam.sdk.io.WriteFilesResult; import org.apache.beam.sdk.transforms.Contextful; -import org.apache.beam.sdk.transforms.SerializableFunction; +import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.windowing.AfterWatermark; import org.apache.beam.sdk.transforms.windowing.FixedWindows; import org.apache.beam.sdk.transforms.windowing.Window; +import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PCollection.IsBounded; import org.apache.beam.sdk.values.PDone; import org.joda.time.Duration; -import feast.ingestion.transform.FeatureIO; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -public class TextFileFeatureIO { - private TextFileFeatureIO() {} +public class TextFileDynamicIO { + + private TextFileDynamicIO() { + } @AllArgsConstructor - public static class Write extends FeatureIO.Write { + public static class Write extends PTransform>, PDone> { + private final FileStoreOptions options; - private final SerializableFunction toTextFunction; private final String suffix; - /** Writes to different file sinks based on a */ + /** + * Writes to different file sinks based on a + */ @Override - public PDone expand(PCollection input) { + public PDone expand(PCollection> input) { final String folderName = options.jobName != null ? options.jobName : "unknown-jobs"; - FileIO.Write write = - FileIO.writeDynamic() - .by((rowExtended) -> rowExtended.getRow().getEntityName()) + FileIO.Write> write = + FileIO.>writeDynamic() + .by(KV::getKey) .withDestinationCoder(StringUtf8Coder.of()) .withNaming( Contextful.fn( - (entityName) -> FileIO.Write.defaultNaming(folderName + "/" + entityName, suffix))) - .via(Contextful.fn(toTextFunction), Contextful.fn((entityName) -> TextIO.sink())) + (key) -> FileIO.Write.defaultNaming(folderName + "/" + key + "/part-", suffix))) + .via(Contextful.fn(KV::getValue), Contextful.fn((entityName) -> TextIO.sink())) .to(options.path); if (input.isBounded().equals(IsBounded.UNBOUNDED)) { - Window minuteWindow = - Window.into(FixedWindows.of(options.getWindowDuration())) + Window> minuteWindow = + Window.>into(FixedWindows.of(options.getWindowDuration())) .triggering(AfterWatermark.pastEndOfWindow()) .discardingFiredPanes() .withAllowedLateness(Duration.ZERO); diff --git a/ingestion/src/main/java/feast/storage/FeatureErrorsStoreFactory.java b/ingestion/src/main/java/feast/store/errors/FeatureErrorsFactory.java similarity index 81% rename from ingestion/src/main/java/feast/storage/FeatureErrorsStoreFactory.java rename to ingestion/src/main/java/feast/store/errors/FeatureErrorsFactory.java index 593b4b02e5..6047e4c088 100644 --- a/ingestion/src/main/java/feast/storage/FeatureErrorsStoreFactory.java +++ b/ingestion/src/main/java/feast/store/errors/FeatureErrorsFactory.java @@ -15,6 +15,8 @@ * */ -package feast.storage; +package feast.store.errors; -public interface FeatureErrorsStoreFactory extends FeatureStoreFactory {} +import feast.store.FeatureStoreFactory; + +public interface FeatureErrorsFactory extends FeatureStoreFactory {} diff --git a/ingestion/src/main/java/feast/storage/service/FeatureWarehouseStoreFactoryService.java b/ingestion/src/main/java/feast/store/errors/FeatureErrorsFactoryService.java similarity index 55% rename from ingestion/src/main/java/feast/storage/service/FeatureWarehouseStoreFactoryService.java rename to ingestion/src/main/java/feast/store/errors/FeatureErrorsFactoryService.java index 9f41ba558d..4f5ad2c50a 100644 --- a/ingestion/src/main/java/feast/storage/service/FeatureWarehouseStoreFactoryService.java +++ b/ingestion/src/main/java/feast/store/errors/FeatureErrorsFactoryService.java @@ -15,39 +15,41 @@ * */ -package feast.storage.service; +package feast.store.errors; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import feast.storage.FeatureWarehouseStoreFactory; import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; import lombok.extern.slf4j.Slf4j; /** - * Service class for fetching all the FeatureWarehouseStoreFactory instances available + * Service class for fetching all the FeatureErrorsFactory instances available */ @Slf4j -public class FeatureWarehouseStoreFactoryService { - private static ServiceLoader serviceLoader = - ServiceLoader.load(FeatureWarehouseStoreFactory.class); - private static List manuallyRegistered = new ArrayList<>(); +public class FeatureErrorsFactoryService { + + private static ServiceLoader serviceLoader = ServiceLoader + .load(FeatureErrorsFactory.class); + private static List manuallyRegistered = new ArrayList<>(); static { - for (FeatureWarehouseStoreFactory store : getAll()) { - log.info("FeatureWarehouseStoreFactory type found: " + store.getType()); + for (FeatureErrorsFactory store : getAll()) { + log.info("FeatureErrorsFactory type found: " + store.getType()); } -} + } - public static List getAll() { + public static List getAll() { return Lists.newArrayList( Iterators.concat(manuallyRegistered.iterator(), serviceLoader.iterator())); } - /** Get store of the given subclass. */ - public static T get(Class clazz) { - for (FeatureWarehouseStoreFactory store : getAll()) { + /** + * Get store of the given subclass. + */ + public static T get(Class clazz) { + for (FeatureErrorsFactory store : getAll()) { if (clazz.isInstance(store)) { //noinspection unchecked return (T) store; @@ -56,7 +58,7 @@ public static T get(Class clazz) { return null; } - public static void register(FeatureWarehouseStoreFactory store) { + public static void register(FeatureErrorsFactory store) { manuallyRegistered.add(store); } - } +} diff --git a/ingestion/src/main/java/feast/storage/noop/NoOpServingStoreFactory.java b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java similarity index 54% rename from ingestion/src/main/java/feast/storage/noop/NoOpServingStoreFactory.java rename to ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java index 7aadb7f961..15d56bcae6 100644 --- a/ingestion/src/main/java/feast/storage/noop/NoOpServingStoreFactory.java +++ b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,24 +15,34 @@ * */ -package feast.storage.noop; +package feast.store.errors.json; import com.google.auto.service.AutoService; import feast.ingestion.model.Specs; import feast.ingestion.transform.FeatureIO.Write; +import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureServingStoreFactory; +import feast.store.FileStoreOptions; +import feast.store.errors.FeatureErrorsFactory; +import lombok.AllArgsConstructor; -@AutoService(FeatureServingStoreFactory.class) -public class NoOpServingStoreFactory implements FeatureServingStoreFactory { +@AutoService(FeatureErrorsFactory.class) +@AllArgsConstructor + +public class JsonFileErrorsFactory implements FeatureErrorsFactory { + + private static final String JSON_FILES_TYPE = "file.json"; @Override public Write create(StorageSpec storageSpec, Specs specs) { - return new NoOpIO.Write(); + FileStoreOptions options = + OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); + options.jobName = specs.getJobName(); + return new JsonFileErrorsWrite(options); } @Override public String getType() { - return "noop"; + return JSON_FILES_TYPE; } } diff --git a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java new file mode 100644 index 0000000000..ec9aa49388 --- /dev/null +++ b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.store.errors.json; + +import static org.apache.beam.sdk.values.TypeDescriptors.kvs; +import static org.apache.beam.sdk.values.TypeDescriptors.strings; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import feast.ingestion.transform.FeatureIO; +import feast.store.FileStoreOptions; +import feast.store.TextFileDynamicIO; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import lombok.AllArgsConstructor; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PDone; + +@AllArgsConstructor +public class JsonFileErrorsWrite extends FeatureIO.Write { + + private FileStoreOptions options; + + @Override + public PDone expand(PCollection input) { + return input.apply("Map to strings", MapElements.into(kvs(strings(), strings())).via( + (rowExtended) -> { + try { + return KV.of( + rowExtended.getRow().getEntityName(), + JsonFormat.printer().omittingInsignificantWhitespace() + .print(rowExtended) + ); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } + )).apply("Write Error Json Files", new TextFileDynamicIO.Write(options, ".json")); + } +} diff --git a/ingestion/src/main/java/feast/storage/stderr/LogIO.java b/ingestion/src/main/java/feast/store/errors/logging/LogIO.java similarity index 91% rename from ingestion/src/main/java/feast/storage/stderr/LogIO.java rename to ingestion/src/main/java/feast/store/errors/logging/LogIO.java index abd6f7c9ff..c79919b594 100644 --- a/ingestion/src/main/java/feast/storage/stderr/LogIO.java +++ b/ingestion/src/main/java/feast/store/errors/logging/LogIO.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.stderr; +package feast.store.errors.logging; import feast.ingestion.transform.FeatureIO; import feast.ingestion.transform.fn.LoggerDoFn; diff --git a/ingestion/src/main/java/feast/storage/stderr/StderrFeatureErrorsStoreFactory.java b/ingestion/src/main/java/feast/store/errors/logging/StderrFeatureErrorsFactory.java similarity index 76% rename from ingestion/src/main/java/feast/storage/stderr/StderrFeatureErrorsStoreFactory.java rename to ingestion/src/main/java/feast/store/errors/logging/StderrFeatureErrorsFactory.java index 43edef4e00..efbe549130 100644 --- a/ingestion/src/main/java/feast/storage/stderr/StderrFeatureErrorsStoreFactory.java +++ b/ingestion/src/main/java/feast/store/errors/logging/StderrFeatureErrorsFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,20 +15,20 @@ * */ -package feast.storage.stderr; +package feast.store.errors.logging; import com.google.auto.service.AutoService; import feast.ingestion.model.Specs; import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureErrorsStoreFactory; +import feast.store.errors.FeatureErrorsFactory; import org.slf4j.event.Level; -@AutoService(FeatureErrorsStoreFactory.class) -public class StderrFeatureErrorsStoreFactory implements FeatureErrorsStoreFactory { +@AutoService(FeatureErrorsFactory.class) +public class StderrFeatureErrorsFactory implements FeatureErrorsFactory { public static final String TYPE_STDERR = "stderr"; - + @Override public Write create(StorageSpec storageSpec, Specs specs) { return new LogIO.Write(Level.ERROR); diff --git a/ingestion/src/main/java/feast/storage/stderr/StdoutFeatureErrorsStoreFactory.java b/ingestion/src/main/java/feast/store/errors/logging/StdoutFeatureErrorsFactory.java similarity index 76% rename from ingestion/src/main/java/feast/storage/stderr/StdoutFeatureErrorsStoreFactory.java rename to ingestion/src/main/java/feast/store/errors/logging/StdoutFeatureErrorsFactory.java index 5a33ab88ce..80191ec1a8 100644 --- a/ingestion/src/main/java/feast/storage/stderr/StdoutFeatureErrorsStoreFactory.java +++ b/ingestion/src/main/java/feast/store/errors/logging/StdoutFeatureErrorsFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,17 +15,17 @@ * */ -package feast.storage.stderr; +package feast.store.errors.logging; import com.google.auto.service.AutoService; import feast.ingestion.model.Specs; import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureErrorsStoreFactory; +import feast.store.errors.FeatureErrorsFactory; import org.slf4j.event.Level; -@AutoService(FeatureErrorsStoreFactory.class) -public class StdoutFeatureErrorsStoreFactory implements FeatureErrorsStoreFactory { +@AutoService(FeatureErrorsFactory.class) +public class StdoutFeatureErrorsFactory implements FeatureErrorsFactory { public static final String TYPE_STDOUT = "stdout"; diff --git a/ingestion/src/main/java/feast/storage/FeatureServingStoreFactory.java b/ingestion/src/main/java/feast/store/serving/FeatureServingFactory.java similarity index 81% rename from ingestion/src/main/java/feast/storage/FeatureServingStoreFactory.java rename to ingestion/src/main/java/feast/store/serving/FeatureServingFactory.java index 178e02bf0b..ca9491973e 100644 --- a/ingestion/src/main/java/feast/storage/FeatureServingStoreFactory.java +++ b/ingestion/src/main/java/feast/store/serving/FeatureServingFactory.java @@ -15,6 +15,8 @@ * */ -package feast.storage; +package feast.store.serving; -public interface FeatureServingStoreFactory extends FeatureStoreFactory {} +import feast.store.FeatureStoreFactory; + +public interface FeatureServingFactory extends FeatureStoreFactory {} diff --git a/ingestion/src/main/java/feast/storage/service/FeatureServingStoreFactoryService.java b/ingestion/src/main/java/feast/store/serving/FeatureServingFactoryService.java similarity index 57% rename from ingestion/src/main/java/feast/storage/service/FeatureServingStoreFactoryService.java rename to ingestion/src/main/java/feast/store/serving/FeatureServingFactoryService.java index 0028ff9dda..d720c0dca1 100644 --- a/ingestion/src/main/java/feast/storage/service/FeatureServingStoreFactoryService.java +++ b/ingestion/src/main/java/feast/store/serving/FeatureServingFactoryService.java @@ -15,38 +15,41 @@ * */ -package feast.storage.service; +package feast.store.serving; import avro.shaded.com.google.common.collect.Lists; import com.google.common.collect.Iterators; -import feast.storage.FeatureServingStoreFactory; import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; import lombok.extern.slf4j.Slf4j; /** - * Service class for fetching all the FeatureServingStoreFactory instances available + * Service class for fetching all the FeatureServingFactory instances available */ @Slf4j -public class FeatureServingStoreFactoryService { - private static ServiceLoader serviceLoader = ServiceLoader.load(FeatureServingStoreFactory.class); - private static List manuallyRegistered = new ArrayList<>(); +public class FeatureServingFactoryService { + + private static ServiceLoader serviceLoader = ServiceLoader + .load(FeatureServingFactory.class); + private static List manuallyRegistered = new ArrayList<>(); static { - for (FeatureServingStoreFactory store : getAll()) { - log.info("FeatureServingStoreFactory type found: " + store.getType()); + for (FeatureServingFactory store : getAll()) { + log.info("FeatureServingFactory type found: " + store.getType()); } } - public static List getAll() { + public static List getAll() { return Lists.newArrayList( Iterators.concat(manuallyRegistered.iterator(), serviceLoader.iterator())); } - /** Get store of the given subclass. */ - public static T get(Class clazz) { - for (FeatureServingStoreFactory store : getAll()) { + /** + * Get store of the given subclass. + */ + public static T get(Class clazz) { + for (FeatureServingFactory store : getAll()) { if (clazz.isInstance(store)) { //noinspection unchecked return (T) store; @@ -55,7 +58,7 @@ public static T get(Class clazz) { return null; } - public static void register(FeatureServingStoreFactory store) { + public static void register(FeatureServingFactory store) { manuallyRegistered.add(store); } } diff --git a/ingestion/src/main/java/feast/storage/FeatureServingStoreClient.java b/ingestion/src/main/java/feast/store/serving/FeatureServingStoreClient.java similarity index 97% rename from ingestion/src/main/java/feast/storage/FeatureServingStoreClient.java rename to ingestion/src/main/java/feast/store/serving/FeatureServingStoreClient.java index af62be1fec..5aa7251b89 100644 --- a/ingestion/src/main/java/feast/storage/FeatureServingStoreClient.java +++ b/ingestion/src/main/java/feast/store/serving/FeatureServingStoreClient.java @@ -15,7 +15,7 @@ * */ -package feast.storage; +package feast.store.serving; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.types.FeatureRowProto.FeatureRow; diff --git a/ingestion/src/main/java/feast/storage/bigtable/BigTableFeatureOptions.java b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableFeatureOptions.java similarity index 95% rename from ingestion/src/main/java/feast/storage/bigtable/BigTableFeatureOptions.java rename to ingestion/src/main/java/feast/store/serving/bigtable/BigTableFeatureOptions.java index 6997ae4397..68bb7e6ec5 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/BigTableFeatureOptions.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableFeatureOptions.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import feast.options.Options; diff --git a/ingestion/src/main/java/feast/storage/bigtable/BigTableServingStoreFactory.java b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableServingStoreFactory.java similarity index 90% rename from ingestion/src/main/java/feast/storage/bigtable/BigTableServingStoreFactory.java rename to ingestion/src/main/java/feast/store/serving/bigtable/BigTableServingStoreFactory.java index 842c5be5cf..d5e8b209fa 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/BigTableServingStoreFactory.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableServingStoreFactory.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; @@ -23,10 +23,11 @@ import feast.ingestion.transform.FeatureIO.Write; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureServingStoreFactory; +import feast.store.serving.FeatureServingFactory; + +@AutoService(FeatureServingFactory.class) +public class BigTableServingStoreFactory implements FeatureServingFactory { -@AutoService(FeatureServingStoreFactory.class) -public class BigTableServingStoreFactory implements FeatureServingStoreFactory { public static String TYPE_BIGTABLE = "bigtable"; @Override diff --git a/ingestion/src/main/java/feast/storage/bigtable/BigTableStoreOptions.java b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableStoreOptions.java similarity index 95% rename from ingestion/src/main/java/feast/storage/bigtable/BigTableStoreOptions.java rename to ingestion/src/main/java/feast/store/serving/bigtable/BigTableStoreOptions.java index fcfcace69b..9e1bbfb8b4 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/BigTableStoreOptions.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableStoreOptions.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import java.io.Serializable; import javax.validation.constraints.NotEmpty; diff --git a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowBigTableIO.java b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowBigTableIO.java similarity index 98% rename from ingestion/src/main/java/feast/storage/bigtable/FeatureRowBigTableIO.java rename to ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowBigTableIO.java index 7437f59c55..9174c2d3f4 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowBigTableIO.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowBigTableIO.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import com.google.cloud.bigtable.beam.CloudBigtableConfiguration; import com.google.cloud.bigtable.beam.CloudBigtableIO; diff --git a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowToBigTableMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowToBigTableMutationDoFn.java similarity index 99% rename from ingestion/src/main/java/feast/storage/bigtable/FeatureRowToBigTableMutationDoFn.java rename to ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowToBigTableMutationDoFn.java index bfd44ceda8..1d14b1201e 100644 --- a/ingestion/src/main/java/feast/storage/bigtable/FeatureRowToBigTableMutationDoFn.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowToBigTableMutationDoFn.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import com.google.common.base.Charsets; import feast.SerializableCache; diff --git a/ingestion/src/main/java/feast/storage/redis/FeatureRowRedisIO.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowRedisIO.java similarity index 94% rename from ingestion/src/main/java/feast/storage/redis/FeatureRowRedisIO.java rename to ingestion/src/main/java/feast/store/serving/redis/FeatureRowRedisIO.java index d58abdc0f5..bdbbea5d96 100644 --- a/ingestion/src/main/java/feast/storage/redis/FeatureRowRedisIO.java +++ b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowRedisIO.java @@ -15,11 +15,11 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import feast.ingestion.model.Specs; import feast.ingestion.transform.FeatureIO; -import feast.storage.redis.RedisCustomIO.RedisMutation; +import feast.store.serving.redis.RedisCustomIO.RedisMutation; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.values.PCollection; @@ -28,6 +28,7 @@ public class FeatureRowRedisIO { public static class Write extends FeatureIO.Write { + private final RedisStoreOptions options; private final Specs specs; diff --git a/ingestion/src/main/java/feast/storage/redis/FeatureRowToRedisMutationDoFn.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java similarity index 96% rename from ingestion/src/main/java/feast/storage/redis/FeatureRowToRedisMutationDoFn.java rename to ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java index dad225d7ba..dbe57bff9f 100644 --- a/ingestion/src/main/java/feast/storage/redis/FeatureRowToRedisMutationDoFn.java +++ b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFn.java @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import static com.google.common.base.Preconditions.checkArgument; @@ -26,8 +26,8 @@ import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.RedisProto.RedisBucketKey; import feast.storage.RedisProto.RedisBucketValue; -import feast.storage.redis.RedisCustomIO.Method; -import feast.storage.redis.RedisCustomIO.RedisMutation; +import feast.store.serving.redis.RedisCustomIO.Method; +import feast.store.serving.redis.RedisCustomIO.RedisMutation; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; diff --git a/ingestion/src/main/java/feast/storage/redis/RedisCustomIO.java b/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java similarity index 99% rename from ingestion/src/main/java/feast/storage/redis/RedisCustomIO.java rename to ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java index d9b8261093..4dc53467c7 100644 --- a/ingestion/src/main/java/feast/storage/redis/RedisCustomIO.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisCustomIO.java @@ -16,7 +16,7 @@ */ // package io.suryawirawan.henry.beam.redis.io; -package feast.storage.redis; +package feast.store.serving.redis; import lombok.Builder; import lombok.Data; diff --git a/ingestion/src/main/java/feast/storage/redis/RedisFeatureOptions.java b/ingestion/src/main/java/feast/store/serving/redis/RedisFeatureOptions.java similarity index 97% rename from ingestion/src/main/java/feast/storage/redis/RedisFeatureOptions.java rename to ingestion/src/main/java/feast/store/serving/redis/RedisFeatureOptions.java index e4654e2a43..cf868cc47b 100644 --- a/ingestion/src/main/java/feast/storage/redis/RedisFeatureOptions.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisFeatureOptions.java @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import com.fasterxml.jackson.annotation.JsonIgnore; import org.joda.time.Duration; diff --git a/ingestion/src/main/java/feast/storage/redis/RedisServingStoreFactory.java b/ingestion/src/main/java/feast/store/serving/redis/RedisServingFactory.java similarity index 86% rename from ingestion/src/main/java/feast/storage/redis/RedisServingStoreFactory.java rename to ingestion/src/main/java/feast/store/serving/redis/RedisServingFactory.java index 6d16e062ba..e044589fbc 100644 --- a/ingestion/src/main/java/feast/storage/redis/RedisServingStoreFactory.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisServingFactory.java @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; @@ -23,10 +23,11 @@ import feast.ingestion.transform.FeatureIO.Write; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureServingStoreFactory; +import feast.store.serving.FeatureServingFactory; + +@AutoService(FeatureServingFactory.class) +public class RedisServingFactory implements FeatureServingFactory { -@AutoService(FeatureServingStoreFactory.class) -public class RedisServingStoreFactory implements FeatureServingStoreFactory { public static final String TYPE_REDIS = "redis"; @Override diff --git a/ingestion/src/main/java/feast/storage/redis/RedisStoreOptions.java b/ingestion/src/main/java/feast/store/serving/redis/RedisStoreOptions.java similarity index 97% rename from ingestion/src/main/java/feast/storage/redis/RedisStoreOptions.java rename to ingestion/src/main/java/feast/store/serving/redis/RedisStoreOptions.java index eecffac5b2..9b257d40f0 100644 --- a/ingestion/src/main/java/feast/storage/redis/RedisStoreOptions.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisStoreOptions.java @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import feast.options.Options; import javax.validation.constraints.NotEmpty; diff --git a/ingestion/src/main/java/feast/storage/FeatureWarehouseStoreFactory.java b/ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactory.java similarity index 80% rename from ingestion/src/main/java/feast/storage/FeatureWarehouseStoreFactory.java rename to ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactory.java index d88350bcae..72b5617962 100644 --- a/ingestion/src/main/java/feast/storage/FeatureWarehouseStoreFactory.java +++ b/ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactory.java @@ -15,6 +15,10 @@ * */ -package feast.storage; +package feast.store.warehouse; -public interface FeatureWarehouseStoreFactory extends FeatureStoreFactory {} +import feast.store.FeatureStoreFactory; + +public interface FeatureWarehouseFactory extends FeatureStoreFactory { + +} diff --git a/ingestion/src/main/java/feast/storage/service/FeatureErrorsStoreFactoryService.java b/ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactoryService.java similarity index 59% rename from ingestion/src/main/java/feast/storage/service/FeatureErrorsStoreFactoryService.java rename to ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactoryService.java index bed7ed9631..06ae723613 100644 --- a/ingestion/src/main/java/feast/storage/service/FeatureErrorsStoreFactoryService.java +++ b/ingestion/src/main/java/feast/store/warehouse/FeatureWarehouseFactoryService.java @@ -15,40 +15,38 @@ * */ -package feast.storage.service; +package feast.store.warehouse; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import feast.storage.FeatureErrorsStoreFactory; -import lombok.extern.slf4j.Slf4j; - import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; +import lombok.extern.slf4j.Slf4j; /** - * Service class for fetching all the FeatureErrorsStoreFactory instances available + * Service class for fetching all the FeatureWarehouseFactory instances available */ @Slf4j -public class FeatureErrorsStoreFactoryService { - - private static ServiceLoader serviceLoader = ServiceLoader.load(FeatureErrorsStoreFactory.class); - private static List manuallyRegistered = new ArrayList<>(); +public class FeatureWarehouseFactoryService { + private static ServiceLoader serviceLoader = + ServiceLoader.load(FeatureWarehouseFactory.class); + private static List manuallyRegistered = new ArrayList<>(); static { - for (FeatureErrorsStoreFactory store : getAll()) { - log.info("FeatureErrorsStoreFactory type found: " + store.getType()); + for (FeatureWarehouseFactory store : getAll()) { + log.info("FeatureWarehouseFactory type found: " + store.getType()); } - } +} - public static List getAll() { + public static List getAll() { return Lists.newArrayList( Iterators.concat(manuallyRegistered.iterator(), serviceLoader.iterator())); } /** Get store of the given subclass. */ - public static T get(Class clazz) { - for (FeatureErrorsStoreFactory store : getAll()) { + public static T get(Class clazz) { + for (FeatureWarehouseFactory store : getAll()) { if (clazz.isInstance(store)) { //noinspection unchecked return (T) store; @@ -57,7 +55,7 @@ public static T get(Class clazz) { return null; } - public static void register(FeatureErrorsStoreFactory store) { + public static void register(FeatureWarehouseFactory store) { manuallyRegistered.add(store); } -} + } diff --git a/ingestion/src/main/java/feast/storage/bigquery/BigQueryStoreOptions.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryStoreOptions.java similarity index 93% rename from ingestion/src/main/java/feast/storage/bigquery/BigQueryStoreOptions.java rename to ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryStoreOptions.java index 95c70be24b..f82efb58e8 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/BigQueryStoreOptions.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryStoreOptions.java @@ -15,9 +15,8 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; -import java.io.Serializable; import javax.validation.constraints.NotEmpty; import feast.options.Options; diff --git a/ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStoreFactory.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryWarehouseFactory.java similarity index 86% rename from ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStoreFactory.java rename to ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryWarehouseFactory.java index 9d4f5285fb..4b4b2139d0 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/BigQueryWarehouseStoreFactory.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryWarehouseFactory.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; @@ -23,10 +23,10 @@ import feast.ingestion.transform.FeatureIO.Write; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureWarehouseStoreFactory; +import feast.store.warehouse.FeatureWarehouseFactory; -@AutoService(FeatureWarehouseStoreFactory.class) -public class BigQueryWarehouseStoreFactory implements FeatureWarehouseStoreFactory { +@AutoService(FeatureWarehouseFactory.class) +public class BigQueryWarehouseFactory implements FeatureWarehouseFactory { public static String TYPE_BIGQUERY = "bigquery"; @Override diff --git a/ingestion/src/main/java/feast/storage/bigquery/FeatureRowBigQueryIO.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java similarity index 99% rename from ingestion/src/main/java/feast/storage/bigquery/FeatureRowBigQueryIO.java rename to ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java index b5e039cc9e..cf0cc5cd4e 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/FeatureRowBigQueryIO.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import com.google.api.services.bigquery.model.TableSchema; import com.google.common.base.Strings; diff --git a/ingestion/src/main/java/feast/storage/bigquery/FeatureRowToBigQueryTableRowDoFn.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowToBigQueryTableRowDoFn.java similarity index 96% rename from ingestion/src/main/java/feast/storage/bigquery/FeatureRowToBigQueryTableRowDoFn.java rename to ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowToBigQueryTableRowDoFn.java index a05569f61d..c666d4d71a 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/FeatureRowToBigQueryTableRowDoFn.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowToBigQueryTableRowDoFn.java @@ -15,13 +15,11 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import com.google.api.services.bigquery.model.TableRow; import feast.ingestion.util.DateUtil; import feast.ingestion.model.Specs; -import feast.ingestion.model.Specs; -import feast.ingestion.util.DateUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.types.FeatureProto.Feature; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; diff --git a/ingestion/src/main/java/feast/storage/bigquery/ValueBigQueryBuilder.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/ValueBigQueryBuilder.java similarity index 99% rename from ingestion/src/main/java/feast/storage/bigquery/ValueBigQueryBuilder.java rename to ingestion/src/main/java/feast/store/warehouse/bigquery/ValueBigQueryBuilder.java index ab292e3e0d..34700dc96b 100644 --- a/ingestion/src/main/java/feast/storage/bigquery/ValueBigQueryBuilder.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/ValueBigQueryBuilder.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.protobuf.ByteString; diff --git a/ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStoreFactory.java b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java similarity index 56% rename from ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStoreFactory.java rename to ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java index dbee993767..a4fa617a82 100644 --- a/ingestion/src/main/java/feast/storage/noop/NoOpWarehouseStoreFactory.java +++ b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,24 +15,33 @@ * */ -package feast.storage.noop; +package feast.store.warehouse.json; import com.google.auto.service.AutoService; import feast.ingestion.model.Specs; import feast.ingestion.transform.FeatureIO.Write; +import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureWarehouseStoreFactory; +import feast.store.FileStoreOptions; +import feast.store.warehouse.FeatureWarehouseFactory; +import lombok.AllArgsConstructor; -@AutoService(FeatureWarehouseStoreFactory.class) -public class NoOpWarehouseStoreFactory implements FeatureWarehouseStoreFactory { +@AutoService(FeatureWarehouseFactory.class) +@AllArgsConstructor +public class JsonFileWarehouseFactory implements FeatureWarehouseFactory { + + private static final String JSON_FILES_TYPE = "file.json"; @Override public Write create(StorageSpec storageSpec, Specs specs) { - return new NoOpIO.Write(); + FileStoreOptions options = + OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); + options.jobName = specs.getJobName(); + return new JsonFileWarehouseWrite(options); } @Override public String getType() { - return "noop"; + return JSON_FILES_TYPE; } } diff --git a/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java new file mode 100644 index 0000000000..bc3c9c3712 --- /dev/null +++ b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java @@ -0,0 +1,56 @@ +/* + * Copyright 2019 The Feast Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package feast.store.warehouse.json; + +import static org.apache.beam.sdk.values.TypeDescriptors.kvs; +import static org.apache.beam.sdk.values.TypeDescriptors.strings; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.util.JsonFormat; +import feast.ingestion.transform.FeatureIO; +import feast.store.FileStoreOptions; +import feast.store.TextFileDynamicIO; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import lombok.AllArgsConstructor; +import org.apache.beam.sdk.transforms.MapElements; +import org.apache.beam.sdk.values.KV; +import org.apache.beam.sdk.values.PCollection; +import org.apache.beam.sdk.values.PDone; + +@AllArgsConstructor +public class JsonFileWarehouseWrite extends FeatureIO.Write { + + private FileStoreOptions options; + + @Override + public PDone expand(PCollection input) { + return input.apply("Map to strings", MapElements.into(kvs(strings(), strings())).via( + (rowExtended) -> { + try { + return KV.of( + rowExtended.getRow().getEntityName(), + JsonFormat.printer().omittingInsignificantWhitespace() + .print(rowExtended.getRow()) + ); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } + )).apply("Write Warehouse Json Files", new TextFileDynamicIO.Write(options, ".json")); + } +} diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java index 1f39c81563..6379e8dfad 100644 --- a/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java +++ b/ingestion/src/test/java/feast/ingestion/ImportJobCSVTest.java @@ -19,7 +19,7 @@ import static feast.FeastMatchers.hasCount; import static feast.NormalizeFeatureRows.normalize; -import static feast.storage.MockFeatureErrorsStoreFactory.MOCK_ERRORS_STORE_TYPE; +import static feast.store.MockFeatureErrorsFactory.MOCK_ERRORS_STORE_TYPE; import static org.junit.Assert.assertEquals; import com.google.common.base.Charsets; @@ -38,12 +38,12 @@ import feast.ingestion.service.SpecRetrievalException; import feast.ingestion.util.ProtoUtil; import feast.specs.ImportSpecProto.ImportSpec; -import feast.storage.MockFeatureErrorsStoreFactory; -import feast.storage.MockServingStoreFactory; -import feast.storage.MockWarehouseStoreFactory; -import feast.storage.service.FeatureErrorsStoreFactoryService; -import feast.storage.service.FeatureServingStoreFactoryService; -import feast.storage.service.FeatureWarehouseStoreFactoryService; +import feast.store.MockFeatureErrorsFactory; +import feast.store.MockServingFactory; +import feast.store.MockWarehouseFactory; +import feast.store.errors.FeatureErrorsFactoryService; +import feast.store.serving.FeatureServingFactoryService; +import feast.store.warehouse.FeatureWarehouseFactoryService; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; import feast.types.GranularityProto.Granularity; @@ -121,16 +121,21 @@ public void testImportCSV() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(FeatureServingStoreFactoryService.get(MockServingStoreFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = PCollectionList.of( - FeatureWarehouseStoreFactoryService.get(MockWarehouseStoreFactory.class).getWrite().getInputs()) + FeatureWarehouseFactoryService.get(MockWarehouseFactory.class).getWrite() + .getInputs()) .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(FeatureErrorsStoreFactoryService.get(MockFeatureErrorsStoreFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); List expectedRows = @@ -212,16 +217,21 @@ public void testImportCSV_withSample1() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(FeatureServingStoreFactoryService.get(MockServingStoreFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = PCollectionList.of( - FeatureWarehouseStoreFactoryService.get(MockWarehouseStoreFactory.class).getWrite().getInputs()) + FeatureWarehouseFactoryService.get(MockWarehouseFactory.class).getWrite() + .getInputs()) .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(FeatureErrorsStoreFactoryService.get(MockFeatureErrorsStoreFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToServing).satisfies(hasCount(1)); @@ -271,16 +281,21 @@ public void testImportCSV_withCoalesceRows() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(FeatureServingStoreFactoryService.get(MockServingStoreFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToWarehouse = PCollectionList.of( - FeatureWarehouseStoreFactoryService.get(MockWarehouseStoreFactory.class).getWrite().getInputs()) + FeatureWarehouseFactoryService.get(MockWarehouseFactory.class).getWrite() + .getInputs()) .apply("flatten warehouse input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(FeatureErrorsStoreFactoryService.get(MockFeatureErrorsStoreFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToErrors).satisfies(hasCount(0)); @@ -394,11 +409,15 @@ public void testImportWithErrors() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(FeatureServingStoreFactoryService.get(MockServingStoreFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(FeatureErrorsStoreFactoryService.get(MockFeatureErrorsStoreFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToErrors) @@ -459,11 +478,15 @@ public void testImportWithoutWarehouseStore() throws IOException { job.expand(); PCollection writtenToServing = - PCollectionList.of(FeatureServingStoreFactoryService.get(MockServingStoreFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureServingFactoryService.get(MockServingFactory.class).getWrite() + .getInputs()) .apply("flatten serving input", Flatten.pCollections()); PCollection writtenToErrors = - PCollectionList.of(FeatureErrorsStoreFactoryService.get(MockFeatureErrorsStoreFactory.class).getWrite().getInputs()) + PCollectionList + .of(FeatureErrorsFactoryService.get(MockFeatureErrorsFactory.class).getWrite() + .getInputs()) .apply("flatten errors input", Flatten.pCollections()); PAssert.that(writtenToErrors) diff --git a/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java b/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java index e5abaf28fa..bddb79515b 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/ErrorsStoreTransformTest.java @@ -18,14 +18,14 @@ package feast.ingestion.transform; import static feast.ingestion.model.Errors.toError; -import static feast.storage.MockFeatureErrorsStoreFactory.MOCK_ERRORS_STORE_TYPE; +import static feast.store.MockFeatureErrorsFactory.MOCK_ERRORS_STORE_TYPE; import feast.ingestion.model.Specs; import feast.ingestion.options.ImportJobPipelineOptions; -import feast.storage.MockFeatureErrorsStoreFactory; -import feast.storage.service.FeatureErrorsStoreFactoryService; -import feast.storage.stderr.StderrFeatureErrorsStoreFactory; -import feast.storage.stderr.StdoutFeatureErrorsStoreFactory; +import feast.store.MockFeatureErrorsFactory; +import feast.store.errors.FeatureErrorsFactoryService; +import feast.store.errors.logging.StderrFeatureErrorsFactory; +import feast.store.errors.logging.StdoutFeatureErrorsFactory; import feast.types.FeatureRowExtendedProto.Attempt; import feast.types.FeatureRowExtendedProto.Error; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; @@ -75,7 +75,7 @@ private FeatureRowExtended errorOf(String transform, Throwable cause) { @Test public void shouldWriteToGivenErrorsStore() { - MockFeatureErrorsStoreFactory mockStore = new MockFeatureErrorsStoreFactory(); + MockFeatureErrorsFactory mockStore = new MockFeatureErrorsFactory(); options.setErrorsStoreType(MOCK_ERRORS_STORE_TYPE); ErrorsStoreTransform transform = new ErrorsStoreTransform(options, specs, Lists.newArrayList(mockStore)); @@ -91,9 +91,9 @@ public void shouldWriteToGivenErrorsStore() { @Test public void logErrorsToStdErr() { - options.setErrorsStoreType(StderrFeatureErrorsStoreFactory.TYPE_STDERR); + options.setErrorsStoreType(StderrFeatureErrorsFactory.TYPE_STDERR); ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, FeatureErrorsStoreFactoryService.getAll()); + new ErrorsStoreTransform(options, specs, FeatureErrorsFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } @@ -101,9 +101,9 @@ public void logErrorsToStdErr() { @Test public void logErrorsToStdOut() { - options.setErrorsStoreType(StdoutFeatureErrorsStoreFactory.TYPE_STDOUT); + options.setErrorsStoreType(StdoutFeatureErrorsFactory.TYPE_STDOUT); ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, FeatureErrorsStoreFactoryService.getAll()); + new ErrorsStoreTransform(options, specs, FeatureErrorsFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } @@ -112,7 +112,7 @@ public void logErrorsToStdOut() { public void logToNull() { //options.setErrorsStoreType(...); // no errors store type set ErrorsStoreTransform transform = - new ErrorsStoreTransform(options, specs, FeatureErrorsStoreFactoryService.getAll()); + new ErrorsStoreTransform(options, specs, FeatureErrorsFactoryService.getAll()); inputs.apply(transform); pipeline.run(); } diff --git a/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java b/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java index ceb9ca642b..bca7f399ee 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java @@ -37,9 +37,9 @@ import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import feast.specs.StorageSpecProto.StorageSpec; -import feast.storage.FeatureStoreFactory; -import feast.storage.MockFeatureStore; -import feast.storage.MockTransforms; +import feast.store.FeatureStoreFactory; +import feast.store.MockFeatureStore; +import feast.store.MockTransforms; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import feast.types.FeatureRowProto.FeatureRow; import java.util.Collections; diff --git a/ingestion/src/test/java/feast/storage/file/FileStoreOptionsTest.java b/ingestion/src/test/java/feast/store/FileStoreOptionsTest.java similarity index 93% rename from ingestion/src/test/java/feast/storage/file/FileStoreOptionsTest.java rename to ingestion/src/test/java/feast/store/FileStoreOptionsTest.java index 4a2be27a70..0d8e06182c 100644 --- a/ingestion/src/test/java/feast/storage/file/FileStoreOptionsTest.java +++ b/ingestion/src/test/java/feast/store/FileStoreOptionsTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,10 +15,11 @@ * */ -package feast.storage.file; +package feast.store; import static org.junit.Assert.assertEquals; +import feast.store.FileStoreOptions; import org.joda.time.Duration; import org.junit.Test; import feast.options.OptionsParser; diff --git a/ingestion/src/test/java/feast/storage/MockFeatureErrorsStoreFactory.java b/ingestion/src/test/java/feast/store/MockFeatureErrorsFactory.java similarity index 67% rename from ingestion/src/test/java/feast/storage/MockFeatureErrorsStoreFactory.java rename to ingestion/src/test/java/feast/store/MockFeatureErrorsFactory.java index eab730d768..49721147a6 100644 --- a/ingestion/src/test/java/feast/storage/MockFeatureErrorsStoreFactory.java +++ b/ingestion/src/test/java/feast/store/MockFeatureErrorsFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,16 +15,17 @@ * */ -package feast.storage; +package feast.store; import com.google.auto.service.AutoService; +import feast.store.errors.FeatureErrorsFactory; -@AutoService(FeatureErrorsStoreFactory.class) -public class MockFeatureErrorsStoreFactory extends MockFeatureStore implements - FeatureErrorsStoreFactory { +@AutoService(FeatureErrorsFactory.class) +public class MockFeatureErrorsFactory extends MockFeatureStore implements + FeatureErrorsFactory { public static final String MOCK_ERRORS_STORE_TYPE = "errors.mock"; - public MockFeatureErrorsStoreFactory() { + public MockFeatureErrorsFactory() { super(MOCK_ERRORS_STORE_TYPE); } } diff --git a/ingestion/src/test/java/feast/storage/MockFeatureStore.java b/ingestion/src/test/java/feast/store/MockFeatureStore.java similarity index 90% rename from ingestion/src/test/java/feast/storage/MockFeatureStore.java rename to ingestion/src/test/java/feast/store/MockFeatureStore.java index f04d187ea2..cdde9e9740 100644 --- a/ingestion/src/test/java/feast/storage/MockFeatureStore.java +++ b/ingestion/src/test/java/feast/store/MockFeatureStore.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage; +package feast.store; import feast.ingestion.model.Specs; import feast.specs.StorageSpecProto.StorageSpec; diff --git a/ingestion/src/test/java/feast/storage/MockServingStoreFactory.java b/ingestion/src/test/java/feast/store/MockServingFactory.java similarity index 67% rename from ingestion/src/test/java/feast/storage/MockServingStoreFactory.java rename to ingestion/src/test/java/feast/store/MockServingFactory.java index 02ec97f382..11195c1385 100644 --- a/ingestion/src/test/java/feast/storage/MockServingStoreFactory.java +++ b/ingestion/src/test/java/feast/store/MockServingFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,16 +15,17 @@ * */ -package feast.storage; +package feast.store; import com.google.auto.service.AutoService; +import feast.store.serving.FeatureServingFactory; -@AutoService(FeatureServingStoreFactory.class) -public class MockServingStoreFactory extends MockFeatureStore implements - FeatureServingStoreFactory { +@AutoService(FeatureServingFactory.class) +public class MockServingFactory extends MockFeatureStore implements + FeatureServingFactory { public static final String MOCK_SERVING_STORE_TYPE = "serving.mock"; - public MockServingStoreFactory() { + public MockServingFactory() { super(MOCK_SERVING_STORE_TYPE); } } diff --git a/ingestion/src/test/java/feast/storage/MockTransforms.java b/ingestion/src/test/java/feast/store/MockTransforms.java similarity index 93% rename from ingestion/src/test/java/feast/storage/MockTransforms.java rename to ingestion/src/test/java/feast/store/MockTransforms.java index 1f5a573495..2e1acc43b2 100644 --- a/ingestion/src/test/java/feast/storage/MockTransforms.java +++ b/ingestion/src/test/java/feast/store/MockTransforms.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage; +package feast.store; import com.google.common.collect.Lists; import feast.ingestion.transform.FeatureIO; diff --git a/ingestion/src/test/java/feast/storage/MockWarehouseStoreFactory.java b/ingestion/src/test/java/feast/store/MockWarehouseFactory.java similarity index 67% rename from ingestion/src/test/java/feast/storage/MockWarehouseStoreFactory.java rename to ingestion/src/test/java/feast/store/MockWarehouseFactory.java index 8bebf5baa0..ddfd06556a 100644 --- a/ingestion/src/test/java/feast/storage/MockWarehouseStoreFactory.java +++ b/ingestion/src/test/java/feast/store/MockWarehouseFactory.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,16 +15,17 @@ * */ -package feast.storage; +package feast.store; import com.google.auto.service.AutoService; +import feast.store.warehouse.FeatureWarehouseFactory; -@AutoService(FeatureWarehouseStoreFactory.class) -public class MockWarehouseStoreFactory extends MockFeatureStore implements - FeatureWarehouseStoreFactory { +@AutoService(FeatureWarehouseFactory.class) +public class MockWarehouseFactory extends MockFeatureStore implements + FeatureWarehouseFactory { public static final String MOCK_WAREHOUSE_STORE_TYPE = "warehouse.mock"; - public MockWarehouseStoreFactory() { + public MockWarehouseFactory() { super(MOCK_WAREHOUSE_STORE_TYPE); } } diff --git a/ingestion/src/test/java/feast/storage/file/TextFileFeatureIOTest.java b/ingestion/src/test/java/feast/store/TextFileDynamicIOTest.java similarity index 56% rename from ingestion/src/test/java/feast/storage/file/TextFileFeatureIOTest.java rename to ingestion/src/test/java/feast/store/TextFileDynamicIOTest.java index 84b87ca671..0a52b4bc78 100644 --- a/ingestion/src/test/java/feast/storage/file/TextFileFeatureIOTest.java +++ b/ingestion/src/test/java/feast/store/TextFileDynamicIOTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,9 +15,9 @@ * */ -package feast.storage.file; +package feast.store; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertThat; import com.google.common.collect.Lists; @@ -27,43 +27,54 @@ import java.nio.file.Path; import java.util.List; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; +import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.PCollection; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import feast.types.FeatureRowProto.FeatureRow; -public class TextFileFeatureIOTest { - @Rule public TemporaryFolder folder = new TemporaryFolder(); +@Slf4j +public class TextFileDynamicIOTest { - @Rule public TestPipeline testPipeline = TestPipeline.create(); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Rule + public TestPipeline testPipeline = TestPipeline.create(); @Test public void testWrite() throws IOException { File path = folder.newFolder(); FileStoreOptions options = new FileStoreOptions(); - options.jobName = testPipeline.getOptions().getJobName(); + options.jobName = "test-job"; options.path = path.getAbsolutePath(); - TextFileFeatureIO.Write write = - new TextFileFeatureIO.Write( - options, (featureRow) -> "output: " + featureRow.getRow().getEntityName(), ".text"); + TextFileDynamicIO.Write write = + new TextFileDynamicIO.Write( + options, ".text"); - PCollection rowExtended = - testPipeline.apply( - Create.of( - FeatureRowExtended.newBuilder() - .setRow(FeatureRow.newBuilder().setEntityName("testEntity")) - .build())); + PCollection> rowExtended = + testPipeline.apply(Create.of( + KV.of("part1", "line1"), + KV.of("part1", "line2"), + KV.of("part2", "line3"), + KV.of("part2", "line4"))); rowExtended.apply(write); testPipeline.run(); - List files = Files.walk(path.toPath()).collect(Collectors.toList()); + List part1 = getAllLines(path.toPath().resolve("test-job/part1")); + List part2 = getAllLines(path.toPath().resolve("test-job/part2")); + assertThat(part1, containsInAnyOrder("line1", "line2")); + assertThat(part2, containsInAnyOrder("line3", "line4")); + } + + List getAllLines(Path path) throws IOException { + List files = Files.walk(path).collect(Collectors.toList()); List lines = Lists.newArrayList(); for (Path file : files) { System.out.println(file); @@ -71,6 +82,6 @@ public void testWrite() throws IOException { lines.addAll(Files.readAllLines(file)); } } - assertThat(lines, equalTo(Lists.newArrayList("output: testEntity"))); + return lines; } } diff --git a/ingestion/src/test/java/feast/storage/bigtable/BigTableFeatureOptionsTest.java b/ingestion/src/test/java/feast/store/serving/bigtable/BigTableFeatureOptionsTest.java similarity index 91% rename from ingestion/src/test/java/feast/storage/bigtable/BigTableFeatureOptionsTest.java rename to ingestion/src/test/java/feast/store/serving/bigtable/BigTableFeatureOptionsTest.java index 8525626583..dfd3b85339 100644 --- a/ingestion/src/test/java/feast/storage/bigtable/BigTableFeatureOptionsTest.java +++ b/ingestion/src/test/java/feast/store/serving/bigtable/BigTableFeatureOptionsTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import org.junit.Assert; import org.junit.Test; diff --git a/ingestion/src/test/java/feast/storage/bigtable/BigTableStoreOptionsTest.java b/ingestion/src/test/java/feast/store/serving/bigtable/BigTableStoreOptionsTest.java similarity index 95% rename from ingestion/src/test/java/feast/storage/bigtable/BigTableStoreOptionsTest.java rename to ingestion/src/test/java/feast/store/serving/bigtable/BigTableStoreOptionsTest.java index a777f50ffd..b9957a2b45 100644 --- a/ingestion/src/test/java/feast/storage/bigtable/BigTableStoreOptionsTest.java +++ b/ingestion/src/test/java/feast/store/serving/bigtable/BigTableStoreOptionsTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.bigtable; +package feast.store.serving.bigtable; import org.junit.Assert; import org.junit.Test; diff --git a/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java similarity index 94% rename from ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java rename to ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java index 74959ccb6c..dfc4907a78 100644 --- a/ingestion/src/test/java/feast/storage/redis/FeatureRowRedisIOWriteTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,10 +15,10 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; -import static feast.storage.redis.FeatureRowToRedisMutationDoFn.getFeatureIdSha1Prefix; -import static feast.storage.redis.FeatureRowToRedisMutationDoFn.getRedisBucketKey; +import static feast.store.serving.redis.FeatureRowToRedisMutationDoFn.getFeatureIdSha1Prefix; +import static feast.store.serving.redis.FeatureRowToRedisMutationDoFn.getRedisBucketKey; import static org.junit.Assert.assertEquals; import com.google.common.io.Resources; @@ -110,7 +110,7 @@ public void testWriteNoneGranularity() throws IOException { Specs specs = getSpecs(); specs.validate(); - new RedisServingStoreFactory().create(specs.getStorageSpec("REDIS1"), specs); + new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowRedisIO.Write write = new FeatureRowRedisIO.Write( RedisStoreOptions.builder().host("localhost").port(REDIS_PORT).build(), specs); @@ -154,7 +154,7 @@ public void testWriteNoneGranularity() throws IOException { @Test public void testWriteNoneGranularityFromOptions() throws IOException { Specs specs = getSpecs(); - FeatureIO.Write write = new RedisServingStoreFactory().create(specs.getStorageSpec("REDIS1"), specs); + FeatureIO.Write write = new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() @@ -195,7 +195,7 @@ public void testWriteNoneGranularityFromOptions() throws IOException { @Test public void testWriteHourGranularity() throws IOException { Specs specs = getSpecs(); - FeatureIO.Write write = new RedisServingStoreFactory().create(specs.getStorageSpec("REDIS1"), specs); + FeatureIO.Write write = new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() diff --git a/ingestion/src/test/java/feast/storage/redis/FeatureRowToRedisMutationDoFnTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java similarity index 93% rename from ingestion/src/test/java/feast/storage/redis/FeatureRowToRedisMutationDoFnTest.java rename to ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java index 242042fb35..64bc401426 100644 --- a/ingestion/src/test/java/feast/storage/redis/FeatureRowToRedisMutationDoFnTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowToRedisMutationDoFnTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import feast.ingestion.util.DateUtil; import feast.storage.RedisProto.RedisBucketKey; diff --git a/ingestion/src/test/java/feast/storage/redis/RedisFeatureOptionsTest.java b/ingestion/src/test/java/feast/store/serving/redis/RedisFeatureOptionsTest.java similarity index 95% rename from ingestion/src/test/java/feast/storage/redis/RedisFeatureOptionsTest.java rename to ingestion/src/test/java/feast/store/serving/redis/RedisFeatureOptionsTest.java index 2cbbffb1db..bf49aca612 100644 --- a/ingestion/src/test/java/feast/storage/redis/RedisFeatureOptionsTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/RedisFeatureOptionsTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import org.joda.time.Duration; import org.junit.Assert; diff --git a/ingestion/src/test/java/feast/storage/redis/RedisStoreOptionsTest.java b/ingestion/src/test/java/feast/store/serving/redis/RedisStoreOptionsTest.java similarity index 92% rename from ingestion/src/test/java/feast/storage/redis/RedisStoreOptionsTest.java rename to ingestion/src/test/java/feast/store/serving/redis/RedisStoreOptionsTest.java index 87b7fc98fa..335a3b1fed 100644 --- a/ingestion/src/test/java/feast/storage/redis/RedisStoreOptionsTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/RedisStoreOptionsTest.java @@ -1,11 +1,11 @@ /* - * Copyright 2018 The Feast Authors + * Copyright 2019 The Feast Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -15,7 +15,7 @@ * */ -package feast.storage.redis; +package feast.store.serving.redis; import static org.junit.Assert.assertEquals; diff --git a/ingestion/src/test/java/feast/storage/bigquery/BigQueryStoreOptionsTest.java b/ingestion/src/test/java/feast/store/warehouse/bigquery/BigQueryStoreOptionsTest.java similarity index 97% rename from ingestion/src/test/java/feast/storage/bigquery/BigQueryStoreOptionsTest.java rename to ingestion/src/test/java/feast/store/warehouse/bigquery/BigQueryStoreOptionsTest.java index 92a30cf56b..4563d3e6fb 100644 --- a/ingestion/src/test/java/feast/storage/bigquery/BigQueryStoreOptionsTest.java +++ b/ingestion/src/test/java/feast/store/warehouse/bigquery/BigQueryStoreOptionsTest.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import org.junit.Assert; import org.junit.Test; diff --git a/ingestion/src/test/java/feast/storage/bigquery/ValueBigQueryBuilderTest.java b/ingestion/src/test/java/feast/store/warehouse/bigquery/ValueBigQueryBuilderTest.java similarity index 98% rename from ingestion/src/test/java/feast/storage/bigquery/ValueBigQueryBuilderTest.java rename to ingestion/src/test/java/feast/store/warehouse/bigquery/ValueBigQueryBuilderTest.java index 4ddc2d8c45..648fa5f151 100644 --- a/ingestion/src/test/java/feast/storage/bigquery/ValueBigQueryBuilderTest.java +++ b/ingestion/src/test/java/feast/store/warehouse/bigquery/ValueBigQueryBuilderTest.java @@ -15,7 +15,7 @@ * */ -package feast.storage.bigquery; +package feast.store.warehouse.bigquery; import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; diff --git a/ingestion/src/test/java/feast/storage/file/json/JsonFileFeatureIOTest.java b/ingestion/src/test/java/feast/store/warehouse/json/JsonFileFeatureIOTest.java similarity index 59% rename from ingestion/src/test/java/feast/storage/file/json/JsonFileFeatureIOTest.java rename to ingestion/src/test/java/feast/store/warehouse/json/JsonFileFeatureIOTest.java index cdc761b4ce..97e09f9e7f 100644 --- a/ingestion/src/test/java/feast/storage/file/json/JsonFileFeatureIOTest.java +++ b/ingestion/src/test/java/feast/store/warehouse/json/JsonFileFeatureIOTest.java @@ -15,56 +15,78 @@ * */ -package feast.storage.file.json; +package feast.store.warehouse.json; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertThat; import com.google.common.collect.Lists; +import feast.store.FileStoreOptions; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; +import feast.types.FeatureRowProto.FeatureRow; import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import org.apache.beam.sdk.testing.TestPipeline; import org.apache.beam.sdk.transforms.Create; import org.apache.beam.sdk.values.PCollection; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import feast.storage.file.FileStoreOptions; -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import feast.types.FeatureRowProto.FeatureRow; +@Slf4j public class JsonFileFeatureIOTest { - @Rule public TemporaryFolder folder = new TemporaryFolder(); - @Rule public TestPipeline testPipeline = TestPipeline.create(); + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + @Rule + public TestPipeline testPipeline = TestPipeline.create(); @Test public void testWrite() throws IOException { File path = folder.newFolder(); FileStoreOptions options = new FileStoreOptions(); - options.jobName = testPipeline.getOptions().getJobName(); + options.jobName = "test-job"; options.path = path.getAbsolutePath(); - JsonFileFeatureIO.Write write = - new JsonFileFeatureIO.Write(options, FeatureRowExtended::getRow); + JsonFileWarehouseWrite write = new JsonFileWarehouseWrite(options); PCollection rowExtended = testPipeline.apply( Create.of( FeatureRowExtended.newBuilder() .setRow( - FeatureRow.newBuilder().setEntityName("testEntity").setEntityKey("1234")) + FeatureRow.newBuilder().setEntityName("testEntity1").setEntityKey("1234")) + .build(), + FeatureRowExtended.newBuilder() + .setRow( + FeatureRow.newBuilder().setEntityName("testEntity2").setEntityKey("1234")) .build())); rowExtended.apply(write); testPipeline.run(); - List files = Files.walk(path.toPath()).collect(Collectors.toList()); + for (Path p : Files.walk(path.toPath()).collect(Collectors.toList())) { + if (p.toFile().isFile()) { + log.debug(p.toString()); + } + } + List part1 = getAllLines(path.toPath().resolve("test-job/testEntity1")); + List part2 = getAllLines(path.toPath().resolve("test-job/testEntity2")); + assertThat(part1, + containsInAnyOrder("{\"entityKey\":\"1234\",\"entityName\":\"testEntity1\"}")); + assertThat(part2, + containsInAnyOrder("{\"entityKey\":\"1234\",\"entityName\":\"testEntity2\"}")); + } + + List getAllLines(Path path) throws IOException { + List files = Files.walk(path).collect(Collectors.toList()); List lines = Lists.newArrayList(); for (Path file : files) { System.out.println(file); @@ -72,8 +94,6 @@ public void testWrite() throws IOException { lines.addAll(Files.readAllLines(file)); } } - assertThat( - lines, - equalTo(Lists.newArrayList("{\"entityKey\":\"1234\",\"entityName\":\"testEntity\"}"))); + return lines; } } From df908fa41ed039fe9783f3c2806f77941f78f1bd Mon Sep 17 00:00:00 2001 From: Tim Sell Date: Mon, 28 Jan 2019 03:22:15 +0800 Subject: [PATCH 3/3] rename FeatureIO.Write to FeatureStoreWrite --- .../transform/ErrorsStoreTransform.java | 6 +-- .../feast/ingestion/transform/FeatureIO.java | 44 ------------------- .../transform/SplitOutputByStore.java | 14 +++--- .../java/feast/store/FeatureStoreFactory.java | 3 +- .../java/feast/store/FeatureStoreWrite.java | 6 +-- .../src/main/java/feast/store/NoOpIO.java | 3 +- .../errors/json/JsonFileErrorsFactory.java | 4 +- .../errors/json/JsonFileErrorsWrite.java | 4 +- .../feast/store/errors/logging/LogIO.java | 4 +- .../logging/StderrFeatureErrorsFactory.java | 4 +- .../logging/StdoutFeatureErrorsFactory.java | 4 +- .../bigtable/BigTableServingStoreFactory.java | 4 +- .../bigtable/FeatureRowBigTableIO.java | 4 +- .../serving/redis/FeatureRowRedisIO.java | 4 +- .../serving/redis/RedisServingFactory.java | 4 +- .../bigquery/BigQueryWarehouseFactory.java | 4 +- .../bigquery/FeatureRowBigQueryIO.java | 4 +- .../json/JsonFileWarehouseFactory.java | 4 +- .../json/JsonFileWarehouseWrite.java | 4 +- .../transform/SplitOutputByStoreTest.java | 6 +-- .../test/java/feast/store/MockTransforms.java | 3 +- .../redis/FeatureRowRedisIOWriteTest.java | 6 +-- 22 files changed, 47 insertions(+), 96 deletions(-) delete mode 100644 ingestion/src/main/java/feast/ingestion/transform/FeatureIO.java diff --git a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java index 8ddd37422b..92590b91df 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java +++ b/ingestion/src/main/java/feast/ingestion/transform/ErrorsStoreTransform.java @@ -23,8 +23,8 @@ import com.google.inject.Inject; import feast.ingestion.model.Specs; import feast.ingestion.options.ImportJobPipelineOptions; -import feast.ingestion.transform.FeatureIO.Write; import feast.specs.StorageSpecProto.StorageSpec; +import feast.store.FeatureStoreWrite; import feast.store.NoOpIO; import feast.store.errors.FeatureErrorsFactory; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; @@ -35,7 +35,7 @@ import org.apache.hadoop.hbase.util.Strings; @Slf4j -public class ErrorsStoreTransform extends FeatureIO.Write { +public class ErrorsStoreTransform extends FeatureStoreWrite { private String errorsStoreType; private StorageSpec errorsStoreSpec; @@ -61,7 +61,7 @@ public ErrorsStoreTransform( @Override public PDone expand(PCollection input) { - Write write; + FeatureStoreWrite write; if (Strings.isEmpty(errorsStoreType)) { write = new NoOpIO.Write(); } else { diff --git a/ingestion/src/main/java/feast/ingestion/transform/FeatureIO.java b/ingestion/src/main/java/feast/ingestion/transform/FeatureIO.java deleted file mode 100644 index c6d9a3b1c0..0000000000 --- a/ingestion/src/main/java/feast/ingestion/transform/FeatureIO.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.ingestion.transform; - -import feast.types.FeatureRowExtendedProto.FeatureRowExtended; -import feast.types.FeatureRowProto.FeatureRow; -import lombok.AllArgsConstructor; -import org.apache.beam.sdk.transforms.PTransform; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PDone; -import org.apache.beam.sdk.values.PInput; - -public class FeatureIO { - - public abstract static class Read extends PTransform> {} - - public abstract static class Write extends PTransform, PDone> {} - - /** Used during setup if read transform can not be determined. */ - @AllArgsConstructor - public static class UnknownRead extends Read { - private String message; - - @Override - public PCollection expand(PInput input) { - throw new IllegalArgumentException(message); - } - } -} diff --git a/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java b/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java index d5eba40444..c7e58892f0 100644 --- a/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java +++ b/ingestion/src/main/java/feast/ingestion/transform/SplitOutputByStore.java @@ -20,11 +20,11 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; import feast.ingestion.transform.SplitFeatures.MultiOutputSplit; import feast.ingestion.values.PFeatureRows; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; +import feast.store.FeatureStoreWrite; import feast.store.FeatureStoreFactory; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.Collection; @@ -52,14 +52,14 @@ public class SplitOutputByStore extends PTransform { @Override public PFeatureRows expand(PFeatureRows input) { - Map transforms = getFeatureStoreTransforms(); + Map transforms = getFeatureStoreTransforms(); Set keys = transforms.keySet(); log.info(String.format("Splitting on keys = [%s]", String.join(",", keys))); MultiOutputSplit splitter = new MultiOutputSplit<>(selector, keys, specs); PCollectionTuple splits = input.getMain().apply(splitter); - Map, Write> taggedTransforms = new HashMap<>(); + Map, FeatureStoreWrite> taggedTransforms = new HashMap<>(); for (String key : transforms.keySet()) { TupleTag tag = splitter.getStrategy().getTag(key); taggedTransforms.put(tag, transforms.get(key)); @@ -80,9 +80,9 @@ private Map getStoresMap() { return storesMap; } - private Map getFeatureStoreTransforms() { + private Map getFeatureStoreTransforms() { Map storesMap = getStoresMap(); - Map transforms = new HashMap<>(); + Map transforms = new HashMap<>(); Map storageSpecs = specs.getStorageSpecs(); for (String storeId : storageSpecs.keySet()) { StorageSpec storageSpec = storageSpecs.get(storeId); @@ -109,14 +109,14 @@ private Map getFeatureStoreTransforms() { public static class WriteTags extends PTransform> { - private Map, Write> transforms; + private Map, FeatureStoreWrite> transforms; private TupleTag mainTag; @Override public PCollection expand(PCollectionTuple tuple) { List> outputList = Lists.newArrayList(); for (TupleTag tag : transforms.keySet()) { - Write write = transforms.get(tag); + FeatureStoreWrite write = transforms.get(tag); Preconditions.checkNotNull(write, String.format("Null transform for tag=%s", tag.getId())); PCollection input = tuple.get(tag); input.apply(String.format("Write to %s", tag.getId()), write); diff --git a/ingestion/src/main/java/feast/store/FeatureStoreFactory.java b/ingestion/src/main/java/feast/store/FeatureStoreFactory.java index 5bb8dc6d6c..998df98b92 100644 --- a/ingestion/src/main/java/feast/store/FeatureStoreFactory.java +++ b/ingestion/src/main/java/feast/store/FeatureStoreFactory.java @@ -18,11 +18,10 @@ package feast.store; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO; import feast.specs.StorageSpecProto.StorageSpec; public interface FeatureStoreFactory { - FeatureIO.Write create(StorageSpec storageSpec, Specs specs); + FeatureStoreWrite create(StorageSpec storageSpec, Specs specs); String getType(); } diff --git a/ingestion/src/main/java/feast/store/FeatureStoreWrite.java b/ingestion/src/main/java/feast/store/FeatureStoreWrite.java index c039ea01ee..759ff3bfa2 100644 --- a/ingestion/src/main/java/feast/store/FeatureStoreWrite.java +++ b/ingestion/src/main/java/feast/store/FeatureStoreWrite.java @@ -17,11 +17,9 @@ package feast.store; -import feast.types.FeatureRowProto.FeatureRow; +import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.values.PCollection; import org.apache.beam.sdk.values.PDone; -public abstract class FeatureStoreWrite extends PTransform, PDone> { - -} +public abstract class FeatureStoreWrite extends PTransform, PDone> {} diff --git a/ingestion/src/main/java/feast/store/NoOpIO.java b/ingestion/src/main/java/feast/store/NoOpIO.java index 8893313c80..65a68118c9 100644 --- a/ingestion/src/main/java/feast/store/NoOpIO.java +++ b/ingestion/src/main/java/feast/store/NoOpIO.java @@ -17,7 +17,6 @@ package feast.store; -import feast.ingestion.transform.FeatureIO; import feast.ingestion.transform.fn.Identity; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import org.apache.beam.sdk.transforms.ParDo; @@ -26,7 +25,7 @@ public class NoOpIO { - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { @Override public PDone expand(PCollection input) { diff --git a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java index 15d56bcae6..3a56382b1f 100644 --- a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java +++ b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsFactory.java @@ -19,7 +19,7 @@ import com.google.auto.service.AutoService; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; import feast.store.FileStoreOptions; @@ -34,7 +34,7 @@ public class JsonFileErrorsFactory implements FeatureErrorsFactory { private static final String JSON_FILES_TYPE = "file.json"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { FileStoreOptions options = OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); options.jobName = specs.getJobName(); diff --git a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java index ec9aa49388..ef95bf634b 100644 --- a/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java +++ b/ingestion/src/main/java/feast/store/errors/json/JsonFileErrorsWrite.java @@ -22,7 +22,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.store.FileStoreOptions; import feast.store.TextFileDynamicIO; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.PDone; @AllArgsConstructor -public class JsonFileErrorsWrite extends FeatureIO.Write { +public class JsonFileErrorsWrite extends FeatureStoreWrite { private FileStoreOptions options; diff --git a/ingestion/src/main/java/feast/store/errors/logging/LogIO.java b/ingestion/src/main/java/feast/store/errors/logging/LogIO.java index c79919b594..45a6c6826a 100644 --- a/ingestion/src/main/java/feast/store/errors/logging/LogIO.java +++ b/ingestion/src/main/java/feast/store/errors/logging/LogIO.java @@ -17,7 +17,7 @@ package feast.store.errors.logging; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.ingestion.transform.fn.LoggerDoFn; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import lombok.AllArgsConstructor; @@ -29,7 +29,7 @@ public class LogIO { @AllArgsConstructor - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { private Level level; diff --git a/ingestion/src/main/java/feast/store/errors/logging/StderrFeatureErrorsFactory.java b/ingestion/src/main/java/feast/store/errors/logging/StderrFeatureErrorsFactory.java index efbe549130..0e904928db 100644 --- a/ingestion/src/main/java/feast/store/errors/logging/StderrFeatureErrorsFactory.java +++ b/ingestion/src/main/java/feast/store/errors/logging/StderrFeatureErrorsFactory.java @@ -19,7 +19,7 @@ import com.google.auto.service.AutoService; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.specs.StorageSpecProto.StorageSpec; import feast.store.errors.FeatureErrorsFactory; import org.slf4j.event.Level; @@ -30,7 +30,7 @@ public class StderrFeatureErrorsFactory implements FeatureErrorsFactory { public static final String TYPE_STDERR = "stderr"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { return new LogIO.Write(Level.ERROR); } diff --git a/ingestion/src/main/java/feast/store/errors/logging/StdoutFeatureErrorsFactory.java b/ingestion/src/main/java/feast/store/errors/logging/StdoutFeatureErrorsFactory.java index 80191ec1a8..afc3a35d1b 100644 --- a/ingestion/src/main/java/feast/store/errors/logging/StdoutFeatureErrorsFactory.java +++ b/ingestion/src/main/java/feast/store/errors/logging/StdoutFeatureErrorsFactory.java @@ -19,7 +19,7 @@ import com.google.auto.service.AutoService; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.specs.StorageSpecProto.StorageSpec; import feast.store.errors.FeatureErrorsFactory; import org.slf4j.event.Level; @@ -30,7 +30,7 @@ public class StdoutFeatureErrorsFactory implements FeatureErrorsFactory { public static final String TYPE_STDOUT = "stdout"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { return new LogIO.Write(Level.INFO); } diff --git a/ingestion/src/main/java/feast/store/serving/bigtable/BigTableServingStoreFactory.java b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableServingStoreFactory.java index d5e8b209fa..9583a5d431 100644 --- a/ingestion/src/main/java/feast/store/serving/bigtable/BigTableServingStoreFactory.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/BigTableServingStoreFactory.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; import feast.store.serving.FeatureServingFactory; @@ -31,7 +31,7 @@ public class BigTableServingStoreFactory implements FeatureServingFactory { public static String TYPE_BIGTABLE = "bigtable"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { Preconditions.checkArgument(storageSpec.getType().equals(getType())); BigTableStoreOptions options = diff --git a/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowBigTableIO.java b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowBigTableIO.java index 9174c2d3f4..4dc289aeb2 100644 --- a/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowBigTableIO.java +++ b/ingestion/src/main/java/feast/store/serving/bigtable/FeatureRowBigTableIO.java @@ -20,7 +20,7 @@ import com.google.cloud.bigtable.beam.CloudBigtableConfiguration; import com.google.cloud.bigtable.beam.CloudBigtableIO; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import java.util.Collections; import lombok.extern.slf4j.Slf4j; @@ -34,7 +34,7 @@ @Slf4j public class FeatureRowBigTableIO { - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { private BigTableStoreOptions bigTableOptions; private Specs specs; diff --git a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowRedisIO.java b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowRedisIO.java index bdbbea5d96..9c5772140b 100644 --- a/ingestion/src/main/java/feast/store/serving/redis/FeatureRowRedisIO.java +++ b/ingestion/src/main/java/feast/store/serving/redis/FeatureRowRedisIO.java @@ -18,7 +18,7 @@ package feast.store.serving.redis; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.store.serving.redis.RedisCustomIO.RedisMutation; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; import org.apache.beam.sdk.transforms.ParDo; @@ -27,7 +27,7 @@ public class FeatureRowRedisIO { - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { private final RedisStoreOptions options; private final Specs specs; diff --git a/ingestion/src/main/java/feast/store/serving/redis/RedisServingFactory.java b/ingestion/src/main/java/feast/store/serving/redis/RedisServingFactory.java index e044589fbc..ca023c616b 100644 --- a/ingestion/src/main/java/feast/store/serving/redis/RedisServingFactory.java +++ b/ingestion/src/main/java/feast/store/serving/redis/RedisServingFactory.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; import feast.store.serving.FeatureServingFactory; @@ -31,7 +31,7 @@ public class RedisServingFactory implements FeatureServingFactory { public static final String TYPE_REDIS = "redis"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { Preconditions.checkArgument( storageSpec.getType().equals(TYPE_REDIS), "Storage spec type was not " + TYPE_REDIS); RedisStoreOptions options = diff --git a/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryWarehouseFactory.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryWarehouseFactory.java index 4b4b2139d0..33ad75a32b 100644 --- a/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryWarehouseFactory.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/BigQueryWarehouseFactory.java @@ -20,7 +20,7 @@ import com.google.auto.service.AutoService; import com.google.common.base.Preconditions; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; import feast.store.warehouse.FeatureWarehouseFactory; @@ -30,7 +30,7 @@ public class BigQueryWarehouseFactory implements FeatureWarehouseFactory { public static String TYPE_BIGQUERY = "bigquery"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { Preconditions.checkArgument( storageSpec.getType().equals(TYPE_BIGQUERY), "Storage spec type was not " + TYPE_BIGQUERY); diff --git a/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java index cf0cc5cd4e..c8ec8a7825 100644 --- a/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java +++ b/ingestion/src/main/java/feast/store/warehouse/bigquery/FeatureRowBigQueryIO.java @@ -21,7 +21,7 @@ import com.google.common.base.Strings; import com.google.inject.Inject; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.ingestion.transform.SplitFeatures.SingleOutputSplit; import feast.specs.EntitySpecProto.EntitySpec; import feast.specs.FeatureSpecProto.FeatureSpec; @@ -45,7 +45,7 @@ @Slf4j public class FeatureRowBigQueryIO { - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { private final BigQueryStoreOptions bigQueryOptions; private final Specs specs; diff --git a/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java index a4fa617a82..5572772fcc 100644 --- a/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java +++ b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseFactory.java @@ -19,7 +19,7 @@ import com.google.auto.service.AutoService; import feast.ingestion.model.Specs; -import feast.ingestion.transform.FeatureIO.Write; +import feast.store.FeatureStoreWrite; import feast.options.OptionsParser; import feast.specs.StorageSpecProto.StorageSpec; import feast.store.FileStoreOptions; @@ -33,7 +33,7 @@ public class JsonFileWarehouseFactory implements FeatureWarehouseFactory { private static final String JSON_FILES_TYPE = "file.json"; @Override - public Write create(StorageSpec storageSpec, Specs specs) { + public FeatureStoreWrite create(StorageSpec storageSpec, Specs specs) { FileStoreOptions options = OptionsParser.parse(storageSpec.getOptionsMap(), FileStoreOptions.class); options.jobName = specs.getJobName(); diff --git a/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java index bc3c9c3712..fd3050a049 100644 --- a/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java +++ b/ingestion/src/main/java/feast/store/warehouse/json/JsonFileWarehouseWrite.java @@ -22,7 +22,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.store.FileStoreOptions; import feast.store.TextFileDynamicIO; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; @@ -33,7 +33,7 @@ import org.apache.beam.sdk.values.PDone; @AllArgsConstructor -public class JsonFileWarehouseWrite extends FeatureIO.Write { +public class JsonFileWarehouseWrite extends FeatureStoreWrite { private FileStoreOptions options; diff --git a/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java b/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java index bca7f399ee..183f117156 100644 --- a/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java +++ b/ingestion/src/test/java/feast/ingestion/transform/SplitOutputByStoreTest.java @@ -26,7 +26,6 @@ import feast.ingestion.model.Specs; import feast.ingestion.model.Values; import feast.ingestion.service.MockSpecService; -import feast.ingestion.transform.FeatureIO.Write; import feast.ingestion.transform.SplitOutputByStore.WriteTags; import feast.ingestion.values.PFeatureRows; import feast.specs.EntitySpecProto.EntitySpec; @@ -37,6 +36,7 @@ import feast.specs.ImportSpecProto.ImportSpec; import feast.specs.ImportSpecProto.Schema; import feast.specs.StorageSpecProto.StorageSpec; +import feast.store.FeatureStoreWrite; import feast.store.FeatureStoreFactory; import feast.store.MockFeatureStore; import feast.store.MockTransforms; @@ -385,7 +385,7 @@ public void testWriteTags() { TupleTag tag3 = new TupleTag<>("TAG3"); TupleTag mainTag = new TupleTag<>("TAG4"); - Map, Write> transforms = ImmutableMap., Write>builder() + Map, FeatureStoreWrite> transforms = ImmutableMap., FeatureStoreWrite>builder() .put(tag1, new MockTransforms.Write()) .put(tag2, new MockTransforms.Write()) // tag3 and mainTag do not have write transforms. @@ -418,7 +418,7 @@ public void testWriteTags() { // input 4 is is returned in the output because it is the main tag. PAssert.that(output).containsInAnyOrder(rowex1, rowex2, rowex4); - // Each non main tagged input should be written to corresponding Write transform + // Each non main tagged input should be written to corresponding FeatureStoreWrite transform PAssert.that(((MockTransforms.Write) transforms.get(tag1)).getInputs().get(0)) .containsInAnyOrder(rowex1); PAssert.that(((MockTransforms.Write) transforms.get(tag2)).getInputs().get(0)) diff --git a/ingestion/src/test/java/feast/store/MockTransforms.java b/ingestion/src/test/java/feast/store/MockTransforms.java index 2e1acc43b2..7d241e77ee 100644 --- a/ingestion/src/test/java/feast/store/MockTransforms.java +++ b/ingestion/src/test/java/feast/store/MockTransforms.java @@ -18,7 +18,6 @@ package feast.store; import com.google.common.collect.Lists; -import feast.ingestion.transform.FeatureIO; import feast.ingestion.transform.fn.Identity; import feast.specs.StorageSpecProto.StorageSpec; import feast.types.FeatureRowExtendedProto.FeatureRowExtended; @@ -31,7 +30,7 @@ public class MockTransforms { @Getter - public static class Write extends FeatureIO.Write { + public static class Write extends FeatureStoreWrite { List> inputs = Lists.newArrayList(); private StorageSpec spec; diff --git a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java index dfc4907a78..3dd1b10112 100644 --- a/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java +++ b/ingestion/src/test/java/feast/store/serving/redis/FeatureRowRedisIOWriteTest.java @@ -29,7 +29,7 @@ import feast.ingestion.model.Values; import feast.ingestion.service.FileSpecService; import feast.ingestion.service.SpecService; -import feast.ingestion.transform.FeatureIO; +import feast.store.FeatureStoreWrite; import feast.ingestion.util.DateUtil; import feast.specs.ImportSpecProto.Field; import feast.specs.ImportSpecProto.ImportSpec; @@ -154,7 +154,7 @@ public void testWriteNoneGranularity() throws IOException { @Test public void testWriteNoneGranularityFromOptions() throws IOException { Specs specs = getSpecs(); - FeatureIO.Write write = new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); + FeatureStoreWrite write = new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder() @@ -195,7 +195,7 @@ public void testWriteNoneGranularityFromOptions() throws IOException { @Test public void testWriteHourGranularity() throws IOException { Specs specs = getSpecs(); - FeatureIO.Write write = new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); + FeatureStoreWrite write = new RedisServingFactory().create(specs.getStorageSpec("REDIS1"), specs); FeatureRowExtended rowExtended = FeatureRowExtended.newBuilder()