diff --git a/.gitignore b/.gitignore index 5468b4d550..86a484646d 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,6 @@ docs/generated # Test artifacts keyFile.json + +# delombok dir +delombok diff --git a/java/pom.xml b/java/pom.xml index 59c997aeed..8b59bd6ca3 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -31,6 +31,9 @@ 5.9.1 2.22.0 4.3.1 + + UTF-8 + ${project.basedir}/delombok @@ -368,6 +371,45 @@ src/test/resources/system.properties + + org.projectlombok + lombok-maven-plugin + ${lombok.version}.0 + + ${project.basedir}/src/main/java + ${delombok.output} + false + + + + + delombok + + + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 3.1.1 + + 1.8 + 1.8 + ${delombok.output} + + + + + attach-javadocs + + jar + + + + + @@ -375,7 +417,6 @@ - Hops diff --git a/java/src/main/java/com/logicalclocks/hsfs/Feature.java b/java/src/main/java/com/logicalclocks/hsfs/Feature.java index 3f6c58dd3b..111876a2e1 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/Feature.java +++ b/java/src/main/java/com/logicalclocks/hsfs/Feature.java @@ -193,7 +193,8 @@ public Filter ge(Feature value) { /** * check if a value exists in a group of values. - * + * @param collection collection of values to filter on SqlFilterCondition.IN + * @return Filter object * @deprecated * `in` method is deprecated. Use `isin` instead. */ diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java b/java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java index 60c597cc68..69a9bb44c2 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java +++ b/java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java @@ -136,11 +136,11 @@ public Dataset read(boolean online, Map readOptions) throws /** * Reads Feature group data at a specific point in time. * - * @param wallclockTime - * @return DataFrame. - * @throws FeatureStoreException - * @throws IOException - * @throws ParseException + * @param wallclockTime point in time + * @return Spark DataFrame containing feature data. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public Dataset read(String wallclockTime) throws FeatureStoreException, IOException, ParseException { return (Dataset) selectAll().asOf(wallclockTime).read(false, null); @@ -149,12 +149,12 @@ public Dataset read(String wallclockTime) throws FeatureStoreException, IOE /** * Reads Feature group data at a specific point in time. * - * @param wallclockTime - * @param readOptions - * @return DataFrame. - * @throws FeatureStoreException - * @throws IOException - * @throws ParseException + * @param wallclockTime point in time + * @param readOptions Additional read options as key-value pairs, defaults to empty Map. + * @return Spark DataFrame containing feature data. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public Dataset read(String wallclockTime, Map readOptions) throws FeatureStoreException, IOException, ParseException { @@ -166,10 +166,10 @@ public Dataset read(String wallclockTime, Map readOptions) * * @param wallclockStartTime start date. * @param wallclockEndTime end date. - * @return DataFrame. - * @throws FeatureStoreException - * @throws IOException - * @throws ParseException + * @return Spark DataFrame containing feature data. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException * * @deprecated */ @@ -182,11 +182,12 @@ public Dataset readChanges(String wallclockStartTime, String wallclockEndTi * `readChanges` method is deprecated. Use `asOf(wallclockEndTime, wallclockStartTime).read(readOptions)` instead. * * @param wallclockStartTime start date. - * @param wallclockEndTime end date. - * @return DataFrame. - * @throws FeatureStoreException - * @throws IOException - * @throws ParseException + * @param wallclockEndTime end date. + * @param readOptions Additional write options as key-value pairs, defaults to empty Map. + * @return Spark DataFrame containing feature data. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException * * @deprecated */ @@ -204,8 +205,8 @@ public Dataset readChanges(String wallclockStartTime, String wallclockEndTi * @param wallclockTime Datetime string. The String should be formatted in one of the * following formats `%Y%m%d`, `%Y%m%d%H`, `%Y%m%d%H%M`, or `%Y%m%d%H%M%S`. * @return Query. The query object with the applied time travel condition - * @throws FeatureStoreException - * @throws ParseException + * @throws FeatureStoreException FeatureStoreException + * @throws ParseException ParseException */ public Query asOf(String wallclockTime) throws FeatureStoreException, ParseException { return selectAll().asOf(wallclockTime); @@ -222,8 +223,8 @@ public Query asOf(String wallclockTime) throws FeatureStoreException, ParseExcep * @param excludeUntil Datetime string. The String should be formatted in one of the * following formats `%Y%m%d`, `%Y%m%d%H`, `%Y%m%d%H%M`, or `%Y%m%d%H%M%S`. * @return Query. The query object with the applied time travel condition - * @throws FeatureStoreException - * @throws ParseException + * @throws FeatureStoreException FeatureStoreException + * @throws ParseException ParseException */ public Query asOf(String wallclockTime, String excludeUntil) throws FeatureStoreException, ParseException { return selectAll().asOf(wallclockTime, excludeUntil); @@ -295,8 +296,9 @@ public void insert(Dataset featureData, boolean overwrite, Map featureData, HudiOperationType operation) throws FeatureStoreException, IOException, ParseException { @@ -330,6 +332,13 @@ public void insert(Dataset featureData, Storage storage, boolean overwrite, /** * insert streaming dataframe in the Feature group. * + * @param featureData Spark dataframe containing feature data + * @return StreamingQuery + * @throws StreamingQueryException StreamingQueryException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws TimeoutException TimeoutException + * @throws ParseException ParseException * @deprecated * insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups. */ @@ -342,6 +351,14 @@ public StreamingQuery insertStream(Dataset featureData) /** * insert streaming dataframe in the Feature group. * + * @param featureData Spark dataframe containing feature data + * @param queryName name of spark StreamingQuery + * @return StreamingQuery + * @throws StreamingQueryException StreamingQueryException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws TimeoutException TimeoutException + * @throws ParseException ParseException * @deprecated * insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups. */ @@ -354,6 +371,15 @@ public StreamingQuery insertStream(Dataset featureData, String queryName) /** * insert streaming dataframe in the Feature group. * + * @param featureData Spark dataframe containing feature data + * @param queryName name of spark StreamingQuery + * @param outputMode outputMode + * @return StreamingQuery + * @throws StreamingQueryException StreamingQueryException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws TimeoutException TimeoutException + * @throws ParseException ParseException * @deprecated * insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups. */ @@ -366,6 +392,17 @@ public StreamingQuery insertStream(Dataset featureData, String queryName, S /** * insert streaming dataframe in the Feature group. * + * @param featureData Spark dataframe containing feature data + * @param queryName name of spark StreamingQuery + * @param outputMode outputMode + * @param awaitTermination whether or not to wait for query Termination + * @param timeout timeout + * @return StreamingQuery + * @throws StreamingQueryException StreamingQueryException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws TimeoutException TimeoutException + * @throws ParseException ParseException * @deprecated * insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups. */ @@ -379,6 +416,17 @@ public StreamingQuery insertStream(Dataset featureData, String queryName, S /** * insert streaming dataframe in the Feature group. * + * @param featureData Spark dataframe containing feature data + * @param queryName name of spark StreamingQuery + * @param outputMode outputMode + * @param awaitTermination whether or not to wait for query Termination + * @param checkpointLocation path to checkpoint location directory + * @return StreamingQuery + * @throws StreamingQueryException StreamingQueryException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws TimeoutException TimeoutException + * @throws ParseException ParseException * @deprecated * insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups. */ @@ -392,6 +440,19 @@ public StreamingQuery insertStream(Dataset featureData, String queryName, S /** * insert streaming dataframe in the Feature group. * + * @param featureData Spark dataframe containing feature data + * @param queryName name of spark StreamingQuery + * @param outputMode outputMode + * @param awaitTermination whether or not to wait for query Termination + * @param timeout timeout + * @param checkpointLocation path to checkpoint location directory + * @param writeOptions Additional write options as key-value pairs, defaults to empty Map. + * @return StreamingQuery + * @throws StreamingQueryException StreamingQueryException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws TimeoutException TimeoutException + * @throws ParseException ParseException * @deprecated * insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups. */ @@ -424,8 +485,10 @@ public void commitDeleteRecord(Dataset featureData, Map wri /** * Return commit details. * - * @throws FeatureStoreException - * @throws IOException + * @return commit details. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public Map> commitDetails() throws IOException, FeatureStoreException, ParseException { return utils.commitDetails(this, null); @@ -435,8 +498,10 @@ public Map> commitDetails() throws IOException, Featur * Return commit details. * * @param limit number of commits to return. - * @throws FeatureStoreException - * @throws IOException + * @return commit details. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public Map> commitDetails(Integer limit) throws IOException, FeatureStoreException, ParseException { @@ -447,8 +512,10 @@ public Map> commitDetails(Integer limit) * Return commit details. * * @param wallclockTime point in time. - * @throws FeatureStoreException - * @throws IOException + * @return commit details. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public Map> commitDetails(String wallclockTime) throws IOException, FeatureStoreException, ParseException { @@ -460,8 +527,10 @@ public Map> commitDetails(String wallclockTime) * * @param wallclockTime point in time. * @param limit number of commits to return. - * @throws FeatureStoreException - * @throws IOException + * @return commit details. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public Map> commitDetails(String wallclockTime, Integer limit) throws IOException, FeatureStoreException, ParseException { @@ -498,8 +567,9 @@ public Schema getDeserializedAvroSchema() throws FeatureStoreException, IOExcept * * @param wallclockTime number of commits to return. * @return statistics object of computed statistics - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public Statistics computeStatistics(String wallclockTime) throws FeatureStoreException, IOException, ParseException { if (statisticsConfig.getEnabled()) { diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java b/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java index df339c21aa..518a4c5ee4 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java +++ b/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java @@ -68,11 +68,11 @@ public FeatureStore() { /** * Get a feature group object from the feature store. * - * @param name the name of the feature group + * @param name the name of the feature group * @param version the version of the feature group * @return FeatureGroup - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public FeatureGroup getFeatureGroup(@NonNull String name, @NonNull Integer version) throws FeatureStoreException, IOException { @@ -84,8 +84,8 @@ public FeatureGroup getFeatureGroup(@NonNull String name, @NonNull Integer versi * * @param name the name of the feature group * @return FeatureGroup - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public FeatureGroup getFeatureGroup(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" @@ -96,10 +96,10 @@ public FeatureGroup getFeatureGroup(String name) throws FeatureStoreException, I /** * Get a list of all versions of a feature group from the feature store. * - * @param name the name of the feature group - * @return FeatureGroup - * @throws FeatureStoreException - * @throws IOException + * @param name the name of the feature group + * @return list of FeatureGroups + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public scala.collection.Seq getFeatureGroups(@NonNull String name) throws FeatureStoreException, IOException { @@ -110,11 +110,11 @@ public scala.collection.Seq getFeatureGroups(@NonNull String name) /** * Get a feature group object from the feature store. * - * @param name the name of the feature group + * @param name the name of the feature group * @param version the version of the feature group * @return FeatureGroup - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public StreamFeatureGroup getStreamFeatureGroup(@NonNull String name, @NonNull Integer version) throws FeatureStoreException, IOException { @@ -126,8 +126,8 @@ public StreamFeatureGroup getStreamFeatureGroup(@NonNull String name, @NonNull I * * @param name the name of the feature group * @return FeatureGroup - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public StreamFeatureGroup getStreamFeatureGroup(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" @@ -141,8 +141,8 @@ public StreamFeatureGroup getStreamFeatureGroup(String name) throws FeatureStore * @param name the name of the feature group * @param version the version of the feature group * @return ExternalFeatureGroup - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public ExternalFeatureGroup getExternalFeatureGroup(@NonNull String name, @NonNull Integer version) throws FeatureStoreException, IOException { @@ -154,8 +154,8 @@ public ExternalFeatureGroup getExternalFeatureGroup(@NonNull String name, @NonNu * * @param name the name of the feature group * @return ExternalFeatureGroup - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public ExternalFeatureGroup getExternalFeatureGroup(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting feature group `" + name + "`, defaulting to `" @@ -167,9 +167,9 @@ public ExternalFeatureGroup getExternalFeatureGroup(String name) throws FeatureS * Get a list of all versions of an external feature group from the feature store. * * @param name the name of the feature group - * @return ExternalFeatureGroup - * @throws FeatureStoreException - * @throws IOException + * @return list of ExternalFeatureGroup objects + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public scala.collection.Seq getExternalFeatureGroups(@NonNull String name) throws FeatureStoreException, IOException { @@ -308,7 +308,6 @@ public StreamFeatureGroup getOrCreateStreamFeatureGroup(String name, Integer ver List partitionKeys, boolean onlineEnabled, String eventTime) throws IOException, FeatureStoreException { - return featureGroupApi.getOrCreateStreamFeatureGroup(this, name, version, null, primaryKeys, partitionKeys, null, onlineEnabled, null, eventTime); } @@ -347,6 +346,8 @@ public FeatureView.FeatureViewBuilder createFeatureView() { * @param query Query object * @param version version of the feature view * @return FeatureView + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public FeatureView getOrCreateFeatureView(String name, Query query, Integer version) throws FeatureStoreException, IOException { @@ -363,6 +364,8 @@ public FeatureView getOrCreateFeatureView(String name, Query query, Integer vers * @param description description of the feature view * @param labels list of label features * @return FeatureView + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public FeatureView getOrCreateFeatureView(String name, Query query, Integer version, String description, List labels) throws FeatureStoreException, IOException { @@ -375,8 +378,8 @@ public FeatureView getOrCreateFeatureView(String name, Query query, Integer vers * @param name name of the feature view * @param version version to get * @return FeatureView - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public FeatureView getFeatureView(@NonNull String name, @NonNull Integer version) throws FeatureStoreException, IOException { @@ -388,8 +391,8 @@ public FeatureView getFeatureView(@NonNull String name, @NonNull Integer version * * @param name name of the feature view * @return FeatureView - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public FeatureView getFeatureView(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting feature view `" + name + "`, defaulting to `" @@ -405,11 +408,11 @@ public TrainingDataset.TrainingDatasetBuilder createTrainingDataset() { /** * Get a training dataset object from the selected feature store. * - * @param name name of the training dataset + * @param name name of the training dataset * @param version version to get * @return TrainingDataset - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public TrainingDataset getTrainingDataset(@NonNull String name, @NonNull Integer version) throws FeatureStoreException, IOException { @@ -421,8 +424,8 @@ public TrainingDataset getTrainingDataset(@NonNull String name, @NonNull Integer * * @param name name of the training dataset * @return TrainingDataset - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public TrainingDataset getTrainingDataset(String name) throws FeatureStoreException, IOException { LOGGER.info("VersionWarning: No version provided for getting training dataset `" + name + "`, defaulting to `" @@ -433,10 +436,10 @@ public TrainingDataset getTrainingDataset(String name) throws FeatureStoreExcept /** * Get all versions of a training dataset object from the selected feature store. * - * @param name name of the training dataset + * @param name name of the training dataset * @return TrainingDataset - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public scala.collection.Seq getTrainingDatasets(@NonNull String name) throws FeatureStoreException, IOException { diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureView.java b/java/src/main/java/com/logicalclocks/hsfs/FeatureView.java index ddd386b67c..d2d75dc7c1 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/FeatureView.java +++ b/java/src/main/java/com/logicalclocks/hsfs/FeatureView.java @@ -106,7 +106,7 @@ public FeatureViewBuilder description(String description) { * Query of a feature view. Note that `as_of` argument in the `Query` will be ignored because feature view does * not support time travel query. * - * @param query + * @param query Query object * @return builder */ public FeatureViewBuilder query(Query query) { @@ -232,8 +232,8 @@ public Dataset getBatchData(String startTime, String endTime, Map getTags() throws FeatureStoreException, IOException { @@ -257,8 +257,8 @@ public Map getTags() throws FeatureStoreException, IOException { * @param name * name of the tag * @return The value of a tag can be any valid json - primitives, arrays or json objects - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Object getTag(String name) throws FeatureStoreException, IOException { @@ -270,8 +270,8 @@ public Object getTag(String name) throws FeatureStoreException, IOException { * * @param name * name of the tag to be deleted - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void deleteTag(String name) throws FeatureStoreException, IOException { featureViewEngine.deleteTag(this, name); @@ -690,12 +690,13 @@ public void deleteAllTrainingDatasets() throws FeatureStoreException, IOExceptio /** * Add name/value tag to the training dataset. * + * @param version tag version * @param name * name of the tag * @param value * value of the tag. The value of a tag can be any valid json - primitives, arrays or json objects - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void addTrainingDatasetTag(Integer version, String name, Object value) throws FeatureStoreException, IOException { @@ -705,9 +706,10 @@ public void addTrainingDatasetTag(Integer version, String name, Object value) th /** * Get all tags of the training dataset. * + * @param version tag version * @return a map of tag name and values. The value of a tag can be any valid json - primitives, arrays or json objects - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Map getTrainingDatasetTags(Integer version) throws FeatureStoreException, IOException { @@ -717,11 +719,12 @@ public Map getTrainingDatasetTags(Integer version) throws Featur /** * Get a single tag value of the training dataset. * + * @param version tag version * @param name * name of the tag * @return The value of a tag can be any valid json - primitives, arrays or json objects - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Object getTrainingDatasetTag(Integer version, String name) throws FeatureStoreException, IOException { @@ -731,10 +734,11 @@ public Object getTrainingDatasetTag(Integer version, String name) throws Feature /** * Delete a tag of the training dataset. * + * @param version tag version * @param name * name of the tag to be deleted - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void deleteTrainingDatasetTag(Integer version, String name) throws FeatureStoreException, IOException { featureViewEngine.deleteTag(this, name, version); @@ -744,10 +748,10 @@ public void deleteTrainingDatasetTag(Integer version, String name) throws Featur * Set of primary key names that is used as keys in input dict object for `get_serving_vector` method. * * @return Set of serving keys - * @throws SQLException - * @throws IOException - * @throws FeatureStoreException - * @throws ClassNotFoundException + * @throws SQLException SQLException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws ClassNotFoundException ClassNotFoundException */ @JsonIgnore public HashSet getPrimaryKeys() diff --git a/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java b/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java index 398b589d98..c127296fe3 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java +++ b/java/src/main/java/com/logicalclocks/hsfs/HopsworksConnection.java @@ -95,8 +95,8 @@ public HopsworksConnection(String host, int port, String project, Region region, * Retrieve the project feature store. * * @return FeatureStore - * @throws IOException - * @throws FeatureStoreException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException */ public FeatureStore getFeatureStore() throws IOException, FeatureStoreException { return getFeatureStore(rewriteFeatureStoreName(project)); @@ -108,8 +108,8 @@ public FeatureStore getFeatureStore() throws IOException, FeatureStoreException * * @param name the name of the feature store to get the handle for * @return FeatureStore - * @throws IOException - * @throws FeatureStoreException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException */ public FeatureStore getFeatureStore(String name) throws IOException, FeatureStoreException { return featureStoreApi.get(projectObj.getProjectId(), rewriteFeatureStoreName(name)); diff --git a/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java b/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java index 0174287c8b..8c77ccb669 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java +++ b/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java @@ -556,6 +556,7 @@ public static class BigqueryConnector extends StorageConnector { /** * Set spark options specific to BigQuery. * @return Map + * @throws IOException IOException */ public Map sparkOptions() throws IOException { Map options = new HashMap<>(); @@ -588,13 +589,13 @@ public Map sparkOptions() throws IOException { /** * If Table options are set in the storage connector, set path to table. * Else use the query argument to set as path. - * @param query - * @param dataFormat - * @param options - * @param path + * @param query query string + * @param dataFormat dataFormat + * @param options options + * @param path path * @return Dataframe - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @Override public Object read(String query, String dataFormat, Map options, String path) diff --git a/java/src/main/java/com/logicalclocks/hsfs/StreamFeatureGroup.java b/java/src/main/java/com/logicalclocks/hsfs/StreamFeatureGroup.java index 3973e40fa7..febe81515b 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/StreamFeatureGroup.java +++ b/java/src/main/java/com/logicalclocks/hsfs/StreamFeatureGroup.java @@ -129,11 +129,9 @@ public StreamFeatureGroup(FeatureStore featureStore, int id) { * Reads Feature group data. * * @return DataFrame. - * @throws FeatureStoreException - * @throws IOException - * @throws ParseException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ - public Object read() throws FeatureStoreException, IOException { return read(false, null); } @@ -149,11 +147,11 @@ public Object read(boolean online, Map readOptions) throws Featu /** * Reads Feature group data at a specific point in time. * - * @param wallclockTime + * @param wallclockTime point in time. * @return DataFrame. - * @throws FeatureStoreException - * @throws IOException - * @throws ParseException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public Object read(String wallclockTime) throws FeatureStoreException, IOException, ParseException { @@ -174,9 +172,9 @@ public Object read(String wallclockTime, Map readOptions) * @param wallclockStartTime start date. * @param wallclockEndTime end date. * @return DataFrame. - * @throws FeatureStoreException - * @throws IOException - * @throws ParseException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ @Deprecated public Object readChanges(String wallclockStartTime, String wallclockEndTime) @@ -184,10 +182,6 @@ public Object readChanges(String wallclockStartTime, String wallclockEndTime) return selectAll().pullChanges(wallclockStartTime, wallclockEndTime).read(false, null); } - /* - * @deprecated - * `readChanges` method is deprecated. Use `asOf(wallclockEndTime, wallclockStartTime).read(readOptions)` instead. - */ @Deprecated public Object readChanges(String wallclockStartTime, String wallclockEndTime, Map readOptions) throws FeatureStoreException, IOException, ParseException { @@ -203,8 +197,8 @@ public Object readChanges(String wallclockStartTime, String wallclockEndTime, Ma * @param wallclockTime Datetime string. The String should be formatted in one of the * following formats `%Y%m%d`, `%Y%m%d%H`, `%Y%m%d%H%M`, or `%Y%m%d%H%M%S`. * @return Query. The query object with the applied time travel condition - * @throws FeatureStoreException - * @throws ParseException + * @throws FeatureStoreException FeatureStoreException + * @throws ParseException ParseException */ public Query asOf(String wallclockTime) throws FeatureStoreException, ParseException { return selectAll().asOf(wallclockTime); @@ -221,17 +215,13 @@ public Query asOf(String wallclockTime) throws FeatureStoreException, ParseExcep * @param excludeUntil Datetime string. The String should be formatted in one of the * following formats `%Y%m%d`, `%Y%m%d%H`, `%Y%m%d%H%M`, or `%Y%m%d%H%M%S`. * @return Query. The query object with the applied time travel condition - * @throws FeatureStoreException - * @throws ParseException + * @throws FeatureStoreException FeatureStoreException + * @throws ParseException ParseException */ public Query asOf(String wallclockTime, String excludeUntil) throws FeatureStoreException, ParseException { return selectAll().asOf(wallclockTime, excludeUntil); } - /* - * @deprecated - * `save` method is deprecated and in the next release it will be replaced by `insert` and `insertStream` methods. - */ @Deprecated public void save(S featureData, Map writeOptions) throws FeatureStoreException, IOException, ParseException { @@ -239,10 +229,6 @@ public void save(S featureData, Map writeOptions) codeEngine.saveCode(this); } - /* - * @deprecated - * In the next release save method will be replaced by insert method. - */ @Deprecated public void save(S featureData, Map writeOptions, JobConfiguration jobConfiguration) throws FeatureStoreException, IOException, ParseException { @@ -354,8 +340,10 @@ public void commitDeleteRecord(S featureData, Map writeOptio /** * Return commit details. * - * @throws FeatureStoreException - * @throws IOException + * @return commit details. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException FeatureStoreException + * @throws ParseException ParseException */ public Map> commitDetails() throws IOException, FeatureStoreException, ParseException { return utils.commitDetails(this, null); @@ -365,8 +353,10 @@ public Map> commitDetails() throws IOException, Featur * Return commit details. * * @param limit number of commits to return. - * @throws FeatureStoreException - * @throws IOException + * @return commit details. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public Map> commitDetails(Integer limit) throws IOException, FeatureStoreException, ParseException { @@ -377,8 +367,10 @@ public Map> commitDetails(Integer limit) * Return commit details. * * @param wallclockTime point in time. - * @throws FeatureStoreException - * @throws IOException + * @return commit details. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public Map> commitDetails(String wallclockTime) throws IOException, FeatureStoreException, ParseException { @@ -390,7 +382,11 @@ public Map> commitDetails(String wallclockTime) * * @param wallclockTime point in time. * @param limit number of commits to return. - */ + * @return commit details. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException + * */ public Map> commitDetails(String wallclockTime, Integer limit) throws IOException, FeatureStoreException, ParseException { return utils.commitDetailsByWallclockTime(this, wallclockTime, limit); diff --git a/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java b/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java index 6f3a218040..64f49ff985 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java +++ b/java/src/main/java/com/logicalclocks/hsfs/TrainingDataset.java @@ -131,7 +131,7 @@ public class TrainingDataset { @Getter @Setter private FilterLogic extraFilter; - + @Getter @Setter private String type = "trainingDatasetDTO"; @@ -224,8 +224,8 @@ private void setValTestSplit(Float valSize, Float testSize) { * Create the training dataset based on the content of the feature store query. * * @param query the query to save as training dataset - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void save(Query query) throws FeatureStoreException, IOException { save(query, null); @@ -234,14 +234,14 @@ public void save(Query query) throws FeatureStoreException, IOException { /** * Create the training dataset based on the content of the feature store query. * - * @param query the query to save as training dataset + * @param query the query to save as training dataset * @param writeOptions options to pass to the Spark write operation - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void save(Query query, Map writeOptions) throws FeatureStoreException, IOException { this.queryInt = query; - TrainingDataset trainingDataset = trainingDatasetEngine.save(this, query, writeOptions, label); + TrainingDataset trainingDataset = trainingDatasetEngine.save(this, query, writeOptions); this.setStorageConnector(trainingDataset.getStorageConnector()); codeEngine.saveCode(this); computeStatistics(); @@ -252,6 +252,7 @@ public void save(Query query, Map writeOptions) throws FeatureSt * * @return Spark Dataset containing the training dataset data * @throws FeatureStoreException if the training dataset has splits and the split was not specified + * @throws IOException IOException */ public Dataset read() throws FeatureStoreException, IOException { return read(""); @@ -263,6 +264,7 @@ public Dataset read() throws FeatureStoreException, IOException { * @param readOptions options to pass to the Spark read operation * @return Spark Dataset containing the training dataset data * @throws FeatureStoreException if the training dataset has splits and the split was not specified + * @throws IOException IOException */ public Dataset read(Map readOptions) throws FeatureStoreException, IOException { return read("", readOptions); @@ -273,12 +275,13 @@ public Dataset read(Map readOptions) throws FeatureStoreExc * * @param split the split name * @return Spark Dataset containing the training dataset data + * @throws FeatureStoreException if the training dataset has splits and the split was not specified + * @throws IOException IOException */ public Dataset read(String split) throws FeatureStoreException, IOException { return read(split, null); } - /** * Read a single split from the training dataset. * @@ -286,6 +289,7 @@ public Dataset read(String split) throws FeatureStoreException, IOException * @param readOptions options to pass to the Spark read operation * @return Spark Dataset containing the training dataset data * @throws FeatureStoreException if the training dataset has splits and the split was not specified + * @throws IOException IOException */ public Dataset read(String split, Map readOptions) throws FeatureStoreException, IOException { if (this.splits != null && !this.splits.isEmpty() && Strings.isNullOrEmpty(split)) { @@ -297,7 +301,9 @@ public Dataset read(String split, Map readOptions) throws F /** * Show numRows from the training dataset (across all splits). * - * @param numRows + * @param numRows number of rows to display + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void show(int numRows) throws FeatureStoreException, IOException { read("").show(numRows); @@ -307,8 +313,8 @@ public void show(int numRows) throws FeatureStoreException, IOException { * Recompute the statistics for the entire training dataset and save them to the feature store. * * @return statistics object of computed statistics - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public Statistics computeStatistics() throws FeatureStoreException, IOException { if (statisticsConfig.getEnabled()) { @@ -326,8 +332,8 @@ public Statistics computeStatistics() throws FeatureStoreException, IOException * Change the `enabled`, `histograms`, `correlations` or `columns` attributes and persist * the changes by calling this method. * - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void updateStatisticsConfig() throws FeatureStoreException, IOException { trainingDatasetEngine.updateStatisticsConfig(this); @@ -337,8 +343,8 @@ public void updateStatisticsConfig() throws FeatureStoreException, IOException { * Get the last statistics commit for the training dataset. * * @return statistics object of latest commit - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Statistics getStatistics() throws FeatureStoreException, IOException { @@ -350,8 +356,8 @@ public Statistics getStatistics() throws FeatureStoreException, IOException { * * @param commitTime commit time in the format "YYYYMMDDhhmmss" * @return statistics object for the commit time - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Statistics getStatistics(String commitTime) throws FeatureStoreException, IOException { @@ -363,8 +369,8 @@ public Statistics getStatistics(String commitTime) throws FeatureStoreException, * * @param name name of the tag * @param value value of the tag. The value of a tag can be any valid json - primitives, arrays or json objects - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void addTag(String name, Object value) throws FeatureStoreException, IOException { trainingDatasetEngine.addTag(this, name, value); @@ -374,8 +380,8 @@ public void addTag(String name, Object value) throws FeatureStoreException, IOEx * Get all tags of the training dataset. * * @return a map of tag name and values. The value of a tag can be any valid json - primitives, arrays or json objects - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Map getTags() throws FeatureStoreException, IOException { @@ -387,8 +393,8 @@ public Map getTags() throws FeatureStoreException, IOException { * * @param name name of the tag * @return The value of a tag can be any valid json - primitives, arrays or json objects - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Object getTag(String name) throws FeatureStoreException, IOException { @@ -399,8 +405,8 @@ public Object getTag(String name) throws FeatureStoreException, IOException { * Delete a tag of the training dataset. * * @param name name of the tag to be deleted - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void deleteTag(String name) throws FeatureStoreException, IOException { trainingDatasetEngine.deleteTag(this, name); @@ -440,9 +446,10 @@ public void setLabel(List label) { /** * Initialise and cache parametrised prepared statement to retrieve feature vector from online feature store. * - * @throws SQLException - * @throws IOException - * @throws FeatureStoreException + * @throws SQLException SQLException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws ClassNotFoundException ClassNotFoundException */ public void initPreparedStatement() throws SQLException, IOException, FeatureStoreException, ClassNotFoundException { initPreparedStatement(false); @@ -451,9 +458,11 @@ public void initPreparedStatement() throws SQLException, IOException, FeatureSto /** * Initialise and cache parametrised prepared statement to retrieve feature vector from online feature store. * - * @throws SQLException - * @throws IOException - * @throws FeatureStoreException + * @param external whether is from external client or not + * @throws SQLException SQLException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws ClassNotFoundException ClassNotFoundException */ public void initPreparedStatement(boolean external) throws SQLException, IOException, FeatureStoreException, ClassNotFoundException { @@ -463,9 +472,12 @@ public void initPreparedStatement(boolean external) /** * Initialise and cache parametrised prepared statement to retrieve batch feature vectors from online feature store. * - * @throws SQLException - * @throws IOException - * @throws FeatureStoreException + * @param external whether is from external client or not + * @param batch whether to initialise feature vector for batch retrieval + * @throws SQLException SQLException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws ClassNotFoundException ClassNotFoundException */ public void initPreparedStatement(boolean external, boolean batch) throws SQLException, IOException, FeatureStoreException, ClassNotFoundException { @@ -477,8 +489,11 @@ public void initPreparedStatement(boolean external, boolean batch) throws SQLExc * * @param entry Map object with kes as primary key names of the training dataset features groups and values as * corresponding ids to retrieve feature vector from online feature store. - * @throws FeatureStoreException - * @throws IOException + * @return list of feature values sorted according to provided primary keys. + * @throws SQLException SQLException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws ClassNotFoundException ClassNotFoundException */ @JsonIgnore public List getServingVector(Map entry) throws SQLException, FeatureStoreException, @@ -493,8 +508,11 @@ public List getServingVector(Map entry) throws SQLExcept * corresponding ids to retrieve feature vector from online feature store. * @param external If true, the connection to the online feature store will be established using the hostname * provided in the hsfs.connection() setup. - * @throws FeatureStoreException - * @throws IOException + * @return list of feature values sorted according to provided primary keys. + * @throws SQLException SQLException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws ClassNotFoundException ClassNotFoundException */ @JsonIgnore public List getServingVector(Map entry, boolean external) @@ -523,8 +541,8 @@ public List> getServingVectors(Map> entry, boo * This operation drops all metadata associated with this version of the * training dataset and and the materialized data in HopsFS. * - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void delete() throws FeatureStoreException, IOException { LOGGER.warn("JobWarning: All jobs associated to training dataset `" + name + "`, version `" @@ -536,10 +554,10 @@ public void delete() throws FeatureStoreException, IOException { * Set of primary key names that is used as keys in input dict object for `get_serving_vector` method. * * @return Set of serving keys - * @throws SQLException - * @throws IOException - * @throws FeatureStoreException - * @throws ClassNotFoundException + * @throws SQLException SQLException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException + * @throws ClassNotFoundException ClassNotFoundException */ @JsonIgnore public HashSet getServingKeys() diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/Query.java b/java/src/main/java/com/logicalclocks/hsfs/constructor/Query.java index f4ebc9a069..2905faf455 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/constructor/Query.java +++ b/java/src/main/java/com/logicalclocks/hsfs/constructor/Query.java @@ -175,8 +175,8 @@ public Query joinFeatures(Query subquery, List leftOn, List ri * * @param wallclockTime point in time * @return Query - * @throws FeatureStoreException - * @throws ParseException + * @throws FeatureStoreException FeatureStoreException + * @throws ParseException ParseException */ public Query asOf(String wallclockTime) throws FeatureStoreException, ParseException { return asOf(wallclockTime, null); @@ -191,8 +191,8 @@ public Query asOf(String wallclockTime) throws FeatureStoreException, ParseExcep * @param wallclockTime point in time * @param excludeUntil point in time * @return Query - * @throws FeatureStoreException - * @throws ParseException + * @throws FeatureStoreException FeatureStoreException + * @throws ParseException ParseException */ public Query asOf(String wallclockTime, String excludeUntil) throws FeatureStoreException, ParseException { Long wallclockTimestamp = utils.getTimeStampFromDateString(wallclockTime); @@ -221,9 +221,8 @@ public Query asOf(String wallclockTime, String excludeUntil) throws FeatureStore * @param wallclockStartTime start date. * @param wallclockEndTime end date. * @return Query - * @throws FeatureStoreException - * @throws IOException - * @throws ParseException + * @throws FeatureStoreException FeatureStoreException + * @throws ParseException ParseException * * @deprecated use asOf(wallclockEndTime, wallclockStartTime) instead */ diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngine.java index 9bc29e80ff..391bbdb9a8 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngine.java +++ b/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngine.java @@ -22,8 +22,6 @@ import com.logicalclocks.hsfs.HudiOperationType; import com.logicalclocks.hsfs.Storage; import com.logicalclocks.hsfs.TimeTravelFormat; -import com.logicalclocks.hsfs.engine.hudi.HudiEngine; -import com.logicalclocks.hsfs.metadata.KafkaApi; import com.logicalclocks.hsfs.metadata.FeatureGroupApi; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; @@ -42,8 +40,6 @@ public class FeatureGroupEngine { private FeatureGroupApi featureGroupApi = new FeatureGroupApi(); - private HudiEngine hudiEngine = new HudiEngine(); - protected KafkaApi kafkaApi = new KafkaApi(); private FeatureGroupUtils utils = new FeatureGroupUtils(); @@ -52,12 +48,19 @@ public class FeatureGroupEngine { /** * Create the metadata and write the data to the online/offline feature store. * - * @param featureGroup - * @param dataset - * @param partitionKeys - * @param writeOptions - * @throws FeatureStoreException - * @throws IOException + * @param featureGroup Feature Group metadata object. + * @param dataset Spark DataFrame or RDD. + * @param partitionKeys A list of feature names to be used as partition key when writing the feature data to the + * offline storage, defaults to empty list. + * @param hudiPrecombineKey A feature name to be used as a precombine key for the `TimeTravelFormat.HUDI` feature + * group. If feature group has `TimeTravelFormat.HUDI` and hudi precombine key was not + * specified then the first primary key of the feature group will be used as hudi precombine + * key. + * @param writeOptions Additional write options as key-value pairs, defaults to empty Map. + * @return Feature Group metadata object + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public FeatureGroup save(FeatureGroup featureGroup, Dataset dataset, List partitionKeys, String hudiPrecombineKey, Map writeOptions) diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/SparkEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/SparkEngine.java index 9b373fa973..fa5953c947 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/SparkEngine.java +++ b/java/src/main/java/com/logicalclocks/hsfs/engine/SparkEngine.java @@ -251,10 +251,14 @@ public void registerHudiTemporaryTable(HudiFeatureGroupAlias hudiFeatureGroupAli /** * Setup Spark to write the data on the File System. * - * @param trainingDataset - * @param query - * @param writeOptions - * @param saveMode + * @param trainingDataset Training Dataset metadata object + * @param query Query Object + * @param queryReadOptions Additional read options as key-value pairs, defaults to empty Map + * @param writeOptions Additional write options as key-value pairs, defaults to empty Map + * @param saveMode org.apache.spark.sql.saveMode: Append, Overwrite, ErrorIfExists, Ignore + * @return Spark dataframe + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public Dataset[] write(TrainingDataset trainingDataset, Query query, Map queryReadOptions, Map writeOptions, SaveMode saveMode) throws FeatureStoreException, IOException { @@ -435,12 +439,12 @@ public Map getReadOptions(Map providedOptions, D /** * Write multiple training dataset splits and name them. * - * @param datasets - * @param dataFormat - * @param writeOptions - * @param saveMode - * @param basePath - * @param splits + * @param datasets Spark DataFrame or RDD. + * @param dataFormat DataFormat: CSV, TSV, PARQUET, AVRO, IMAGE, ORC, TFRECORDS, TFRECORD + * @param writeOptions Additional write options as key-value pairs, defaults to empty Map + * @param saveMode saveMode org.apache.spark.sql.saveMode: Append, Overwrite, ErrorIfExists, Ignore + * @param basePath Base path to training dataset file + * @param splits list of training dataset Split metadata objects */ private void writeSplits(Dataset[] datasets, DataFormat dataFormat, Map writeOptions, SaveMode saveMode, String basePath, List splits) { @@ -453,11 +457,11 @@ private void writeSplits(Dataset[] datasets, DataFormat dataFormat, Map dataset, DataFormat dataFormat, Map writeOptions, SaveMode saveMode, String path) { @@ -500,15 +504,6 @@ public Dataset read(StorageConnector storageConnector, String dataFormat, return reader.load(); } - /** - * Writes feature group dataframe to kafka for online-fs ingestion. - * - * @param featureGroupBase - * @param dataset - * @param writeOptions - * @throws FeatureStoreException - * @throws IOException - */ public void writeOnlineDataframe(FeatureGroupBase featureGroupBase, S dataset, String onlineTopicName, Map writeOptions) throws FeatureStoreException, IOException { @@ -563,9 +558,11 @@ public StreamingQuery writeStreamDataframe(FeatureGroupBase featureGroupBase /** * Encodes all complex type features to binary using their avro type as schema. * - * @param featureGroupBase - * @param dataset - * @return + * @param featureGroupBase FeatureGroupBase Feature Group base metadata object + * @param dataset Spark DataFrame or RDD. + * @return Spark DataFrame. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public Dataset encodeComplexFeatures(FeatureGroupBase featureGroupBase, Dataset dataset) throws FeatureStoreException, IOException { @@ -584,11 +581,11 @@ public Dataset encodeComplexFeatures(FeatureGroupBase featureGroupBase, Dat /** * Serializes dataframe to two binary columns, one avro serialized key and one avro serialized value column. * - * @param featureGroupBase - * @param dataset - * @return dataset - * @throws FeatureStoreException - * @throws IOException + * @param featureGroupBase FeatureGroupBase Feature Group base metadata object + * @param dataset Spark DataFrame or RDD. + * @return Spark DataFrame. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ private Dataset onlineFeatureGroupToAvro(FeatureGroupBase featureGroupBase, Dataset dataset) throws FeatureStoreException, IOException { diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/StreamFeatureGroupEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/StreamFeatureGroupEngine.java index 717974c32e..c7ef1fef0b 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/StreamFeatureGroupEngine.java +++ b/java/src/main/java/com/logicalclocks/hsfs/engine/StreamFeatureGroupEngine.java @@ -46,17 +46,6 @@ public class StreamFeatureGroupEngine { private static final Logger LOGGER = LoggerFactory.getLogger(FeatureGroupEngine.class); - /** - * Create the metadata and write the data to the online/offline feature store. - * - * @param featureGroup - * @param dataset - * @param partitionKeys - * @param writeOptions - * @param sparkJobConfiguration - * @throws FeatureStoreException - * @throws IOException - */ public StreamFeatureGroup save(StreamFeatureGroup featureGroup, S dataset, List partitionKeys, String hudiPrecombineKey, Map writeOptions, JobConfiguration sparkJobConfiguration) diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/TrainingDatasetEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/TrainingDatasetEngine.java index 7c07b3eda9..5c2579a26e 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/engine/TrainingDatasetEngine.java +++ b/java/src/main/java/com/logicalclocks/hsfs/engine/TrainingDatasetEngine.java @@ -34,7 +34,6 @@ import java.io.IOException; import java.util.HashMap; -import java.util.List; import java.util.Map; public class TrainingDatasetEngine { @@ -49,14 +48,15 @@ public class TrainingDatasetEngine { /** * Make a REST call to Hopsworks to create the metadata and write the data on the File System. * - * @param trainingDataset - * @param query - * @param userWriteOptions - * @throws FeatureStoreException - * @throws IOException + * @param trainingDataset Training Dataset metadata object + * @param query Query Object + * @param userWriteOptions Additional write options as key-value pairs, defaults to empty Map + * @return Training Dataset metadata object + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public TrainingDataset save(TrainingDataset trainingDataset, Query query, - Map userWriteOptions, List labels) + Map userWriteOptions) throws FeatureStoreException, IOException { // Make the rest call to create the training dataset metadata @@ -138,6 +138,4 @@ public void updateStatisticsConfig(TrainingDataset trainingDataset) throws Featu public void delete(TrainingDataset trainingDataset) throws FeatureStoreException, IOException { trainingDatasetApi.delete(trainingDataset); } - - } diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupBase.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupBase.java index dd261b065e..16b347c389 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupBase.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupBase.java @@ -176,8 +176,8 @@ public T read() throws FeatureStoreException, IOException { * * @param name name of the tag * @param value value of the tag. The value of a tag can be any valid json - primitives, arrays or json objects. - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void addTag(String name, Object value) throws FeatureStoreException, IOException { featureGroupBaseEngine.addTag(this, name, value); @@ -187,8 +187,8 @@ public void addTag(String name, Object value) throws FeatureStoreException, IOEx * Get all tags of the feature group. * * @return map of tag name and values. The value of a tag can be any valid json - primitives, arrays or json objects - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Map getTags() throws FeatureStoreException, IOException { @@ -200,8 +200,8 @@ public Map getTags() throws FeatureStoreException, IOException { * * @param name name of tha tag * @return The value of a tag can be any valid json - primitives, arrays or json objects - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Object getTag(String name) throws FeatureStoreException, IOException { @@ -212,8 +212,8 @@ public Object getTag(String name) throws FeatureStoreException, IOException { * Delete a tag of the feature group. * * @param name name of the tag to be deleted - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void deleteTag(String name) throws FeatureStoreException, IOException { featureGroupBaseEngine.deleteTag(this, name); @@ -222,9 +222,9 @@ public void deleteTag(String name) throws FeatureStoreException, IOException { /** * Update the description of the feature group. * - * @param description - * @throws FeatureStoreException - * @throws IOException + * @param description feature group description. + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void updateDescription(String description) throws FeatureStoreException, IOException { featureGroupBaseEngine.updateDescription(this, description, this.getClass()); @@ -233,10 +233,10 @@ public void updateDescription(String description) throws FeatureStoreException, /** * Update the description of a single feature. * - * @param featureName - * @param description - * @throws FeatureStoreException - * @throws IOException + * @param featureName Name of the feature + * @param description Description of the feature + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void updateFeatureDescription(String featureName, String description) throws FeatureStoreException, IOException { @@ -249,9 +249,10 @@ public void updateFeatureDescription(String featureName, String description) * Update the metadata of multiple features. * Currently only feature description updates are supported. * - * @param features - * @throws FeatureStoreException - * @throws IOException + * @param features List of Feature metadata objects + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public void updateFeatures(List features) throws FeatureStoreException, IOException, ParseException { featureGroupBaseEngine.appendFeatures(this, features, this.getClass()); @@ -261,9 +262,10 @@ public void updateFeatures(List features) throws FeatureStoreException, * Update the metadata of multiple features. * Currently only feature description updates are supported. * - * @param feature - * @throws FeatureStoreException - * @throws IOException + * @param feature Feature metadata object + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public void updateFeatures(Feature feature) throws FeatureStoreException, IOException, ParseException { featureGroupBaseEngine.appendFeatures(this, Collections.singletonList(feature), this.getClass()); @@ -273,9 +275,10 @@ public void updateFeatures(Feature feature) throws FeatureStoreException, IOExce * Append features to the schema of the feature group. * It is only possible to append features to a feature group. Removing features is considered a breaking change. * - * @param features - * @throws FeatureStoreException - * @throws IOException + * @param features list of Feature metadata objects + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public void appendFeatures(List features) throws FeatureStoreException, IOException, ParseException { featureGroupBaseEngine.appendFeatures(this, new ArrayList<>(features), this.getClass()); @@ -285,9 +288,10 @@ public void appendFeatures(List features) throws FeatureStoreException, * Append a single feature to the schema of the feature group. * It is only possible to append features to a feature group. Removing features is considered a breaking change. * - * @param features - * @throws FeatureStoreException - * @throws IOException + * @param features List of Feature metadata objects + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException + * @throws ParseException ParseException */ public void appendFeatures(Feature features) throws FeatureStoreException, IOException, ParseException { List featureList = new ArrayList<>(); @@ -300,8 +304,8 @@ public void appendFeatures(Feature features) throws FeatureStoreException, IOExc * Change the `enabled`, `histograms`, `correlations` or `columns` attributes and persist * the changes by calling this method. * - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public void updateStatisticsConfig() throws FeatureStoreException, IOException { featureGroupBaseEngine.updateStatisticsConfig(this, this.getClass()); @@ -311,8 +315,8 @@ public void updateStatisticsConfig() throws FeatureStoreException, IOException { * Recompute the statistics for the feature group and save them to the feature store. * * @return statistics object of computed statistics - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public Statistics computeStatistics() throws FeatureStoreException, IOException { if (statisticsConfig.getEnabled()) { @@ -328,8 +332,8 @@ public Statistics computeStatistics() throws FeatureStoreException, IOException * Get the last statistics commit for the feature group. * * @return statistics object of latest commit - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Statistics getStatistics() throws FeatureStoreException, IOException { @@ -341,8 +345,8 @@ public Statistics getStatistics() throws FeatureStoreException, IOException { * * @param commitTime commit time in the format "YYYYMMDDhhmmss" * @return statistics object for the commit time - * @throws FeatureStoreException - * @throws IOException + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ @JsonIgnore public Statistics getStatistics(String commitTime) throws FeatureStoreException, IOException { @@ -365,10 +369,10 @@ public void unloadSubject() { /** * Filter the query based on a condition for a feature or a conjunction of multiple filters. * - * @param filter - * @return Query - * @throws FeatureStoreException - * @throws IOException + * @param filter Filter metadata object + * @return Query object + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public Query filter(Filter filter) throws FeatureStoreException, IOException { return this.selectAll().filter(filter); @@ -377,10 +381,10 @@ public Query filter(Filter filter) throws FeatureStoreException, IOException { /** * Filter the query based on a condition for a feature or a conjunction of multiple filters. * - * @param filter - * @return Query - * @throws FeatureStoreException - * @throws IOException + * @param filter Filter metadata object + * @return Query object + * @throws FeatureStoreException FeatureStoreException + * @throws IOException IOException */ public Query filter(FilterLogic filter) throws FeatureStoreException, IOException { return this.selectAll().filter(filter); @@ -389,9 +393,9 @@ public Query filter(FilterLogic filter) throws FeatureStoreException, IOExceptio /** * Retrieve a feature of the feature group by name. * - * @param name - * @return Feature - * @throws FeatureStoreException + * @param name feature name + * @return Feature metadata object + * @throws FeatureStoreException FeatureStoreException */ @JsonIgnore public Feature getFeature(String name) throws FeatureStoreException { diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java index 781eaa51cc..d2164d7385 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java +++ b/java/src/main/java/com/logicalclocks/hsfs/metadata/HopsworksExternalClient.java @@ -192,12 +192,12 @@ private static String readCertKey(String materialPwd) { * - AWS Secrets manager * - FIle on the local file system * - * @param secretStore - * @param region - * @param apiKeyFilepath + * @param secretStore SecretStore PARAMETER_STORE or SECRET_MANAGER + * @param region AWS regions + * @param apiKeyFilepath path to API key file * @return String - * @throws IOException - * @throws FeatureStoreException + * @throws IOException IOException + * @throws FeatureStoreException FeatureStoreException */ public String readApiKey(SecretStore secretStore, Region region, String apiKeyFilepath) throws IOException, FeatureStoreException {