Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
2329257
remove unused import
davitbzh Jul 13, 2022
e38ca9c
Merge remote-tracking branch 'upstream/master'
davitbzh Jul 13, 2022
10ad8b9
Merge remote-tracking branch 'upstream/master'
davitbzh Jul 13, 2022
16b2e52
Merge remote-tracking branch 'upstream/master'
davitbzh Jul 15, 2022
63b812b
Merge remote-tracking branch 'upstream/master'
davitbzh Jul 18, 2022
9fdd925
Merge remote-tracking branch 'upstream/master'
davitbzh Aug 4, 2022
3be3d58
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 7, 2022
6873412
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 14, 2022
c033dad
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 15, 2022
1b3d076
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 15, 2022
738ce21
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 19, 2022
afe2459
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 20, 2022
f972c0d
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 22, 2022
3145a76
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 26, 2022
fc39ad2
Merge remote-tracking branch 'upstream/master'
davitbzh Sep 28, 2022
b750dd4
Merge remote-tracking branch 'upstream/master'
davitbzh Oct 3, 2022
941d0e3
Merge remote-tracking branch 'upstream/master'
davitbzh Oct 3, 2022
3fbbe22
Merge remote-tracking branch 'upstream/master'
davitbzh Oct 11, 2022
75b9d89
Merge remote-tracking branch 'upstream/master'
davitbzh Oct 25, 2022
ac281ff
Merge remote-tracking branch 'upstream/master'
davitbzh Oct 26, 2022
2a493ea
Merge remote-tracking branch 'upstream/master'
davitbzh Nov 2, 2022
5a70a55
Merge remote-tracking branch 'upstream/master'
davitbzh Nov 6, 2022
20812f4
Merge remote-tracking branch 'upstream/master'
davitbzh Nov 8, 2022
435faec
Merge remote-tracking branch 'upstream/master'
davitbzh Nov 8, 2022
5d378a1
Merge remote-tracking branch 'upstream/master'
davitbzh Nov 8, 2022
dad216f
Merge remote-tracking branch 'upstream/master'
davitbzh Nov 14, 2022
866d575
Merge remote-tracking branch 'upstream/master'
davitbzh Nov 16, 2022
ca44cda
Merge remote-tracking branch 'upstream/master'
davitbzh Nov 25, 2022
42c8771
Merge remote-tracking branch 'upstream/master'
davitbzh Nov 30, 2022
3b4466b
Merge remote-tracking branch 'upstream/master'
davitbzh Dec 4, 2022
1b7245b
Merge remote-tracking branch 'upstream/master'
davitbzh Dec 5, 2022
eba5166
Merge remote-tracking branch 'upstream/master'
davitbzh Dec 13, 2022
5ba28f6
Merge remote-tracking branch 'upstream/master'
davitbzh Jan 9, 2023
04f0cc8
Merge remote-tracking branch 'upstream/master'
davitbzh Jan 24, 2023
a3261a4
javadocs build
davitbzh Jan 26, 2023
84ef547
maven javadoc plugin
davitbzh Jan 26, 2023
7dc7914
Merge branch 'master' into java_doc_build
moritzmeister Feb 8, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,6 @@ docs/generated

# Test artifacts
keyFile.json

# delombok dir
delombok
43 changes: 42 additions & 1 deletion java/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
<junit.version>5.9.1</junit.version>
<surefire-plugin.version>2.22.0</surefire-plugin.version>
<mockito.version>4.3.1</mockito.version>

<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<delombok.output>${project.basedir}/delombok</delombok.output>
</properties>

<dependencies>
Expand Down Expand Up @@ -368,14 +371,52 @@
<systemPropertiesFile>src/test/resources/system.properties</systemPropertiesFile>
</configuration>
</plugin>
<plugin>
<groupId>org.projectlombok</groupId>
<artifactId>lombok-maven-plugin</artifactId>
<version>${lombok.version}.0</version>
<configuration>
<sourceDirectory>${project.basedir}/src/main/java</sourceDirectory>
<outputDirectory>${delombok.output}</outputDirectory>
<addOutputDirectory>false</addOutputDirectory>
</configuration>
<executions>
<execution>
<goals>
<goal>delombok</goal>
</goals>
</execution>
</executions>
</plugin>


<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>3.1.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<sourcepath>${delombok.output}</sourcepath>
</configuration>

<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>

</plugins>
<testResources>
<testResource>
<directory>src/test/resources</directory>
</testResource>
</testResources>
</build>

<repositories>
<repository>
<id>Hops</id>
Expand Down
3 changes: 2 additions & 1 deletion java/src/main/java/com/logicalclocks/hsfs/Feature.java
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,8 @@ public Filter ge(Feature value) {

/**
* check if a value exists in a group of values.
*
* @param collection collection of values to filter on SqlFilterCondition.IN
* @return Filter object
* @deprecated
* `in` method is deprecated. Use `isin` instead.
*/
Expand Down
142 changes: 106 additions & 36 deletions java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java
Original file line number Diff line number Diff line change
Expand Up @@ -136,11 +136,11 @@ public Dataset<Row> read(boolean online, Map<String, String> readOptions) throws
/**
* Reads Feature group data at a specific point in time.
*
* @param wallclockTime
* @return DataFrame.
* @throws FeatureStoreException
* @throws IOException
* @throws ParseException
* @param wallclockTime point in time
* @return Spark DataFrame containing feature data.
* @throws FeatureStoreException FeatureStoreException
* @throws IOException IOException
* @throws ParseException ParseException
*/
public Dataset<Row> read(String wallclockTime) throws FeatureStoreException, IOException, ParseException {
return (Dataset<Row>) selectAll().asOf(wallclockTime).read(false, null);
Expand All @@ -149,12 +149,12 @@ public Dataset<Row> read(String wallclockTime) throws FeatureStoreException, IOE
/**
* Reads Feature group data at a specific point in time.
*
* @param wallclockTime
* @param readOptions
* @return DataFrame.
* @throws FeatureStoreException
* @throws IOException
* @throws ParseException
* @param wallclockTime point in time
* @param readOptions Additional read options as key-value pairs, defaults to empty Map.
* @return Spark DataFrame containing feature data.
* @throws FeatureStoreException FeatureStoreException
* @throws IOException IOException
* @throws ParseException ParseException
*/
public Dataset<Row> read(String wallclockTime, Map<String, String> readOptions)
throws FeatureStoreException, IOException, ParseException {
Expand All @@ -166,10 +166,10 @@ public Dataset<Row> read(String wallclockTime, Map<String, String> readOptions)
*
* @param wallclockStartTime start date.
* @param wallclockEndTime end date.
* @return DataFrame.
* @throws FeatureStoreException
* @throws IOException
* @throws ParseException
* @return Spark DataFrame containing feature data.
* @throws FeatureStoreException FeatureStoreException
* @throws IOException IOException
* @throws ParseException ParseException
*
* @deprecated
*/
Expand All @@ -182,11 +182,12 @@ public Dataset<Row> readChanges(String wallclockStartTime, String wallclockEndTi
* `readChanges` method is deprecated. Use `asOf(wallclockEndTime, wallclockStartTime).read(readOptions)` instead.
*
* @param wallclockStartTime start date.
* @param wallclockEndTime end date.
* @return DataFrame.
* @throws FeatureStoreException
* @throws IOException
* @throws ParseException
* @param wallclockEndTime end date.
* @param readOptions Additional write options as key-value pairs, defaults to empty Map.
* @return Spark DataFrame containing feature data.
* @throws FeatureStoreException FeatureStoreException
* @throws IOException IOException
* @throws ParseException ParseException
*
* @deprecated
*/
Expand All @@ -204,8 +205,8 @@ public Dataset<Row> readChanges(String wallclockStartTime, String wallclockEndTi
* @param wallclockTime Datetime string. The String should be formatted in one of the
* following formats `%Y%m%d`, `%Y%m%d%H`, `%Y%m%d%H%M`, or `%Y%m%d%H%M%S`.
* @return Query. The query object with the applied time travel condition
* @throws FeatureStoreException
* @throws ParseException
* @throws FeatureStoreException FeatureStoreException
* @throws ParseException ParseException
*/
public Query asOf(String wallclockTime) throws FeatureStoreException, ParseException {
return selectAll().asOf(wallclockTime);
Expand All @@ -222,8 +223,8 @@ public Query asOf(String wallclockTime) throws FeatureStoreException, ParseExcep
* @param excludeUntil Datetime string. The String should be formatted in one of the
* following formats `%Y%m%d`, `%Y%m%d%H`, `%Y%m%d%H%M`, or `%Y%m%d%H%M%S`.
* @return Query. The query object with the applied time travel condition
* @throws FeatureStoreException
* @throws ParseException
* @throws FeatureStoreException FeatureStoreException
* @throws ParseException ParseException
*/
public Query asOf(String wallclockTime, String excludeUntil) throws FeatureStoreException, ParseException {
return selectAll().asOf(wallclockTime, excludeUntil);
Expand Down Expand Up @@ -295,8 +296,9 @@ public void insert(Dataset<Row> featureData, boolean overwrite, Map<String, Stri
*
* @param featureData dataframe to be committed.
* @param operation commit operation type, INSERT or UPSERT.
* @throws FeatureStoreException
* @throws IOException
* @throws FeatureStoreException FeatureStoreException
* @throws IOException IOException
* @throws ParseException ParseException
*/
public void insert(Dataset<Row> featureData, HudiOperationType operation)
throws FeatureStoreException, IOException, ParseException {
Expand Down Expand Up @@ -330,6 +332,13 @@ public void insert(Dataset<Row> featureData, Storage storage, boolean overwrite,
/**
* insert streaming dataframe in the Feature group.
*
* @param featureData Spark dataframe containing feature data
* @return StreamingQuery
* @throws StreamingQueryException StreamingQueryException
* @throws IOException IOException
* @throws FeatureStoreException FeatureStoreException
* @throws TimeoutException TimeoutException
* @throws ParseException ParseException
* @deprecated
* insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
*/
Expand All @@ -342,6 +351,14 @@ public StreamingQuery insertStream(Dataset<Row> featureData)
/**
* insert streaming dataframe in the Feature group.
*
* @param featureData Spark dataframe containing feature data
* @param queryName name of spark StreamingQuery
* @return StreamingQuery
* @throws StreamingQueryException StreamingQueryException
* @throws IOException IOException
* @throws FeatureStoreException FeatureStoreException
* @throws TimeoutException TimeoutException
* @throws ParseException ParseException
* @deprecated
* insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
*/
Expand All @@ -354,6 +371,15 @@ public StreamingQuery insertStream(Dataset<Row> featureData, String queryName)
/**
* insert streaming dataframe in the Feature group.
*
* @param featureData Spark dataframe containing feature data
* @param queryName name of spark StreamingQuery
* @param outputMode outputMode
* @return StreamingQuery
* @throws StreamingQueryException StreamingQueryException
* @throws IOException IOException
* @throws FeatureStoreException FeatureStoreException
* @throws TimeoutException TimeoutException
* @throws ParseException ParseException
* @deprecated
* insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
*/
Expand All @@ -366,6 +392,17 @@ public StreamingQuery insertStream(Dataset<Row> featureData, String queryName, S
/**
* insert streaming dataframe in the Feature group.
*
* @param featureData Spark dataframe containing feature data
* @param queryName name of spark StreamingQuery
* @param outputMode outputMode
* @param awaitTermination whether or not to wait for query Termination
* @param timeout timeout
* @return StreamingQuery
* @throws StreamingQueryException StreamingQueryException
* @throws IOException IOException
* @throws FeatureStoreException FeatureStoreException
* @throws TimeoutException TimeoutException
* @throws ParseException ParseException
* @deprecated
* insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
*/
Expand All @@ -379,6 +416,17 @@ public StreamingQuery insertStream(Dataset<Row> featureData, String queryName, S
/**
* insert streaming dataframe in the Feature group.
*
* @param featureData Spark dataframe containing feature data
* @param queryName name of spark StreamingQuery
* @param outputMode outputMode
* @param awaitTermination whether or not to wait for query Termination
* @param checkpointLocation path to checkpoint location directory
* @return StreamingQuery
* @throws StreamingQueryException StreamingQueryException
* @throws IOException IOException
* @throws FeatureStoreException FeatureStoreException
* @throws TimeoutException TimeoutException
* @throws ParseException ParseException
* @deprecated
* insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
*/
Expand All @@ -392,6 +440,19 @@ public StreamingQuery insertStream(Dataset<Row> featureData, String queryName, S
/**
* insert streaming dataframe in the Feature group.
*
* @param featureData Spark dataframe containing feature data
* @param queryName name of spark StreamingQuery
* @param outputMode outputMode
* @param awaitTermination whether or not to wait for query Termination
* @param timeout timeout
* @param checkpointLocation path to checkpoint location directory
* @param writeOptions Additional write options as key-value pairs, defaults to empty Map.
* @return StreamingQuery
* @throws StreamingQueryException StreamingQueryException
* @throws IOException IOException
* @throws FeatureStoreException FeatureStoreException
* @throws TimeoutException TimeoutException
* @throws ParseException ParseException
* @deprecated
* insertStream method is deprecated FeatureGroups. Full capability insertStream is available for StreamFeatureGroups.
*/
Expand Down Expand Up @@ -424,8 +485,10 @@ public void commitDeleteRecord(Dataset<Row> featureData, Map<String, String> wri
/**
* Return commit details.
*
* @throws FeatureStoreException
* @throws IOException
* @return commit details.
* @throws FeatureStoreException FeatureStoreException
* @throws IOException IOException
* @throws ParseException ParseException
*/
public Map<Long, Map<String, String>> commitDetails() throws IOException, FeatureStoreException, ParseException {
return utils.commitDetails(this, null);
Expand All @@ -435,8 +498,10 @@ public Map<Long, Map<String, String>> commitDetails() throws IOException, Featur
* Return commit details.
*
* @param limit number of commits to return.
* @throws FeatureStoreException
* @throws IOException
* @return commit details.
* @throws FeatureStoreException FeatureStoreException
* @throws IOException IOException
* @throws ParseException ParseException
*/
public Map<Long, Map<String, String>> commitDetails(Integer limit)
throws IOException, FeatureStoreException, ParseException {
Expand All @@ -447,8 +512,10 @@ public Map<Long, Map<String, String>> commitDetails(Integer limit)
* Return commit details.
*
* @param wallclockTime point in time.
* @throws FeatureStoreException
* @throws IOException
* @return commit details.
* @throws FeatureStoreException FeatureStoreException
* @throws IOException IOException
* @throws ParseException ParseException
*/
public Map<Long, Map<String, String>> commitDetails(String wallclockTime)
throws IOException, FeatureStoreException, ParseException {
Expand All @@ -460,8 +527,10 @@ public Map<Long, Map<String, String>> commitDetails(String wallclockTime)
*
* @param wallclockTime point in time.
* @param limit number of commits to return.
* @throws FeatureStoreException
* @throws IOException
* @return commit details.
* @throws FeatureStoreException FeatureStoreException
* @throws IOException IOException
* @throws ParseException ParseException
*/
public Map<Long, Map<String, String>> commitDetails(String wallclockTime, Integer limit)
throws IOException, FeatureStoreException, ParseException {
Expand Down Expand Up @@ -498,8 +567,9 @@ public Schema getDeserializedAvroSchema() throws FeatureStoreException, IOExcept
*
* @param wallclockTime number of commits to return.
* @return statistics object of computed statistics
* @throws FeatureStoreException
* @throws IOException
* @throws FeatureStoreException FeatureStoreException
* @throws IOException IOException
* @throws ParseException ParseException
*/
public Statistics computeStatistics(String wallclockTime) throws FeatureStoreException, IOException, ParseException {
if (statisticsConfig.getEnabled()) {
Expand Down
Loading