From 2329257698ad1b71d4e5723971c7e713b89ded28 Mon Sep 17 00:00:00 2001 From: davitbzh Date: Wed, 13 Jul 2022 15:39:24 +0200 Subject: [PATCH 1/2] remove unused import --- .../main/java/com/logicalclocks/hsfs/constructor/FsQuery.java | 1 - 1 file changed, 1 deletion(-) diff --git a/java/src/main/java/com/logicalclocks/hsfs/constructor/FsQuery.java b/java/src/main/java/com/logicalclocks/hsfs/constructor/FsQuery.java index 5caf7d71dc..0140a7d7ad 100644 --- a/java/src/main/java/com/logicalclocks/hsfs/constructor/FsQuery.java +++ b/java/src/main/java/com/logicalclocks/hsfs/constructor/FsQuery.java @@ -22,7 +22,6 @@ import com.logicalclocks.hsfs.ExternalFeatureGroup; import com.logicalclocks.hsfs.Storage; import com.logicalclocks.hsfs.engine.SparkEngine; -import com.logicalclocks.hsfs.metadata.FeatureGroupBase; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; From 7e4eb4c9bea24622e803f63621bae47f586741f9 Mon Sep 17 00:00:00 2001 From: davitbzh Date: Wed, 5 Apr 2023 16:50:57 +0200 Subject: [PATCH 2/2] fix delta streamer config --- .../hsfs/spark/engine/hudi/HudiEngine.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java index afddae7fb7..b80f70a486 100644 --- a/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java +++ b/java/spark/src/main/java/com/logicalclocks/hsfs/spark/engine/hudi/HudiEngine.java @@ -112,16 +112,16 @@ public class HudiEngine { protected static final String COMMIT_METADATA_KEYPREFIX_OPT_KEY = "hoodie.datasource.write.commitmeta.key.prefix"; protected static final String DELTASTREAMER_CHECKPOINT_KEY = "deltastreamer.checkpoint.key"; protected static final String INITIAL_CHECKPOINT_STRING = "initialCheckPointString"; - protected static final String FEATURE_GROUP_SCHEMA = "StreamFeatureGroup.avroSchema"; + protected static final String FEATURE_GROUP_SCHEMA = "com.logicalclocks.hsfs.spark.StreamFeatureGroup.avroSchema"; protected static final String FEATURE_GROUP_ENCODED_SCHEMA = - "StreamFeatureGroup.encodedAvroSchema"; + "com.logicalclocks.hsfs.spark.StreamFeatureGroup.encodedAvroSchema"; protected static final String FEATURE_GROUP_COMPLEX_FEATURES = - "StreamFeatureGroup.complexFeatures"; - protected static final String KAFKA_SOURCE = "DeltaStreamerKafkaSource"; + "com.logicalclocks.hsfs.spark.StreamFeatureGroup.complexFeatures"; + protected static final String KAFKA_SOURCE = "com.logicalclocks.hsfs.spark.engine.hudi.DeltaStreamerKafkaSource"; protected static final String SCHEMA_PROVIDER = - "DeltaStreamerSchemaProvider"; + "com.logicalclocks.hsfs.spark.engine.hudi.DeltaStreamerSchemaProvider"; protected static final String DELTA_STREAMER_TRANSFORMER = - "DeltaStreamerTransformer"; + "com.logicalclocks.hsfs.spark.engine.hudi.DeltaStreamerTransformer"; protected static final String DELTA_SOURCE_ORDERING_FIELD_OPT_KEY = "sourceOrderingField"; protected static final String MIN_SYNC_INTERVAL_SECONDS = "minSyncIntervalSeconds";