Skip to content
Permalink
Browse files
[HUDI-3771] flink supports sync table information to aws glue (#5202)
  • Loading branch information
todd5167 committed Apr 2, 2022
1 parent 020786a commit eef3f9c74acfe0ebec77694044b416696cfc7c2d
Show file tree
Hide file tree
Showing 4 changed files with 12 additions and 2 deletions.
@@ -45,7 +45,7 @@ public AwsGlueCatalogSyncTool(TypedProperties props, Configuration conf, FileSys
super(props, new HiveConf(conf, HiveConf.class), fs);
}

private AwsGlueCatalogSyncTool(HiveSyncConfig hiveSyncConfig, HiveConf hiveConf, FileSystem fs) {
public AwsGlueCatalogSyncTool(HiveSyncConfig hiveSyncConfig, HiveConf hiveConf, FileSystem fs) {
super(hiveSyncConfig, hiveConf, fs);
}

@@ -18,10 +18,12 @@

package org.apache.hudi.sink.utils;

import org.apache.hudi.aws.sync.AwsGlueCatalogSyncTool;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.hive.HiveSyncConfig;
import org.apache.hudi.hive.HiveSyncTool;
import org.apache.hudi.hive.ddl.HiveSyncMode;
import org.apache.hudi.table.format.FilePathUtils;
import org.apache.hudi.util.StreamerUtil;

@@ -48,6 +50,10 @@ private HiveSyncContext(HiveSyncConfig syncConfig, HiveConf hiveConf, FileSystem
}

public HiveSyncTool hiveSyncTool() {
HiveSyncMode syncMode = HiveSyncMode.of(syncConfig.syncMode);
if (syncMode == HiveSyncMode.GLUE) {
return new AwsGlueCatalogSyncTool(this.syncConfig, this.hiveConf, this.fs);
}
return new HiveSyncTool(this.syncConfig, this.hiveConf, this.fs);
}

@@ -55,7 +55,7 @@ public class HiveSyncConfig extends HoodieSyncConfig {
@Parameter(names = {"--use-jdbc"}, description = "Hive jdbc connect url")
public Boolean useJdbc;

@Parameter(names = {"--sync-mode"}, description = "Mode to choose for Hive ops. Valid values are hms, jdbc and hiveql")
@Parameter(names = {"--sync-mode"}, description = "Mode to choose for Hive ops. Valid values are hms,glue,jdbc and hiveql")
public String syncMode;

@Parameter(names = {"--auto-create-database"}, description = "Auto create hive database")
@@ -26,6 +26,10 @@ public enum HiveSyncMode {
* The HMS mode use the hive meta client to sync metadata.
*/
HMS,
/**
* The GLUE mode use the glue client to sync metadata.
*/
GLUE,
/**
* The HIVEQL mode execute hive ql to sync metadata.
*/

0 comments on commit eef3f9c

Please sign in to comment.