Skip to content
Permalink
Browse files
[HUDI-2747] support set --sparkMaster for MDT cli (#4964)
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
  • Loading branch information
zhangyue19921010 and yuezhang committed Mar 7, 2022
1 parent a66fd40 commit 53826d69e4120fd2a67986237fa85795209d9dab
Showing 1 changed file with 13 additions and 8 deletions.
@@ -108,7 +108,9 @@ public String set(@CliOption(key = {"metadataDir"},
}

@CliCommand(value = "metadata create", help = "Create the Metadata Table if it does not exist")
public String create() throws IOException {
public String create(
@CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master
) throws IOException {
HoodieCLI.getTableMetaClient();
Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath));
try {
@@ -123,7 +125,7 @@ public String create() throws IOException {

HoodieTimer timer = new HoodieTimer().startTimer();
HoodieWriteConfig writeConfig = getWriteConfig();
initJavaSparkContext();
initJavaSparkContext(Option.of(master));
SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc));
return String.format("Created Metadata Table in %s (duration=%.2f secs)", metadataPath, timer.endTimer() / 1000.0);
}
@@ -145,7 +147,8 @@ public String delete() throws Exception {
}

@CliCommand(value = "metadata init", help = "Update the metadata table from commits since the creation")
public String init(@CliOption(key = {"readonly"}, unspecifiedDefaultValue = "false",
public String init(@CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master,
@CliOption(key = {"readonly"}, unspecifiedDefaultValue = "false",
help = "Open in read-only mode") final boolean readOnly) throws Exception {
HoodieCLI.getTableMetaClient();
Path metadataPath = new Path(getMetadataTableBasePath(HoodieCLI.basePath));
@@ -159,7 +162,7 @@ public String init(@CliOption(key = {"readonly"}, unspecifiedDefaultValue = "fal
HoodieTimer timer = new HoodieTimer().startTimer();
if (!readOnly) {
HoodieWriteConfig writeConfig = getWriteConfig();
initJavaSparkContext();
initJavaSparkContext(Option.of(master));
SparkHoodieBackedTableMetadataWriter.create(HoodieCLI.conf, writeConfig, new HoodieSparkEngineContext(jsc));
}

@@ -191,9 +194,11 @@ public String stats() throws IOException {
}

@CliCommand(value = "metadata list-partitions", help = "List all partitions from metadata")
public String listPartitions() throws IOException {
public String listPartitions(
@CliOption(key = "sparkMaster", unspecifiedDefaultValue = SparkUtil.DEFAULT_SPARK_MASTER, help = "Spark master") final String master
) throws IOException {
HoodieCLI.getTableMetaClient();
initJavaSparkContext();
initJavaSparkContext(Option.of(master));
HoodieMetadataConfig config = HoodieMetadataConfig.newBuilder().enable(true).build();
HoodieBackedTableMetadata metadata = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(jsc), config,
HoodieCLI.basePath, "/tmp");
@@ -357,9 +362,9 @@ private HoodieWriteConfig getWriteConfig() {
.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).build()).build();
}

private void initJavaSparkContext() {
private void initJavaSparkContext(Option<String> userDefinedMaster) {
if (jsc == null) {
jsc = SparkUtil.initJavaSparkConf(SparkUtil.getDefaultConf("HoodieCLI", Option.empty()));
jsc = SparkUtil.initJavaSparkConf(SparkUtil.getDefaultConf("HoodieCLI", userDefinedMaster));
}
}
}

0 comments on commit 53826d6

Please sign in to comment.