Skip to content

KYLIN-4712 Optimize CubeMetaIngester.java CLI #1413

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
hit-lacus merged 1 commit into from
Sep 20, 2020
Merged

KYLIN-4712 Optimize CubeMetaIngester.java CLI #1413

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions tool/src/main/java/org/apache/kylin/tool/CubeMetaIngester.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import org.apache.kylin.common.persistence.RootPersistentEntity;
import org.apache.kylin.common.util.AbstractApplication;
import org.apache.kylin.common.util.OptionsHelper;
import org.apache.kylin.common.util.RandomUtil;
import org.apache.kylin.common.util.ZipFileUtils;
import org.apache.kylin.cube.CubeDescManager;
import org.apache.kylin.cube.CubeInstance;
Expand Down Expand Up @@ -75,12 +76,15 @@ public class CubeMetaIngester extends AbstractApplication {
@SuppressWarnings("static-access")
private static final Option OPTION_OVERWRITE_TABLES = OptionBuilder.withArgName("overwriteTables").hasArg().isRequired(false).withDescription("If table meta conflicts, overwrite the one in metadata store with the one in srcPath. Use in caution because it might break existing cubes! Suggest to backup metadata store first").create("overwriteTables");

@SuppressWarnings("static-access")
private static final Option OPTION_CREATE_PROJECT = OptionBuilder.withArgName("createProjectIfNotExists").hasArg().isRequired(false).withDescription("If project is not exists, kylin will create it.").create("createProjectIfNotExists");
private KylinConfig kylinConfig;

Set<String> requiredResources = Sets.newLinkedHashSet();
private String targetProjectName;
private boolean overwriteTables = false;
private boolean forceIngest = false;
private boolean createProjectIfNotExists = false;

@Override
protected Options getOptions() {
Expand All @@ -89,6 +93,7 @@ protected Options getOptions() {
options.addOption(OPTION_PROJECT);
options.addOption(OPTION_FORCE_INGEST);
options.addOption(OPTION_OVERWRITE_TABLES);
options.addOption(OPTION_CREATE_PROJECT);
return options;
}

Expand All @@ -104,6 +109,10 @@ protected void execute(OptionsHelper optionsHelper) throws Exception {
overwriteTables = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_OVERWRITE_TABLES));
}

if (optionsHelper.hasOption(OPTION_CREATE_PROJECT)) {
createProjectIfNotExists = Boolean.parseBoolean(optionsHelper.getOptionValue(OPTION_CREATE_PROJECT));
}

targetProjectName = optionsHelper.getOptionValue(OPTION_PROJECT);

String srcPath = optionsHelper.getOptionValue(OPTION_SRC);
Expand Down Expand Up @@ -154,15 +163,19 @@ private void injest(File metaRoot) throws IOException {

}

private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) throws IOException {
if (srcHybridManager.listHybridInstances().size() > 0) {
throw new IllegalStateException("Does not support ingest hybrid yet");
}

ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
ProjectInstance targetProject = projectManager.getProject(targetProjectName);
if (targetProject == null) {
throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
if (createProjectIfNotExists) {
projectManager.createProject(targetProjectName, null, "This is a project automatically added when ingest cube", null);
} else {
throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
}
}

TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
Expand All @@ -179,6 +192,9 @@ private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelMana
logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity());
}
}
tableDesc.setUuid(RandomUtil.randomUUID().toString());
tableDesc.setLastModified(0);
metadataManager.saveSourceTable(tableDesc, targetProjectName);
requiredResources.add(tableDesc.getResourcePath());
}

Expand Down