Skip to content

Commit

Permalink
0004206: Disable initial load in background if max batch size is 1
Browse files Browse the repository at this point in the history
  • Loading branch information
erilong committed Dec 13, 2019
1 parent e09292f commit 7c7a182
Showing 1 changed file with 19 additions and 1 deletion.
Expand Up @@ -354,16 +354,34 @@ public void processTableRequestLoads(Node source, ProcessInfo processInfo, Map<
List<TableReloadRequest> loadsToProcess = engine.getDataService().getTableReloadRequestToProcess(source.getNodeId());
if (loadsToProcess.size() > 0) {
processInfo.setStatus(ProcessInfo.ProcessStatus.CREATING);
log.info("Found " + loadsToProcess.size() + " table reload requests to process.");

boolean useExtractJob = parameterService.is(ParameterConstants.INITIAL_LOAD_USE_EXTRACT_JOB, true);
boolean streamToFile = parameterService.is(ParameterConstants.STREAM_TO_FILE_ENABLED, false);

if (useExtractJob) {
Map<String, Channel> channels = engine.getConfigurationService().getChannels(false);
boolean isError = false;
for (Channel channel : channels.values()) {
if (channel.isReloadFlag() && channel.getMaxBatchSize() == 1) {
log.error("Max batch size must be greater than 1 for '{}' channel", channel.getChannelId());
isError = true;
}
}
if (isError) {
log.error("Initial loads are disabled until max batch size is corrected or {} is set to false",
ParameterConstants.INITIAL_LOAD_USE_EXTRACT_JOB);
return;
}
}

log.info("Found " + loadsToProcess.size() + " table reload requests to process.");

Map<String, List<TableReloadRequest>> requestsSplitByLoad = new HashMap<String, List<TableReloadRequest>>();
Map<Integer, ExtractRequest> extractRequests = null;

for (TableReloadRequest load : loadsToProcess) {
Node targetNode = engine.getNodeService().findNode(load.getTargetNodeId(), true);

if (!useExtractJob || streamToFile) {
if (load.isFullLoadRequest() && isValidLoadTarget(load.getTargetNodeId())) {
List<TableReloadRequest> fullLoad = new ArrayList<TableReloadRequest>();
Expand Down

0 comments on commit 7c7a182

Please sign in to comment.