Skip to content
Permalink
Browse files
Pre defining download path for parsing to rectify helix context issue
  • Loading branch information
DImuthuUpe committed Sep 3, 2021
1 parent d8d0867 commit d3bf8766eb0d5e2ca0f4a8f0a5579fdecd12741c
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 8 deletions.
@@ -41,6 +41,7 @@
import org.springframework.beans.factory.annotation.Autowired;

import javax.script.*;
import java.io.File;
import java.util.*;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -177,12 +178,15 @@ public void submitDataParsingWorkflow(WorkflowInvocationRequest request) throws

for(String parserId: parserInputMappings.keySet()) {

String parserWorkingDir = "/tmp/" + UUID.randomUUID();

GenericDataParsingTask dataParsingTask = new GenericDataParsingTask();
dataParsingTask.setTaskId("DPT-" + UUID.randomUUID().toString());
dataParsingTask.setParserId(parserId);
dataParsingTask.setParserServiceHost(orchHost);
dataParsingTask.setParserServicePort(orchPort);
dataParsingTask.setInputMapping(parserInputMappings.get(parserId));
dataParsingTask.setWorkingDirectory(parserWorkingDir);
taskMap.put(dataParsingTask.getTaskId(), dataParsingTask);

OutPort outPort = new OutPort();
@@ -208,8 +212,8 @@ public void submitDataParsingWorkflow(WorkflowInvocationRequest request) throws
mpt.setServiceAccountKey(mftClientId);
mpt.setServiceAccountSecret(mftClientSecret);
mpt.setResourceId(sourceResourceId);
mpt.setJsonFile("$" + dataParsingTask.getTaskId() +
"-" + dataParserOutputInterface.getOutputName());
mpt.setJsonFile(parserWorkingDir +
File.separator + "outputs" + File.separator + dataParserOutputInterface.getOutputName());
OutPort dpOut = new OutPort();
dpOut.setNextTaskId(mpt.getTaskId());
dataParsingTask.addOutPort(dpOut);
@@ -69,6 +69,9 @@ public class GenericDataParsingTask extends BlockingTask {
@TaskParam(name = "ParserServicePort")
private ThreadLocal<Integer> parserServicePort = new ThreadLocal<>();

@TaskParam(name = "WorkingDirectory")
private ThreadLocal<String> workingDirectory = new ThreadLocal<>();

@Override
public TaskResult runBlockingCode() {

@@ -92,16 +95,15 @@ public TaskResult runBlockingCode() {
DataParser parser = parserFetchResponse.getParser();
List<DataParserInputInterface> inputInterfaces = parser.getInputInterfacesList();

String tempWorkDir = "/tmp/" + UUID.randomUUID();
String tempInputDir = tempWorkDir + File.separator + "inputs";
String tempOutputDir = tempWorkDir + File.separator + "outputs";
logger.info("Using temp working directory {}", tempWorkDir);
String tempInputDir = getWorkingDirectory() + File.separator + "inputs";
String tempOutputDir = getWorkingDirectory() + File.separator + "outputs";
logger.info("Using temp working directory {}", getWorkingDirectory());
try {
Files.createDirectory(Paths.get(tempWorkDir));
Files.createDirectory(Paths.get(getWorkingDirectory()));
Files.createDirectory(Paths.get(tempInputDir));
Files.createDirectory(Paths.get(tempOutputDir));
} catch (IOException e) {
logger.error("Failed to create temp working directories in {}", tempWorkDir, e);
logger.error("Failed to create temp working directories in {}", getWorkingDirectory(), e);
return new TaskResult(TaskResult.Status.FAILED, "Failed to create temp working directories");
}

@@ -240,6 +242,14 @@ public void onComplete() {
}
}

public String getWorkingDirectory() {
return workingDirectory.get();
}

public void setWorkingDirectory(String workingDirectory) {
this.workingDirectory.set(workingDirectory);
}

public String getParserId() {
return parserId.get();
}

0 comments on commit d3bf876

Please sign in to comment.