Skip to content

Commit

Permalink
Fix null in distributed load cmd output
Browse files Browse the repository at this point in the history
### What changes are proposed in this pull request?
fix null in distributedLoad when cmd is not using the batched job. And
also put guard when we are not able to find filePath.
Before this:
```
enterprise git:(enterprise-2.8.0-2.x) ✗ bin/alluxio fs distributedLoad
/1/dict2.pickle
Please wait for command submission to finish..
Submitted successfully, jobControlId = 1667603624290
Waiting for the command to finish ...
Get command status information below:
Successfully loaded path /1/dict2.pickle
Total completed file count is 1, failed file count is 1
Finished running the command, jobControlId = 1667603624290
Here are failed files:
null,
Check out ./logs/user/distributedLoad_1_dict2.pickle_failures.csv for
full list of failed files.
```
After this change:
```
➜ enterprise git:(enterprise-2.8.0-2.x) ✗ bin/alluxio fs
distributedLoad /1/dict3.pickle
Please wait for command submission to finish..
Submitted successfully, jobControlId = 1667856918305
Waiting for the command to finish ...
Get command status information below:
Total completed file count is 0, failed file count is 1
Finished running the command, jobControlId = 1667856918305
Here are failed files:
/1/dict3.pickle,
Check out ./logs/user/distributedLoad_1_dict3.pickle_failures.csv for
full list of failed files.

```

### Why are the changes needed?

bug fix

### Does this PR introduce any user facing changes?

na

pr-link: #16486
change-id: cid-f30ece47cbce6c66e27f96fef98f100c88a70050
  • Loading branch information
jja725 committed Nov 8, 2022
1 parent 0df477c commit 3df5641
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 3 deletions.
15 changes: 13 additions & 2 deletions job/server/src/main/java/alluxio/job/plan/load/LoadDefinition.java
Expand Up @@ -139,7 +139,7 @@ && isEmptySet(config.getExcludedLocalityIds()))) {
String address = workersWithoutBlock.get(i).getNetAddress().getHost();
WorkerInfo jobWorker = jobWorkersByAddress.get(address);
assignments.put(jobWorker, new LoadTask(blockInfo.getBlockInfo().getBlockId(),
workersWithoutBlock.get(i).getNetAddress()));
workersWithoutBlock.get(i).getNetAddress(), config.getFilePath()));
}
}

Expand Down Expand Up @@ -203,15 +203,18 @@ private boolean isEmptySet(Set s) {
public static class LoadTask implements Serializable {
private static final long serialVersionUID = 2028545900913354425L;
final long mBlockId;
final String mFilePath;
final WorkerNetAddress mWorkerNetAddress;

/**
* @param blockId the id of the block to load
* @param workerNetAddress worker net address
* @param filePath file path for task
*/
public LoadTask(long blockId, WorkerNetAddress workerNetAddress) {
public LoadTask(long blockId, WorkerNetAddress workerNetAddress, String filePath) {
mBlockId = blockId;
mWorkerNetAddress = workerNetAddress;
mFilePath = filePath;
}

/**
Expand All @@ -221,6 +224,13 @@ public long getBlockId() {
return mBlockId;
}

/**
* @return the file path
*/
public String getFilePath() {
return mFilePath;
}

/**
* @return worker net address
*/
Expand All @@ -231,6 +241,7 @@ public WorkerNetAddress getWorkerNetAddress() {
@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("FilePath", mFilePath) // first letter has to be Upper case for parsing
.add("blockId", mBlockId)
.add("workerNetAddress", mWorkerNetAddress)
.toString();
Expand Down
Expand Up @@ -188,7 +188,11 @@ public Status checkJobStatus() {
Set<JobInfo> failed = jobInfo.getChildren().stream()
.filter(child -> child.getStatus() == Status.FAILED).collect(Collectors.toSet());
for (JobInfo task : failed) {
mFailedFiles.add(StringUtils.substringBetween(task.getDescription(), prefix, ","));
String filePath = StringUtils.substringBetween(task.getDescription(), prefix, ",");
if (filePath == null) {
filePath = String.format("unknown filePath for task: %s", task.getId());
}
mFailedFiles.add(filePath);
}
}
return jobInfo.getStatus();
Expand Down

0 comments on commit 3df5641

Please sign in to comment.