From 35d06d9c8cd27064b35cc6c1641e1f080829019e Mon Sep 17 00:00:00 2001
From: Akshay Rai <akshayrai09@gmail.com>
Date: Fri, 17 Nov 2017 12:27:58 +0530
Subject: [PATCH] Dr. Elephant should check for finished directory before
 listing

---
 .../fetchers/MapReduceFSFetcherHadoop2.java   | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFSFetcherHadoop2.java b/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFSFetcherHadoop2.java
index 5b1cab68a..ec025927d 100644
--- a/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFSFetcherHadoop2.java
+++ b/app/com/linkedin/drelephant/mapreduce/fetchers/MapReduceFSFetcherHadoop2.java
@@ -153,14 +153,17 @@ private DataFiles getHistoryFiles(AnalyticJob job) throws IOException {
 
     // Search files in done dir
     String jobHistoryDirPath = getHistoryDir(job);
-    RemoteIterator<LocatedFileStatus> it = _fs.listFiles(new Path(jobHistoryDirPath), false);
-    while (it.hasNext() && (jobConfPath == null || jobHistPath == null)) {
-      String name = it.next().getPath().getName();
-      if (name.contains(jobId)) {
-        if (name.endsWith("_conf.xml")) {
-          jobConfPath = jobHistoryDirPath + name;
-        } else if (name.endsWith(".jhist")) {
-          jobHistPath = jobHistoryDirPath + name;
+
+    if (_fs.exists(new Path(jobHistoryDirPath))) {
+      RemoteIterator<LocatedFileStatus> it = _fs.listFiles(new Path(jobHistoryDirPath), false);
+      while (it.hasNext() && (jobConfPath == null || jobHistPath == null)) {
+        String name = it.next().getPath().getName();
+        if (name.contains(jobId)) {
+          if (name.endsWith("_conf.xml")) {
+            jobConfPath = jobHistoryDirPath + name;
+          } else if (name.endsWith(".jhist")) {
+            jobHistPath = jobHistoryDirPath + name;
+          }
         }
       }
     }
@@ -178,7 +181,7 @@ private DataFiles getHistoryFiles(AnalyticJob job) throws IOException {
     }
     if (jobHistPath == null) {
       try {
-        it = _fs.listFiles(new Path(intermediateDirPath), false);
+        RemoteIterator<LocatedFileStatus> it = _fs.listFiles(new Path(intermediateDirPath), false);
         while (it.hasNext()) {
           String name = it.next().getPath().getName();
           if (name.contains(jobId) && name.endsWith(".jhist")) {