GRIFFIN-184 - service for download miss records

service/src/main/java/org/apache/griffin/core/job/JobController.java

@@ -87,8 +87,9 @@ public JobHealth getHealthInfo() {
     }
 
     @RequestMapping(path = "/jobs/download", method = RequestMethod.GET)
-    public ResponseEntity<Resource> download(@RequestParam("hdfsPath") String hdfsPath) throws IOException {
-        InputStreamResource resource = new InputStreamResource(FSUtil.getSampleInputStream(hdfsPath));
+    public ResponseEntity<Resource> download(@RequestParam("jobName") String jobName ,@RequestParam("ts") long timestamp) throws Exception {
+        String path = jobService.getJobHdfsPersistPath(jobName,timestamp);
+        InputStreamResource resource = new InputStreamResource(FSUtil.getMissSampleInputStream(path));
         return ResponseEntity.ok().
                 header("content-disposition", "attachment; filename = sampleMissingData.json")
                 .contentType(MediaType.APPLICATION_OCTET_STREAM)

service/src/main/java/org/apache/griffin/core/job/JobService.java

@@ -42,4 +42,6 @@ public interface JobService {
     List<JobInstanceBean> findInstancesOfJob(Long jobId, int page, int size);
 
     JobHealth getHealthInfo();
+
+    String getJobHdfsPersistPath(String jobName, long timestamp);
 }

service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java

@@ -20,6 +20,7 @@ Licensed to the Apache Software Foundation (ASF) under one
 package org.apache.griffin.core.job;
 
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.lang.StringUtils;
 import org.apache.griffin.core.exception.GriffinException;
 import org.apache.griffin.core.job.entity.*;
@@ -33,6 +34,8 @@ Licensed to the Apache Software Foundation (ASF) under one
 import org.apache.griffin.core.measure.repo.GriffinMeasureRepo;
 import org.apache.griffin.core.util.JsonUtil;
 import org.apache.griffin.core.util.YarnNetUtil;
+import org.json.JSONArray;
+import org.json.JSONObject;
 import org.quartz.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -55,6 +58,8 @@ Licensed to the Apache Software Foundation (ASF) under one
 import java.util.TimeZone;
 
 import static java.util.TimeZone.getTimeZone;
+import static org.apache.griffin.core.config.EnvConfig.ENV_BATCH;
+import static org.apache.griffin.core.config.EnvConfig.ENV_STREAMING;
 import static org.apache.griffin.core.exception.GriffinExceptionMessage.*;
 import static org.apache.griffin.core.job.entity.LivySessionStates.State.*;
 import static org.apache.griffin.core.job.entity.LivySessionStates.isActive;
@@ -511,4 +516,34 @@ public Boolean isJobHealthy(Long jobId) {
         List<JobInstanceBean> instances = instanceRepo.findByJobId(jobId, pageable);
         return !CollectionUtils.isEmpty(instances) && LivySessionStates.isHealthy(instances.get(0).getState());
     }
+
+    @Override
+    public String getJobHdfsPersistPath(String jobName, long timestamp) {
+        List<AbstractJob> jobList = jobRepo.findByJobNameAndDeleted(jobName, false);
+        if (jobList.size() == 0) {
+            return null;
+        }
+        if (jobList.get(0).getType().toLowerCase().equals("batch")) {
+            return getPersistPath(ENV_BATCH) + "/" + jobName + "/" + timestamp + "";
+        }
+
+        return getPersistPath(ENV_STREAMING) + "/" + jobName + "/" + timestamp + "";
+    }
+
+    private String getPersistPath(String jsonString) {
+        try {
+            JSONObject obj = new JSONObject(jsonString);
+            JSONArray persistArray = obj.getJSONArray("persist");
+            for (int i = 0; i < persistArray.length(); i++) {
+                if (persistArray.getJSONObject(i).get("type").equals("hdfs")) {
+                    return persistArray.getJSONObject(i).getJSONObject("config").getString("path");
+                }
+            }
+
+            return null;
+        } catch (Exception ex) {
+            LOGGER.error("Fail to get Persist path from {}", jsonString, ex);
+            return null;
+        }
+    }
 }

service/src/main/java/org/apache/griffin/core/util/FSUtil.java

@@ -27,7 +27,6 @@ Licensed to the Apache Software Foundation (ASF) under one
 import org.apache.hadoop.fs.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStream;
@@ -177,4 +176,23 @@ private static void checkHDFSConf() {
         }
     }
 
+    public static String getFirstMissRecordPath(String hdfsDir) throws Exception{
+        List<FileStatus> fileList = listFileStatus(hdfsDir);
+        for(int i=0; i<fileList.size();i++){
+            if(fileList.get(i).getPath().toUri().toString().toLowerCase().contains("missrecord")){
+                return fileList.get(i).getPath().toUri().toString();
+            }
+        }
+        return null;
+    }
+
+    public static InputStream getMissSampleInputStream(String path) throws Exception {
+        List<String> subDirList = listSubDir(path);
+        //FIXME: only handle 1-sub dir here now
+        for(int i=0; i< subDirList.size();i++){
+            return getSampleInputStream(getFirstMissRecordPath(subDirList.get(i)));
+        }
+        return getSampleInputStream(getFirstMissRecordPath(path));
+    }
+
 }

ui/angular/src/app/app.component.html

@@ -43,6 +43,7 @@
               <li><a href="#!/undercons"><i class="fa fa-user fa-fw"></i> User Profile</a></li>
               <li><a href="#!/undercons"><i class="fa fa-gear fa-fw"></i> Settings</a></li>
               <li class="divider"></li>
+              <li><a href="https://griffin.incubator.apache.org/" target="_blank"><i class="fa fa-home fa-fw"></i> Home Page</a></li>
               <li><a href="https://github.com/apache/incubator-griffin/blob/master/griffin-doc/service/api-guide.md" target="_blank"><i class="fa fa-book fa-fw"></i> API DOCs</a></li>
               <li><a href="https://github.com/apache/incubator-griffin/blob/master/griffin-doc/ui/user-guide.md" target="_blank"><i class="fa fa-question-circle fa-fw"></i> User Guide</a></li>
               <li><a href="mailto:dev@griffin.incubator.apache.org"><i class="fa fa-envelope fa-fw"></i> Contact us</a></li>