From 5b9f07980f21fab7c40033a0a76682fa0bbf82d0 Mon Sep 17 00:00:00 2001 From: "Li, Juan" Date: Tue, 17 Jul 2018 21:31:23 +0800 Subject: [PATCH 1/6] add home page link under user setting --- ui/angular/src/app/app.component.html | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/angular/src/app/app.component.html b/ui/angular/src/app/app.component.html index d5b2951b4..84b64136f 100644 --- a/ui/angular/src/app/app.component.html +++ b/ui/angular/src/app/app.component.html @@ -43,6 +43,7 @@
  • User Profile
  • Settings
  • +
  • Home Page
  • API DOCs
  • User Guide
  • Contact us
    • From 0cc51bc300b74957f51fb48e6a4b0a2702a29638 Mon Sep 17 00:00:00 2001 From: "Li, Juan" Date: Fri, 20 Jul 2018 23:47:13 +0800 Subject: [PATCH 2/6] [Service] download missRecord service part --- .../griffin/core/job/JobController.java | 5 +-- .../apache/griffin/core/job/JobService.java | 2 ++ .../griffin/core/job/JobServiceImpl.java | 35 +++++++++++++++++++ .../org/apache/griffin/core/util/FSUtil.java | 22 +++++++++++- 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/service/src/main/java/org/apache/griffin/core/job/JobController.java b/service/src/main/java/org/apache/griffin/core/job/JobController.java index 1d0a8aca4..4b630ec4c 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobController.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobController.java @@ -87,8 +87,9 @@ public JobHealth getHealthInfo() { } @RequestMapping(path = "/jobs/download", method = RequestMethod.GET) - public ResponseEntity download(@RequestParam("hdfsPath") String hdfsPath) throws IOException { - InputStreamResource resource = new InputStreamResource(FSUtil.getSampleInputStream(hdfsPath)); + public ResponseEntity download(@RequestParam("jobId") long jobId ,@RequestParam("ts") long timestamp) throws Exception { + String path = jobService.getJobHdfsPersistPath(jobId,timestamp); + InputStreamResource resource = new InputStreamResource(FSUtil.getMissSampleInputStream(path)); return ResponseEntity.ok(). header("content-disposition", "attachment; filename = sampleMissingData.json") .contentType(MediaType.APPLICATION_OCTET_STREAM) diff --git a/service/src/main/java/org/apache/griffin/core/job/JobService.java b/service/src/main/java/org/apache/griffin/core/job/JobService.java index 7539cea2f..3cb197e7a 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobService.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobService.java @@ -42,4 +42,6 @@ public interface JobService { List findInstancesOfJob(Long jobId, int page, int size); JobHealth getHealthInfo(); + + String getJobHdfsPersistPath(Long jobId, long timestamp); } diff --git a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java index 8e68567c2..47f3aa293 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java @@ -20,6 +20,7 @@ Licensed to the Apache Software Foundation (ASF) under one package org.apache.griffin.core.job; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang.StringUtils; import org.apache.griffin.core.exception.GriffinException; import org.apache.griffin.core.job.entity.*; @@ -33,6 +34,8 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.griffin.core.measure.repo.GriffinMeasureRepo; import org.apache.griffin.core.util.JsonUtil; import org.apache.griffin.core.util.YarnNetUtil; +import org.json.JSONArray; +import org.json.JSONObject; import org.quartz.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,6 +58,8 @@ Licensed to the Apache Software Foundation (ASF) under one import java.util.TimeZone; import static java.util.TimeZone.getTimeZone; +import static org.apache.griffin.core.config.EnvConfig.ENV_BATCH; +import static org.apache.griffin.core.config.EnvConfig.ENV_STREAMING; import static org.apache.griffin.core.exception.GriffinExceptionMessage.*; import static org.apache.griffin.core.job.entity.LivySessionStates.State.*; import static org.apache.griffin.core.job.entity.LivySessionStates.isActive; @@ -511,4 +516,34 @@ public Boolean isJobHealthy(Long jobId) { List instances = instanceRepo.findByJobId(jobId, pageable); return !CollectionUtils.isEmpty(instances) && LivySessionStates.isHealthy(instances.get(0).getState()); } + + @Override + public String getJobHdfsPersistPath(Long jobId, long timestamp) { + AbstractJob job = jobRepo.findByIdAndDeleted(jobId, false); + if (job == null) { + return null; + } + if (job.getType().toLowerCase().equals("batch")) { + return getPersistPath(ENV_BATCH) + "/" + job.getJobName() + "/" + timestamp + ""; + } + + return getPersistPath(ENV_STREAMING) + "/" + job.getJobName() + "/" + timestamp + ""; + } + + private String getPersistPath(String jsonString) { + try { + JSONObject obj = new JSONObject(jsonString); + JSONArray persistArray = obj.getJSONArray("persist"); + for (int i = 0; i < persistArray.length(); i++) { + if (persistArray.getJSONObject(i).get("type").equals("hdfs")) { + return persistArray.getJSONObject(i).getJSONObject("config").getString("path"); + } + } + + return null; + } catch (Exception ex) { + LOGGER.error("Fail to get Persist path from {}", jsonString, ex); + return null; + } + } } diff --git a/service/src/main/java/org/apache/griffin/core/util/FSUtil.java b/service/src/main/java/org/apache/griffin/core/util/FSUtil.java index ed40ee285..6e2f1e95d 100644 --- a/service/src/main/java/org/apache/griffin/core/util/FSUtil.java +++ b/service/src/main/java/org/apache/griffin/core/util/FSUtil.java @@ -27,7 +27,6 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.hadoop.fs.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -36,6 +35,7 @@ Licensed to the Apache Software Foundation (ASF) under one import java.util.List; import static org.apache.griffin.core.exception.GriffinExceptionMessage.HDFS_FILE_NOT_EXIST; +import static org.apache.griffin.core.exception.GriffinExceptionMessage.STREAMING_JOB_IS_STOPPED; public class FSUtil { @@ -54,6 +54,7 @@ private static FileSystem getFileSystem() { private static void initFileSystem() { Configuration conf = new Configuration(); + conf.set("fs.defaultFS", "hdfs://localhost:8020"); if (StringUtils.isEmpty(conf.get("fs.hdfs.impl"))) { LOGGER.info("Setting fs.hdfs.impl:{}", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); @@ -177,4 +178,23 @@ private static void checkHDFSConf() { } } + public static String getFirstMissRecordPath(String hdfsDir) throws Exception{ + List fileList = listFileStatus(hdfsDir); + for(int i=0; i subDirList = listSubDir(path); + //FIXME: only handle 1-sub dir here now + for(int i=0; i< subDirList.size();i++){ + return getSampleInputStream(getFirstMissRecordPath(subDirList.get(i))); + } + return getSampleInputStream(getFirstMissRecordPath(path)); + } + } From f1da2d557eb1ba34d316b3680df28db1f07872e5 Mon Sep 17 00:00:00 2001 From: "Li, Juan" Date: Sun, 22 Jul 2018 12:16:57 +0800 Subject: [PATCH 3/6] Refine service --- .../org/apache/griffin/core/job/JobController.java | 4 ++-- .../java/org/apache/griffin/core/job/JobService.java | 2 +- .../org/apache/griffin/core/job/JobServiceImpl.java | 12 ++++++------ .../java/org/apache/griffin/core/util/FSUtil.java | 1 - 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/service/src/main/java/org/apache/griffin/core/job/JobController.java b/service/src/main/java/org/apache/griffin/core/job/JobController.java index 4b630ec4c..2f65b6336 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobController.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobController.java @@ -87,8 +87,8 @@ public JobHealth getHealthInfo() { } @RequestMapping(path = "/jobs/download", method = RequestMethod.GET) - public ResponseEntity download(@RequestParam("jobId") long jobId ,@RequestParam("ts") long timestamp) throws Exception { - String path = jobService.getJobHdfsPersistPath(jobId,timestamp); + public ResponseEntity download(@RequestParam("jobName") String jobName ,@RequestParam("ts") long timestamp) throws Exception { + String path = jobService.getJobHdfsPersistPath(jobName,timestamp); InputStreamResource resource = new InputStreamResource(FSUtil.getMissSampleInputStream(path)); return ResponseEntity.ok(). header("content-disposition", "attachment; filename = sampleMissingData.json") diff --git a/service/src/main/java/org/apache/griffin/core/job/JobService.java b/service/src/main/java/org/apache/griffin/core/job/JobService.java index 3cb197e7a..269f00476 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobService.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobService.java @@ -43,5 +43,5 @@ public interface JobService { JobHealth getHealthInfo(); - String getJobHdfsPersistPath(Long jobId, long timestamp); + String getJobHdfsPersistPath(String jobName, long timestamp); } diff --git a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java index 47f3aa293..29cb2491d 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java @@ -518,16 +518,16 @@ public Boolean isJobHealthy(Long jobId) { } @Override - public String getJobHdfsPersistPath(Long jobId, long timestamp) { - AbstractJob job = jobRepo.findByIdAndDeleted(jobId, false); - if (job == null) { + public String getJobHdfsPersistPath(String jobName, long timestamp) { + List jobList = jobRepo.findByJobNameAndDeleted(jobName, false); + if (jobList.size() == 0) { return null; } - if (job.getType().toLowerCase().equals("batch")) { - return getPersistPath(ENV_BATCH) + "/" + job.getJobName() + "/" + timestamp + ""; + if (jobList.get(0).getType().toLowerCase().equals("batch")) { + return getPersistPath(ENV_BATCH) + "/" + jobName + "/" + timestamp + ""; } - return getPersistPath(ENV_STREAMING) + "/" + job.getJobName() + "/" + timestamp + ""; + return getPersistPath(ENV_STREAMING) + "/" + jobName + "/" + timestamp + ""; } private String getPersistPath(String jsonString) { diff --git a/service/src/main/java/org/apache/griffin/core/util/FSUtil.java b/service/src/main/java/org/apache/griffin/core/util/FSUtil.java index 6e2f1e95d..c0c63eecf 100644 --- a/service/src/main/java/org/apache/griffin/core/util/FSUtil.java +++ b/service/src/main/java/org/apache/griffin/core/util/FSUtil.java @@ -35,7 +35,6 @@ Licensed to the Apache Software Foundation (ASF) under one import java.util.List; import static org.apache.griffin.core.exception.GriffinExceptionMessage.HDFS_FILE_NOT_EXIST; -import static org.apache.griffin.core.exception.GriffinExceptionMessage.STREAMING_JOB_IS_STOPPED; public class FSUtil { From 6e1ace4bfa2afbc09f73f815ca44238e7efeffc2 Mon Sep 17 00:00:00 2001 From: "Li, Juan" Date: Sun, 22 Jul 2018 12:23:44 +0800 Subject: [PATCH 4/6] UI part for feature of downloading miss record sample --- .../detail-metric.component.html | 45 ++++++++++++++ .../detail-metric/detail-metric.component.ts | 59 +++++++++++++++++-- ui/angular/src/app/service/service.service.ts | 5 +- 3 files changed, 103 insertions(+), 6 deletions(-) diff --git a/ui/angular/src/app/metric/detail-metric/detail-metric.component.html b/ui/angular/src/app/metric/detail-metric/detail-metric.component.html index 3d112b6a2..f99e5fed1 100644 --- a/ui/angular/src/app/metric/detail-metric/detail-metric.component.html +++ b/ui/angular/src/app/metric/detail-metric/detail-metric.component.html @@ -19,9 +19,13 @@
    No content
    +