From adc5aa71d2551b3c6b67765a4f3063ba0d7daca3 Mon Sep 17 00:00:00 2001 From: He Wang Date: Fri, 2 Feb 2018 13:07:29 +0800 Subject: [PATCH 1/2] add dqType as 'type' field of Metric DTO --- .../core/measure/entity/GriffinMeasure.java | 12 ------ .../griffin/core/measure/entity/Measure.java | 20 +++++++-- .../core/metric/MetricServiceImpl.java | 2 +- .../griffin/core/metric/MetricStoreImpl.java | 43 +++++++++++-------- .../griffin/core/metric/model/Metric.java | 12 +++++- .../core/metric/MetricControllerTest.java | 2 +- .../core/metric/MetricStoreImplTest.java | 19 ++++++++ 7 files changed, 73 insertions(+), 37 deletions(-) diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java b/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java index 6b060b53f..8576b86ed 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java +++ b/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java @@ -34,8 +34,6 @@ Licensed to the Apache Software Foundation (ASF) under one @Entity public class GriffinMeasure extends Measure { - private String dqType; - private String processType; @Transient @@ -52,16 +50,6 @@ public class GriffinMeasure extends Measure { @JoinColumn(name = "evaluate_rule_id") private EvaluateRule evaluateRule; - @JsonProperty("dq.type") - public String getDqType() { - return dqType; - } - - @JsonProperty("dq.type") - public void setDqType(String dqType) { - this.dqType = dqType; - } - @JsonProperty("process.type") public String getProcessType() { return processType; diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java b/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java index a5c97a1d9..4ccc75f12 100644 --- a/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java +++ b/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java @@ -38,13 +38,15 @@ public abstract class Measure extends AbstractAuditableEntity { @NotNull protected String name; - protected String description; + protected String owner; - protected String organization; + private String dqType; - protected String owner; + private String description; - protected Boolean deleted = false; + private String organization; + + private Boolean deleted = false; public String getName() { return name; @@ -54,6 +56,16 @@ public void setName(String name) { this.name = name; } + @JsonProperty("dq.type") + public String getDqType() { + return dqType; + } + + @JsonProperty("dq.type") + public void setDqType(String dqType) { + this.dqType = dqType; + } + public String getDescription() { return description; } diff --git a/service/src/main/java/org/apache/griffin/core/metric/MetricServiceImpl.java b/service/src/main/java/org/apache/griffin/core/metric/MetricServiceImpl.java index b09c0502b..77cc168c8 100644 --- a/service/src/main/java/org/apache/griffin/core/metric/MetricServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/metric/MetricServiceImpl.java @@ -71,7 +71,7 @@ public Map> getAllMetrics() { List metrics = new ArrayList<>(); for (AbstractJob job : jobList) { List metricValues = getMetricValues(job.getMetricName(), 0, 300); - metrics.add(new Metric(job.getMetricName(), measure.getOwner(), metricValues)); + metrics.add(new Metric(job.getMetricName(), measure.getDqType(), measure.getOwner(), metricValues)); } metricMap.put(measure.getName(), metrics); diff --git a/service/src/main/java/org/apache/griffin/core/metric/MetricStoreImpl.java b/service/src/main/java/org/apache/griffin/core/metric/MetricStoreImpl.java index b641cc7fd..360d4ec3d 100644 --- a/service/src/main/java/org/apache/griffin/core/metric/MetricStoreImpl.java +++ b/service/src/main/java/org/apache/griffin/core/metric/MetricStoreImpl.java @@ -33,6 +33,7 @@ Licensed to the Apache Software Foundation (ASF) under one import org.apache.http.nio.entity.NStringEntity; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.springframework.beans.factory.annotation.Value; @@ -43,7 +44,6 @@ Licensed to the Apache Software Foundation (ASF) under one import org.springframework.stereotype.Component; import java.io.IOException; -import java.nio.charset.Charset; import java.util.*; @Component @@ -51,14 +51,14 @@ public class MetricStoreImpl implements MetricStore { private static final String INDEX = "griffin"; private static final String TYPE = "accuracy"; - private static final String URL_BASE = "/griffin/accuracy"; private RestClient client; private HttpHeaders responseHeaders; - private String url_get; - private String url_delete; - private String url_post; + private String urlGet; + private String urlDelete; + private String urlPost; private ObjectMapper mapper; + private String indexMetaData; public MetricStoreImpl(@Value("${elasticsearch.host}") String host, @Value("${elasticsearch.port}") int port, @@ -73,21 +73,30 @@ public MetricStoreImpl(@Value("${elasticsearch.host}") String host, new BasicHeader(org.apache.http.HttpHeaders.AUTHORIZATION, encodedAuth)}; builder.setDefaultHeaders(requestHeaders); } - client = builder.build(); + this.client = builder.build(); HttpHeaders responseHeaders = new HttpHeaders(); responseHeaders.setContentType(MediaType.APPLICATION_JSON); this.responseHeaders = responseHeaders; - this.url_get = URL_BASE + "/_search?filter_path=hits.hits._source"; - this.url_post = URL_BASE + "/_bulk"; - this.url_delete = URL_BASE + "/_delete_by_query"; + String urlBase = String.format("/%s/%s", INDEX, TYPE); + this.urlGet = urlBase.concat("/_search?filter_path=hits.hits._source"); + this.urlPost = urlBase.concat("/_bulk"); + this.urlDelete = urlBase.concat("/_delete_by_query"); + this.indexMetaData = String.format("{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }\n", INDEX, TYPE); this.mapper = new ObjectMapper(); } @Override public List getMetricValues(String metricName, int from, int size) throws IOException { HttpEntity entity = getHttpEntityForSearch(metricName, from, size); - Response response = client.performRequest("GET", url_get, Collections.emptyMap(), entity); - return getMetricValuesFromResponse(response); + try { + Response response = client.performRequest("GET", urlGet, Collections.emptyMap(), entity); + return getMetricValuesFromResponse(response); + } catch (ResponseException e) { + if (e.getResponse().getStatusLine().getStatusCode() == 404) { + return Collections.emptyList(); + } + throw e; + } } private HttpEntity getHttpEntityForSearch(String metricName, int from, int size) throws JsonProcessingException { @@ -119,16 +128,15 @@ private List getMetricValuesFromResponse(Response response) throws public ResponseEntity addMetricValues(List metricValues) throws IOException { String bulkRequestBody = getBulkRequestBody(metricValues); HttpEntity entity = new NStringEntity(bulkRequestBody, ContentType.APPLICATION_JSON); - Response response = client.performRequest("POST", url_post, Collections.emptyMap(), entity); + Response response = client.performRequest("POST", urlPost, Collections.emptyMap(), entity); return getResponseEntityFromResponse(response); } private String getBulkRequestBody(List metricValues) throws JsonProcessingException { - String actionMetaData = String.format("{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }\n", INDEX, TYPE); StringBuilder bulkRequestBody = new StringBuilder(); for (MetricValue metricValue : metricValues) { - bulkRequestBody.append(actionMetaData); + bulkRequestBody.append(indexMetaData); bulkRequestBody.append(JsonUtil.toJson(metricValue)); bulkRequestBody.append("\n"); } @@ -141,7 +149,7 @@ public ResponseEntity deleteMetricValues(String metricName) throws IOException { Map param = Collections.singletonMap("query", Collections.singletonMap("term", Collections.singletonMap("name.keyword", metricName))); HttpEntity entity = new NStringEntity(JsonUtil.toJson(param), ContentType.APPLICATION_JSON); - Response response = client.performRequest("POST", url_delete, Collections.emptyMap(), entity); + Response response = client.performRequest("POST", urlDelete, Collections.emptyMap(), entity); return getResponseEntityFromResponse(response); } @@ -151,9 +159,8 @@ private ResponseEntity getResponseEntityFromResponse(Response response) throws I return new ResponseEntity<>(body, responseHeaders, status); } - private static String buildBasicAuthString (String user, String password) { + private static String buildBasicAuthString(String user, String password) { String auth = user + ":" + password; - String encodedAuth = "Basic " + Base64.getEncoder().encodeToString(auth.getBytes()); - return encodedAuth; + return "Basic " + Base64.getEncoder().encodeToString(auth.getBytes()); } } diff --git a/service/src/main/java/org/apache/griffin/core/metric/model/Metric.java b/service/src/main/java/org/apache/griffin/core/metric/model/Metric.java index 6de7a33b5..ed336586e 100644 --- a/service/src/main/java/org/apache/griffin/core/metric/model/Metric.java +++ b/service/src/main/java/org/apache/griffin/core/metric/model/Metric.java @@ -24,14 +24,16 @@ Licensed to the Apache Software Foundation (ASF) under one public class Metric { private String name; + private String type; private String owner; private List metricValues; public Metric() { } - public Metric(String name, String owner, List metricValues) { + public Metric(String name, String type, String owner, List metricValues) { this.name = name; + this.type = type; this.owner = owner; this.metricValues = metricValues; } @@ -44,6 +46,14 @@ public void setName(String name) { this.name = name; } + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + public String getOwner() { return owner; } diff --git a/service/src/test/java/org/apache/griffin/core/metric/MetricControllerTest.java b/service/src/test/java/org/apache/griffin/core/metric/MetricControllerTest.java index 97ca3b6a2..84c8fb381 100644 --- a/service/src/test/java/org/apache/griffin/core/metric/MetricControllerTest.java +++ b/service/src/test/java/org/apache/griffin/core/metric/MetricControllerTest.java @@ -73,7 +73,7 @@ public void setup() { @Test public void testGetAllMetricsSuccess() throws Exception { - Metric metric = new Metric("metricName", "owner", Collections.emptyList()); + Metric metric = new Metric("metricName", "accuracy", "owner", Collections.emptyList()); given(service.getAllMetrics()).willReturn( Collections.singletonMap("measureName", Collections.singletonList(metric))); diff --git a/service/src/test/java/org/apache/griffin/core/metric/MetricStoreImplTest.java b/service/src/test/java/org/apache/griffin/core/metric/MetricStoreImplTest.java index 2ff5a3fa9..79f873e6f 100644 --- a/service/src/test/java/org/apache/griffin/core/metric/MetricStoreImplTest.java +++ b/service/src/test/java/org/apache/griffin/core/metric/MetricStoreImplTest.java @@ -1,3 +1,22 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + package org.apache.griffin.core.metric; import org.junit.Test; From 9af36ef162137d38b460ac168a31e2fb08aa344d Mon Sep 17 00:00:00 2001 From: He Wang Date: Fri, 2 Feb 2018 13:19:53 +0800 Subject: [PATCH 2/2] change response of addJob method --- .../java/org/apache/griffin/core/job/JobController.java | 7 +++++-- .../main/java/org/apache/griffin/core/job/JobService.java | 7 +++++-- .../java/org/apache/griffin/core/job/JobServiceImpl.java | 5 ++--- .../org/apache/griffin/core/job/JobControllerTest.java | 5 ++--- .../org/apache/griffin/core/job/JobServiceImplTest.java | 4 ++-- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/service/src/main/java/org/apache/griffin/core/job/JobController.java b/service/src/main/java/org/apache/griffin/core/job/JobController.java index 7705e43a9..3f03f8ced 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobController.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobController.java @@ -19,7 +19,10 @@ Licensed to the Apache Software Foundation (ASF) under one package org.apache.griffin.core.job; -import org.apache.griffin.core.job.entity.*; +import org.apache.griffin.core.job.entity.JobDataBean; +import org.apache.griffin.core.job.entity.JobHealth; +import org.apache.griffin.core.job.entity.JobInstanceBean; +import org.apache.griffin.core.job.entity.JobSchedule; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.*; @@ -40,7 +43,7 @@ public List getJobs() { @RequestMapping(value = "/jobs", method = RequestMethod.POST) @ResponseStatus(HttpStatus.CREATED) - public GriffinJob addJob(@RequestBody JobSchedule jobSchedule) throws Exception { + public JobSchedule addJob(@RequestBody JobSchedule jobSchedule) throws Exception { return jobService.addJob(jobSchedule); } diff --git a/service/src/main/java/org/apache/griffin/core/job/JobService.java b/service/src/main/java/org/apache/griffin/core/job/JobService.java index 8c2c197db..3c15f8f74 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobService.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobService.java @@ -19,7 +19,10 @@ Licensed to the Apache Software Foundation (ASF) under one package org.apache.griffin.core.job; -import org.apache.griffin.core.job.entity.*; +import org.apache.griffin.core.job.entity.JobDataBean; +import org.apache.griffin.core.job.entity.JobHealth; +import org.apache.griffin.core.job.entity.JobInstanceBean; +import org.apache.griffin.core.job.entity.JobSchedule; import org.quartz.SchedulerException; import java.util.List; @@ -28,7 +31,7 @@ public interface JobService { List getAliveJobs(); - GriffinJob addJob(JobSchedule jobSchedule) throws Exception; + JobSchedule addJob(JobSchedule jobSchedule) throws Exception; void pauseJob(String group, String name) throws SchedulerException; diff --git a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java index 226ad6432..aa84f115b 100644 --- a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java +++ b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java @@ -137,7 +137,7 @@ private void setTriggerTime(Trigger trigger, JobDataBean jobBean) { @Override @Transactional(rollbackFor = Exception.class) - public GriffinJob addJob(JobSchedule js) throws Exception { + public JobSchedule addJob(JobSchedule js) throws Exception { Long measureId = js.getMeasureId(); GriffinMeasure measure = getMeasureIfValid(measureId); checkJobScheduleParams(js, measure); @@ -151,8 +151,7 @@ public GriffinJob addJob(JobSchedule js) throws Exception { job = jobRepo.save(job); js = jobScheduleRepo.save(js); addJob(triggerKey, js, job); - return job; - + return js; } private void addJob(TriggerKey triggerKey, JobSchedule js, GriffinJob job) throws Exception { diff --git a/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java b/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java index 02903068a..90a522b07 100644 --- a/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java +++ b/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java @@ -40,7 +40,6 @@ Licensed to the Apache Software Foundation (ASF) under one import static org.apache.griffin.core.exception.GriffinExceptionMessage.JOB_ID_DOES_NOT_EXIST; import static org.apache.griffin.core.exception.GriffinExceptionMessage.JOB_NAME_DOES_NOT_EXIST; -import static org.apache.griffin.core.util.EntityHelper.createGriffinJob; import static org.apache.griffin.core.util.EntityHelper.createJobSchedule; import static org.hamcrest.CoreMatchers.is; import static org.mockito.BDDMockito.given; @@ -84,8 +83,8 @@ public void testGetJobs() throws Exception { @Test public void testAddJobForSuccess() throws Exception { JobSchedule jobSchedule = createJobSchedule(); - GriffinJob job = createGriffinJob(); - given(service.addJob(jobSchedule)).willReturn(job); + jobSchedule.setId(1L); + given(service.addJob(jobSchedule)).willReturn(jobSchedule); mvc.perform(post(URLHelper.API_VERSION_PATH + "/jobs") .contentType(MediaType.APPLICATION_JSON) diff --git a/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java b/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java index a5beb51ed..ba2afaadf 100644 --- a/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java +++ b/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java @@ -146,8 +146,8 @@ public void testAddJobForSuccess() throws Exception { given(jobScheduleRepo.save(js)).willReturn(js); given(jobRepo.save(Matchers.any(GriffinJob.class))).willReturn(job); - GriffinJob griffinJob = service.addJob(js); - assertEquals(griffinJob.getJobName(), job.getJobName()); + JobSchedule createdJs = service.addJob(js); + assertEquals(js.getJobName(), createdJs.getJobName()); } @Test(expected = GriffinException.BadRequestException.class)