From e4619635cee9500d971206447b6f40321dc6ebf9 Mon Sep 17 00:00:00 2001 From: Puneet Jaiswal Date: Mon, 9 Jan 2017 10:31:11 -0800 Subject: [PATCH 1/3] Pushing daily email generator script as a backend service --- .../anomaly/alert/AlertTaskRunner.java | 7 +- .../alert/{ => util}/AlertFilterHelper.java | 2 +- .../alert/util/AnomalyReportGenerator.java | 249 ++++++++++++++++++ .../anomaly/alert/{ => util}/EmailHelper.java | 2 +- .../dashboard/resources/EmailResource.java | 77 ++++++ .../bao/MergedAnomalyResultManager.java | 5 +- .../datalayer/bao/MetricConfigManager.java | 1 + .../jdbc/MergedAnomalyResultManagerImpl.java | 2 - .../bao/jdbc/MetricConfigManagerImpl.java | 9 + .../detector/custom-anomaly-report.ftl | 5 +- .../tools/RunAdhocDatabaseQueriesTool.java | 8 - .../anomaly/report/AnomalyReportConfig.java | 9 + .../anomaly/report/GenerateAnomalyReport.java | 2 +- 13 files changed, 358 insertions(+), 20 deletions(-) rename thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/{ => util}/AlertFilterHelper.java (92%) create mode 100644 thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AnomalyReportGenerator.java rename thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/{ => util}/EmailHelper.java (99%) diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/AlertTaskRunner.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/AlertTaskRunner.java index c98c7b9847f9..7661d437b10c 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/AlertTaskRunner.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/AlertTaskRunner.java @@ -1,6 +1,8 @@ package com.linkedin.thirdeye.anomaly.alert; import com.linkedin.thirdeye.anomaly.alert.template.pojo.MetricDimensionReport; +import com.linkedin.thirdeye.anomaly.alert.util.AlertFilterHelper; +import com.linkedin.thirdeye.anomaly.alert.util.EmailHelper; import com.linkedin.thirdeye.api.DimensionMap; import com.linkedin.thirdeye.client.DAORegistry; import com.linkedin.thirdeye.dashboard.views.contributor.ContributorViewResponse; @@ -48,7 +50,7 @@ import freemarker.template.TemplateModelException; import freemarker.template.TemplateNumberModel; -import static com.linkedin.thirdeye.anomaly.alert.AlertFilterHelper.FILTER_TYPE_KEY; +import static com.linkedin.thirdeye.anomaly.alert.util.AlertFilterHelper.FILTER_TYPE_KEY; public class AlertTaskRunner implements TaskRunner { @@ -243,7 +245,8 @@ private void sendAlertForAnomalies(String collectionAlias, List metricDimensionValueReports; List reports = new ArrayList<>(); for (String dimension : alertConfig.getDimensions()) { - ContributorViewResponse report = EmailHelper.getContributorData(collectionAlias, alertConfig.getMetric(), Arrays.asList(dimension)); + ContributorViewResponse report = EmailHelper + .getContributorData(collectionAlias, alertConfig.getMetric(), Arrays.asList(dimension)); if(report != null) { reports.add(report); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/AlertFilterHelper.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AlertFilterHelper.java similarity index 92% rename from thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/AlertFilterHelper.java rename to thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AlertFilterHelper.java index 15ade8df1f27..b90ff3735f71 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/AlertFilterHelper.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AlertFilterHelper.java @@ -1,4 +1,4 @@ -package com.linkedin.thirdeye.anomaly.alert; +package com.linkedin.thirdeye.anomaly.alert.util; import com.linkedin.thirdeye.detector.email.filter.AlertFilter; import com.linkedin.thirdeye.detector.email.filter.AlphaBetaAlertFilter; diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AnomalyReportGenerator.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AnomalyReportGenerator.java new file mode 100644 index 000000000000..9f2ba9d1c575 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AnomalyReportGenerator.java @@ -0,0 +1,249 @@ +package com.linkedin.thirdeye.anomaly.alert.util; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.google.common.base.Throwables; +import com.linkedin.thirdeye.anomaly.SmtpConfiguration; +import com.linkedin.thirdeye.anomaly.ThirdEyeAnomalyConfiguration; +import com.linkedin.thirdeye.anomaly.alert.AlertTaskRunner; +import com.linkedin.thirdeye.client.DAORegistry; +import com.linkedin.thirdeye.constant.AnomalyFeedbackType; +import com.linkedin.thirdeye.datalayer.bao.MergedAnomalyResultManager; +import com.linkedin.thirdeye.datalayer.bao.MetricConfigManager; +import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO; +import com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO; +import freemarker.template.Configuration; +import freemarker.template.Template; +import freemarker.template.TemplateExceptionHandler; +import java.io.ByteArrayOutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.commons.mail.HtmlEmail; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AnomalyReportGenerator { + // private static final DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ"); + + private static final Logger LOG = LoggerFactory.getLogger(AnomalyReportGenerator.class); + + private static final AnomalyReportGenerator INSTANCE = new AnomalyReportGenerator(); + public static AnomalyReportGenerator getInstance() { + return INSTANCE; + } + + MergedAnomalyResultManager anomalyResultManager = DAORegistry.getInstance().getMergedAnomalyResultDAO(); + MetricConfigManager metricConfigManager = DAORegistry.getInstance().getMetricConfigDAO(); + + public List getAnomaliesForDatasets(List collections, long startTime, long endTime) { + List anomalies = new ArrayList<>(); + for (String collection : collections) { + anomalies.addAll(anomalyResultManager.findByCollectionTime(collection, startTime, endTime, false)); + } + return anomalies; + } + + public List getAnomaliesForMetrics(List metrics, long startTime, long endTime) { + List anomalies = new ArrayList<>(); + for (String metric : metrics) { + MetricConfigDTO metricConfigDTO = metricConfigManager.findByMetricName(metric); + anomalies.addAll(anomalyResultManager.findByCollectionMetricTime(metricConfigDTO.getDataset(), metric, startTime, endTime, false)); + } + return anomalies; + } + + public void buildReport(long startTime, long endTime, List anomalies, + ThirdEyeAnomalyConfiguration configuration, boolean includeSentAnomaliesOnly, + String emailRecipients) { + if (anomalies == null || anomalies.size() == 0) { + LOG.info("No anomalies found, please check the parameters.. exiting"); + } else { + Set metrics = new HashSet<>(); + int alertedAnomalies = 0; + int feedbackCollected = 0; + int trueAlert = 0; + int falseAlert = 0; + int nonActionable = 0; + + List anomalyReportDTOList = new ArrayList<>(); + + for (MergedAnomalyResultDTO anomaly : anomalies) { + metrics.add(anomaly.getMetric()); + if (anomaly.getFeedback() != null) { + feedbackCollected++; + if (anomaly.getFeedback().getFeedbackType().equals(AnomalyFeedbackType.ANOMALY)) { + trueAlert++; + } else if (anomaly.getFeedback().getFeedbackType() + .equals(AnomalyFeedbackType.NOT_ANOMALY)) { + falseAlert++; + } else { + nonActionable++; + } + } + String feedbackVal = getFeedback( + anomaly.getFeedback() == null ? "NA" : anomaly.getFeedback().getFeedbackType().name()); + + AnomalyReportDTO anomalyReportDTO = + new AnomalyReportDTO(String.valueOf(anomaly.getId()), feedbackVal, + String.format("%+.2f", anomaly.getWeight()), anomaly.getMetric(), + new Date(anomaly.getStartTime()).toString(), String + .format("%.2f", getTimeDiffInHours(anomaly.getStartTime(), anomaly.getEndTime())), + getAnomalyURL(anomaly, configuration.getDashboardHost())); + + if (anomaly.isNotified()) { + alertedAnomalies++; + } + // include notified alerts only in the email + if (includeSentAnomaliesOnly) { + if (anomaly.isNotified()) { + anomalyReportDTOList.add(anomalyReportDTO); + } + } else { + anomalyReportDTOList.add(anomalyReportDTO); + } + } + + Map templateData = new HashMap<>(); + templateData.put("startTime", new Date(startTime)); + templateData.put("endTime", new Date(endTime)); + templateData.put("anomalyCount", anomalies.size()); + templateData.put("metricsCount", metrics.size()); + templateData.put("notifiedCount", alertedAnomalies); + templateData.put("feedbackCount", feedbackCollected); + templateData.put("trueAlertCount", trueAlert); + templateData.put("falseAlertCount", falseAlert); + templateData.put("nonActionableCount", nonActionable); + templateData.put("anomalyDetails", anomalyReportDTOList); + buildEmailTemplateAndSendAlert(templateData, configuration.getSmtpConfiguration(), + emailRecipients); + } + } + + void buildEmailTemplateAndSendAlert(Map paramMap, + SmtpConfiguration smtpConfiguration, String emailRecipients) { + HtmlEmail email = new HtmlEmail(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (Writer out = new OutputStreamWriter(baos, AlertTaskRunner.CHARSET)) { + Configuration freemarkerConfig = new Configuration(Configuration.VERSION_2_3_21); + freemarkerConfig.setClassForTemplateLoading(getClass(), "/com/linkedin/thirdeye/detector"); + freemarkerConfig.setDefaultEncoding(AlertTaskRunner.CHARSET); + freemarkerConfig.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER); + Template template = freemarkerConfig.getTemplate("custom-anomaly-report.ftl"); + template.process(paramMap, out); + + String alertEmailSubject = "Thirdeye : Daily anomaly report"; + String alertEmailHtml = new String(baos.toByteArray(), AlertTaskRunner.CHARSET); + EmailHelper.sendEmailWithHtml(email, smtpConfiguration, alertEmailSubject, alertEmailHtml, + "thirdeye-dev@linkedin.com", emailRecipients); + } catch (Exception e) { + Throwables.propagate(e); + } + } + + double getTimeDiffInHours(long start, long end) { + return Double.valueOf((end - start) / 1000) / 3600; + } + + String getFeedback(String feedbackType) { + switch (feedbackType) { + case "ANOMALY": + return "Confirmed Anomaly"; + case "NOT_ANOMALY": + return "False Alarm"; + case "ANOMALY_NO_ACTION": + return "Not Actionable"; + } + return "NA"; + } + + String getAnomalyURL(MergedAnomalyResultDTO anomalyResultDTO, String dashboardUrl) { + String urlPart = "#view=anomalies&dataset=%s&metrics=%s¤tStart=%s¤tEnd=%s"; + return dashboardUrl + String + .format(urlPart, anomalyResultDTO.getCollection(), anomalyResultDTO.getMetric(), + anomalyResultDTO.getStartTime(), anomalyResultDTO.getEndTime()); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static class AnomalyReportDTO { + String metric; + String startDateTime; + String windowSize; + String lift; + String feedback; + String anomalyId; + String anomalyURL; + + public AnomalyReportDTO(String anomalyId, String feedback, String lift, String metric, + String startDateTime, String windowSize, String anomalyURL) { + this.anomalyId = anomalyId; + this.feedback = feedback; + this.lift = lift; + this.metric = metric; + this.startDateTime = startDateTime; + this.windowSize = windowSize; + this.anomalyURL = anomalyURL; + } + + public String getAnomalyId() { + return anomalyId; + } + + public void setAnomalyId(String anomalyId) { + this.anomalyId = anomalyId; + } + + public String getFeedback() { + return feedback; + } + + public void setFeedback(String feedback) { + this.feedback = feedback; + } + + public String getLift() { + return lift; + } + + public void setLift(String lift) { + this.lift = lift; + } + + public String getMetric() { + return metric; + } + + public void setMetric(String metric) { + this.metric = metric; + } + + public String getStartDateTime() { + return startDateTime; + } + + public void setStartDateTime(String startDateTime) { + this.startDateTime = startDateTime; + } + + public String getWindowSize() { + return windowSize; + } + + public void setWindowSize(String windowSize) { + this.windowSize = windowSize; + } + + public String getAnomalyURL() { + return anomalyURL; + } + + public void setAnomalyURL(String anomalyURL) { + this.anomalyURL = anomalyURL; + } + } +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/EmailHelper.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/EmailHelper.java similarity index 99% rename from thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/EmailHelper.java rename to thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/EmailHelper.java index 04cd135df186..da5fa240685e 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/EmailHelper.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/EmailHelper.java @@ -1,4 +1,4 @@ -package com.linkedin.thirdeye.anomaly.alert; +package com.linkedin.thirdeye.anomaly.alert.util; import com.google.common.cache.LoadingCache; import com.linkedin.thirdeye.anomaly.SmtpConfiguration; diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/EmailResource.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/EmailResource.java index b61163c15a41..70566d3c9805 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/EmailResource.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/EmailResource.java @@ -1,5 +1,11 @@ package com.linkedin.thirdeye.dashboard.resources; +import com.google.common.base.Strings; +import com.linkedin.thirdeye.anomaly.SmtpConfiguration; +import com.linkedin.thirdeye.anomaly.ThirdEyeAnomalyConfiguration; +import com.linkedin.thirdeye.anomaly.alert.util.AnomalyReportGenerator; +import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO; +import java.util.Arrays; import java.util.List; import javax.ws.rs.DELETE; @@ -9,6 +15,7 @@ import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -114,4 +121,74 @@ public Response deleteByEmail(@PathParam("emailId") Long emailId) { public List findEmailIdsByFunction(@PathParam("functionId") Long functionId) { return emailDAO.findByFunctionId(functionId); } + + @GET + @Path("generate/datasets/{startTime}/{endTime}") + public Response generateAndSendAlertForDatasets(@PathParam("startTime") Long startTime, + @PathParam("endTime") Long endTime, @QueryParam("datasets") String datasets, + @QueryParam("from") String fromAddr, @QueryParam("to") String toAddr, + @QueryParam("includeSentAnomaliesOnly") boolean includeSentAnomaliesOnly, + @QueryParam("teHost") String teHost, @QueryParam("smtpHost") String smtpHost, @QueryParam("smtpPort") int smtpPort) { + AnomalyReportGenerator anomalyReportGenerator = AnomalyReportGenerator.getInstance(); + String [] dataSetArr = datasets.split(","); + if (dataSetArr.length == 0) { + throw new WebApplicationException("Datasets empty : " + datasets); + } + if (Strings.isNullOrEmpty(toAddr)) { + throw new WebApplicationException("Empty : list of recipients" + toAddr); + } + if(Strings.isNullOrEmpty(teHost)) { + throw new WebApplicationException("Invalid TE host" + teHost); + } + if (Strings.isNullOrEmpty(smtpHost)) { + throw new WebApplicationException("invalid smtp host" + smtpHost); + } + List anomalies = anomalyReportGenerator + .getAnomaliesForDatasets(Arrays.asList(dataSetArr), startTime, endTime); + ThirdEyeAnomalyConfiguration configuration = new ThirdEyeAnomalyConfiguration(); + SmtpConfiguration smtpConfiguration = new SmtpConfiguration(); + smtpConfiguration.setSmtpHost(smtpHost); + smtpConfiguration.setSmtpPort(smtpPort); + + configuration.setSmtpConfiguration(smtpConfiguration); + configuration.setDashboardHost(teHost); + + anomalyReportGenerator.buildReport(startTime, endTime, anomalies, configuration, includeSentAnomaliesOnly, toAddr); + return Response.ok().build(); + } + + @GET + @Path("generate/metrics/{startTime}/{endTime}") + public Response generateAndSendAlertForMetrics(@PathParam("startTime") Long startTime, + @PathParam("endTime") Long endTime, @QueryParam("metrics") String metrics, + @QueryParam("from") String fromAddr, @QueryParam("to") String toAddr, + @QueryParam("includeSentAnomaliesOnly") boolean includeSentAnomaliesOnly, + @QueryParam("teHost") String teHost, @QueryParam("smtpHost") String smtpHost, @QueryParam("smtpPort") int smtpPort) { + AnomalyReportGenerator anomalyReportGenerator = AnomalyReportGenerator.getInstance(); + String [] metricsArr = metrics.split(","); + if (metricsArr.length == 0) { + throw new WebApplicationException("metrics empty : " + metricsArr); + } + if (Strings.isNullOrEmpty(toAddr)) { + throw new WebApplicationException("Empty : list of recipients" + toAddr); + } + if(Strings.isNullOrEmpty(teHost)) { + throw new WebApplicationException("Invalid TE host" + teHost); + } + if (Strings.isNullOrEmpty(smtpHost)) { + throw new WebApplicationException("invalid smtp host" + smtpHost); + } + List anomalies = anomalyReportGenerator + .getAnomaliesForMetrics(Arrays.asList(metricsArr), startTime, endTime); + ThirdEyeAnomalyConfiguration configuration = new ThirdEyeAnomalyConfiguration(); + SmtpConfiguration smtpConfiguration = new SmtpConfiguration(); + smtpConfiguration.setSmtpHost(smtpHost); + smtpConfiguration.setSmtpPort(smtpPort); + + configuration.setSmtpConfiguration(smtpConfiguration); + configuration.setDashboardHost(teHost); + + anomalyReportGenerator.buildReport(startTime, endTime, anomalies, configuration, includeSentAnomaliesOnly, toAddr); + return Response.ok().build(); + } } diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MergedAnomalyResultManager.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MergedAnomalyResultManager.java index 25144887fb91..a88a7d01ec42 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MergedAnomalyResultManager.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MergedAnomalyResultManager.java @@ -18,8 +18,9 @@ List getAllByTimeEmailIdAndNotifiedFalse(long startTime, List findByCollectionMetricDimensionsTime(String collection, String metric, String dimensions, long startTime, long endTime, boolean loadRawAnomalies); - List findByCollectionMetricTime(String collection, String metric, - long startTime, long endTime, boolean loadRawAnomalies); + List findByCollectionMetricTime(String collection, String metric, long startTime, long endTime, boolean loadRawAnomalies); + + // TODO : add findByMetricId - currently we are not updating metricId in table. List findByCollectionTime(String collection, long startTime, long endTime, boolean loadRawAnomalies); diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MetricConfigManager.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MetricConfigManager.java index afef5b7dade4..2537bfa47569 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MetricConfigManager.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MetricConfigManager.java @@ -11,4 +11,5 @@ public interface MetricConfigManager extends AbstractManager { MetricConfigDTO findByMetricAndDataset(String metricName, String dataset); MetricConfigDTO findByAliasAndDataset(String alias, String dataset); List findActiveByDataset(String dataset); + MetricConfigDTO findByMetricName(String metricName); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java index 39e96f6c1771..72e11bd37061 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MergedAnomalyResultManagerImpl.java @@ -171,7 +171,6 @@ public List findByCollectionMetricDimensionsTime(String return batchConvertMergedAnomalyBean2DTO(list, loadRawAnomalies); } - @Override public List findByCollectionMetricTime(String collection, String metric, long startTime, long endTime, boolean loadRawAnomalies) { @@ -215,7 +214,6 @@ public MergedAnomalyResultDTO findLatestConflictByFunctionIdDimensions(Long func MergedAnomalyResultBean mostRecentConflictMergedAnomalyResultBean = list.get(0); return convertMergedAnomalyBean2DTO(mostRecentConflictMergedAnomalyResultBean, true); } - return null; } diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MetricConfigManagerImpl.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MetricConfigManagerImpl.java index 1ae209fed5a5..7d9600adc304 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MetricConfigManagerImpl.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MetricConfigManagerImpl.java @@ -56,6 +56,15 @@ public MetricConfigDTO findByMetricAndDataset(String metricName, String dataset) return result; } + public MetricConfigDTO findByMetricName(String metricName) { + Predicate metricNamePredicate = Predicate.EQ("name", metricName); + List list = genericPojoDao.get(metricNamePredicate, MetricConfigBean.class); + MetricConfigDTO result = null; + if (CollectionUtils.isNotEmpty(list)) { + result = MODEL_MAPPER.map(list.get(0), MetricConfigDTO.class); + } + return result; + } @Override public MetricConfigDTO findByAliasAndDataset(String alias, String dataset) { diff --git a/thirdeye/thirdeye-pinot/src/main/resources/com/linkedin/thirdeye/detector/custom-anomaly-report.ftl b/thirdeye/thirdeye-pinot/src/main/resources/com/linkedin/thirdeye/detector/custom-anomaly-report.ftl index 4d12d7a42b2b..aee0b6d1ba98 100644 --- a/thirdeye/thirdeye-pinot/src/main/resources/com/linkedin/thirdeye/detector/custom-anomaly-report.ftl +++ b/thirdeye/thirdeye-pinot/src/main/resources/com/linkedin/thirdeye/detector/custom-anomaly-report.ftl @@ -10,9 +10,8 @@ -

Datasets : [${datasets}]

-

Analysis Start : ${startTime}

-

Analysis End : ${endTime}

+

Analysis Start : ${startTime?date}

+

Analysis End : ${endTime?date}

diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/RunAdhocDatabaseQueriesTool.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/RunAdhocDatabaseQueriesTool.java index eb89e377d069..a0f58e404a8d 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/RunAdhocDatabaseQueriesTool.java +++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/RunAdhocDatabaseQueriesTool.java @@ -1,18 +1,11 @@ package com.linkedin.thirdeye.tools; -import com.linkedin.thirdeye.anomaly.alert.AlertFilterHelper; -import com.linkedin.thirdeye.anomaly.override.OverrideConfigHelper; import com.linkedin.thirdeye.datalayer.bao.OverrideConfigManager; import com.linkedin.thirdeye.datalayer.dto.OverrideConfigDTO; -import com.linkedin.thirdeye.detector.email.filter.AlphaBetaAlertFilter; import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; import java.util.List; import java.util.Map; -import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,7 +20,6 @@ import com.linkedin.thirdeye.datalayer.dto.DashboardConfigDTO; import com.linkedin.thirdeye.datalayer.dto.EmailConfigurationDTO; import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO; -import com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO; import com.linkedin.thirdeye.datalayer.util.DaoProviderUtil; import com.linkedin.thirdeye.util.ThirdEyeUtils; diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/anomaly/report/AnomalyReportConfig.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/anomaly/report/AnomalyReportConfig.java index 0bfb68ebab08..925940474f0e 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/anomaly/report/AnomalyReportConfig.java +++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/anomaly/report/AnomalyReportConfig.java @@ -7,6 +7,7 @@ public class AnomalyReportConfig { private String datasets; private String teBaseUrl; private String emailRecipients; + private boolean includeNotifiedOnly = true; public String getEndTimeIso() { return endTimeIso; @@ -55,4 +56,12 @@ public String getEmailRecipients() { public void setEmailRecipients(String emailRecipients) { this.emailRecipients = emailRecipients; } + + public boolean isIncludeNotifiedOnly() { + return includeNotifiedOnly; + } + + public void setIncludeNotifiedOnly(boolean includeNotifiedOnly) { + this.includeNotifiedOnly = includeNotifiedOnly; + } } diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/anomaly/report/GenerateAnomalyReport.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/anomaly/report/GenerateAnomalyReport.java index 2f4355c926ab..57ce928ab3d6 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/anomaly/report/GenerateAnomalyReport.java +++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/tools/anomaly/report/GenerateAnomalyReport.java @@ -6,7 +6,7 @@ import com.linkedin.thirdeye.anomaly.SmtpConfiguration; import com.linkedin.thirdeye.anomaly.ThirdEyeAnomalyConfiguration; import com.linkedin.thirdeye.anomaly.alert.AlertTaskRunner; -import com.linkedin.thirdeye.anomaly.alert.EmailHelper; +import com.linkedin.thirdeye.anomaly.alert.util.EmailHelper; import com.linkedin.thirdeye.constant.AnomalyFeedbackType; import com.linkedin.thirdeye.datalayer.bao.EmailConfigurationManager; import com.linkedin.thirdeye.datalayer.bao.MergedAnomalyResultManager; From 0fd0aedd3c44d7294e1f71e68a9554dc986cf494 Mon Sep 17 00:00:00 2001 From: Puneet Jaiswal Date: Mon, 9 Jan 2017 11:00:59 -0800 Subject: [PATCH 2/3] adding multiple metrics in the email report api --- .../anomaly/alert/util/AnomalyReportGenerator.java | 11 +++++++++-- .../thirdeye/dashboard/resources/EmailResource.java | 10 ++++++++-- .../thirdeye/datalayer/bao/MetricConfigManager.java | 2 +- .../datalayer/bao/jdbc/MetricConfigManagerImpl.java | 10 +++++----- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AnomalyReportGenerator.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AnomalyReportGenerator.java index 9f2ba9d1c575..f47c34aa892e 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AnomalyReportGenerator.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AnomalyReportGenerator.java @@ -51,9 +51,16 @@ public List getAnomaliesForDatasets(List collect public List getAnomaliesForMetrics(List metrics, long startTime, long endTime) { List anomalies = new ArrayList<>(); + LOG.info("fetching anomalies for metrics : " + metrics); for (String metric : metrics) { - MetricConfigDTO metricConfigDTO = metricConfigManager.findByMetricName(metric); - anomalies.addAll(anomalyResultManager.findByCollectionMetricTime(metricConfigDTO.getDataset(), metric, startTime, endTime, false)); + List metricConfigDTOList = metricConfigManager.findByMetricName(metric); + for (MetricConfigDTO metricConfigDTO : metricConfigDTOList) { + List results = anomalyResultManager + .findByCollectionMetricTime(metricConfigDTO.getDataset(), metric, startTime, endTime, + false); + LOG.info("Found {} result for metric {}", results.size(), metric); + anomalies.addAll(results); + } } return anomalies; } diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/EmailResource.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/EmailResource.java index 70566d3c9805..9ec6050fae57 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/EmailResource.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/EmailResource.java @@ -129,7 +129,9 @@ public Response generateAndSendAlertForDatasets(@PathParam("startTime") Long sta @QueryParam("from") String fromAddr, @QueryParam("to") String toAddr, @QueryParam("includeSentAnomaliesOnly") boolean includeSentAnomaliesOnly, @QueryParam("teHost") String teHost, @QueryParam("smtpHost") String smtpHost, @QueryParam("smtpPort") int smtpPort) { - AnomalyReportGenerator anomalyReportGenerator = AnomalyReportGenerator.getInstance(); + if (Strings.isNullOrEmpty(datasets)) { + throw new WebApplicationException("datasets null or empty : " + datasets); + } String [] dataSetArr = datasets.split(","); if (dataSetArr.length == 0) { throw new WebApplicationException("Datasets empty : " + datasets); @@ -143,6 +145,7 @@ public Response generateAndSendAlertForDatasets(@PathParam("startTime") Long sta if (Strings.isNullOrEmpty(smtpHost)) { throw new WebApplicationException("invalid smtp host" + smtpHost); } + AnomalyReportGenerator anomalyReportGenerator = AnomalyReportGenerator.getInstance(); List anomalies = anomalyReportGenerator .getAnomaliesForDatasets(Arrays.asList(dataSetArr), startTime, endTime); ThirdEyeAnomalyConfiguration configuration = new ThirdEyeAnomalyConfiguration(); @@ -164,7 +167,9 @@ public Response generateAndSendAlertForMetrics(@PathParam("startTime") Long star @QueryParam("from") String fromAddr, @QueryParam("to") String toAddr, @QueryParam("includeSentAnomaliesOnly") boolean includeSentAnomaliesOnly, @QueryParam("teHost") String teHost, @QueryParam("smtpHost") String smtpHost, @QueryParam("smtpPort") int smtpPort) { - AnomalyReportGenerator anomalyReportGenerator = AnomalyReportGenerator.getInstance(); + if (Strings.isNullOrEmpty(metrics)) { + throw new WebApplicationException("metrics null or empty: " + metrics); + } String [] metricsArr = metrics.split(","); if (metricsArr.length == 0) { throw new WebApplicationException("metrics empty : " + metricsArr); @@ -178,6 +183,7 @@ public Response generateAndSendAlertForMetrics(@PathParam("startTime") Long star if (Strings.isNullOrEmpty(smtpHost)) { throw new WebApplicationException("invalid smtp host" + smtpHost); } + AnomalyReportGenerator anomalyReportGenerator = AnomalyReportGenerator.getInstance(); List anomalies = anomalyReportGenerator .getAnomaliesForMetrics(Arrays.asList(metricsArr), startTime, endTime); ThirdEyeAnomalyConfiguration configuration = new ThirdEyeAnomalyConfiguration(); diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MetricConfigManager.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MetricConfigManager.java index 2537bfa47569..dc098aeeb026 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MetricConfigManager.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/MetricConfigManager.java @@ -11,5 +11,5 @@ public interface MetricConfigManager extends AbstractManager { MetricConfigDTO findByMetricAndDataset(String metricName, String dataset); MetricConfigDTO findByAliasAndDataset(String alias, String dataset); List findActiveByDataset(String dataset); - MetricConfigDTO findByMetricName(String metricName); + List findByMetricName(String metricName); } diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MetricConfigManagerImpl.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MetricConfigManagerImpl.java index 7d9600adc304..3fd414444625 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MetricConfigManagerImpl.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/MetricConfigManagerImpl.java @@ -42,7 +42,6 @@ public List findActiveByDataset(String dataset) { return result; } - @Override public MetricConfigDTO findByMetricAndDataset(String metricName, String dataset) { Predicate datasetPredicate = Predicate.EQ("dataset", dataset); @@ -56,12 +55,13 @@ public MetricConfigDTO findByMetricAndDataset(String metricName, String dataset) return result; } - public MetricConfigDTO findByMetricName(String metricName) { + public List findByMetricName(String metricName) { Predicate metricNamePredicate = Predicate.EQ("name", metricName); List list = genericPojoDao.get(metricNamePredicate, MetricConfigBean.class); - MetricConfigDTO result = null; - if (CollectionUtils.isNotEmpty(list)) { - result = MODEL_MAPPER.map(list.get(0), MetricConfigDTO.class); + List result = new ArrayList<>(); + for (MetricConfigBean abstractBean : list) { + MetricConfigDTO dto = MODEL_MAPPER.map(abstractBean, MetricConfigDTO.class); + result.add(dto); } return result; } From b3b977b0fcb23cb062407fbb18b04420214bab07 Mon Sep 17 00:00:00 2001 From: Puneet Jaiswal Date: Tue, 10 Jan 2017 13:15:40 -0800 Subject: [PATCH 3/3] adding AlertConfiguration data objects --- .../datalayer/bao/AlertConfigManager.java | 8 ++ .../bao/EmailConfigurationManager.java | 1 + .../bao/jdbc/AlertConfigManagerImpl.java | 23 ++++ .../jdbc/EmailConfigurationManagerImpl.java | 1 + .../datalayer/dao/GenericPojoDao.java | 9 +- .../datalayer/dto/AlertConfigDTO.java | 7 ++ .../datalayer/dto/EmailConfigurationDTO.java | 1 + .../datalayer/entity/AlertConfigIndex.java | 22 ++++ .../entity/EmailConfigurationIndex.java | 1 + .../datalayer/pojo/AlertConfigBean.java | 113 ++++++++++++++++++ .../pojo/EmailConfigurationBean.java | 2 +- .../datalayer/util/DaoProviderUtil.java | 3 + .../bao/AbstractManagerTestBase.java | 61 +++++----- .../bao/TestAnomalyConfigManager.java | 39 ++++++ .../test/resources/schema/create-schema.sql | 11 ++ .../src/test/resources/schema/drop-tables.sql | 1 + 16 files changed, 272 insertions(+), 31 deletions(-) create mode 100644 thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/AlertConfigManager.java create mode 100644 thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/AlertConfigManagerImpl.java create mode 100644 thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dto/AlertConfigDTO.java create mode 100644 thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/entity/AlertConfigIndex.java create mode 100644 thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/pojo/AlertConfigBean.java create mode 100644 thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/datalayer/bao/TestAnomalyConfigManager.java diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/AlertConfigManager.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/AlertConfigManager.java new file mode 100644 index 000000000000..198917f430cd --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/AlertConfigManager.java @@ -0,0 +1,8 @@ +package com.linkedin.thirdeye.datalayer.bao; + +import com.linkedin.thirdeye.datalayer.dto.AlertConfigDTO; +import java.util.List; + +public interface AlertConfigManager extends AbstractManager { + List findByActive(boolean active); +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/EmailConfigurationManager.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/EmailConfigurationManager.java index 07c16b011943..ed469314a2a5 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/EmailConfigurationManager.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/EmailConfigurationManager.java @@ -4,6 +4,7 @@ import com.linkedin.thirdeye.datalayer.dto.EmailConfigurationDTO; +@Deprecated public interface EmailConfigurationManager extends AbstractManager { List findByFunctionId(Long id); diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/AlertConfigManagerImpl.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/AlertConfigManagerImpl.java new file mode 100644 index 000000000000..1a079c584926 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/AlertConfigManagerImpl.java @@ -0,0 +1,23 @@ +package com.linkedin.thirdeye.datalayer.bao.jdbc; + +import com.linkedin.thirdeye.datalayer.bao.AlertConfigManager; +import com.linkedin.thirdeye.datalayer.dto.AlertConfigDTO; +import com.linkedin.thirdeye.datalayer.pojo.AlertConfigBean; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class AlertConfigManagerImpl extends AbstractManagerImpl + implements AlertConfigManager { + + public AlertConfigManagerImpl() { + super(AlertConfigDTO.class, AlertConfigBean.class); + } + + @Override + public List findByActive(boolean active) { + Map filters = new HashMap<>(); + filters.put("active", active); + return super.findByParams(filters); + } +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/EmailConfigurationManagerImpl.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/EmailConfigurationManagerImpl.java index f7c24a593fb2..a2ac72bd1b10 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/EmailConfigurationManagerImpl.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/bao/jdbc/EmailConfigurationManagerImpl.java @@ -10,6 +10,7 @@ import com.linkedin.thirdeye.datalayer.pojo.EmailConfigurationBean; import com.linkedin.thirdeye.datalayer.util.Predicate; +@Deprecated public class EmailConfigurationManagerImpl extends AbstractManagerImpl implements EmailConfigurationManager { diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dao/GenericPojoDao.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dao/GenericPojoDao.java index 0302fc546bde..7ce6f53b96a2 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dao/GenericPojoDao.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dao/GenericPojoDao.java @@ -1,6 +1,8 @@ package com.linkedin.thirdeye.datalayer.dao; +import com.linkedin.thirdeye.datalayer.entity.AlertConfigIndex; import com.linkedin.thirdeye.datalayer.entity.OverrideConfigIndex; +import com.linkedin.thirdeye.datalayer.pojo.AlertConfigBean; import com.linkedin.thirdeye.datalayer.pojo.OverrideConfigBean; import java.sql.Connection; import java.sql.PreparedStatement; @@ -102,6 +104,9 @@ public class GenericPojoDao { pojoInfoMap.put(OverrideConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, OverrideConfigIndex.class)); + + pojoInfoMap + .put(AlertConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, AlertConfigIndex.class)); } private static PojoInfo newPojoInfo(String baseTableName, @@ -124,12 +129,14 @@ private static PojoInfo newPojoInfo(String baseTableName, static ModelMapper MODEL_MAPPER = new ModelMapper(); static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - public GenericPojoDao() {} + public GenericPojoDao() { + } /** * Use at your own risk!!! Ensure to close the connection after using it or it can cause a leak. * * @return + * * @throws SQLException */ public Connection getConnection() throws SQLException { diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dto/AlertConfigDTO.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dto/AlertConfigDTO.java new file mode 100644 index 000000000000..7ac5d02a37d5 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dto/AlertConfigDTO.java @@ -0,0 +1,7 @@ +package com.linkedin.thirdeye.datalayer.dto; + +import com.linkedin.thirdeye.datalayer.pojo.AlertConfigBean; + +public class AlertConfigDTO extends AlertConfigBean { + +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dto/EmailConfigurationDTO.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dto/EmailConfigurationDTO.java index e3a0081cdc4a..0da67804e0b3 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dto/EmailConfigurationDTO.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/dto/EmailConfigurationDTO.java @@ -5,6 +5,7 @@ import com.linkedin.thirdeye.datalayer.pojo.EmailConfigurationBean; +@Deprecated public class EmailConfigurationDTO extends EmailConfigurationBean { private List functions = new ArrayList<>(); diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/entity/AlertConfigIndex.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/entity/AlertConfigIndex.java new file mode 100644 index 000000000000..17b37ea53c44 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/entity/AlertConfigIndex.java @@ -0,0 +1,22 @@ +package com.linkedin.thirdeye.datalayer.entity; + +public class AlertConfigIndex extends AbstractIndexEntity { + String name; + boolean active; + + public boolean isActive() { + return active; + } + + public void setActive(boolean active) { + this.active = active; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/entity/EmailConfigurationIndex.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/entity/EmailConfigurationIndex.java index d937dddff8ea..e8dc41a8910f 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/entity/EmailConfigurationIndex.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/entity/EmailConfigurationIndex.java @@ -1,5 +1,6 @@ package com.linkedin.thirdeye.datalayer.entity; +@Deprecated public class EmailConfigurationIndex extends AbstractIndexEntity { String collection; String metric; diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/pojo/AlertConfigBean.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/pojo/AlertConfigBean.java new file mode 100644 index 000000000000..33714146bc27 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/pojo/AlertConfigBean.java @@ -0,0 +1,113 @@ +package com.linkedin.thirdeye.datalayer.pojo; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class AlertConfigBean extends AbstractBean { + String name; + String cronExpression; + boolean active; + EmailConfig emailConfig; + ReportConfig reportConfig; + + public String getCronExpression() { + return cronExpression; + } + + public void setCronExpression(String cronExpression) { + this.cronExpression = cronExpression; + } + + public EmailConfig getEmailConfig() { + return emailConfig; + } + + public void setEmailConfig(EmailConfig emailConfig) { + this.emailConfig = emailConfig; + } + + public boolean isActive() { + return active; + } + + public void setActive(boolean active) { + this.active = active; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public ReportConfig getReportConfig() { + return reportConfig; + } + + public void setReportConfig(ReportConfig reportConfig) { + this.reportConfig = reportConfig; + } + + public static class EmailConfig { + boolean sendAlertOnZeroAnomaly; + long lastNotifiedAnomalyId; + List functionIds; + + public List getFunctionIds() { + return functionIds; + } + + public void setFunctionIds(List functionIds) { + this.functionIds = functionIds; + } + + public long getLastNotifiedAnomalyId() { + return lastNotifiedAnomalyId; + } + + public void setLastNotifiedAnomalyId(long lastNotifiedAnomalyId) { + this.lastNotifiedAnomalyId = lastNotifiedAnomalyId; + } + + public boolean isSendAlertOnZeroAnomaly() { + return sendAlertOnZeroAnomaly; + } + + public void setSendAlertOnZeroAnomaly(boolean sendAlertOnZeroAnomaly) { + this.sendAlertOnZeroAnomaly = sendAlertOnZeroAnomaly; + } + } + + public static class ReportConfig { + boolean enabled = true; + List metricIds; + List> metricDimensions; + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public List> getMetricDimensions() { + return metricDimensions; + } + + public void setMetricDimensions(List> metricDimensions) { + this.metricDimensions = metricDimensions; + } + + public List getMetricIds() { + return metricIds; + } + + public void setMetricIds(List metricIds) { + this.metricIds = metricIds; + } + } +} diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/pojo/EmailConfigurationBean.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/pojo/EmailConfigurationBean.java index 572427ee2b1f..18c0c789d3ba 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/pojo/EmailConfigurationBean.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/pojo/EmailConfigurationBean.java @@ -10,7 +10,7 @@ import org.hibernate.validator.constraints.Email; import com.google.common.base.MoreObjects; -@JsonIgnoreProperties(ignoreUnknown=true) +@JsonIgnoreProperties(ignoreUnknown = true) public class EmailConfigurationBean extends AbstractBean { private List functionIds; diff --git a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/util/DaoProviderUtil.java b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/util/DaoProviderUtil.java index c3860e46fae0..243db5ef693d 100644 --- a/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/util/DaoProviderUtil.java +++ b/thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/datalayer/util/DaoProviderUtil.java @@ -1,5 +1,6 @@ package com.linkedin.thirdeye.datalayer.util; +import com.linkedin.thirdeye.datalayer.entity.AlertConfigIndex; import com.linkedin.thirdeye.datalayer.entity.OverrideConfigIndex; import io.dropwizard.configuration.ConfigurationFactory; import io.dropwizard.jackson.Jackson; @@ -119,6 +120,8 @@ static class DataSourceModule extends AbstractModule { convertCamelCaseToUnderscore(IngraphMetricConfigIndex.class.getSimpleName())); entityMappingHolder.register(conn, OverrideConfigIndex.class, convertCamelCaseToUnderscore(OverrideConfigIndex.class.getSimpleName())); + entityMappingHolder.register(conn, AlertConfigIndex.class, + convertCamelCaseToUnderscore(AlertConfigIndex.class.getSimpleName())); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/datalayer/bao/AbstractManagerTestBase.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/datalayer/bao/AbstractManagerTestBase.java index 1726b015f83a..455beac42bb9 100644 --- a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/datalayer/bao/AbstractManagerTestBase.java +++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/datalayer/bao/AbstractManagerTestBase.java @@ -3,6 +3,19 @@ import com.google.common.collect.Lists; import com.linkedin.thirdeye.anomaly.override.OverrideConfigHelper; import com.linkedin.thirdeye.api.DimensionMap; +import com.linkedin.thirdeye.datalayer.bao.jdbc.AlertConfigManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.DashboardConfigManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.DatasetConfigManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.EmailConfigurationManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.IngraphDashboardConfigManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.IngraphMetricConfigManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.JobManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.MergedAnomalyResultManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.MetricConfigManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.OverrideConfigManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.RawAnomalyResultManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.TaskManagerImpl; +import com.linkedin.thirdeye.datalayer.bao.jdbc.WebappConfigManagerImpl; import com.linkedin.thirdeye.datalayer.dto.OverrideConfigDTO; import com.linkedin.thirdeye.datalayer.util.DaoProviderUtil; @@ -53,6 +66,7 @@ public abstract class AbstractManagerTestBase { protected IngraphDashboardConfigManager ingraphDashboardConfigDAO; protected IngraphMetricConfigManager ingraphMetricConfigDAO; protected OverrideConfigManager overrideConfigDAO; + protected AlertConfigManager alertConfigManager; private ManagerProvider managerProvider; private PersistenceConfig configuration; @@ -60,7 +74,8 @@ public abstract class AbstractManagerTestBase { private DataSource ds; private String dbId = System.currentTimeMillis() + "" + Math.random(); - @BeforeClass(alwaysRun = true) public void init() throws Exception { + @BeforeClass(alwaysRun = true) + public void init() throws Exception { URL url = AbstractManagerTestBase.class.getResource("/persistence-local.yml"); File configFile = new File(url.toURI()); configuration = DaoProviderUtil.createConfiguration(configFile); @@ -120,35 +135,23 @@ public void cleanUpJDBC() throws Exception { public void initManagers() throws Exception { managerProvider = new ManagerProvider(ds); - Class c = - com.linkedin.thirdeye.datalayer.bao.jdbc.AnomalyFunctionManagerImpl.class; + Class c = AnomalyFunctionManagerImpl.class; System.out.println(c); - anomalyFunctionDAO = (AnomalyFunctionManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.AnomalyFunctionManagerImpl.class); - rawResultDAO = (RawAnomalyResultManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.RawAnomalyResultManagerImpl.class); - jobDAO = (JobManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.JobManagerImpl.class); - taskDAO = (TaskManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.TaskManagerImpl.class); - emailConfigurationDAO = (EmailConfigurationManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.EmailConfigurationManagerImpl.class); - mergedResultDAO = (MergedAnomalyResultManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.MergedAnomalyResultManagerImpl.class); - webappConfigDAO = (WebappConfigManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.WebappConfigManagerImpl.class); - datasetConfigDAO = (DatasetConfigManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.DatasetConfigManagerImpl.class); - metricConfigDAO = (MetricConfigManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.MetricConfigManagerImpl.class); - dashboardConfigDAO = (DashboardConfigManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.DashboardConfigManagerImpl.class); - ingraphDashboardConfigDAO = (IngraphDashboardConfigManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.IngraphDashboardConfigManagerImpl.class); - ingraphMetricConfigDAO = (IngraphMetricConfigManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.IngraphMetricConfigManagerImpl.class); - overrideConfigDAO = (OverrideConfigManager) managerProvider - .getInstance(com.linkedin.thirdeye.datalayer.bao.jdbc.OverrideConfigManagerImpl.class); + anomalyFunctionDAO = managerProvider.getInstance(AnomalyFunctionManagerImpl.class); + rawResultDAO = managerProvider.getInstance(RawAnomalyResultManagerImpl.class); + jobDAO = managerProvider.getInstance(JobManagerImpl.class); + taskDAO = managerProvider.getInstance(TaskManagerImpl.class); + emailConfigurationDAO = managerProvider.getInstance(EmailConfigurationManagerImpl.class); + mergedResultDAO = managerProvider.getInstance(MergedAnomalyResultManagerImpl.class); + webappConfigDAO = managerProvider.getInstance(WebappConfigManagerImpl.class); + datasetConfigDAO = managerProvider.getInstance(DatasetConfigManagerImpl.class); + metricConfigDAO = managerProvider.getInstance(MetricConfigManagerImpl.class); + dashboardConfigDAO = managerProvider.getInstance(DashboardConfigManagerImpl.class); + ingraphDashboardConfigDAO = + managerProvider.getInstance(IngraphDashboardConfigManagerImpl.class); + ingraphMetricConfigDAO = managerProvider.getInstance(IngraphMetricConfigManagerImpl.class); + overrideConfigDAO = managerProvider.getInstance(OverrideConfigManagerImpl.class); + alertConfigManager = managerProvider.getInstance(AlertConfigManagerImpl.class); } @AfterClass(alwaysRun = true) diff --git a/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/datalayer/bao/TestAnomalyConfigManager.java b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/datalayer/bao/TestAnomalyConfigManager.java new file mode 100644 index 000000000000..b864d9ccf468 --- /dev/null +++ b/thirdeye/thirdeye-pinot/src/test/java/com/linkedin/thirdeye/datalayer/bao/TestAnomalyConfigManager.java @@ -0,0 +1,39 @@ +package com.linkedin.thirdeye.datalayer.bao; + +import com.linkedin.thirdeye.datalayer.dto.AlertConfigDTO; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class TestAnomalyConfigManager extends AbstractManagerTestBase { + + Long alertConfigid; + + @Test + public void testCreateAlertConfig() { + AlertConfigDTO request = new AlertConfigDTO(); + request.setActive(true); + request.setName("my alert config"); + alertConfigid = alertConfigManager.save(request); + Assert.assertTrue(alertConfigid > 0); + + } + + @Test (dependsOnMethods = {"testCreateAlertConfig"}) + public void testFetchAlertConfig() { + // find by id + AlertConfigDTO response = alertConfigManager.findById(alertConfigid); + Assert.assertNotNull(response); + Assert.assertEquals(response.getId(), alertConfigid); + Assert.assertEquals(alertConfigManager.findAll().size(), 1); + } + + @Test (dependsOnMethods = {"testFetchAlertConfig"}) + public void testDeleteAlertConfig() { + alertConfigManager.deleteById(alertConfigid); + Assert.assertEquals(alertConfigManager.findAll().size(), 0); + } + + // TODO: add tests for Email Config + + // TODO: add tests for Report Config +} diff --git a/thirdeye/thirdeye-pinot/src/test/resources/schema/create-schema.sql b/thirdeye/thirdeye-pinot/src/test/resources/schema/create-schema.sql index c78425f45bd2..9eb9c1a15bf4 100644 --- a/thirdeye/thirdeye-pinot/src/test/resources/schema/create-schema.sql +++ b/thirdeye/thirdeye-pinot/src/test/resources/schema/create-schema.sql @@ -211,3 +211,14 @@ create table if not exists override_config_index ( ) ENGINE=InnoDB; create index override_config_target_entity_idx on override_config_index(target_entity); create index override_config_target_start_time_idx on override_config_index(start_time); + + +create table if not exists alert_config_index ( + active boolean, + name varchar(500) not null, + base_id bigint(20) not null, + create_time timestamp, + update_time timestamp default current_timestamp, + version int(10) +) ENGINE=InnoDB; + diff --git a/thirdeye/thirdeye-pinot/src/test/resources/schema/drop-tables.sql b/thirdeye/thirdeye-pinot/src/test/resources/schema/drop-tables.sql index d1abafe3ec41..ed3112fdc6ff 100644 --- a/thirdeye/thirdeye-pinot/src/test/resources/schema/drop-tables.sql +++ b/thirdeye/thirdeye-pinot/src/test/resources/schema/drop-tables.sql @@ -13,4 +13,5 @@ DROP TABLE if EXISTS ingraph_dashboard_config_index; DROP TABLE if EXISTS ingraph_metric_config_index; DROP TABLE if EXISTS webapp_config_index; DROP TABLE if EXISTS override_config_index; +DROP TABLE if EXISTS alert_config_index; SET FOREIGN_KEY_CHECKS = 1;