Adding new AlertConfiguration, API for sending adhoc anomaly report emails.

thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/AlertTaskRunner.java

@@ -1,6 +1,8 @@
 package com.linkedin.thirdeye.anomaly.alert;
 
 import com.linkedin.thirdeye.anomaly.alert.template.pojo.MetricDimensionReport;
+import com.linkedin.thirdeye.anomaly.alert.util.AlertFilterHelper;
+import com.linkedin.thirdeye.anomaly.alert.util.EmailHelper;
 import com.linkedin.thirdeye.api.DimensionMap;
 import com.linkedin.thirdeye.client.DAORegistry;
 import com.linkedin.thirdeye.dashboard.views.contributor.ContributorViewResponse;
@@ -48,7 +50,7 @@
 import freemarker.template.TemplateModelException;
 import freemarker.template.TemplateNumberModel;
 
-import static com.linkedin.thirdeye.anomaly.alert.AlertFilterHelper.FILTER_TYPE_KEY;
+import static com.linkedin.thirdeye.anomaly.alert.util.AlertFilterHelper.FILTER_TYPE_KEY;
 
 public class AlertTaskRunner implements TaskRunner {
 
@@ -243,7 +245,8 @@ private void sendAlertForAnomalies(String collectionAlias, List<MergedAnomalyRes
         List<MetricDimensionReport> metricDimensionValueReports;
         List<ContributorViewResponse> reports = new ArrayList<>();
         for (String dimension : alertConfig.getDimensions()) {
-          ContributorViewResponse report = EmailHelper.getContributorData(collectionAlias, alertConfig.getMetric(), Arrays.asList(dimension));
+          ContributorViewResponse report = EmailHelper
+              .getContributorData(collectionAlias, alertConfig.getMetric(), Arrays.asList(dimension));
           if(report != null) {
             reports.add(report);
           }

thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/AlertFilterHelper.java → thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AlertFilterHelper.java

@@ -1,4 +1,4 @@
-package com.linkedin.thirdeye.anomaly.alert;
+package com.linkedin.thirdeye.anomaly.alert.util;
 
 import com.linkedin.thirdeye.detector.email.filter.AlertFilter;
 import com.linkedin.thirdeye.detector.email.filter.AlphaBetaAlertFilter;

thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/AnomalyReportGenerator.java

@@ -0,0 +1,256 @@
+package com.linkedin.thirdeye.anomaly.alert.util;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.google.common.base.Throwables;
+import com.linkedin.thirdeye.anomaly.SmtpConfiguration;
+import com.linkedin.thirdeye.anomaly.ThirdEyeAnomalyConfiguration;
+import com.linkedin.thirdeye.anomaly.alert.AlertTaskRunner;
+import com.linkedin.thirdeye.client.DAORegistry;
+import com.linkedin.thirdeye.constant.AnomalyFeedbackType;
+import com.linkedin.thirdeye.datalayer.bao.MergedAnomalyResultManager;
+import com.linkedin.thirdeye.datalayer.bao.MetricConfigManager;
+import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import com.linkedin.thirdeye.datalayer.dto.MetricConfigDTO;
+import freemarker.template.Configuration;
+import freemarker.template.Template;
+import freemarker.template.TemplateExceptionHandler;
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.commons.mail.HtmlEmail;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AnomalyReportGenerator {
+  // private static final DateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ");
+
+  private static final Logger LOG = LoggerFactory.getLogger(AnomalyReportGenerator.class);
+
+  private static  final AnomalyReportGenerator INSTANCE = new AnomalyReportGenerator();
+  public static AnomalyReportGenerator getInstance() {
+    return INSTANCE;
+  }
+
+  MergedAnomalyResultManager anomalyResultManager = DAORegistry.getInstance().getMergedAnomalyResultDAO();
+  MetricConfigManager metricConfigManager = DAORegistry.getInstance().getMetricConfigDAO();
+
+  public List<MergedAnomalyResultDTO> getAnomaliesForDatasets(List<String> collections, long startTime, long endTime) {
+    List<MergedAnomalyResultDTO> anomalies = new ArrayList<>();
+    for (String collection : collections) {
+      anomalies.addAll(anomalyResultManager.findByCollectionTime(collection, startTime, endTime, false));
+    }
+    return anomalies;
+  }
+
+  public List<MergedAnomalyResultDTO> getAnomaliesForMetrics(List<String> metrics, long startTime, long endTime) {
+    List<MergedAnomalyResultDTO> anomalies = new ArrayList<>();
+    LOG.info("fetching anomalies for metrics : " + metrics);
+    for (String metric : metrics) {
+      List<MetricConfigDTO> metricConfigDTOList = metricConfigManager.findByMetricName(metric);
+      for (MetricConfigDTO metricConfigDTO : metricConfigDTOList) {
+        List<MergedAnomalyResultDTO> results = anomalyResultManager
+            .findByCollectionMetricTime(metricConfigDTO.getDataset(), metric, startTime, endTime,
+                false);
+        LOG.info("Found {} result for metric {}", results.size(), metric);
+        anomalies.addAll(results);
+      }
+    }
+    return anomalies;
+  }
+
+ public void buildReport(long startTime, long endTime, List<MergedAnomalyResultDTO> anomalies,
+      ThirdEyeAnomalyConfiguration configuration, boolean includeSentAnomaliesOnly,
+      String emailRecipients) {
+    if (anomalies == null || anomalies.size() == 0) {
+      LOG.info("No anomalies found, please check the parameters.. exiting");
+    } else {
+      Set<String> metrics = new HashSet<>();
+      int alertedAnomalies = 0;
+      int feedbackCollected = 0;
+      int trueAlert = 0;
+      int falseAlert = 0;
+      int nonActionable = 0;
+
+      List<AnomalyReportDTO> anomalyReportDTOList = new ArrayList<>();
+
+      for (MergedAnomalyResultDTO anomaly : anomalies) {
+        metrics.add(anomaly.getMetric());
+        if (anomaly.getFeedback() != null) {
+          feedbackCollected++;
+          if (anomaly.getFeedback().getFeedbackType().equals(AnomalyFeedbackType.ANOMALY)) {
+            trueAlert++;
+          } else if (anomaly.getFeedback().getFeedbackType()
+              .equals(AnomalyFeedbackType.NOT_ANOMALY)) {
+            falseAlert++;
+          } else {
+            nonActionable++;
+          }
+        }
+        String feedbackVal = getFeedback(
+            anomaly.getFeedback() == null ? "NA" : anomaly.getFeedback().getFeedbackType().name());
+
+        AnomalyReportDTO anomalyReportDTO =
+            new AnomalyReportDTO(String.valueOf(anomaly.getId()), feedbackVal,
+                String.format("%+.2f", anomaly.getWeight()), anomaly.getMetric(),
+                new Date(anomaly.getStartTime()).toString(), String
+                .format("%.2f", getTimeDiffInHours(anomaly.getStartTime(), anomaly.getEndTime())),
+                getAnomalyURL(anomaly, configuration.getDashboardHost()));
+
+        if (anomaly.isNotified()) {
+          alertedAnomalies++;
+        }
+        // include notified alerts only in the email
+        if (includeSentAnomaliesOnly) {
+          if (anomaly.isNotified()) {
+            anomalyReportDTOList.add(anomalyReportDTO);
+          }
+        } else {
+          anomalyReportDTOList.add(anomalyReportDTO);
+        }
+      }
+
+      Map<String, Object> templateData = new HashMap<>();
+      templateData.put("startTime", new Date(startTime));
+      templateData.put("endTime", new Date(endTime));
+      templateData.put("anomalyCount", anomalies.size());
+      templateData.put("metricsCount", metrics.size());
+      templateData.put("notifiedCount", alertedAnomalies);
+      templateData.put("feedbackCount", feedbackCollected);
+      templateData.put("trueAlertCount", trueAlert);
+      templateData.put("falseAlertCount", falseAlert);
+      templateData.put("nonActionableCount", nonActionable);
+      templateData.put("anomalyDetails", anomalyReportDTOList);
+      buildEmailTemplateAndSendAlert(templateData, configuration.getSmtpConfiguration(),
+          emailRecipients);
+    }
+  }
+
+  void buildEmailTemplateAndSendAlert(Map<String, Object> paramMap,
+      SmtpConfiguration smtpConfiguration, String emailRecipients) {
+    HtmlEmail email = new HtmlEmail();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    try (Writer out = new OutputStreamWriter(baos, AlertTaskRunner.CHARSET)) {
+      Configuration freemarkerConfig = new Configuration(Configuration.VERSION_2_3_21);
+      freemarkerConfig.setClassForTemplateLoading(getClass(), "/com/linkedin/thirdeye/detector");
+      freemarkerConfig.setDefaultEncoding(AlertTaskRunner.CHARSET);
+      freemarkerConfig.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER);
+      Template template = freemarkerConfig.getTemplate("custom-anomaly-report.ftl");
+      template.process(paramMap, out);
+
+      String alertEmailSubject = "Thirdeye : Daily anomaly report";
+      String alertEmailHtml = new String(baos.toByteArray(), AlertTaskRunner.CHARSET);
+      EmailHelper.sendEmailWithHtml(email, smtpConfiguration, alertEmailSubject, alertEmailHtml,
+          "thirdeye-dev@linkedin.com", emailRecipients);
+    } catch (Exception e) {
+      Throwables.propagate(e);
+    }
+  }
+
+  double getTimeDiffInHours(long start, long end) {
+    return Double.valueOf((end - start) / 1000) / 3600;
+  }
+
+  String getFeedback(String feedbackType) {
+    switch (feedbackType) {
+    case "ANOMALY":
+      return "Confirmed Anomaly";
+    case "NOT_ANOMALY":
+      return "False Alarm";
+    case "ANOMALY_NO_ACTION":
+      return "Not Actionable";
+    }
+    return "NA";
+  }
+
+  String getAnomalyURL(MergedAnomalyResultDTO anomalyResultDTO, String dashboardUrl) {
+    String urlPart = "#view=anomalies&dataset=%s&metrics=%s&currentStart=%s&currentEnd=%s";
+    return dashboardUrl + String
+        .format(urlPart, anomalyResultDTO.getCollection(), anomalyResultDTO.getMetric(),
+            anomalyResultDTO.getStartTime(), anomalyResultDTO.getEndTime());
+  }
+
+  @JsonIgnoreProperties(ignoreUnknown = true)
+  public static class AnomalyReportDTO {
+    String metric;
+    String startDateTime;
+    String windowSize;
+    String lift;
+    String feedback;
+    String anomalyId;
+    String anomalyURL;
+
+    public AnomalyReportDTO(String anomalyId, String feedback, String lift, String metric,
+        String startDateTime, String windowSize, String anomalyURL) {
+      this.anomalyId = anomalyId;
+      this.feedback = feedback;
+      this.lift = lift;
+      this.metric = metric;
+      this.startDateTime = startDateTime;
+      this.windowSize = windowSize;
+      this.anomalyURL = anomalyURL;
+    }
+
+    public String getAnomalyId() {
+      return anomalyId;
+    }
+
+    public void setAnomalyId(String anomalyId) {
+      this.anomalyId = anomalyId;
+    }
+
+    public String getFeedback() {
+      return feedback;
+    }
+
+    public void setFeedback(String feedback) {
+      this.feedback = feedback;
+    }
+
+    public String getLift() {
+      return lift;
+    }
+
+    public void setLift(String lift) {
+      this.lift = lift;
+    }
+
+    public String getMetric() {
+      return metric;
+    }
+
+    public void setMetric(String metric) {
+      this.metric = metric;
+    }
+
+    public String getStartDateTime() {
+      return startDateTime;
+    }
+
+    public void setStartDateTime(String startDateTime) {
+      this.startDateTime = startDateTime;
+    }
+
+    public String getWindowSize() {
+      return windowSize;
+    }
+
+    public void setWindowSize(String windowSize) {
+      this.windowSize = windowSize;
+    }
+
+    public String getAnomalyURL() {
+      return anomalyURL;
+    }
+
+    public void setAnomalyURL(String anomalyURL) {
+      this.anomalyURL = anomalyURL;
+    }
+  }
+}

thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/EmailHelper.java → thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/anomaly/alert/util/EmailHelper.java

@@ -1,4 +1,4 @@
-package com.linkedin.thirdeye.anomaly.alert;
+package com.linkedin.thirdeye.anomaly.alert.util;
 
 import com.google.common.cache.LoadingCache;
 import com.linkedin.thirdeye.anomaly.SmtpConfiguration;

thirdeye/thirdeye-pinot/src/main/java/com/linkedin/thirdeye/dashboard/resources/EmailResource.java

@@ -1,5 +1,11 @@
 package com.linkedin.thirdeye.dashboard.resources;
 
+import com.google.common.base.Strings;
+import com.linkedin.thirdeye.anomaly.SmtpConfiguration;
+import com.linkedin.thirdeye.anomaly.ThirdEyeAnomalyConfiguration;
+import com.linkedin.thirdeye.anomaly.alert.util.AnomalyReportGenerator;
+import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO;
+import java.util.Arrays;
 import java.util.List;
 
 import javax.ws.rs.DELETE;
@@ -9,6 +15,7 @@
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
 
@@ -114,4 +121,80 @@ public Response deleteByEmail(@PathParam("emailId") Long emailId) {
   public List<EmailConfigurationDTO> findEmailIdsByFunction(@PathParam("functionId") Long functionId) {
     return emailDAO.findByFunctionId(functionId);
   }
+
+  @GET
+  @Path("generate/datasets/{startTime}/{endTime}")
+  public Response generateAndSendAlertForDatasets(@PathParam("startTime") Long startTime,
+      @PathParam("endTime") Long endTime, @QueryParam("datasets") String datasets,
+      @QueryParam("from") String fromAddr, @QueryParam("to") String toAddr,
+      @QueryParam("includeSentAnomaliesOnly") boolean includeSentAnomaliesOnly,
+      @QueryParam("teHost") String teHost,  @QueryParam("smtpHost") String smtpHost, @QueryParam("smtpPort") int smtpPort) {
+    if (Strings.isNullOrEmpty(datasets)) {
+      throw new WebApplicationException("datasets null or empty : " + datasets);
+    }
+    String [] dataSetArr = datasets.split(",");
+    if (dataSetArr.length == 0) {
+      throw new WebApplicationException("Datasets empty : " + datasets);
+    }
+    if (Strings.isNullOrEmpty(toAddr)) {
+      throw new WebApplicationException("Empty : list of recipients" + toAddr);
+    }
+    if(Strings.isNullOrEmpty(teHost)) {
+      throw new WebApplicationException("Invalid TE host" + teHost);
+    }
+    if (Strings.isNullOrEmpty(smtpHost)) {
+      throw new WebApplicationException("invalid smtp host" + smtpHost);
+    }
+    AnomalyReportGenerator anomalyReportGenerator = AnomalyReportGenerator.getInstance();
+    List<MergedAnomalyResultDTO> anomalies = anomalyReportGenerator
+        .getAnomaliesForDatasets(Arrays.asList(dataSetArr), startTime, endTime);
+    ThirdEyeAnomalyConfiguration configuration = new ThirdEyeAnomalyConfiguration();
+    SmtpConfiguration smtpConfiguration = new SmtpConfiguration();
+    smtpConfiguration.setSmtpHost(smtpHost);
+    smtpConfiguration.setSmtpPort(smtpPort);
+
+    configuration.setSmtpConfiguration(smtpConfiguration);
+    configuration.setDashboardHost(teHost);
+
+    anomalyReportGenerator.buildReport(startTime, endTime, anomalies, configuration, includeSentAnomaliesOnly, toAddr);
+    return Response.ok().build();
+  }
+
+  @GET
+  @Path("generate/metrics/{startTime}/{endTime}")
+  public Response generateAndSendAlertForMetrics(@PathParam("startTime") Long startTime,
+      @PathParam("endTime") Long endTime, @QueryParam("metrics") String metrics,
+      @QueryParam("from") String fromAddr, @QueryParam("to") String toAddr,
+      @QueryParam("includeSentAnomaliesOnly") boolean includeSentAnomaliesOnly,
+      @QueryParam("teHost") String teHost,  @QueryParam("smtpHost") String smtpHost, @QueryParam("smtpPort") int smtpPort) {
+    if (Strings.isNullOrEmpty(metrics)) {
+      throw new WebApplicationException("metrics null or empty: " + metrics);
+    }
+    String [] metricsArr = metrics.split(",");
+    if (metricsArr.length == 0) {
+      throw new WebApplicationException("metrics empty : " + metricsArr);
+    }
+    if (Strings.isNullOrEmpty(toAddr)) {
+      throw new WebApplicationException("Empty : list of recipients" + toAddr);
+    }
+    if(Strings.isNullOrEmpty(teHost)) {
+      throw new WebApplicationException("Invalid TE host" + teHost);
+    }
+    if (Strings.isNullOrEmpty(smtpHost)) {
+      throw new WebApplicationException("invalid smtp host" + smtpHost);
+    }
+    AnomalyReportGenerator anomalyReportGenerator = AnomalyReportGenerator.getInstance();
+    List<MergedAnomalyResultDTO> anomalies = anomalyReportGenerator
+        .getAnomaliesForMetrics(Arrays.asList(metricsArr), startTime, endTime);
+    ThirdEyeAnomalyConfiguration configuration = new ThirdEyeAnomalyConfiguration();
+    SmtpConfiguration smtpConfiguration = new SmtpConfiguration();
+    smtpConfiguration.setSmtpHost(smtpHost);
+    smtpConfiguration.setSmtpPort(smtpPort);
+
+    configuration.setSmtpConfiguration(smtpConfiguration);
+    configuration.setDashboardHost(teHost);
+
+    anomalyReportGenerator.buildReport(startTime, endTime, anomalies, configuration, includeSentAnomaliesOnly, toAddr);
+    return Response.ok().build();
+  }
 }