AMBARI-22348 : Metric Definition Service V1 Implementation. (avijayan)

ambari-metrics-anomaly-detection-service/pom.xml

@@ -446,5 +446,11 @@
       <artifactId>metrics-core</artifactId>
       <version>3.2.5</version>
     </dependency>
+    <dependency>
+      <groupId>org.easymock</groupId>
+      <artifactId>easymock</artifactId>
+      <version>2.5</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>

ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/common/StatisticUtils.java

@@ -18,10 +18,7 @@
 package org.apache.ambari.metrics.adservice.prototype.common;
 
 
-import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
 
 public class StatisticUtils {
 

ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/MetricSparkConsumer.java

@@ -35,10 +35,15 @@
 import org.apache.spark.streaming.kafka.KafkaUtils;
 import scala.Tuple2;
 
-import java.util.*;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 

ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/PointInTimeADSystem.java

@@ -17,8 +17,8 @@
  */
 package org.apache.ambari.metrics.adservice.prototype.core;
 
-import org.apache.ambari.metrics.adservice.prototype.common.ResultSet;
 import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
+import org.apache.ambari.metrics.adservice.prototype.common.ResultSet;
 import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaModel;
 import org.apache.ambari.metrics.adservice.prototype.methods.ema.EmaTechnique;
 import org.apache.commons.logging.Log;

ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/core/TrendADSystem.java

@@ -17,9 +17,9 @@
  */
 package org.apache.ambari.metrics.adservice.prototype.core;
 
+import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
 import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
 import org.apache.ambari.metrics.adservice.prototype.methods.hsdev.HsdevTechnique;
-import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
 import org.apache.ambari.metrics.adservice.prototype.methods.kstest.KSTechnique;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;

ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/MetricAnomaly.java

@@ -18,8 +18,6 @@
 package org.apache.ambari.metrics.adservice.prototype.methods;
 
 import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
 
 public class MetricAnomaly implements Serializable{
 

ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/hsdev/HsdevTechnique.java

@@ -21,14 +21,15 @@
 import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import static org.apache.ambari.metrics.adservice.prototype.common.StatisticUtils.median;
-import static org.apache.ambari.metrics.adservice.prototype.common.StatisticUtils.sdev;
 
 import java.io.Serializable;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.apache.ambari.metrics.adservice.prototype.common.StatisticUtils.median;
+import static org.apache.ambari.metrics.adservice.prototype.common.StatisticUtils.sdev;
+
 public class HsdevTechnique implements Serializable {
 
   private Map<String, Double> hsdevMap;

ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/methods/kstest/KSTechnique.java

@@ -20,8 +20,8 @@
 
 import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
 import org.apache.ambari.metrics.adservice.prototype.common.ResultSet;
-import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
 import org.apache.ambari.metrics.adservice.prototype.core.RFunctionInvoker;
+import org.apache.ambari.metrics.adservice.prototype.methods.MetricAnomaly;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 

ambari-metrics-anomaly-detection-service/src/main/java/org/apache/ambari/metrics/adservice/prototype/testing/utilities/MetricAnomalyTester.java

@@ -18,23 +18,6 @@
 
 package org.apache.ambari.metrics.adservice.prototype.testing.utilities;
 
-import org.apache.ambari.metrics.adservice.prototype.common.DataSeries;
-import org.apache.ambari.metrics.adservice.prototype.common.ResultSet;
-import org.apache.ambari.metrics.adservice.prototype.core.MetricsCollectorInterface;
-import org.apache.ambari.metrics.adservice.prototype.core.RFunctionInvoker;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TreeMap;
-
 /**
  * Class which was originally used to send test series from AMS to Spark through Kafka.
  */

ambari-metrics-anomaly-detection-service/src/main/resources/config.yml

@@ -1,3 +1,15 @@
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 server:
   applicationConnectors:
    - type: http
@@ -7,3 +19,18 @@ server:
 
 logging:
   type: external
+
+metricManagerService:
+  inputDefinitionDirectory: /etc/adservice/conf/input-definitions-directory
+
+metricsCollector:
+  hostPortList: host1:6188,host2:6188
+  metadataEndpoint: /v1/timeline/metrics/metadata/keys
+
+adQueryService:
+  anomalyDataTtl: 604800
+
+#subsystemService:
+#  spark:
+#  pointInTime:
+#  trend:

ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionApp.scala

@@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.{ObjectMapper, SerializationFeature}
 import com.fasterxml.jackson.datatype.joda.JodaModule
 import com.fasterxml.jackson.jaxrs.json.JacksonJaxbJsonProvider
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
+
 import io.dropwizard.Application
 import io.dropwizard.setup.Environment
 

ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppConfig.scala

@@ -1,7 +1,3 @@
-package org.apache.ambari.metrics.adservice.app
-
-import io.dropwizard.Configuration
-
 /**
   * Licensed to the Apache Software Foundation (ASF) under one
   * or more contributor license agreements.  See the NOTICE file
@@ -19,6 +15,55 @@ import io.dropwizard.Configuration
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
+
+package org.apache.ambari.metrics.adservice.app
+
+import javax.validation.Valid
+
+import org.apache.ambari.metrics.adservice.configuration.{AdServiceConfiguration, HBaseConfiguration, MetricCollectorConfiguration, MetricManagerServiceConfiguration}
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+import io.dropwizard.Configuration
+
+/**
+  * Top Level AD System Manager config items.
+  */
 class AnomalyDetectionAppConfig extends Configuration {
 
+  /*
+   Metric Definition Service configuration
+    */
+  @Valid
+  private val metricManagerServiceConfiguration = new MetricManagerServiceConfiguration
+
+  @Valid
+  private val metricCollectorConfiguration = new MetricCollectorConfiguration
+
+  /*
+   Anomaly Service configuration
+    */
+  @Valid
+  private val adServiceConfiguration = new AdServiceConfiguration
+
+  /*
+   HBase Conf
+    */
+  def getHBaseConf : org.apache.hadoop.conf.Configuration = {
+    HBaseConfiguration.getHBaseConf
+  }
+
+  @JsonProperty("metricManagerService")
+  def getMetricManagerServiceConfiguration: MetricManagerServiceConfiguration = {
+    metricManagerServiceConfiguration
+  }
+
+  @JsonProperty("adQueryService")
+  def getAdServiceConfiguration: AdServiceConfiguration = {
+    adServiceConfiguration
+  }
+
+  @JsonProperty("metricsCollector")
+  def getMetricCollectorConfiguration: MetricCollectorConfiguration = metricCollectorConfiguration
+
 }

ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/app/AnomalyDetectionAppModule.scala

@@ -23,6 +23,7 @@ import org.apache.ambari.metrics.adservice.service.{ADQueryService, ADQueryServi
 import com.codahale.metrics.health.HealthCheck
 import com.google.inject.AbstractModule
 import com.google.inject.multibindings.Multibinder
+
 import io.dropwizard.setup.Environment
 
 class AnomalyDetectionAppModule(config: AnomalyDetectionAppConfig, env: Environment) extends AbstractModule {

ambari-metrics-anomaly-detection-service/src/main/scala/org/apache/ambari/metrics/adservice/common/Range.scala

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.metrics.adservice.common
+
+/**
+  * Class to capture a Range in a Season.
+  * For example Monday - Wednesday is a 'Range' in a DAY Season.
+  * @param lower lower end
+  * @param higher higher end
+  */
+case class Range (lower: Int, higher: Int) {
+
+  def withinRange(value: Int) : Boolean = {
+    if (lower <= higher) {
+      (value >= lower) && (value <= higher)
+    } else {
+      !(value > higher) && (value < lower)
+    }
+  }
+
+  @Override
+  override def equals(obj: scala.Any): Boolean = {
+    if (obj == null) {
+      return false
+    }
+    val that : Range = obj.asInstanceOf[Range]
+    (lower == that.lower) && (higher == that.higher)
+  }
+}