Skip to content
This repository has been archived by the owner. It is now read-only.
Permalink
Browse files
AMBARI-22437 : Create an 'AD Manager' component in Ambari Metrics Ser…
…vice stack side. (avijayan)
  • Loading branch information
Aravindan Vijayan committed Apr 1, 2018
1 parent b866169 commit 1336d891fbb8a04c312af5b91b19e004bb3c3f12
Showing 19 changed files with 422 additions and 50 deletions.
@@ -0,0 +1,194 @@
#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific

PIDFILE=/var/run//var/run/ambari-metrics-anomaly-detection/ambari-metrics-admanager.pid
OUTFILE=/var/log/ambari-metrics-anomaly-detection/ambari-metrics-admanager.out

CONF_DIR=/etc/ambari-metrics-anomaly-detection/conf
DAEMON_NAME=ams_admanager

STOP_TIMEOUT=5

function write_pidfile
{
local pidfile="$1"
echo $! > "${pidfile}" 2>/dev/null
if [[ $? -gt 0 ]]; then
echo "ERROR: Cannot write pid ${pidfile}." | tee -a $STARTUPFILE
exit 1;
fi
}

function java_setup
{
# Bail if we did not detect it
if [[ -z "${JAVA_HOME}" ]]; then
echo "ERROR: JAVA_HOME is not set and could not be found."
exit 1
fi

if [[ ! -d "${JAVA_HOME}" ]]; then
echo "ERROR: JAVA_HOME ${JAVA_HOME} does not exist."
exit 1
fi

JAVA="${JAVA_HOME}/bin/java"

if [[ ! -x "$JAVA" ]]; then
echo "ERROR: $JAVA is not executable."
exit 1
fi
}

function daemon_status()
{
#
# LSB 4.1.0 compatible status command (1)
#
# 0 = program is running
# 1 = dead, but still a pid (2)
# 2 = (not used by us)
# 3 = not running
#
# 1 - this is not an endorsement of the LSB
#
# 2 - technically, the specification says /var/run/pid, so
# we should never return this value, but we're giving
# them the benefit of a doubt and returning 1 even if
# our pid is not in in /var/run .
#

local pidfile="$1"
shift

local pid

if [[ -f "${pidfile}" ]]; then
pid=$(cat "${pidfile}")
if ps -p "${pid}" > /dev/null 2>&1; then
return 0
fi
return 1
fi
return 3
}

function start()
{
java_setup

daemon_status "${PIDFILE}"
if [[ $? == 0 ]]; then
echo "AMS AD Manager is running as process $(cat "${PIDFILE}"). Exiting" | tee -a $STARTUPFILE
exit 0
else
# stale pid file, so just remove it and continue on
rm -f "${PIDFILE}" >/dev/null 2>&1
fi

nohup "${JAVA}" "-Xms$AMS_AD_HEAPSIZE" "-Xmx$AMS_AD_HEAPSIZE" ${AMS_AD_OPTS} "-Dlog4j.configuration=file://$CONF_DIR/log4j.properties" "-jar" "/usr/lib/ambari-metrics-anomaly-detection/ambari-metrics-anomaly-detection-service.jar" "server" "${CONF_DIR}/config.yaml" "$@" > $OUTFILE 2>&1 &
PID=$!
write_pidfile "${PIDFILE}"
sleep 2

echo "Verifying ${DAEMON_NAME} process status..."
if [ -z "`ps ax -o pid | grep ${PID}`" ]; then
if [ -s ${OUTFILE} ]; then
echo "ERROR: ${DAEMON_NAME} start failed. For more details, see ${OUTFILE}:"
echo "===================="
tail -n 10 ${OUTFILE}
echo "===================="
else
echo "ERROR: ${DAEMON_NAME} start failed"
rm -f ${PIDFILE}
fi
echo "Anomaly Detection Manager out at: ${OUTFILE}"
exit -1
fi

rm -f $STARTUPFILE #Deleting startup file
echo "Anomaly Detection Manager successfully started."
}

function stop()
{
pidfile=${PIDFILE}

if [[ -f "${pidfile}" ]]; then
pid=$(cat "$pidfile")

kill "${pid}" >/dev/null 2>&1
sleep "${STOP_TIMEOUT}"

if kill -0 "${pid}" > /dev/null 2>&1; then
echo "WARNING: ${DAEMON_NAME} did not stop gracefully after ${STOP_TIMEOUT} seconds: Trying to kill with kill -9"
kill -9 "${pid}" >/dev/null 2>&1
fi

if ps -p "${pid}" > /dev/null 2>&1; then
echo "ERROR: Unable to kill ${pid}"
else
rm -f "${pidfile}" >/dev/null 2>&1
fi
fi
}

# execute ams-env.sh
if [[ -f "${CONF_DIR}/ams-admanager-env.sh" ]]; then
. "${CONF_DIR}/ams-admanager-env.sh"
else
echo "ERROR: Cannot execute ${CONF_DIR}/ams-admanager-env.sh." 2>&1
exit 1
fi

# set these env variables only if they were not set by ams-env.sh
: ${AMS_AD_LOG_DIR:=/var/log/ambari-metrics-anomaly-detection}

# set pid dir path
if [[ -n "${AMS_AD_PID_DIR}" ]]; then
PIDFILE=${AMS_AD_PID_DIR}/admanager.pid
fi

# set out file path
if [[ -n "${AMS_AD_LOG_DIR}" ]]; then
OUTFILE=${AMS_AD_LOG_DIR}/ambari-metrics-admanager.out
fi

#TODO manage 3 hbase daemons for start/stop/status
case "$1" in

start)
start

;;
stop)
stop

;;
status)
daemon_status "${PIDFILE}"
if [[ $? == 0 ]]; then
echo "AMS AD Manager is running as process $(cat "${PIDFILE}")."
else
echo "AMS AD Manager is not running."
fi
;;
restart)
stop
start
;;

esac
@@ -0,0 +1,31 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Define some default values that can be overridden by system properties
# Root logger option
log4j.rootLogger=INFO,file

# Direct log messages to a log file
log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.File=/var/log/ambari-metrics-anomaly-detection/ambari-metrics-admanager.log
log4j.appender.file.MaxFileSize=80MB
log4j.appender.file.MaxBackupIndex=60
log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{ABSOLUTE} %5p [%t] %c{1}:%L - %m%n


@@ -135,7 +135,7 @@
<version>3.1.0</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<minimizeJar>true</minimizeJar>
<!--<minimizeJar>true</minimizeJar>-->
<filters>
<filter>
<artifact>*:*</artifact>
@@ -231,6 +231,12 @@
<groupId>org.apache.kafka</groupId>
<artifactId>connect-json</artifactId>
<version>0.10.1.0</version>
<exclusions>
<exclusion>
<artifactId>jackson-databind</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -262,6 +268,10 @@
<artifactId>jersey-json</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
@@ -307,7 +317,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>commons-el</groupId>
@@ -446,7 +455,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>21.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.dropwizard.metrics</groupId>
@@ -21,10 +21,12 @@ logging:
type: external

metricDefinitionService:
inputDefinitionDirectory: /etc/ambari-metrics-anomaly-detection/conf
inputDefinitionDirectory: /etc/ambari-metrics-anomaly-detection/conf/definitionDirectory

metricsCollector:
hostPortList: host1:6188,host2:6188
hosts: host1,host2
port: 6188
protocol: http
metadataEndpoint: /v1/timeline/metrics/metadata/keys

adQueryService:
@@ -20,14 +20,16 @@ package org.apache.ambari.metrics.adservice.app

import javax.validation.Valid

import org.apache.ambari.metrics.adservice.configuration._
import org.apache.ambari.metrics.adservice.configuration.{HBaseConfiguration, _}

import com.fasterxml.jackson.annotation.{JsonIgnore, JsonIgnoreProperties, JsonProperty}

import com.fasterxml.jackson.annotation.JsonProperty
import io.dropwizard.Configuration

/**
* Top Level AD System Manager config items.
*/
@JsonIgnoreProperties(ignoreUnknown=true)
class AnomalyDetectionAppConfig extends Configuration {

/*
@@ -54,6 +56,7 @@ class AnomalyDetectionAppConfig extends Configuration {
/*
HBase Conf
*/
@JsonIgnore
def getHBaseConf : org.apache.hadoop.conf.Configuration = {
HBaseConfiguration.getHBaseConf
}
@@ -17,14 +17,16 @@
*/
package org.apache.ambari.metrics.adservice.app

import org.apache.ambari.metrics.adservice.db.MetadataDatasource
import org.apache.ambari.metrics.adservice.db.{AdMetadataStoreAccessor, LevelDbStoreAccessor, MetadataDatasource}
import org.apache.ambari.metrics.adservice.leveldb.LevelDBDataSource
import org.apache.ambari.metrics.adservice.resource.{AnomalyResource, RootResource}
import org.apache.ambari.metrics.adservice.metadata.{MetricDefinitionService, MetricDefinitionServiceImpl}
import org.apache.ambari.metrics.adservice.resource.{AnomalyResource, MetricDefinitionResource, RootResource}
import org.apache.ambari.metrics.adservice.service.{ADQueryService, ADQueryServiceImpl}

import com.codahale.metrics.health.HealthCheck
import com.google.inject.AbstractModule
import com.google.inject.multibindings.Multibinder

import io.dropwizard.setup.Environment

class AnomalyDetectionAppModule(config: AnomalyDetectionAppConfig, env: Environment) extends AbstractModule {
@@ -34,8 +36,11 @@ class AnomalyDetectionAppModule(config: AnomalyDetectionAppConfig, env: Environm
val healthCheckBinder = Multibinder.newSetBinder(binder(), classOf[HealthCheck])
healthCheckBinder.addBinding().to(classOf[DefaultHealthCheck])
bind(classOf[AnomalyResource])
bind(classOf[MetricDefinitionResource])
bind(classOf[RootResource])
bind(classOf[AdMetadataStoreAccessor]).to(classOf[LevelDbStoreAccessor])
bind(classOf[ADQueryService]).to(classOf[ADQueryServiceImpl])
bind(classOf[MetricDefinitionService]).to(classOf[MetricDefinitionServiceImpl])
bind(classOf[MetadataDatasource]).to(classOf[LevelDBDataSource])
}
}
@@ -19,12 +19,14 @@ package org.apache.ambari.metrics.adservice.configuration
import java.net.{MalformedURLException, URISyntaxException}

import org.apache.hadoop.conf.Configuration
import org.slf4j.{Logger, LoggerFactory}

object HBaseConfiguration {

val HBASE_SITE_CONFIGURATION_FILE: String = "hbase-site.xml"
val hbaseConf: org.apache.hadoop.conf.Configuration = new Configuration(true)
var isInitialized: Boolean = false
val LOG : Logger = LoggerFactory.getLogger("HBaseConfiguration")

def initConfigs(): Unit = {
if (!isInitialized) {
@@ -28,13 +28,25 @@ import com.fasterxml.jackson.annotation.JsonProperty
class MetricCollectorConfiguration {

@NotNull
private var hostPortList: String = _
private var hosts: String = _

@NotNull
private var port: String = _

@NotNull
private var protocol: String = _

@NotNull
private var metadataEndpoint: String = _

@JsonProperty
def getHostPortList: String = hostPortList
def getHosts: String = hosts

@JsonProperty
def getPort: String = port

@JsonProperty
def getProtocol: String = protocol

@JsonProperty
def getMetadataEndpoint: String = metadataEndpoint

0 comments on commit 1336d89

Please sign in to comment.