Skip to content
This repository has been archived by the owner on Feb 20, 2023. It is now read-only.

Commit

Permalink
Make @sla work in combination with @timed and @exception metered
Browse files Browse the repository at this point in the history
  • Loading branch information
Felix Barnsteiner committed Apr 24, 2016
1 parent 01f6b6d commit bf6dea2
Show file tree
Hide file tree
Showing 17 changed files with 215 additions and 74 deletions.
2 changes: 1 addition & 1 deletion stagemonitor-alerting/src/main/java/org/stagemonitor/alerting/AlertingPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ public void initializePlugin(StagemonitorPlugin.InitArguments initArguments) thr

thresholdMonitoringReporter = new ThresholdMonitoringReporter(initArguments.getMetricRegistry(), alertingPlugin, alertSender, incidentRepository, initArguments.getMeasurementSession());
thresholdMonitoringReporter.start(alertingPlugin.checkFrequency.getValue(), TimeUnit.SECONDS);
SlaCheckCreatingClassPathScanner.onStart();
SlaCheckCreatingClassPathScanner.onStart(initArguments.getMeasurementSession());
}

@Override
Expand Down
6 changes: 5 additions & 1 deletion stagemonitor-alerting/src/main/java/org/stagemonitor/alerting/annotation/SLA.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,16 @@

import org.stagemonitor.alerting.check.CheckResult;
import org.stagemonitor.alerting.check.Threshold;
import org.stagemonitor.requestmonitor.MonitorRequests;

/**
* This annotation lets you define service level agreements within the code.
* <p/>
* It automatically creates checks for the annotated method.
* The method also has to be annotated with @{@link org.stagemonitor.requestmonitor.MonitorRequests}
* The method also has to be annotated either with @{@link org.stagemonitor.requestmonitor.MonitorRequests}
* ({@link MonitorRequests#resolveNameAtRuntime()} must be set to <code>false</code> then) or
* with @{@link com.codahale.metrics.annotation.Timed} for response time SLAs
* and @{@link com.codahale.metrics.annotation.ExceptionMetered} for {@link #errorRateThreshold()}s.
*/
@Retention(RetentionPolicy.RUNTIME)
@Target(ElementType.METHOD)
Expand Down
101 changes: 71 additions & 30 deletions .../src/main/java/org/stagemonitor/alerting/annotation/SlaCheckCreatingClassPathScanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import java.util.List;
import java.util.regex.Pattern;

import com.codahale.metrics.annotation.ExceptionMetered;
import com.codahale.metrics.annotation.Timed;
import net.bytebuddy.description.annotation.AnnotationList;
import net.bytebuddy.description.method.MethodDescription;
import net.bytebuddy.matcher.ElementMatcher;
Expand All @@ -16,19 +18,21 @@
import org.stagemonitor.alerting.check.Check;
import org.stagemonitor.alerting.check.MetricCategory;
import org.stagemonitor.alerting.check.Threshold;
import org.stagemonitor.core.CorePlugin;
import org.stagemonitor.core.Stagemonitor;
import org.stagemonitor.core.MeasurementSession;
import org.stagemonitor.core.instrument.AbstractClassPathScanner;
import org.stagemonitor.core.metrics.annotations.ExceptionMeteredTransformer;
import org.stagemonitor.core.metrics.annotations.TimedTransformer;
import org.stagemonitor.core.metrics.metrics2.MetricName;
import org.stagemonitor.requestmonitor.AbstractMonitorRequestsTransformer;
import org.stagemonitor.requestmonitor.MonitorRequests;
import org.stagemonitor.requestmonitor.RequestMonitor;
import org.stagemonitor.requestmonitor.RequestMonitorPlugin;

public class SlaCheckCreatingClassPathScanner extends AbstractClassPathScanner {

private static final Logger logger = LoggerFactory.getLogger(SlaCheckCreatingClassPathScanner.class);

private static List<Check> checksCreatedBeforeMeasurementStarted = new LinkedList<Check>();
private static MeasurementSession measurementSession;

@Override
protected ElementMatcher.Junction<MethodDescription.InDefinedShape> getExtraMethodElementMatcher() {
Expand All @@ -37,46 +41,78 @@ protected ElementMatcher.Junction<MethodDescription.InDefinedShape> getExtraMeth

@Override
protected void onMethodMatch(MethodDescription.InDefinedShape methodDescription) {
final String requestName = configuration.getConfig(RequestMonitorPlugin.class).getBusinessTransactionNamingStrategy()
.getBusinessTransationName(methodDescription.getDeclaringType().getSimpleName(), methodDescription.getName());
final String fullMethodSignature = methodDescription.toString();
final AnnotationList declaredAnnotations = methodDescription.getDeclaredAnnotations();

if (!declaredAnnotations.isAnnotationPresent(MonitorRequests.class)) {
logger.warn("To create an SLA for the method {}, it also has to be annotated with @MonitorRequests",
fullMethodSignature);
final TimerNames timerNames = getTimerNames(methodDescription, declaredAnnotations);

createChecks(fullMethodSignature, declaredAnnotations, timerNames);
}

private static class TimerNames {
private MetricName timerMetricName;
private String timerName;
private String errorRequestName;
private MetricName errorMetricName;
}

private TimerNames getTimerNames(MethodDescription.InDefinedShape methodDescription, AnnotationList declaredAnnotations) {
TimerNames timerNames = new TimerNames();
if (declaredAnnotations.isAnnotationPresent(MonitorRequests.class)) {
timerNames.timerName = AbstractMonitorRequestsTransformer.getRequestName(methodDescription);
if (timerNames.timerName != null) {
timerNames.timerMetricName = RequestMonitor.getTimerMetricName(timerNames.timerName);
timerNames.errorRequestName = timerNames.timerName;
timerNames.errorMetricName = RequestMonitor.getErrorMetricName(timerNames.timerName);
}
} else {
if (declaredAnnotations.isAnnotationPresent(SLA.class)) {
createSlaCheck(declaredAnnotations.ofType(SLA.class).loadSilent(), fullMethodSignature, requestName);
if (declaredAnnotations.isAnnotationPresent(Timed.class)) {
timerNames.timerName = new TimedTransformer.TimedSignatureDynamicValue().getRequestName(methodDescription);
timerNames.timerMetricName = TimedTransformer.getTimerName(timerNames.timerName);
}
if (declaredAnnotations.isAnnotationPresent(SLAs.class)) {
for (SLA sla : declaredAnnotations.ofType(SLAs.class).loadSilent().value()) {
createSlaCheck(sla, fullMethodSignature, requestName);
}
if (declaredAnnotations.isAnnotationPresent(ExceptionMetered.class)) {
timerNames.errorRequestName = new ExceptionMeteredTransformer.ExceptionMeteredSignatureDynamicValue().getRequestName(methodDescription);
timerNames.errorMetricName = ExceptionMeteredTransformer.getMetricName(timerNames.errorRequestName);
}
}
return timerNames;
}

private static void createSlaCheck(SLA slaAnnotation, String fullMethodSignature, String requestName) {
private void createChecks(String fullMethodSignature, AnnotationList declaredAnnotations, TimerNames timerNames) {
if (declaredAnnotations.isAnnotationPresent(SLA.class)) {
createSlaCheck(declaredAnnotations.ofType(SLA.class).loadSilent(), fullMethodSignature, timerNames);
}
if (declaredAnnotations.isAnnotationPresent(SLAs.class)) {
for (SLA sla : declaredAnnotations.ofType(SLAs.class).loadSilent().value()) {
createSlaCheck(sla, fullMethodSignature, timerNames);
}
}
}

private static void createSlaCheck(SLA slaAnnotation, String fullMethodSignature, TimerNames timerNames) {
if (slaAnnotation.metric().length > 0) {
addResponseTimeCheck(slaAnnotation, fullMethodSignature, requestName);
addResponseTimeCheck(slaAnnotation, fullMethodSignature, timerNames);
}
if (slaAnnotation.errorRateThreshold() >= 0) {
addErrorRateCheck(slaAnnotation, fullMethodSignature, requestName);
addErrorRateCheck(slaAnnotation, fullMethodSignature, timerNames);
}
}

private static void addResponseTimeCheck(SLA slaAnnotation, String fullMethodSignature, String requestName) {
private static void addResponseTimeCheck(SLA slaAnnotation, String fullMethodSignature, TimerNames timerNames) {
SLA.Metric[] metrics = slaAnnotation.metric();
double[] thresholdValues = slaAnnotation.threshold();
if (metrics.length != thresholdValues.length) {
logger.warn("The number of provided metrics don't match the number of provided thresholds in @SLA {}", fullMethodSignature);
return;
}
if (timerNames.timerName == null) {
logger.warn("To create a timer SLA for the method {}, it also has to be annotated with @MonitorRequests or " +
" @Timed. When using @MonitorRequests, resolveNameAtRuntime must not be set to true.", fullMethodSignature);
return;
}

final MetricName timerMetricName = RequestMonitor.getTimerMetricName(requestName);
Check check = createCheck(slaAnnotation, fullMethodSignature, requestName, MetricCategory.TIMER, timerMetricName,
" (response time)", "responseTime");
Check check = createCheck(slaAnnotation, fullMethodSignature, timerNames.timerName, MetricCategory.TIMER,
timerNames.timerMetricName, " (response time)", "responseTime");

final List<Threshold> thresholds = check.getThresholds(slaAnnotation.severity());
for (int i = 0; i < metrics.length; i++) {
Expand All @@ -86,9 +122,13 @@ private static void addResponseTimeCheck(SLA slaAnnotation, String fullMethodSig
addCheckIfStarted(check);
}

private static void addErrorRateCheck(SLA slaAnnotation, String fullMethodSignature, String requestName) {
final MetricName errorMetricName = RequestMonitor.getErrorMetricName(requestName);
final Check check = createCheck(slaAnnotation, fullMethodSignature, requestName, MetricCategory.METER, errorMetricName, " (errors)", "errors");
private static void addErrorRateCheck(SLA slaAnnotation, String fullMethodSignature, TimerNames timerNames) {
if (timerNames.errorRequestName == null) {
logger.warn("To create an error SLA for the method {}, it also has to be annotated with @MonitorRequests or " +
" @ExceptionMetered. When using @MonitorRequests, resolveNameAtRuntime must not be set to true.", fullMethodSignature);
return;
}
final Check check = createCheck(slaAnnotation, fullMethodSignature, timerNames.errorRequestName, MetricCategory.METER, timerNames.errorMetricName, " (errors)", "errors");
final Threshold t = new Threshold(SLA.Metric.M1_RATE.getValue(), Threshold.Operator.GREATER_EQUAL, slaAnnotation.errorRateThreshold());
check.getThresholds(slaAnnotation.severity()).add(t);
addCheckIfStarted(check);
Expand All @@ -106,25 +146,26 @@ private static Check createCheck(SLA slaAnnotation, String fullMethodSignature,
}

private static void addCheckIfStarted(Check check) {
if (Stagemonitor.isStarted()) {
addCheck(check);
if (measurementSession != null) {
addCheck(check, measurementSession);
} else {
checksCreatedBeforeMeasurementStarted.add(check);
}
}

private static void addCheck(Check check) {
check.setApplication(configuration.getConfig(CorePlugin.class).getMeasurementSession().getApplicationName());
private static void addCheck(Check check, MeasurementSession measurementSession) {
check.setApplication(measurementSession.getApplicationName());
try {
configuration.getConfig(AlertingPlugin.class).addCheck(check);
} catch (IOException e) {
logger.warn(e.getMessage(), e);
}
}

public static void onStart() {
public static void onStart(MeasurementSession measurementSession) {
SlaCheckCreatingClassPathScanner.measurementSession = measurementSession;
for (Check check : checksCreatedBeforeMeasurementStarted) {
addCheck(check);
addCheck(check, measurementSession);
}
checksCreatedBeforeMeasurementStarted.clear();
}
Expand Down
91 changes: 83 additions & 8 deletions .../test/java/org/stagemonitor/alerting/annotation/SlaCheckCreatingClassPathScannerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;

import java.util.List;
import java.util.Map;

import com.codahale.metrics.annotation.ExceptionMetered;
import com.codahale.metrics.annotation.Timed;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
Expand All @@ -22,6 +25,7 @@
public class SlaCheckCreatingClassPathScannerTest {

private final AlertingPlugin alertingPlugin = Stagemonitor.getPlugin(AlertingPlugin.class);
private Map<String, Check> checks;

private static class SlaTestClass {
@SLAs({
Expand All @@ -42,6 +46,36 @@ void monitorRequestsAnnotationMissing() {
void tooFewThresholds() {
}

@MonitorRequests(resolveNameAtRuntime = true)
@SLA(metric = {SLA.Metric.P95, SLA.Metric.MAX}, threshold = {0, 0})
void slaMonitorRequestsResolveAtRuntime() {
}

@MonitorRequests(requestName = "monitor requests custom name")
@SLA(metric = {SLA.Metric.P95, SLA.Metric.MAX}, threshold = {0, 0})
void slaMonitorRequestsCustomName() {
}

@Timed(name = "timed custom name", absolute = true)
@SLA(metric = {SLA.Metric.P95, SLA.Metric.MAX}, threshold = {0, 0})
void slaTimedCustomName() {
}

@Timed
@SLA(metric = {SLA.Metric.P95, SLA.Metric.MAX}, threshold = {0, 0})
void slaOnTimed() {
}

@ExceptionMetered
@SLA(errorRateThreshold = 0)
void slaOnExceptionMetered() {
}

@Timed
@SLA(errorRateThreshold = 0)
void slaMissingExceptionMetered() {
}

static void makeSureClassIsLoaded() {
}
}
Expand All @@ -55,25 +89,32 @@ public static void init() throws Exception {

@Before
public void setUp() throws Exception {
checks = alertingPlugin.getChecks();
}

@Test
public void testSlaCreatesChecks() throws Exception {
final Map<String, Check> checks = alertingPlugin.getChecks();
public void testSlaMonitorRequests() throws Exception {
testErrorRateCheck("void org.stagemonitor.alerting.annotation.SlaCheckCreatingClassPathScannerTest$SlaTestClass.monitorSla().errors",
"\\Qerror_rate_server.Monitor-Sla.All\\E");

final Check errorRateCheck = checks
.get("void org.stagemonitor.alerting.annotation.SlaCheckCreatingClassPathScannerTest$SlaTestClass.monitorSla().errors");
testResponseTimeCheck("void org.stagemonitor.alerting.annotation.SlaCheckCreatingClassPathScannerTest$SlaTestClass.monitorSla().responseTime",
"\\Qresponse_time_server.Monitor-Sla.All\\E");
}

private void testErrorRateCheck(String checkId, String checkTargetRegex) {
final Check errorRateCheck = checks.get(checkId);
assertNotNull(checks.keySet().toString(), errorRateCheck);
assertEquals("Alerting-Test", errorRateCheck.getApplication());
assertEquals(MetricCategory.METER, errorRateCheck.getMetricCategory());
assertEquals("\\Qerror_rate_server.Monitor-Sla.All\\E", errorRateCheck.getTarget().toString());
assertEquals(checkTargetRegex, errorRateCheck.getTarget().toString());
}

final Check responseTimeChek = checks
.get("void org.stagemonitor.alerting.annotation.SlaCheckCreatingClassPathScannerTest$SlaTestClass.monitorSla().responseTime");
private void testResponseTimeCheck(String checkId, String checkTargetRegex) {
final Check responseTimeChek = checks.get(checkId);
assertNotNull(checks.keySet().toString(), responseTimeChek);
assertEquals("Alerting-Test", responseTimeChek.getApplication());
assertEquals(MetricCategory.TIMER, responseTimeChek.getMetricCategory());
assertEquals("\\Qresponse_time_server.Monitor-Sla.All\\E", responseTimeChek.getTarget().toString());
assertEquals(checkTargetRegex, responseTimeChek.getTarget().toString());
final List<Threshold> thresholds = responseTimeChek.getThresholds(CheckResult.Status.ERROR);
final Threshold p95 = thresholds.get(0);
assertEquals("p95", p95.getMetric());
Expand All @@ -85,6 +126,40 @@ public void testSlaCreatesChecks() throws Exception {
assertEquals(0, max.getThresholdValue(), 0);
}

@Test
public void testSlaCustomName() throws Exception {
testResponseTimeCheck("void org.stagemonitor.alerting.annotation.SlaCheckCreatingClassPathScannerTest$SlaTestClass.slaMonitorRequestsCustomName().responseTime",
"\\Qresponse_time_server.monitor-requests-custom-name.All\\E");
}

@Test
public void testTimedCustomName() throws Exception {
testResponseTimeCheck("void org.stagemonitor.alerting.annotation.SlaCheckCreatingClassPathScannerTest$SlaTestClass.slaTimedCustomName().responseTime",
"\\Qtimer.timed-custom-name\\E");
}

@Test
public void testSlaTimed() throws Exception {
testResponseTimeCheck("void org.stagemonitor.alerting.annotation.SlaCheckCreatingClassPathScannerTest$SlaTestClass.slaOnTimed().responseTime",
"\\Qtimer.SlaCheckCreatingClassPathScannerTest$SlaTestClass#slaOnTimed\\E");
}

@Test
public void testSlaExceptionMetered() throws Exception {
testErrorRateCheck("void org.stagemonitor.alerting.annotation.SlaCheckCreatingClassPathScannerTest$SlaTestClass.slaOnExceptionMetered().errors",
"\\Qexception_rate.SlaCheckCreatingClassPathScannerTest$SlaTestClass#slaOnExceptionMetered\\E");
}

@Test
public void testSlaMissingExceptionMetered() throws Exception {
assertNull(checks.get("void org.stagemonitor.alerting.annotation.SlaCheckCreatingClassPathScannerTest$SlaTestClass.slaMissingExceptionMetered().errors"));
}

@Test
public void testSlaMonitorRequestsResolveAtRuntime() throws Exception {
assertNull(checks.get("void org.stagemonitor.alerting.annotation.SlaCheckCreatingClassPathScannerTest$SlaTestClass.slaMonitorRequestsResolveAtRuntime().responseTime"));
}

@Test
public void testTriggersResponseTimeIncident() throws Exception {
try {
Expand Down
7 changes: 6 additions & 1 deletion .../src/main/java/org/stagemonitor/core/metrics/annotations/ExceptionMeteredTransformer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import net.bytebuddy.matcher.ElementMatcher;
import org.stagemonitor.core.Stagemonitor;
import org.stagemonitor.core.instrument.StagemonitorByteBuddyTransformer;
import org.stagemonitor.core.metrics.metrics2.MetricName;

public class ExceptionMeteredTransformer extends StagemonitorByteBuddyTransformer {

Expand All @@ -29,10 +30,14 @@ protected ElementMatcher.Junction<MethodDescription.InDefinedShape> getExtraMeth
@Advice.OnMethodExit(onThrowable = Exception.class, inline = false)
public static void meterException(@ExceptionMeteredSignature String signature, @MeterExceptionsFor Class<? extends Exception> cause, @Advice.Thrown Throwable e) {
if (e != null && cause.isInstance(e)) {
Stagemonitor.getMetric2Registry().meter(name("exception_rate").tag("signature", signature).build()).mark();
Stagemonitor.getMetric2Registry().meter(getMetricName(signature)).mark();
}
}

public static MetricName getMetricName(String signature) {
return name("exception_rate").tag("signature", signature).build();
}

@Override
protected List<StagemonitorDynamicValue<?>> getDynamicValues() {
return Arrays.asList(new ExceptionMeteredTransformer.ExceptionMeteredSignatureDynamicValue(),
Expand Down
3 changes: 1 addition & 2 deletions ...onitor-core/src/main/java/org/stagemonitor/core/metrics/annotations/GaugeTransformer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,12 @@ public static void gauges(@Advice.This Object thiz) {
}

public static void monitorGauges(Object object) {
String simpleClassName = object.getClass().getSimpleName();
for (final Method method : object.getClass().getDeclaredMethods()) {
final Gauge gaugeAnnotation = method.getAnnotation(Gauge.class);
// only create gauge, if method takes no parameters and is non-void
if (gaugeAnnotation != null && methodTakesNoParamsAndIsNonVoid(method)) {
method.setAccessible(true);
final String signature = SignatureUtils.getSignature(simpleClassName, method.getName(),
final String signature = SignatureUtils.getSignature(object.getClass().getName(), method.getName(),
gaugeAnnotation.name(), gaugeAnnotation.absolute());

registerGauge(object, method, signature);
Expand Down

0 comments on commit bf6dea2

Please sign in to comment.