/
EmitToGraphiteLogbackAppender.java
191 lines (170 loc) · 8.22 KB
/
EmitToGraphiteLogbackAppender.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
/*
* Copyright 2018 Expedia, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.expedia.www.haystack.metrics.appenders.logback;
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.spi.ILoggingEvent;
import ch.qos.logback.core.AppenderBase;
import com.expedia.www.haystack.metrics.GraphiteConfigImpl;
import com.expedia.www.haystack.metrics.MetricObjects;
import com.expedia.www.haystack.metrics.MetricPublishing;
import com.netflix.servo.monitor.Counter;
import com.netflix.servo.util.VisibleForTesting;
import java.util.Map;
import java.util.Timer;
import java.util.concurrent.ConcurrentHashMap;
import static ch.qos.logback.classic.Level.ERROR;
/**
* A logback appender that sends an error count to a graphite endpoint.
*/
@SuppressWarnings("WeakerAccess") // for the setter methods that need to be public to be used by other packages
public class EmitToGraphiteLogbackAppender extends AppenderBase<ILoggingEvent> {
@VisibleForTesting
static final String ERRORS_METRIC_GROUP = "errors";
@VisibleForTesting
static final Map<String, Counter> ERRORS_COUNTERS = new ConcurrentHashMap<>();
private final MetricPublishing metricPublishing;
private final MetricObjects metricObjects;
private final Factory factory;
// These attributes need to be configured
private String host = "haystack.local"; // this is the value used by Minikube
private String subsystem;
// These attributes have sensible default values and don't need to be configured
private boolean enabled = true;
private int port = 2003;
private int pollintervalseconds = 60;
private int queuesize = 10;
private boolean sendasrate = false;
// This attribute is not set until the appender starts
private StartUpMetric startUpMetric;
/**
* The default constructor, used by logback. Logback configuration uses setters, but of the six values needed
* (host, subsystem, port, poll interval, send as rate, and queue size), all but host and subsystem are set to
* sensible values and probably don't need to be configured. The host should be set to the DNS name or IP host of
* the Graphite endpoint you wish to receive counts of errors.
*/
public EmitToGraphiteLogbackAppender() {
this(new MetricPublishing(), new MetricObjects(), new Factory());
}
@VisibleForTesting
EmitToGraphiteLogbackAppender(MetricPublishing metricPublishing, MetricObjects metricObjects, Factory factory) {
this.metricPublishing = metricPublishing;
this.metricObjects = metricObjects;
this.factory = factory;
}
// Setters are used by logback to configure the Appender.
public void setEnabled(boolean enabled) {
this.enabled = enabled;
}
public void setHost(String host) {
this.host = host;
}
public void setPort(int port) {
this.port = port;
}
public void setPollintervalseconds(int pollintervalseconds) {
this.pollintervalseconds = pollintervalseconds;
}
public void setQueuesize(int queuesize) {
this.queuesize = queuesize;
}
public void setSendasrate(boolean sendasrate) {
this.sendasrate = sendasrate;
}
public void setSubsystem(String subsystem) {
this.subsystem = subsystem;
}
/**
* Starts the appender by starting a background thread to poll the error counters and publish them to Graphite.
* Multiple instances of this EmitToGraphiteLogbackAppender will only start one background thread.
* This method also starts the heartbeat metric background thread.
*/
@Override
public void start() {
super.start();
// If disabled we do not create a publisher to graphite but error counts are still collected.
if(enabled) {
metricPublishing.start(new GraphiteConfigImpl(host, port, pollintervalseconds, queuesize, sendasrate));
}
this.startUpMetric = factory.createStartUpMetric(metricObjects, subsystem, new Timer());
startUpMetric.start();
}
/**
* Stops the appender, shutting down the background polling thread to ensure that the connection to the metrics
* database is closed. This method also stops the heartbeat method background thread.
*/
@Override
public void stop() {
if(enabled) {
metricPublishing.stop();
}
if(startUpMetric != null) {
startUpMetric.stop();
}
super.stop();
}
@Override
protected void append(ILoggingEvent logEvent) {
final Level level = logEvent.getLevel();
if (isLevelSevereEnoughToCount(level)) {
final StackTraceElement[] stackTraceElements = logEvent.getCallerData();
final StackTraceElement stackTraceElement = stackTraceElements[0];
getCounter(level, stackTraceElement).increment();
}
}
@VisibleForTesting
boolean isLevelSevereEnoughToCount(Level level) {
return level == ERROR;
}
// From https://github.com/ExpediaDotCom/haystack-logback-metrics-appender/issues/28
// Scaling issues with InfluxDb have led us to change the way that Graphite messages are changed into tagged metrics
// in InfluxDb; in particular, the InfluxDb template for the error metrics was changed from
// "haystack.errors.* system.metricGroup.subsystem.fqClass.host.lineNumber.measurement*" to
// "haystack.errors.* measurement.measurement.measurement.fqClass.host.field*". As a result, the presence of line
// number in the metric needs to be removed. In the interest of simplicity, I will comment out the code that inserts
// line number into the Graphite metric, to facilitate a potential setting-based change in the future to allow this
// package to create both types of Graphite metrics.
@VisibleForTesting
Counter getCounter(Level level, StackTraceElement stackTraceElement) {
final String fullyQualifiedClassName = changePeriodsToDashes(stackTraceElement.getClassName());
//final String lineNumber = Integer.toString(stackTraceElement.getLineNumber());
@SuppressWarnings("UnnecessaryLocalVariable")
final String key = fullyQualifiedClassName/* + ':' + lineNumber*/;
if (!ERRORS_COUNTERS.containsKey(key)) {
final Counter counter = factory.createCounter(
metricObjects, subsystem, fullyQualifiedClassName, /*lineNumber, */level.toString());
// It is possible but highly unlikely that two threads are in this if() block at the same time; if that
// occurs, only one of the calls to ERRORS_COUNTERS.putIfAbsent(hashCode, counter) in the next line of code
// will succeed, but the increment of the thread whose call did not succeed will not be lost, because the
// value returned by this method will be the Counter put successfully by the other thread.
ERRORS_COUNTERS.putIfAbsent(key, counter);
}
return ERRORS_COUNTERS.get(key);
}
static String changePeriodsToDashes(String fullyQualifiedClassName) {
return fullyQualifiedClassName.replace('.', '-');
}
@VisibleForTesting
static class Factory {
Counter createCounter(MetricObjects metricObjects, String subsystem, String fullyQualifiedClassName,
/*String lineNumber, */String counterName) {
return metricObjects.createAndRegisterResettingCounter(
ERRORS_METRIC_GROUP, subsystem, fullyQualifiedClassName, /*lineNumber, */counterName);
}
StartUpMetric createStartUpMetric(MetricObjects metricObjects, String subsystem, Timer timer) {
return new StartUpMetric(timer, new StartUpMetric.Factory(), metricObjects, subsystem);
}
}
}