Skip to content

Commit

Permalink
Global unhealthy stats
Browse files Browse the repository at this point in the history 
[#151023785] https://www.pivotaltracker.com/story/show/151257999
[#151023775] https://www.pivotaltracker.com/story/show/151023775
  • Loading branch information
@fhanik
fhanik committed Sep 21, 2017
1 parent b522cab commit 4820174
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 24 deletions.
9 changes: 6 additions & 3 deletions statsd/src/main/java/org/cloudfoundry/identity/statsd/UaaMetricsEmitter.java
View file
Expand Up @@ -65,9 +65,12 @@ public void emitGlobalRequestMetrics() throws Exception {
String json = (String) uaaMetricsMap.get("globals");
if (json != null) {
MetricsQueue globals = JsonUtils.readValue(json, MetricsQueue.class);
String prefix = "requests.global.completed.";
statsDClient.gauge(prefix + "time", (long) globals.getTotals().getAverageTime());
statsDClient.gauge(prefix + "count", globals.getTotals().getCount());
String prefix = "requests.global.";
statsDClient.gauge(prefix + "completed.time", (long) globals.getTotals().getAverageTime());
statsDClient.gauge(prefix + "completed.count", globals.getTotals().getCount());
//unhealthy
statsDClient.gauge(prefix + "unhealthy.count", globals.getTotals().getIntolerableCount());
statsDClient.gauge(prefix + "unhealthy.time", (long)globals.getTotals().getAverageIntolerableTime());
}
}
//server statistics
Expand Down
9 changes: 7 additions & 2 deletions statsd/src/test/java/org/cloudfoundry/identity/statsd/UaaMetricsEmitterTests.java
View file
Expand Up @@ -57,6 +57,7 @@ public void setUp() {

serverRequestsBeanMap = new MBeanMap();
serverRequestsBeanMap.put("globals", globalsJson);
serverRequestsBeanMap.put("inflight.count", 3l);


mBeanMap3 = new HashMap();
Expand Down Expand Up @@ -86,6 +87,10 @@ public void requestCount_metrics_emitted() throws Exception {

uaaMetricsEmitter.emitGlobalRequestMetrics();
Mockito.verify(statsDClient).gauge("requests.global.completed.count", 18l);
Mockito.verify(statsDClient).gauge("requests.global.completed.time", 67l);
Mockito.verify(statsDClient).gauge("server.inflight.count", 3l);
Mockito.verify(statsDClient).gauge("requests.global.unhealthy.count", 1l);
Mockito.verify(statsDClient).gauge("requests.global.unhealthy.time", (long)3.25);
}

@Test
Expand Down Expand Up @@ -164,8 +169,8 @@ public void test(Collection<?> c) {
" \"200\":{\n" +
" \"count\":14,\n" +
" \"averageTime\":74.21428571428572,\n" +
" \"intolerableCount\":0,\n" +
" \"averageIntolerableTime\":0.0,\n" +
" \"intolerableCount\":1,\n" +
" \"averageIntolerableTime\":3.25,\n" +
" \"databaseQueryCount\":113,\n" +
" \"averageDatabaseQueryTime\":0.03539823008849556,\n" +
" \"databaseFailedQueryCount\":0,\n" +
Expand Down
59 changes: 40 additions & 19 deletions statsd/src/test/java/org/cloudfoundry/identity/statsd/integration/UaaMetricsEmitterIT.java
View file
Expand Up @@ -27,6 +27,10 @@
import java.net.DatagramPacket;
import java.net.DatagramSocket;
import java.net.SocketTimeoutException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.cloudfoundry.identity.statsd.integration.IntegrationTestUtils.TEST_PASSWORD;
import static org.cloudfoundry.identity.statsd.integration.IntegrationTestUtils.TEST_USERNAME;
Expand All @@ -38,6 +42,7 @@
import static org.junit.Assert.assertThat;

public class UaaMetricsEmitterIT {
public static final int WAIT_FOR_MESSAGE = 5500;
private static DatagramSocket serverSocket;
private static byte[] receiveData;
private static DatagramPacket receivePacket;
Expand Down Expand Up @@ -77,54 +82,70 @@ public void testStatsDClientEmitsMetricsCollectedFromUAA() throws InterruptedExc
new HttpEntity<>(body, headers),
String.class);
assertEquals(HttpStatus.FOUND, loginResponse.getStatusCode());
assertNotNull(getMessage("uaa.audit_service.user.authentication.count:", 5000));
assertNotNull(getMessage("uaa.audit_service.user.authentication.count:", WAIT_FOR_MESSAGE));
}

@Test
public void global_completed_count() throws IOException {
String message = getMessage("uaa.requests.global.completed.count", 5000);
long previousValue = IntegrationTestUtils.getGaugeValueFromMessage(message);
public void global_counts() throws IOException {
List<String> fragments = Arrays.asList(
"uaa.requests.global.completed.count",
"uaa.requests.global.completed.count",
"uaa.requests.global.unhealthy.time",
"uaa.requests.global.unhealthy.count"
);
Map<String,String> firstBatch = getMessages(fragments, WAIT_FOR_MESSAGE);
performSimpleGet();
message = getMessage("uaa.requests.global.completed.count", 5000);
Map<String,String> secondBatch = getMessages(fragments, WAIT_FOR_MESSAGE);

String message = firstBatch.get("uaa.requests.global.completed.count");
long previousValue = IntegrationTestUtils.getGaugeValueFromMessage(message);

message = secondBatch.get("uaa.requests.global.completed.count");
long nextValue = IntegrationTestUtils.getGaugeValueFromMessage(message);
assertThat(nextValue, greaterThan(previousValue));
}

@Test
public void global_completed_time() throws IOException {
performSimpleGet();
String message = getMessage("uaa.requests.global.completed.time", 5000);
long nextValue = IntegrationTestUtils.getGaugeValueFromMessage(message);
assertThat(nextValue, greaterThan(0l));
message = firstBatch.get("uaa.requests.global.unhealthy.time");
long value = IntegrationTestUtils.getGaugeValueFromMessage(message);
assertThat(value, greaterThanOrEqualTo(0l));

message = firstBatch.get("uaa.requests.global.unhealthy.count");
value = IntegrationTestUtils.getGaugeValueFromMessage(message);
assertThat(value, greaterThanOrEqualTo(0l));
}

@Test
public void server_inflight_count() throws IOException {
performSimpleGet();
String message = getMessage("uaa.server.inflight.count", 5000);
String message = getMessage("uaa.server.inflight.count", WAIT_FOR_MESSAGE);
long nextValue = IntegrationTestUtils.getGaugeValueFromMessage(message);
assertThat(nextValue, greaterThanOrEqualTo(0l));
}


protected String getMessage(String fragment, int timeout) throws IOException {
return getMessages(Arrays.asList(fragment), timeout).get(fragment);
}

protected Map<String,String> getMessages(List<String> fragments, int timeout) throws IOException {
long startTime = System.currentTimeMillis();
String found = null;
Map<String,String> results = new HashMap<>();
do {
receiveData = new byte[65535];
receivePacket.setData(receiveData);
try {
serverSocket.receive(receivePacket);
String message = new String(receivePacket.getData()).trim();
System.out.println("message = " + message);
if (message.startsWith(fragment)) {
found = message;
}
fragments.stream().forEach(fragment -> {
if (message.startsWith(fragment)) {
results.put(fragment, message);
}
});
} catch (SocketTimeoutException e) {
//expected so that we keep looping
}
} while (found == null && (System.currentTimeMillis() < (startTime + timeout)));
return found;
} while (results.size()<fragments.size() && (System.currentTimeMillis() < (startTime + timeout)));
return results;
}

public void performSimpleGet() {
Expand Down

0 comments on commit 4820174

Please sign in to comment.