Skip to content

Commit

Permalink
[WFLY-19351] Add Micrometer test to verify that applications with sha…
Browse files Browse the repository at this point in the history
…red metrics names are not merged when exported

Add test with multiple war deployments
Add test with ear with multiple war subdeployments
Rework how Prometheus metrics are retrieved and handled
Add test to verify that metrics with identical names are exported separately
  • Loading branch information
jasondlee committed Jun 13, 2024
1 parent 155419c commit 5c3ae15
Show file tree
Hide file tree
Showing 12 changed files with 424 additions and 66 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,17 @@
*/
package org.wildfly.test.integration.observability.container;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.ClientBuilder;
import jakarta.ws.rs.client.WebTarget;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.MountableFile;
Expand Down Expand Up @@ -86,4 +94,60 @@ public String getPrometheusUrl() {
public List<JaegerTrace> getTraces(String serviceName) throws InterruptedException {
return (jaegerContainer != null ? jaegerContainer.getTraces(serviceName) : Collections.emptyList());
}

public List<PrometheusMetric> fetchMetrics(String nameToMonitor) throws InterruptedException {
String body = "";
try (Client client = ClientBuilder.newClient()) {
WebTarget target = client.target(OpenTelemetryCollectorContainer.getInstance().getPrometheusUrl());

int attemptCount = 0;
boolean found = false;

// Request counts can vary. Setting high to help ensure test stability
while (!found && attemptCount < 30) {
// Wait to give Micrometer time to export
Thread.sleep(1000);

body = target.request().get().readEntity(String.class);
found = body.contains(nameToMonitor);
attemptCount++;
}
}

return buildPrometheusMetrics(body);
}

private List<PrometheusMetric> buildPrometheusMetrics(String body) {
String[] entries = body.split("\n");
Map<String, String> help = new HashMap<>();
Map<String, String> type = new HashMap<>();
List<PrometheusMetric> metrics = new LinkedList<>();
Arrays.stream(entries).forEach(e -> {
if (e.startsWith("# HELP")) {
extractMetadata(help, e);
} else if (e.startsWith("# TYPE")) {
extractMetadata(type, e);
} else {
String[] parts = e.split("[{}]");
String key = parts[0];
Map<String, String> tags = Arrays.stream(parts[1].split(","))
.map(t -> t.split("="))
.collect(Collectors.toMap(i -> i[0],
i -> i[1]
.replaceAll("^\"", "")
.replaceAll("\"$", "")
));
metrics.add(new PrometheusMetric(key, tags, parts[2].trim(), type.get(key), help.get(key)));
}
});

return metrics;
}

private void extractMetadata(Map<String, String> target, String source) {
String[] parts = source.split(" ");
target.put(parts[2],
Arrays.stream(Arrays.copyOfRange(parts, 3, parts.length))
.reduce("", (total, element) -> total + " " + element));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Copyright The WildFly Authors
* SPDX-License-Identifier: Apache-2.0
*/
package org.wildfly.test.integration.observability.container;

import java.util.Collections;
import java.util.Map;

public class PrometheusMetric {
private final String key;
private final Map<String, String> tags;
private final String value;
private final String type;
private final String help;

public PrometheusMetric(String key,
Map<String, String> tags,
String value,
String type,
String help) {
this.key = key;
this.tags = Collections.unmodifiableMap(tags);
this.value = value;
this.type = type;
this.help = help;
}

public String getKey() {
return key;
}

public Map<String, String> getTags() {
return tags;
}

public String getValue() {
return value;
}

public String getType() {
return type;
}

public String getHelp() {
return help;
}

@Override
public String toString() {
return "PrometheusMetric{" +
"key='" + key + '\'' +
", tags=" + tags +
", value='" + value + '\'' +
", type='" + type + '\'' +
", help='" + help + '\'' +
'}';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,16 @@

package org.wildfly.test.integration.observability.micrometer;

import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.Timer;
import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.RequestScoped;
import jakarta.inject.Inject;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;

import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.Timer;

/**
* @author <a href="mailto:jasondlee@redhat.com">Jason Lee</a>
*/
Expand All @@ -38,7 +37,7 @@ public double getCount() {

timer.record(() -> {
try {
Thread.sleep((long) (Math.random() * 1000L));
Thread.sleep((long) (Math.random() * 100L));
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,18 @@
*/
package org.wildfly.test.integration.observability.micrometer;


import java.net.URISyntaxException;
import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import jakarta.inject.Inject;
import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.ClientBuilder;
import jakarta.ws.rs.client.WebTarget;

import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
import org.jboss.arquillian.container.test.api.Deployment;
import org.jboss.arquillian.container.test.api.RunAsClient;
import org.jboss.arquillian.junit.Arquillian;
Expand All @@ -35,12 +34,11 @@
import org.junit.Test;
import org.junit.runner.RunWith;
import org.wildfly.test.integration.observability.container.OpenTelemetryCollectorContainer;
import org.wildfly.test.integration.observability.container.PrometheusMetric;

@RunWith(Arquillian.class)
@ServerSetup(MicrometerSetupTask.class)
public class MicrometerOtelIntegrationTestCase {
protected static boolean dockerAvailable = AssumeTestGroupUtil.isDockerAvailable();

public static final int REQUEST_COUNT = 5;
@ArquillianResource
private URL url;
Expand All @@ -61,14 +59,13 @@ public class MicrometerOtelIntegrationTestCase {

@Deployment
public static Archive<?> deploy() {
return dockerAvailable ?
ShrinkWrap.create(WebArchive.class, "micrometer-test.war")
.addClasses(ServerSetupTask.class,
MetricResource.class,
AssumeTestGroupUtil.class)
.addAsWebInfResource(new StringAsset(WEB_XML), "web.xml")
.addAsWebInfResource(CdiUtils.createBeansXml(), "beans.xml") :
AssumeTestGroupUtil.emptyWar();
return ShrinkWrap.create(WebArchive.class, "micrometer-test.war")
.addClasses(ServerSetupTask.class,
MetricResource.class,
AssumeTestGroupUtil.class,
PrometheusMetric.class)
.addAsWebInfResource(new StringAsset(WEB_XML), "web.xml")
.addAsWebInfResource(CdiUtils.createBeansXml(), "beans.xml");
}

// The @ServerSetup(MicrometerSetupTask.class) requires Docker to be available.
Expand Down Expand Up @@ -112,6 +109,7 @@ public void checkCounter() {
public void getMetrics() throws InterruptedException {
List<String> metricsToTest = Arrays.asList(
"demo_counter",
"demo_timer",
"memory_used_heap",
"cpu_available_processors",
"classloader_loaded_classes_count",
Expand All @@ -121,8 +119,9 @@ public void getMetrics() throws InterruptedException {
"undertow_bytes_received"
);

final String response = fetchMetrics(metricsToTest.get(0));
metricsToTest.forEach(n -> Assert.assertTrue("Missing metric: " + n, response.contains(n)));
final List<PrometheusMetric> metrics = OpenTelemetryCollectorContainer.getInstance().fetchMetrics(metricsToTest.get(0));
metricsToTest.forEach(n -> Assert.assertTrue("Missing metric: " + n,
metrics.stream().anyMatch(m -> m.getKey().startsWith(n))));
}

@Test
Expand All @@ -134,50 +133,19 @@ public void testJmxMetrics() throws InterruptedException {
"classloader_loaded_classes",
"cpu_system_load_average",
"cpu_process_cpu_time",
"classloader_unloaded_classes",
"classloader_loaded_classes_count",
"thread_count",
"thread_daemon_count",
"cpu_available_processors"
);
final String response = fetchMetrics(metricsToTest.get(0));
Map<String, String> metrics = Arrays.stream(response.split("\n"))
.filter(s -> !s.startsWith("#"))
.map(this::splitMetric)
.collect(Collectors.toMap(e -> e[0], e -> e[1]));
final List<PrometheusMetric> metrics = OpenTelemetryCollectorContainer.getInstance().fetchMetrics(metricsToTest.get(0));

metricsToTest.forEach(m -> {
Assert.assertNotEquals("Metric value should be non-zero: " + m,
"0", metrics.get(m + "{job=\"wildfly\"}")); // Add the metrics tags to complete the key
"0", metrics.stream().filter(e -> e.getKey().startsWith(m))
.findFirst()
.orElseThrow()
.getValue()); // Add the metrics tags to complete the key
});
}

private String[] splitMetric(String entry) {
int index = entry.lastIndexOf(" ");
return new String[]{
entry.substring(0, index),
entry.substring(index + 1)
};
}

private String fetchMetrics(String nameToMonitor) throws InterruptedException {
String body = "";
try (Client client = ClientBuilder.newClient()) {
WebTarget target = client.target(OpenTelemetryCollectorContainer.getInstance().getPrometheusUrl());

int attemptCount = 0;
boolean found = false;

// Request counts can vary. Setting high to help ensure test stability
while (!found && attemptCount < 30) {
// Wait to give Micrometer time to export
Thread.sleep(1000);

body = target.request().get().readEntity(String.class);
found = body.contains(nameToMonitor);
attemptCount++;
}
}

return body;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ public class MicrometerSetupTask extends AbstractSetupTask {

@Override
public void setup(final ManagementClient managementClient, String containerId) throws Exception {
AssumeTestGroupUtil.assumeDockerAvailable();

executeOp(managementClient, writeAttribute("undertow", STATISTICS_ENABLED, "true"));

if (!Operations.isSuccessfulOutcome(executeRead(managementClient, micrometerExtension))) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Copyright The WildFly Authors
* SPDX-License-Identifier: Apache-2.0
*/
package org.wildfly.test.integration.observability.micrometer.multiple;

import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.stream.Collectors;

import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.ClientBuilder;
import jakarta.ws.rs.client.WebTarget;
import org.jboss.as.test.shared.util.AssumeTestGroupUtil;
import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.wildfly.test.integration.observability.container.PrometheusMetric;

public abstract class BaseMultipleTestCase {
protected static final String SERVICE_ONE = "service-one";
protected static final String SERVICE_TWO = "service-two";
protected static final int REQUEST_COUNT = 5;

// The @ServerSetup(MicrometerSetupTask.class) requires Docker to be available.
// Otherwise the org.wildfly.extension.micrometer.registry.NoOpRegistry is installed which will result in 0 counters,
// and cause the test fail seemingly intermittently on machines with broken Docker setup.
@BeforeClass
public static void checkForDocker() {
AssumeTestGroupUtil.assumeDockerAvailable();
}

protected void makeRequests(URI service) throws URISyntaxException {
try (Client client = ClientBuilder.newClient()) {
WebTarget target = client.target(service);
for (int i = 0; i < REQUEST_COUNT; i++) {
Assert.assertEquals(200, target.request().get().getStatus());
}
}
}

protected @NotNull List<PrometheusMetric> getMetricsByName(List<PrometheusMetric> metrics, String key) {
return metrics.stream()
.filter(m -> m.getKey().equals(key))
.collect(Collectors.toList());
}
}
Loading

0 comments on commit 5c3ae15

Please sign in to comment.