Skip to content

Commit

Permalink
[WFLY-19351] Add Micrometer test to verify that applications with sha…
Browse files Browse the repository at this point in the history
…red metrics names are not merged when exported

Add test with multiple war deployments
Add test with ear with multiple war subdeployments
Rework how Prometheus metrics are retrieved and handled
Add test to verify that metrics with identical names are exported separately
  • Loading branch information
jasondlee committed Jun 11, 2024
1 parent 5e16342 commit 3ed7fba
Show file tree
Hide file tree
Showing 10 changed files with 420 additions and 51 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,17 @@
*/
package org.wildfly.test.integration.observability.container;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.ClientBuilder;
import jakarta.ws.rs.client.WebTarget;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.utility.MountableFile;
Expand Down Expand Up @@ -86,4 +94,60 @@ public String getPrometheusUrl() {
public List<JaegerTrace> getTraces(String serviceName) throws InterruptedException {
return (jaegerContainer != null ? jaegerContainer.getTraces(serviceName) : Collections.emptyList());
}

public List<PrometheusMetric> fetchMetrics(String nameToMonitor) throws InterruptedException {
String body = "";
try (Client client = ClientBuilder.newClient()) {
WebTarget target = client.target(OpenTelemetryCollectorContainer.getInstance().getPrometheusUrl());

int attemptCount = 0;
boolean found = false;

// Request counts can vary. Setting high to help ensure test stability
while (!found && attemptCount < 30) {
// Wait to give Micrometer time to export
Thread.sleep(1000);

body = target.request().get().readEntity(String.class);
found = body.contains(nameToMonitor);
attemptCount++;
}
}

return buildPrometheusMetrics(body);
}

private List<PrometheusMetric> buildPrometheusMetrics(String body) {
String[] entries = body.split("\n");
Map<String, String> help = new HashMap<>();
Map<String, String> type = new HashMap<>();
List<PrometheusMetric> metrics = new LinkedList<>();
Arrays.stream(entries).forEach(e -> {
if (e.startsWith("# HELP")) {
extractMetadata(help, e);
} else if (e.startsWith("# TYPE")) {
extractMetadata(type, e);
} else {
String[] parts = e.split("[{}]");
String key = parts[0];
Map<String, String> tags = Arrays.stream(parts[1].split(","))
.map(t -> t.split("="))
.collect(Collectors.toMap(i -> i[0],
i -> i[1]
.replaceAll("^\"", "")
.replaceAll("\"$", "")
));
metrics.add(new PrometheusMetric(key, tags, parts[2].trim(), type.get(key), help.get(key)));
}
});

return metrics;
}

private void extractMetadata(Map<String, String> target, String source) {
String[] parts = source.split(" ");
target.put(parts[2],
Arrays.stream(Arrays.copyOfRange(parts, 3, parts.length))
.reduce("", (total, element) -> total + " " + element));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Copyright The WildFly Authors
* SPDX-License-Identifier: Apache-2.0
*/
package org.wildfly.test.integration.observability.container;

import java.util.Collections;
import java.util.Map;

public class PrometheusMetric {
private final String key;
private final Map<String, String> tags;
private final String value;
private final String type;
private final String help;

public PrometheusMetric(String key,
Map<String, String> tags,
String value,
String type,
String help) {
this.key = key;
this.tags = Collections.unmodifiableMap(tags);
this.value = value;
this.type = type;
this.help = help;
}

public String getKey() {
return key;
}

public Map<String, String> getTags() {
return tags;
}

public String getValue() {
return value;
}

public String getType() {
return type;
}

public String getHelp() {
return help;
}

@Override
public String toString() {
return "PrometheusMetric{" +
"key='" + key + '\'' +
", tags=" + tags +
", value='" + value + '\'' +
", type='" + type + '\'' +
", help='" + help + '\'' +
'}';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public double getCount() {

timer.record(() -> {
try {
Thread.sleep((long) (Math.random() * 1000L));
Thread.sleep((long) (Math.random() * 100L));
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,6 @@
*/
package org.wildfly.test.integration.observability.micrometer;

import java.net.URISyntaxException;
import java.net.URL;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Inject;
import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.ClientBuilder;
import jakarta.ws.rs.client.WebTarget;

import io.micrometer.core.instrument.Counter;
import io.micrometer.core.instrument.MeterRegistry;
Expand All @@ -35,6 +25,16 @@
import org.junit.Test;
import org.junit.runner.RunWith;
import org.wildfly.test.integration.observability.container.OpenTelemetryCollectorContainer;
import org.wildfly.test.integration.observability.container.PrometheusMetric;

import jakarta.inject.Inject;
import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.ClientBuilder;
import jakarta.ws.rs.client.WebTarget;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Arrays;
import java.util.List;

@RunWith(Arquillian.class)
@ServerSetup(MicrometerSetupTask.class)
Expand Down Expand Up @@ -65,7 +65,8 @@ public static Archive<?> deploy() {
ShrinkWrap.create(WebArchive.class, "micrometer-test.war")
.addClasses(ServerSetupTask.class,
MetricResource.class,
AssumeTestGroupUtil.class)
AssumeTestGroupUtil.class,
PrometheusMetric.class)
.addAsWebInfResource(new StringAsset(WEB_XML), "web.xml")
.addAsWebInfResource(CdiUtils.createBeansXml(), "beans.xml") :
AssumeTestGroupUtil.emptyWar();
Expand Down Expand Up @@ -112,6 +113,7 @@ public void checkCounter() {
public void getMetrics() throws InterruptedException {
List<String> metricsToTest = Arrays.asList(
"demo_counter",
"demo_timer",
"memory_used_heap",
"cpu_available_processors",
"classloader_loaded_classes_count",
Expand All @@ -121,8 +123,9 @@ public void getMetrics() throws InterruptedException {
"undertow_bytes_received"
);

final String response = fetchMetrics(metricsToTest.get(0));
metricsToTest.forEach(n -> Assert.assertTrue("Missing metric: " + n, response.contains(n)));
final List<PrometheusMetric> metrics = OpenTelemetryCollectorContainer.getInstance().fetchMetrics(metricsToTest.get(0));
metricsToTest.forEach(n -> Assert.assertTrue("Missing metric: " + n,
metrics.stream().anyMatch(m -> m.getKey().startsWith(n))));
}

@Test
Expand All @@ -134,50 +137,19 @@ public void testJmxMetrics() throws InterruptedException {
"classloader_loaded_classes",
"cpu_system_load_average",
"cpu_process_cpu_time",
"classloader_unloaded_classes",
"classloader_loaded_classes_count",
"thread_count",
"thread_daemon_count",
"cpu_available_processors"
);
final String response = fetchMetrics(metricsToTest.get(0));
Map<String, String> metrics = Arrays.stream(response.split("\n"))
.filter(s -> !s.startsWith("#"))
.map(this::splitMetric)
.collect(Collectors.toMap(e -> e[0], e -> e[1]));
final List<PrometheusMetric> metrics = OpenTelemetryCollectorContainer.getInstance().fetchMetrics(metricsToTest.get(0));

metricsToTest.forEach(m -> {
Assert.assertNotEquals("Metric value should be non-zero: " + m,
"0", metrics.get(m + "{job=\"wildfly\"}")); // Add the metrics tags to complete the key
"0", metrics.stream().filter(e -> e.getKey().startsWith(m))
.findFirst()
.orElseThrow()
.getValue()); // Add the metrics tags to complete the key
});
}

private String[] splitMetric(String entry) {
int index = entry.lastIndexOf(" ");
return new String[]{
entry.substring(0, index),
entry.substring(index + 1)
};
}

private String fetchMetrics(String nameToMonitor) throws InterruptedException {
String body = "";
try (Client client = ClientBuilder.newClient()) {
WebTarget target = client.target(OpenTelemetryCollectorContainer.getInstance().getPrometheusUrl());

int attemptCount = 0;
boolean found = false;

// Request counts can vary. Setting high to help ensure test stability
while (!found && attemptCount < 30) {
// Wait to give Micrometer time to export
Thread.sleep(1000);

body = target.request().get().readEntity(String.class);
found = body.contains(nameToMonitor);
attemptCount++;
}
}

return body;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Copyright The WildFly Authors
* SPDX-License-Identifier: Apache-2.0
*/
package org.wildfly.test.integration.observability.micrometer.multiple;

import org.jboss.as.test.shared.util.AssumeTestGroupUtil;
import org.jetbrains.annotations.NotNull;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.wildfly.test.integration.observability.container.PrometheusMetric;

import jakarta.ws.rs.client.Client;
import jakarta.ws.rs.client.ClientBuilder;
import jakarta.ws.rs.client.WebTarget;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import java.util.stream.Collectors;

public abstract class BaseMultipleTestCase {
protected static final String SERVICE_ONE = "service-one";
protected static final String SERVICE_TWO = "service-two";
protected static final int REQUEST_COUNT = 5;

// The @ServerSetup(MicrometerSetupTask.class) requires Docker to be available.
// Otherwise the org.wildfly.extension.micrometer.registry.NoOpRegistry is installed which will result in 0 counters,
// and cause the test fail seemingly intermittently on machines with broken Docker setup.
@BeforeClass
public static void checkForDocker() {
AssumeTestGroupUtil.assumeDockerAvailable();
}

protected void makeRequests(URI service) throws URISyntaxException {
try (Client client = ClientBuilder.newClient()) {
WebTarget target = client.target(service);
for (int i = 0; i < REQUEST_COUNT; i++) {
Assert.assertEquals(200, target.request().get().getStatus());
}
}
}

protected @NotNull List<PrometheusMetric> getMetricsByName(List<PrometheusMetric> metrics, String key) {
return metrics.stream()
.filter(m -> m.getKey().equals(key))
.collect(Collectors.toList());
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Copyright The WildFly Authors
* SPDX-License-Identifier: Apache-2.0
*/
package org.wildfly.test.integration.observability.micrometer.multiple;

import org.jboss.arquillian.container.test.api.Deployment;
import org.jboss.arquillian.container.test.api.OperateOnDeployment;
import org.jboss.arquillian.container.test.api.RunAsClient;
import org.jboss.arquillian.junit.Arquillian;
import org.jboss.arquillian.test.api.ArquillianResource;
import org.jboss.as.arquillian.api.ServerSetup;
import org.jboss.shrinkwrap.api.ShrinkWrap;
import org.jboss.shrinkwrap.api.asset.StringAsset;
import org.jboss.shrinkwrap.api.spec.EnterpriseArchive;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.wildfly.test.integration.observability.container.OpenTelemetryCollectorContainer;
import org.wildfly.test.integration.observability.container.PrometheusMetric;
import org.wildfly.test.integration.observability.micrometer.MicrometerSetupTask;
import org.wildfly.test.integration.observability.micrometer.multiple.application.DuplicateMetricResource1;
import org.wildfly.test.integration.observability.micrometer.multiple.application.DuplicateMetricResource2;

import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.List;

@RunWith(Arquillian.class)
@ServerSetup(MicrometerSetupTask.class)
public class EarDeploymentTestCase extends BaseMultipleTestCase {
protected static final String ENTERPRISE_APP = "enterprise-app";

@Deployment(name = ENTERPRISE_APP)
public static EnterpriseArchive createDeployment() {
return ShrinkWrap.create(EnterpriseArchive.class, ENTERPRISE_APP + ".ear")
.addAsModule(MultipleWarTestCase.createDeployment1())
.addAsModule(MultipleWarTestCase.createDeployment2())
.setApplicationXML(new StringAsset("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ "<application xmlns=\"https://jakarta.ee/xml/ns/jakartaee\" " +
" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" " +
" xsi:schemaLocation=\"https://jakarta.ee/xml/ns/jakartaee https://jakarta.ee/xml/ns/jakartaee/application_10.xsd\" " +
" version=\"10\">\n"
+ " <display-name>metrics</display-name>\n"
+ " <module>\n"
+ " <web>\n"
+ " <web-uri>" + SERVICE_ONE + ".war</web-uri>\n"
+ " </web>\n"
+ " </module>\n"
+ " <module>\n"
+ " <web>\n"
+ " <web-uri>" + SERVICE_TWO + ".war</web-uri>\n"
+ " </web>\n"
+ " </module>\n"
+ "</application>"));
}

@Test
@RunAsClient
public void dataTest(@ArquillianResource @OperateOnDeployment(ENTERPRISE_APP) URL earUrl)
throws URISyntaxException, InterruptedException {
makeRequests(new URI(String.format("%s/%s/%s/%s", earUrl, ENTERPRISE_APP, SERVICE_ONE, DuplicateMetricResource1.TAG)));
makeRequests(new URI(String.format("%s/%s/%s/%s", earUrl, ENTERPRISE_APP, SERVICE_TWO, DuplicateMetricResource2.TAG)));

List<PrometheusMetric> results = getMetricsByName(
OpenTelemetryCollectorContainer.getInstance().fetchMetrics(DuplicateMetricResource1.METER_NAME),
DuplicateMetricResource1.METER_NAME + "_total"); // Adjust for Prometheus naming conventions

Assert.assertEquals(2, results.size());
results.forEach(r -> Assert.assertEquals("" + REQUEST_COUNT, r.getValue()));
}
}
Loading

0 comments on commit 3ed7fba

Please sign in to comment.