diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 63f1cad..d19d951 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -2,7 +2,7 @@ [![Coverage Status](https://coveralls.io/repos/github/skuzzle/gh-prom-exporter/badge.svg?branch=master)](https://coveralls.io/github/skuzzle/gh-prom-exporter?branch=master) [![Twitter Follow](https://img.shields.io/twitter/follow/skuzzleOSS.svg?style=social)](https://twitter.com/skuzzleOSS) -* Display application version +* Support open metrics format (`Accept: application/openmetrics-text`) ``` docker pull ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.7-SNAPSHOT diff --git a/readme/RELEASE_NOTES.md b/readme/RELEASE_NOTES.md index d6f87ff..703e095 100644 --- a/readme/RELEASE_NOTES.md +++ b/readme/RELEASE_NOTES.md @@ -2,7 +2,7 @@ [![Coverage Status](https://coveralls.io/repos/github/${github.user}/${github.name}/badge.svg?branch=${github.main-branch})](https://coveralls.io/github/${github.user}/${github.name}?branch=${github.main-branch}) [![Twitter Follow](https://img.shields.io/twitter/follow/skuzzleOSS.svg?style=social)](https://twitter.com/skuzzleOSS) -* Display application version +* Support open metrics format (`Accept: application/openmetrics-text`) ``` docker pull ${docker.image.name}:${project.version} diff --git a/src/main/java/de/skuzzle/ghpromexporter/web/MultipleRepositories.java b/src/main/java/de/skuzzle/ghpromexporter/web/MultipleRepositories.java index 97b5c61..932ad07 100644 --- a/src/main/java/de/skuzzle/ghpromexporter/web/MultipleRepositories.java +++ b/src/main/java/de/skuzzle/ghpromexporter/web/MultipleRepositories.java @@ -25,4 +25,9 @@ Flux requests() { return Flux.fromStream(repositories.stream() .map(repository -> ScrapeRepositoryRequest.of(owner, repository))); } + + @Override + public String toString() { + return "owner=%s, repositories=%s".formatted(owner, repositories); + } } diff --git a/src/main/java/de/skuzzle/ghpromexporter/web/PromController.java b/src/main/java/de/skuzzle/ghpromexporter/web/PromController.java index 910ce92..8bba075 100644 --- a/src/main/java/de/skuzzle/ghpromexporter/web/PromController.java +++ b/src/main/java/de/skuzzle/ghpromexporter/web/PromController.java @@ -2,6 +2,8 @@ import java.net.InetAddress; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -25,6 +27,8 @@ record PromController( AbuseLimiter abuseLimiter, WebProperties properties) { + private static final Logger log = LoggerFactory.getLogger(PromController.class); + @GetMapping(path = "{owner}/{repositories}") public Mono> createStats( @PathVariable String owner, @@ -39,9 +43,10 @@ public Mono> createStats( } final InetAddress origin = request.getRemoteAddress().getAddress(); - final MediaType contentType = MediaType.TEXT_PLAIN;// determineContentType(request); + final MediaType contentType = determineContentType(request); final MultipleRepositories multipleRepositories = MultipleRepositories.parse(owner, repositories); + log.info("Request from '{}' to scrape '{}'", gitHubAuthentication, multipleRepositories); return abuseLimiter.blockAbusers(origin) .flatMap(__ -> freshResponse(gitHubAuthentication, multipleRepositories, contentType)) @@ -53,6 +58,10 @@ public Mono> createStats( .body("Your IP '%s' has exceeded the abuse limit\n".formatted(origin)))); } + private MediaType determineContentType(ServerHttpRequest request) { + return serializer.determineMediaType(request.getHeaders().getAccept()); + } + private Mono> freshResponse(GitHubAuthentication authentication, MultipleRepositories repositories, MediaType contentType) { diff --git a/src/main/java/de/skuzzle/ghpromexporter/web/RegistrySerializer.java b/src/main/java/de/skuzzle/ghpromexporter/web/RegistrySerializer.java index 6dd9e21..87f2baa 100644 --- a/src/main/java/de/skuzzle/ghpromexporter/web/RegistrySerializer.java +++ b/src/main/java/de/skuzzle/ghpromexporter/web/RegistrySerializer.java @@ -2,30 +2,91 @@ import java.io.IOException; import java.io.StringWriter; +import java.io.Writer; +import java.util.Collection; +import java.util.Enumeration; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; import org.springframework.stereotype.Component; +import io.prometheus.client.Collector; +import io.prometheus.client.Collector.MetricFamilySamples; import io.prometheus.client.CollectorRegistry; import io.prometheus.client.exporter.common.TextFormat; @Component class RegistrySerializer { - private static final MediaType OPEN_METRICS = MediaType + private static final Logger log = LoggerFactory.getLogger(RegistrySerializer.class); + + static final MediaType OPEN_METRICS = MediaType .parseMediaType("application/openmetrics-text; version=1.0.0; charset=utf-8"); + static final MediaType FORMAT_004 = MediaType.parseMediaType("text/plain; version=0.0.4; charset=utf-8"); + + private final List serializers = List.of(new V004(), new OpenMetrics()); + + public MediaType determineMediaType(Collection acceptibleMediaTypes) { + final MediaType result = serializers.stream() + .filter(serializer -> acceptibleMediaTypes.stream() + .anyMatch(acceptible -> serializer.supportedMediaType().isCompatibleWith(acceptible))) + .findFirst() + .map(Serializer::supportedMediaType) + .orElse(FORMAT_004); + + log.debug("Chose '{}' from acceptible meda types {}", result, acceptibleMediaTypes); + return result; + } + public String serializeRegistry(CollectorRegistry registry, MediaType mediaType) { try (final var stringWriter = new StringWriter()) { - if (mediaType.isCompatibleWith(OPEN_METRICS)) { - TextFormat.writeOpenMetrics100(stringWriter, registry.metricFamilySamples()); - } else { - TextFormat.write004(stringWriter, registry.metricFamilySamples()); - } + serializers.stream() + .filter(serializer -> serializer.supportedMediaType().equals(mediaType)) + .findFirst() + .orElseThrow() + .write(stringWriter, registry.metricFamilySamples()); + return stringWriter.toString(); } catch (final IOException e) { throw new IllegalStateException("Error while serializing registry", e); } } + private interface Serializer { + + MediaType supportedMediaType(); + + void write(Writer writer, Enumeration mfs) throws IOException; + } + + private static class OpenMetrics implements Serializer { + + @Override + public MediaType supportedMediaType() { + return OPEN_METRICS; + } + + @Override + public void write(Writer writer, Enumeration mfs) throws IOException { + TextFormat.writeOpenMetrics100(writer, mfs); + } + + } + + private static final class V004 implements Serializer { + + @Override + public MediaType supportedMediaType() { + return FORMAT_004; + } + + @Override + public void write(Writer writer, Enumeration mfs) throws IOException { + TextFormat.write004(writer, mfs); + } + + } } diff --git a/src/test/java/de/skuzzle/ghpromexporter/web/PromControllerTest.java b/src/test/java/de/skuzzle/ghpromexporter/web/PromControllerTest.java index cc40dc0..195a560 100644 --- a/src/test/java/de/skuzzle/ghpromexporter/web/PromControllerTest.java +++ b/src/test/java/de/skuzzle/ghpromexporter/web/PromControllerTest.java @@ -6,23 +6,21 @@ import static de.skuzzle.ghpromexporter.web.CanonicalPrometheusRegistrySerializer.canonicalPrometheusRegistry; import static org.assertj.core.api.Assertions.assertThat; +import java.util.Map; + import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.context.SpringBootTest.WebEnvironment; -import org.springframework.boot.web.server.LocalServerPort; import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.reactive.function.client.WebClient; import de.skuzzle.ghpromexporter.github.GitHubAuthentication; import de.skuzzle.test.snapshots.SnapshotAssertions; import de.skuzzle.test.snapshots.SnapshotDsl.Snapshot; -import reactor.core.publisher.Mono; import reactor.test.StepVerifier; -@SnapshotAssertions(forceUpdateSnapshots = false) +@SnapshotAssertions @SpringBootTest(webEnvironment = WebEnvironment.RANDOM_PORT, properties = "web.abuseCache.expireAfterWrite=1s") public class PromControllerTest { @@ -32,8 +30,8 @@ public class PromControllerTest { private WebProperties webProperties; @Autowired private AbuseLimiter abuseLimiter; - @LocalServerPort - private int localPort; + @Autowired + private TestClient testClient; @AfterEach void cleanup() { @@ -41,22 +39,9 @@ void cleanup() { abuseLimiter.unblockAll(); } - private WebClient client() { - return WebClient.builder() - .baseUrl("http://localhost:" + localPort) - .build(); - } - - private Mono> getStatsFor(String owner, String repository) { - return client().get().uri("/{owner}/{repository}", owner, repository) - .retrieve() - .onStatus(HttpStatus::is4xxClientError, response -> Mono.empty()) - .toEntity(String.class); - } - @Test void scrape_anonymously_forbidden() throws Exception { - final var serviceCall = getStatsFor("skuzzle", "test-repo"); + final var serviceCall = testClient.getStatsFor("skuzzle", "test-repo"); final GitHubAuthentication gitHubAuthentication = successfulAuthenticationForRepository( withName("skuzzle", "test-repo") .withStargazerCount(1337)) @@ -64,15 +49,14 @@ void scrape_anonymously_forbidden() throws Exception { authentication.with(gitHubAuthentication, () -> { StepVerifier.create(serviceCall) - .assertNext( - response -> assertThat(response.getStatusCode()).isEqualTo(HttpStatus.UNAUTHORIZED)) + .assertNext(response -> assertThat(response.getStatusCode()).isEqualTo(HttpStatus.UNAUTHORIZED)) .verifyComplete(); }); } @Test void scrape_multiple_repositories(Snapshot snapshot) throws Exception { - final var serviceCall = getStatsFor("skuzzle", "test-repo1,test-repo2"); + final var serviceCall = testClient.getStatsFor("skuzzle", "test-repo1,test-repo2"); final var gitHubAuthentication = successfulAuthenticationForRepository( withName("skuzzle", "test-repo1") .withForkCount(5)); @@ -89,7 +73,7 @@ void scrape_multiple_repositories(Snapshot snapshot) throws Exception { @Test void test_successful_initial_scrape(Snapshot snapshot) throws Exception { - final var serviceCall = getStatsFor("skuzzle", "test-repo"); + final var serviceCall = testClient.getStatsFor("skuzzle", "test-repo"); final GitHubAuthentication gitHubAuthentication = successfulAuthenticationForRepository( withName("skuzzle", "test-repo") .withStargazerCount(1337) @@ -104,16 +88,50 @@ void test_successful_initial_scrape(Snapshot snapshot) throws Exception { authentication.with(gitHubAuthentication, () -> { StepVerifier.create(serviceCall) - .assertNext(response -> snapshot.assertThat(response.getBody()) - .as(canonicalPrometheusRegistry()) - .matchesSnapshotText()) + .assertNext(response -> { + assertThat(response.getHeaders().getContentType()).isEqualTo(RegistrySerializer.FORMAT_004); + + snapshot.assertThat(response.getBody()) + .as(canonicalPrometheusRegistry()) + .matchesSnapshotText(); + }) + .verifyComplete(); + }); + } + + @Test + void test_successful_scrape_open_metrics(Snapshot snapshot) throws Exception { + final var serviceCall = testClient.getStatsFor("skuzzle", "test-repo", + Map.of("Accept", "application/openmetrics-text")); + + final GitHubAuthentication gitHubAuthentication = successfulAuthenticationForRepository( + withName("skuzzle", "test-repo") + .withStargazerCount(1337) + .withForkCount(5) + .withOpenIssueCount(2) + .withWatchersCount(1) + .withSubscriberCount(4) + .withAdditions(50) + .withDeletions(-20) + .withSizeInKb(127)); + + authentication.with(gitHubAuthentication, () -> { + + StepVerifier.create(serviceCall) + .assertNext(response -> { + assertThat(response.getHeaders().getContentType()).isEqualTo(RegistrySerializer.OPEN_METRICS); + + snapshot.assertThat(response.getBody()) + .as(canonicalPrometheusRegistry()) + .matchesSnapshotText(); + }) .verifyComplete(); }); } @Test void test_successful_anonymous_scrape(Snapshot snapshot) throws Exception { - final var serviceCall = getStatsFor("skuzzle", "test-repo"); + final var serviceCall = testClient.getStatsFor("skuzzle", "test-repo"); webProperties.setAllowAnonymousScrape(true); final GitHubAuthentication gitHubAuthentication = successfulAuthenticationForRepository( withName("skuzzle", "test-repo") @@ -138,7 +156,7 @@ void test_successful_anonymous_scrape(Snapshot snapshot) throws Exception { @Test void client_should_be_blocked_when_abuse_limit_is_exceeded() throws Exception { - final var serviceCall = getStatsFor("skuzzle", "test-repo"); + final var serviceCall = testClient.getStatsFor("skuzzle", "test-repo"); authentication.with(failingAuthentication(), () -> { for (int i = 0; i < webProperties.abuseLimit(); ++i) { @@ -157,7 +175,7 @@ void client_should_be_blocked_when_abuse_limit_is_exceeded() throws Exception { @Test void client_should_be_unblocked_after_a_while() throws Exception { - final var serviceCall = getStatsFor("skuzzle", "test-repo"); + final var serviceCall = testClient.getStatsFor("skuzzle", "test-repo"); authentication.with(failingAuthentication(), () -> { for (int i = 0; i < webProperties.abuseLimit(); ++i) { diff --git a/src/test/java/de/skuzzle/ghpromexporter/web/TestClient.java b/src/test/java/de/skuzzle/ghpromexporter/web/TestClient.java new file mode 100644 index 0000000..5df49f5 --- /dev/null +++ b/src/test/java/de/skuzzle/ghpromexporter/web/TestClient.java @@ -0,0 +1,41 @@ +package de.skuzzle.ghpromexporter.web; + +import java.util.Map; + +import org.springframework.boot.web.server.LocalServerPort; +import org.springframework.context.annotation.Lazy; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; + +import reactor.core.publisher.Mono; + +@Component +@Lazy(true) +class TestClient { + + @LocalServerPort + private int localPort; + + private WebClient client() { + return WebClient.builder() + .baseUrl("http://localhost:" + localPort) + .build(); + } + + public Mono> getStatsFor(String owner, String repository, Map extraHeaders) { + return client().get().uri("/{owner}/{repository}", owner, repository) + .headers(requestHeaders -> extraHeaders.forEach(requestHeaders::add)) + .retrieve() + .onStatus(HttpStatus::is4xxClientError, response -> Mono.empty()) + .toEntity(String.class); + } + + public Mono> getStatsFor(String owner, String repository) { + return client().get().uri("/{owner}/{repository}", owner, repository) + .retrieve() + .onStatus(HttpStatus::is4xxClientError, response -> Mono.empty()) + .toEntity(String.class); + } +} diff --git a/src/test/resources/de/skuzzle/ghpromexporter/web/PromControllerTest_snapshots/test_successful_scrape_open_metrics_0.snapshot b/src/test/resources/de/skuzzle/ghpromexporter/web/PromControllerTest_snapshots/test_successful_scrape_open_metrics_0.snapshot new file mode 100644 index 0000000..56cba30 --- /dev/null +++ b/src/test/resources/de/skuzzle/ghpromexporter/web/PromControllerTest_snapshots/test_successful_scrape_open_metrics_0.snapshot @@ -0,0 +1,28 @@ +# EOF +# HELP github_additions Sum of additions over the last 52 weeks +# HELP github_deletions Negative sum of deletions over the last 52 weeks +# HELP github_forks The repository's fork count +# HELP github_open_issues The repository's open issue count +# HELP github_scrape_duration Duration of a single scrape +# HELP github_size The repository's size in KB +# HELP github_stargazers The repository's stargazer count +# HELP github_subscribers The repository's subscriber count +# HELP github_watchers The repository's watcher count +# TYPE github_additions counter +# TYPE github_deletions counter +# TYPE github_forks counter +# TYPE github_open_issues counter +# TYPE github_scrape_duration summary +# TYPE github_size counter +# TYPE github_stargazers counter +# TYPE github_subscribers counter +# TYPE github_watchers counter +github_additions_total{owner="skuzzle",repository="test-repo"} 50.0 +github_deletions_total{owner="skuzzle",repository="test-repo"} 20.0 +github_forks_total{owner="skuzzle",repository="test-repo"} 5.0 +github_open_issues_total{owner="skuzzle",repository="test-repo"} 2.0 +github_scrape_duration_count{owner="skuzzle",repository="test-repo"} 1.0 +github_size_total{owner="skuzzle",repository="test-repo"} 127.0 +github_stargazers_total{owner="skuzzle",repository="test-repo"} 1337.0 +github_subscribers_total{owner="skuzzle",repository="test-repo"} 4.0 +github_watchers_total{owner="skuzzle",repository="test-repo"} 1.0 \ No newline at end of file