diff --git a/src/main/java/me/itzg/helpers/http/FailedRequestException.java b/src/main/java/me/itzg/helpers/http/FailedRequestException.java index 94f963c4..9a6ff760 100644 --- a/src/main/java/me/itzg/helpers/http/FailedRequestException.java +++ b/src/main/java/me/itzg/helpers/http/FailedRequestException.java @@ -17,7 +17,7 @@ public class FailedRequestException extends RuntimeException { */ public FailedRequestException(HttpResponseException e, URI uri) { super( - String.format("HTTP request failed uri: %s %s", uri, e.getMessage()) + String.format("HTTP request of %s failed with %d: %s", uri, e.getStatusCode(), e.getMessage()) ); this.uri = uri; this.statusCode = e.getStatusCode(); @@ -28,7 +28,7 @@ public FailedRequestException(HttpResponseException e, URI uri) { */ public FailedRequestException(HttpResponseStatus status, URI uri, String msg) { super( - String.format("HTTP request failed uri: %s %s", uri, msg) + String.format("HTTP request of %s failed with %s: %s", uri, status, msg) ); this.uri = uri; this.statusCode = status.code(); diff --git a/src/main/java/me/itzg/helpers/http/FetchBuilderBase.java b/src/main/java/me/itzg/helpers/http/FetchBuilderBase.java index d22e4ada..ba4cca00 100644 --- a/src/main/java/me/itzg/helpers/http/FetchBuilderBase.java +++ b/src/main/java/me/itzg/helpers/http/FetchBuilderBase.java @@ -1,24 +1,27 @@ package me.itzg.helpers.http; +import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE; + import com.fasterxml.jackson.databind.ObjectMapper; +import io.netty.handler.codec.http.HttpStatusClass; import java.io.IOException; import java.net.URI; import java.nio.file.Path; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.function.BiConsumer; import lombok.extern.slf4j.Slf4j; +import me.itzg.helpers.errors.GenericException; +import me.itzg.helpers.json.ObjectMappers; import org.apache.hc.client5.http.HttpResponseException; -import org.apache.hc.client5.http.classic.HttpClient; -import org.apache.hc.client5.http.classic.methods.HttpGet; -import org.apache.hc.client5.http.classic.methods.HttpHead; -import org.apache.hc.core5.http.HttpHeaders; -import org.apache.hc.core5.http.message.BasicHttpRequest; import org.slf4j.Logger; +import reactor.core.publisher.Mono; import reactor.netty.Connection; import reactor.netty.http.client.HttpClientRequest; +import reactor.netty.http.client.HttpClientResponse; @Slf4j public class FetchBuilderBase> { @@ -58,30 +61,24 @@ public OutputToDirectoryFetchBuilder toDirectory(Path directory) { return new OutputToDirectoryFetchBuilder(this.state, directory); } + /** + * NOTE: this will set expected content types to application/json + */ public ObjectFetchBuilder toObject(Class type) { - return new ObjectFetchBuilder<>(this.state, type); + acceptContentTypes(Collections.singletonList("application/json")); + return new ObjectFetchBuilder<>(this.state, type, false, ObjectMappers.defaultMapper()); } + /** + * NOTE: this will set expected content types to application/json + */ public ObjectListFetchBuilder toObjectList(Class type) { - return new ObjectListFetchBuilder<>(this.state, type); + acceptContentTypes(Collections.singletonList("application/json")); + return new ObjectListFetchBuilder<>(this.state, type, ObjectMappers.defaultMapper()); } public ObjectFetchBuilder toObject(Class type, ObjectMapper objectMapper) { - return new ObjectFetchBuilder<>(this.state, type, objectMapper); - } - - protected HttpGet get() throws IOException { - final HttpGet request = new HttpGet(state.uri); - configureRequest(request); - return request; - } - - protected HttpHead head(boolean withConfigure) throws IOException { - final HttpHead request = new HttpHead(state.uri); - if (withConfigure) { - configureRequest(request); - } - return request; + return new ObjectFetchBuilder<>(this.state, type, false, objectMapper); } protected URI uri() { @@ -117,14 +114,8 @@ protected interface PreparedFetchUser { R use(SharedFetch sharedFetch) throws IOException; } - protected interface ClientUser { - R use(HttpClient client) throws IOException; - } - /** * Intended to be called by subclass specific execute methods. - * @param user provided either a multi-request {@link SharedFetch} or an instance scoped to this call. - * Either way, {@link SharedFetch#getClient()} can be used to execute requests. */ protected R usePreparedFetch(PreparedFetchUser user) throws IOException { if (state.sharedFetch != null) { @@ -147,43 +138,55 @@ protected R usePreparedFetch(PreparedFetchUser user) throws IOException { } } + protected static BiConsumer debugLogRequest( + Logger log, String operation + ) { + return (req, connection) -> + log.debug("{}: uri={} headers={}", + operation.toUpperCase(), req.resourceUrl(), req.requestHeaders() + ); + } + + protected Mono failedRequestMono(HttpClientResponse resp, String description) { + return Mono.error(new FailedRequestException(resp.status(), uri(), description)); + } + + protected static boolean notSuccess(HttpClientResponse resp) { + return HttpStatusClass.valueOf(resp.status().code()) != HttpStatusClass.SUCCESS; + } + /** - * Intended to be called by subclass specific execute methods. - * This is a convenience version of {@link #usePreparedFetch(PreparedFetchUser)} - * that provides just the {@link HttpClient} to execute requests. + * @return false if response content type is not one of the expected content types, + * but true if no expected content types */ - protected R useClient(ClientUser user) throws IOException { - return usePreparedFetch(sharedFetch -> - user.use(sharedFetch.getClient()) - ); - } + protected boolean notExpectedContentType(HttpClientResponse resp) { + final List contentTypes = getAcceptContentTypes(); + if (contentTypes != null && !contentTypes.isEmpty()) { + final List respTypes = resp.responseHeaders() + .getAll(CONTENT_TYPE); - protected void configureRequest(BasicHttpRequest request) throws IOException { - if (state.acceptContentTypes != null) { - for (final String type : state.acceptContentTypes) { - request.addHeader(HttpHeaders.ACCEPT, type); - } + return respTypes.stream().noneMatch(contentTypes::contains); } + return false; + } - for (final Entry entry : state.requestHeaders.entrySet()) { - request.addHeader(entry.getKey(), entry.getValue()); - } - // and apply shared headers that weren't overridden by per-request - if (state.sharedFetch != null) { - for (final Entry entry : state.sharedFetch.getHeaders().entrySet()) { - if (!state.requestHeaders.containsKey(entry.getKey())) { - request.addHeader(entry.getKey(), entry.getValue()); - } - } - } + protected Mono failedContentTypeMono(HttpClientResponse resp) { + return Mono.error(new GenericException( + String.format("Unexpected content type in response. Expected '%s' but got '%s'", + getAcceptContentTypes(), resp.responseHeaders() + .getAll(CONTENT_TYPE) + ))); } - protected static BiConsumer debugLogRequest( - Logger log, String operation - ) { - return (req, connection) -> - log.debug("{}: uri={} headers={}", - operation.toUpperCase(), req.resourceUrl(), req.requestHeaders() + protected void applyHeaders(io.netty.handler.codec.http.HttpHeaders headers) { + final List contentTypes = getAcceptContentTypes(); + if (contentTypes != null && !contentTypes.isEmpty()) { + headers.set( + ACCEPT.toString(), + contentTypes ); + } + + state.requestHeaders.forEach(headers::set); } } diff --git a/src/main/java/me/itzg/helpers/http/ObjectFetchBuilder.java b/src/main/java/me/itzg/helpers/http/ObjectFetchBuilder.java index 1cc7d8de..03e9ad47 100644 --- a/src/main/java/me/itzg/helpers/http/ObjectFetchBuilder.java +++ b/src/main/java/me/itzg/helpers/http/ObjectFetchBuilder.java @@ -1,74 +1,88 @@ package me.itzg.helpers.http; import com.fasterxml.jackson.databind.ObjectMapper; -import io.netty.handler.codec.http.HttpHeaderNames; +import com.fasterxml.jackson.databind.ObjectReader; import java.io.IOException; +import java.util.List; import lombok.extern.slf4j.Slf4j; import me.itzg.helpers.errors.GenericException; -import me.itzg.helpers.json.ObjectMappers; -import org.apache.hc.core5.http.HttpHeaders; -import org.apache.hc.core5.http.message.BasicHttpRequest; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; +import reactor.netty.ByteBufMono; +import reactor.netty.http.client.HttpClientResponse; @Slf4j public class ObjectFetchBuilder extends FetchBuilderBase> { private final Class type; - private final ObjectMapper objectMapper; + private final boolean listOf; + private final ObjectReader reader; - ObjectFetchBuilder(State state, Class type, ObjectMapper objectMapper) { + protected ObjectFetchBuilder(State state, Class type, boolean listOf, ObjectMapper objectMapper) { super(state); this.type = type; - this.objectMapper = objectMapper; + this.listOf = listOf; + if (listOf) { + reader = objectMapper.readerForListOf(type); + } + else { + reader = objectMapper.readerFor(type); + } } - ObjectFetchBuilder(State state, Class type) { - this(state, type, ObjectMappers.defaultMapper()); + public T execute() throws IOException { + return assemble().block(); } - @Override - protected void configureRequest(BasicHttpRequest request) throws IOException { - super.configureRequest(request); - request.addHeader(HttpHeaders.ACCEPT, "application/json"); + public Mono assemble() throws IOException { + return assembleCommon(); } - public T execute() throws IOException { - return assemble().block(); + protected Mono> assembleToList() throws IOException { + return assembleCommon(); } - public Mono assemble() throws IOException { + private Mono assembleCommon() throws IOException { return usePreparedFetch(sharedFetch -> sharedFetch.getReactiveClient() - .headers(headers -> - headers.set(HttpHeaderNames.ACCEPT, "application/json") - ) + .headers(this::applyHeaders) .followRedirect(true) .doOnRequest(debugLogRequest(log, "json fetch")) .get() .uri(uri()) - .responseContent() - .aggregate() - .asInputStream() - .publishOn(Schedulers.boundedElastic()) - .flatMap(inputStream -> { + .responseSingle(this::handleResponse) + ); + } + + private Mono handleResponse(HttpClientResponse resp, ByteBufMono bodyMono) { + if (notSuccess(resp)) { + return failedRequestMono(resp, "Fetching object content"); + } + if (notExpectedContentType(resp)) { + return failedContentTypeMono(resp); + } + + return bodyMono.asInputStream() + .publishOn(Schedulers.boundedElastic()) + .flatMap(inputStream -> { + try { try { - try { - return Mono.just(objectMapper.readValue(inputStream, type)); - } catch (IOException e) { - return Mono.error(new GenericException("Failed to parse response body into " + type, e)); - } + return Mono.just(reader.readValue(inputStream)); + } catch (IOException e) { + return Mono.error(new GenericException( + "Failed to parse response body into " + + (listOf ? "list of " + type : type), + e + )); } - finally { - try { - //noinspection BlockingMethodInNonBlockingContext - inputStream.close(); - } catch (IOException e) { - log.warn("Unable to close body input stream", e); - } + } finally { + try { + //noinspection BlockingMethodInNonBlockingContext + inputStream.close(); + } catch (IOException e) { + log.warn("Unable to close body input stream", e); } - }) - ); + } + }); } - } diff --git a/src/main/java/me/itzg/helpers/http/ObjectListFetchBuilder.java b/src/main/java/me/itzg/helpers/http/ObjectListFetchBuilder.java index 156aef4f..44b131f0 100644 --- a/src/main/java/me/itzg/helpers/http/ObjectListFetchBuilder.java +++ b/src/main/java/me/itzg/helpers/http/ObjectListFetchBuilder.java @@ -3,35 +3,24 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.util.List; -import me.itzg.helpers.json.ObjectMappers; -import org.apache.hc.core5.http.HttpHeaders; -import org.apache.hc.core5.http.message.BasicHttpRequest; +import reactor.core.publisher.Mono; public class ObjectListFetchBuilder extends FetchBuilderBase> { - private final Class type; - private final ObjectMapper objectMapper; + private final ObjectFetchBuilder delegate; ObjectListFetchBuilder(State state, Class type, ObjectMapper objectMapper) { super(state); - this.type = type; - this.objectMapper = objectMapper; - } - - ObjectListFetchBuilder(State state, Class type) { - this(state, type, ObjectMappers.defaultMapper()); - } - @Override - protected void configureRequest(BasicHttpRequest request) throws IOException { - super.configureRequest(request); - request.addHeader(HttpHeaders.ACCEPT, "application/json"); + delegate = new ObjectFetchBuilder<>(state, type, true, objectMapper); } public List execute() throws IOException { - return useClient(client -> - client.execute(get(), new ObjectListMapperHandler<>(type, objectMapper)) - ); + return delegate.assembleToList() + .block(); } + public Mono> assemble() throws IOException { + return delegate.assembleToList(); + } } diff --git a/src/main/java/me/itzg/helpers/http/ObjectListMapperHandler.java b/src/main/java/me/itzg/helpers/http/ObjectListMapperHandler.java deleted file mode 100644 index 0c319814..00000000 --- a/src/main/java/me/itzg/helpers/http/ObjectListMapperHandler.java +++ /dev/null @@ -1,24 +0,0 @@ -package me.itzg.helpers.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import java.io.IOException; -import java.util.List; -import org.apache.hc.core5.http.HttpEntity; - -public class ObjectListMapperHandler extends LoggingResponseHandler> { - - private final ObjectMapper objectMapper; - private final Class type; - - public ObjectListMapperHandler(Class type, ObjectMapper objectMapper) { - this.type = type; - this.objectMapper = objectMapper; - } - - @Override - public List handleEntity(HttpEntity entity) throws IOException { - final ObjectReader objectReader = objectMapper.readerForListOf(type); - return objectReader.readValue(entity.getContent()); - } -} diff --git a/src/main/java/me/itzg/helpers/http/ObjectMapperHandler.java b/src/main/java/me/itzg/helpers/http/ObjectMapperHandler.java deleted file mode 100644 index 57a457f6..00000000 --- a/src/main/java/me/itzg/helpers/http/ObjectMapperHandler.java +++ /dev/null @@ -1,21 +0,0 @@ -package me.itzg.helpers.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import org.apache.hc.core5.http.HttpEntity; - -public class ObjectMapperHandler extends LoggingResponseHandler { - - private final ObjectMapper objectMapper; - private final Class type; - - public ObjectMapperHandler(Class type, ObjectMapper objectMapper) { - this.type = type; - this.objectMapper = objectMapper; - } - - @Override - public T handleEntity(HttpEntity entity) throws IOException { - return objectMapper.readValue(entity.getContent(), type); - } -} diff --git a/src/main/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilder.java b/src/main/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilder.java index 305fba43..8b453e37 100644 --- a/src/main/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilder.java +++ b/src/main/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilder.java @@ -4,11 +4,9 @@ import io.netty.handler.codec.http.HttpHeaderNames; import java.io.IOException; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.util.function.Function; import lombok.Setter; import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; @@ -62,8 +60,10 @@ public Mono assemble() throws IOException { .head() .uri(uri()) .response() - .map(OutputToDirectoryFetchBuilder::extractFilename) - .map(outputDirectory::resolve) + .flatMap(resp -> + notSuccess(resp) ? failedRequestMono(resp, "Extracting filename") + : Mono.just(outputDirectory.resolve(extractFilename(resp))) + ) .flatMap(outputFile -> assembleFileDownload(sharedFetch, outputFile) ) @@ -83,23 +83,28 @@ private Mono assembleFileDownload(SharedFetch sharedFetch, Path outputFile .doOnRequest(debugLogRequest(log, "file fetch")) .get() .uri(uri()) - .responseContent().aggregate() - .asInputStream() - .publishOn(Schedulers.boundedElastic()) - .flatMap((Function>) inputStream -> { - try { - @SuppressWarnings("BlockingMethodInNonBlockingContext") // false warning, see above - final long size = Files.copy(inputStream, outputFile, StandardCopyOption.REPLACE_EXISTING); - statusHandler.call(FileDownloadStatus.DOWNLOADED, uri(), outputFile); - downloadedHandler.call(uri(), outputFile, size); - return Mono.just(outputFile); - } catch (IOException e) { - return Mono.error(e); + .responseSingle((resp, byteBufMono) -> { + if (notSuccess(resp)) { + return failedRequestMono(resp, "Downloading file"); } + + return byteBufMono.asInputStream() + .publishOn(Schedulers.boundedElastic()) + .flatMap(inputStream -> { + try { + @SuppressWarnings("BlockingMethodInNonBlockingContext") // false warning, see above + final long size = Files.copy(inputStream, outputFile, StandardCopyOption.REPLACE_EXISTING); + statusHandler.call(FileDownloadStatus.DOWNLOADED, uri(), outputFile); + downloadedHandler.call(uri(), outputFile, size); + return Mono.just(outputFile); + } catch (IOException e) { + return Mono.error(e); + } + }); }); } - private static String extractFilename(HttpClientResponse resp) { + private String extractFilename(HttpClientResponse resp) { final String contentDisposition = resp.responseHeaders().get(HttpHeaderNames.CONTENT_DISPOSITION); final String dispositionFilename = FilenameExtractor.filenameFromContentDisposition(contentDisposition); if (dispositionFilename != null) { diff --git a/src/main/java/me/itzg/helpers/http/SaveToFileHandler.java b/src/main/java/me/itzg/helpers/http/SaveToFileHandler.java deleted file mode 100644 index 04d6bc49..00000000 --- a/src/main/java/me/itzg/helpers/http/SaveToFileHandler.java +++ /dev/null @@ -1,49 +0,0 @@ -package me.itzg.helpers.http; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.apache.hc.core5.http.ClassicHttpResponse; -import org.apache.hc.core5.http.HttpEntity; - -@Slf4j -public class SaveToFileHandler extends LoggingResponseHandler implements OutputResponseHandler { - - private final Path outputFile; - private final boolean logProgressEach; - private ContentTypeValidator contentTypeValidator; - - public SaveToFileHandler(Path outputFile, boolean logProgressEach) { - this.outputFile = outputFile; - this.logProgressEach = logProgressEach; - } - - @Override - public void setExpectedContentTypes(List contentTypes) { - if (contentTypes != null) { - contentTypeValidator = new ContentTypeValidator(contentTypes); - } - } - - @Override - public Path handleResponse(ClassicHttpResponse response) throws IOException { - if (contentTypeValidator != null) { - contentTypeValidator.validate(response); - } - return super.handleResponse(response); - } - - @Override - public Path handleEntity(HttpEntity entity) throws IOException { - try (OutputStream out = Files.newOutputStream(outputFile)) { - entity.writeTo(out); - } - if (logProgressEach) { - log.info("Downloaded {}", outputFile); - } - return outputFile; - } -} diff --git a/src/main/java/me/itzg/helpers/http/SharedFetch.java b/src/main/java/me/itzg/helpers/http/SharedFetch.java index b513e06f..40f5fd5d 100644 --- a/src/main/java/me/itzg/helpers/http/SharedFetch.java +++ b/src/main/java/me/itzg/helpers/http/SharedFetch.java @@ -1,19 +1,13 @@ package me.itzg.helpers.http; import io.netty.handler.codec.http.HttpHeaderNames; -import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.UUID; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import me.itzg.helpers.McImageHelper; -import me.itzg.helpers.get.ExtendedRequestRetryStrategy; -import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.HttpClients; import reactor.netty.http.client.HttpClient; /** @@ -27,9 +21,6 @@ @Slf4j public class SharedFetch implements AutoCloseable { - @Getter - private final CloseableHttpClient client; - @Getter private final Map headers = new HashMap<>(); @Getter @@ -39,39 +30,22 @@ public class SharedFetch implements AutoCloseable { private final HttpClient reactiveClient; public SharedFetch(String forCommand) { - this(forCommand, 5, 2); - } - - public SharedFetch(String forCommand, int retryCount, int retryDelaySeconds) { final String userAgent = String.format("%s/%s (cmd=%s)", "mc-image-helper", McImageHelper.getVersion(), forCommand != null ? forCommand : "unspecified" ); + final String fetchSessionId = UUID.randomUUID().toString(); + reactiveClient = HttpClient.create() .headers(headers -> - headers.set(HttpHeaderNames.USER_AGENT.toString(), userAgent) + headers + .set(HttpHeaderNames.USER_AGENT.toString(), userAgent) + .set("x-fetch-session", fetchSessionId) ); - this.client = HttpClients.custom() - .addRequestInterceptorFirst((request, entity, context) -> { - try { - log.debug("Request: {} {} with headers {}", - request.getMethod(), request.getUri(), Arrays.toString(request.getHeaders())); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - }) - .addExecInterceptorFirst("latchingUris", latchingUrisInterceptor) - .useSystemProperties() - .setUserAgent(userAgent) - .setRetryStrategy( - new ExtendedRequestRetryStrategy(retryCount, retryDelaySeconds) - ) - .build(); - - headers.put("x-fetch-session", UUID.randomUUID().toString()); + headers.put("x-fetch-session", fetchSessionId); } public FetchBuilderBase fetch(URI uri) { @@ -85,7 +59,6 @@ public SharedFetch addHeader(String name, String value) { } @Override - public void close() throws IOException { - client.close(); + public void close() { } } diff --git a/src/main/java/me/itzg/helpers/http/SpecificFileFetchBuilder.java b/src/main/java/me/itzg/helpers/http/SpecificFileFetchBuilder.java index f3cf4e64..b7be174d 100644 --- a/src/main/java/me/itzg/helpers/http/SpecificFileFetchBuilder.java +++ b/src/main/java/me/itzg/helpers/http/SpecificFileFetchBuilder.java @@ -1,11 +1,10 @@ package me.itzg.helpers.http; -import static io.netty.handler.codec.http.HttpHeaderNames.*; +import static io.netty.handler.codec.http.HttpHeaderNames.IF_MODIFIED_SINCE; import static io.netty.handler.codec.http.HttpResponseStatus.NOT_MODIFIED; import static java.util.Objects.requireNonNull; import io.netty.handler.codec.http.HttpResponseStatus; -import io.netty.handler.codec.http.HttpStatusClass; import java.io.IOException; import java.net.URI; import java.nio.file.Files; @@ -14,13 +13,15 @@ import java.nio.file.attribute.FileTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.List; +import lombok.Setter; +import lombok.experimental.Accessors; import lombok.extern.slf4j.Slf4j; import me.itzg.helpers.errors.GenericException; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; @Slf4j +@Accessors(fluent = true) public class SpecificFileFetchBuilder extends FetchBuilderBase { private final static DateTimeFormatter httpDateTimeFormatter = @@ -32,7 +33,9 @@ public class SpecificFileFetchBuilder extends FetchBuilderBase assemble() { } } - final List contentTypes = getAcceptContentTypes(); - if (contentTypes != null && !contentTypes.isEmpty()) { - headers.set( - ACCEPT.toString(), - contentTypes - ); - } + + applyHeaders(headers); }) .followRedirect(true) .doOnRequest(debugLogRequest(log, "file fetch")) @@ -113,25 +101,16 @@ public Mono assemble() { .responseSingle((resp, bodyMono) -> { final HttpResponseStatus status = resp.status(); - if (useIfModifiedSince && status.equals(NOT_MODIFIED)) { + if (useIfModifiedSince && status == NOT_MODIFIED) { return Mono.just(file); } - if (HttpStatusClass.valueOf(status.code()) != HttpStatusClass.SUCCESS) { - return Mono.error(new FailedRequestException(status, uri, "Trying to retrieve file")); + if (notSuccess(resp)) { + return failedRequestMono(resp, "Trying to retrieve file"); } - final List contentTypes = getAcceptContentTypes(); - if (contentTypes != null && !contentTypes.isEmpty()) { - final List respTypes = resp.responseHeaders() - .getAll(CONTENT_TYPE); - if (respTypes.stream() - .noneMatch(contentTypes::contains)) { - return Mono.error(new GenericException( - String.format("Unexpected content type in response. Expected '%s' but got '%s'", - contentTypes, respTypes - ))); - } + if (notExpectedContentType(resp)) { + return failedContentTypeMono(resp); } return bodyMono.asInputStream() diff --git a/src/test/java/me/itzg/helpers/http/ObjectFetchBuilderTest.java b/src/test/java/me/itzg/helpers/http/ObjectFetchBuilderTest.java new file mode 100644 index 00000000..e2b9444d --- /dev/null +++ b/src/test/java/me/itzg/helpers/http/ObjectFetchBuilderTest.java @@ -0,0 +1,87 @@ +package me.itzg.helpers.http; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static me.itzg.helpers.http.Fetch.fetch; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.URI; +import lombok.Data; +import org.junit.jupiter.api.Test; + +@WireMockTest +class ObjectFetchBuilderTest { + + @Data + static class Content { + private String name; + private int count; + } + + @Test + void basicScenario(WireMockRuntimeInfo wm) throws IOException { + stubFor( + get("/content") + .withHeader("accept", WireMock.equalTo("application/json")) + .willReturn( + jsonResponse( + "{\n" + + " \"name\": \"alpha\",\n" + + " \"count\": 5\n" + + "}", + 200) + ) + ); + + final Content result = fetch(URI.create(wm.getHttpBaseUrl() + "/content")) + .toObject(Content.class) + .assemble() + .block(); + + assertThat(result) + .extracting("name", "count") + .contains("alpha", 5); + } + + @Test + void handlesNotFound(WireMockRuntimeInfo wm) { + stubFor(get(anyUrl()) + .willReturn(notFound()) + ); + + assertThatThrownBy(() -> + fetch(URI.create(wm.getHttpBaseUrl())) + .toObject(String.class) + .assemble() + .block() + ) + .isInstanceOf(FailedRequestException.class) + .hasMessageContaining("404"); + } + + @Test + void verifyAllExpectedHeaders(WireMockRuntimeInfo wm) throws IOException { + stubFor(get(anyUrl()) + .willReturn(jsonResponse("{}", 200)) + ); + + final Content result = fetch(URI.create(wm.getHttpBaseUrl() + "/")) + .header("x-custom", "customValue") + .toObject(Content.class) + .execute(); + + assertThat(result).isNotNull(); + + verify( + getRequestedFor(urlEqualTo("/")) + .withHeader("accept", WireMock.equalTo("application/json")) + .withHeader("x-custom", WireMock.equalTo("customValue")) + .withHeader("user-agent", WireMock.containing("mc-image-helper")) + .withHeader("x-fetch-session", WireMock.matching("[a-z0-9-]+")) + ); + } +} \ No newline at end of file diff --git a/src/test/java/me/itzg/helpers/http/ObjectListFetchBuilderTest.java b/src/test/java/me/itzg/helpers/http/ObjectListFetchBuilderTest.java new file mode 100644 index 00000000..b5765c21 --- /dev/null +++ b/src/test/java/me/itzg/helpers/http/ObjectListFetchBuilderTest.java @@ -0,0 +1,52 @@ +package me.itzg.helpers.http; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static me.itzg.helpers.http.Fetch.fetch; +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.URI; +import java.util.List; +import lombok.Data; +import org.junit.jupiter.api.Test; + +@WireMockTest +class ObjectListFetchBuilderTest { + + @Data + static class Entry { + private String name; + } + + @Test + void testBasicScenario(WireMockRuntimeInfo wm) throws IOException { + stubFor( + get("/content") + .withHeader("accept", WireMock.equalTo("application/json")) + .willReturn( + jsonResponse( + "[\n" + + " {\n" + + " \"name\": \"alpha\"\n" + + " },\n" + + " {\n" + + " \"name\": \"beta\"\n" + + " }\n" + + "]", 200) + ) + ); + + final List results = fetch(URI.create(wm.getHttpBaseUrl() + "/content")) + .toObjectList(Entry.class) + .assemble() + .block(); + + assertThat(results) + .hasSize(2) + .extracting("name") + .contains("alpha", "beta"); + } +} \ No newline at end of file diff --git a/src/test/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilderTest.java b/src/test/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilderTest.java new file mode 100644 index 00000000..5932d03c --- /dev/null +++ b/src/test/java/me/itzg/helpers/http/OutputToDirectoryFetchBuilderTest.java @@ -0,0 +1,46 @@ +package me.itzg.helpers.http; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static me.itzg.helpers.http.Fetch.fetch; +import static org.assertj.core.api.Assertions.assertThat; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +@WireMockTest +class OutputToDirectoryFetchBuilderTest { + + @Test + void basicScenario(WireMockRuntimeInfo wm, @TempDir Path tempDir) throws IOException { + stubFor( + head(WireMock.urlPathEqualTo("/file")) + .willReturn( + ok() + .withHeader("content-disposition", "attachment; filename=\"actual.txt\"") + ) + ); + stubFor( + get("/file") + .willReturn( + ok("content of actual.txt") + ) + ); + + final Path result = fetch(URI.create(wm.getHttpBaseUrl() + "/file")) + .toDirectory(tempDir) + .execute(); + + final Path expectedFile = tempDir.resolve("actual.txt"); + + assertThat(result) + .isEqualTo(expectedFile) + .exists() + .hasContent("content of actual.txt"); + } +} \ No newline at end of file diff --git a/src/test/java/me/itzg/helpers/http/SpecificFileFetchBuilderTest.java b/src/test/java/me/itzg/helpers/http/SpecificFileFetchBuilderTest.java new file mode 100644 index 00000000..48d6b2ab --- /dev/null +++ b/src/test/java/me/itzg/helpers/http/SpecificFileFetchBuilderTest.java @@ -0,0 +1,93 @@ +package me.itzg.helpers.http; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static me.itzg.helpers.http.Fetch.fetch; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +@WireMockTest +class SpecificFileFetchBuilderTest { + + @Test + void handlesNotFound(WireMockRuntimeInfo wm, @TempDir Path tempDir) { + stubFor( + get(anyUrl()) + .willReturn( + notFound() + ) + ); + + final Path requestedOutputFile = tempDir.resolve("downloaded"); + + assertThatThrownBy( + fetch(URI.create(wm.getHttpBaseUrl() + "/file")) + .toFile(requestedOutputFile)::execute + ) + .isInstanceOf(FailedRequestException.class) + .hasMessageContaining("404"); + + assertThat(requestedOutputFile) + .doesNotExist(); + } + + @Test + void overwritesWhenNoConstraints(WireMockRuntimeInfo wm, @TempDir Path tempDir) throws IOException { + final Path requestedOutputFile = tempDir.resolve("downloaded.txt"); + + Files.write(requestedOutputFile, Collections.singletonList("original content")); + + stubFor( + get("/requested.txt") + .willReturn( + ok("new content") + ) + ); + + final Path result = fetch(URI.create(wm.getHttpBaseUrl() + "/requested.txt")) + .toFile(requestedOutputFile) + .execute(); + + assertThat(result).isEqualTo(requestedOutputFile); + + assertThat(result) + .exists() + .hasContent("new content"); + } + + @Test + void whenRequestSkipNotExists_butExists(WireMockRuntimeInfo wm, @TempDir Path tempDir) throws IOException { + final Path requestedOutputFile = tempDir.resolve("downloaded.txt"); + + Files.write(requestedOutputFile, Collections.singletonList("original content")); + + stubFor( + get("/requested.txt") + .willReturn( + ok("new content") + ) + ); + + final Path result = fetch(URI.create(wm.getHttpBaseUrl() + "/requested.txt")) + .toFile(requestedOutputFile) + .skipExisting(true) + .execute(); + + assertThat(result).isEqualTo(requestedOutputFile); + + assertThat(result) + .exists() + .hasContent("original content"); + } + + +} \ No newline at end of file diff --git a/src/test/java/me/itzg/helpers/lombok.config b/src/test/java/me/itzg/helpers/lombok.config new file mode 100644 index 00000000..96f4d9b0 --- /dev/null +++ b/src/test/java/me/itzg/helpers/lombok.config @@ -0,0 +1,2 @@ +config.stopbubbling=true +lombok.accessors.chain=true \ No newline at end of file