diff --git a/metafacture-io/build.gradle b/metafacture-io/build.gradle index 3de8b53c..e9947499 100644 --- a/metafacture-io/build.gradle +++ b/metafacture-io/build.gradle @@ -23,7 +23,9 @@ dependencies { implementation 'commons-io:commons-io:2.5' implementation 'org.apache.commons:commons-compress:1.21' runtimeOnly 'org.tukaani:xz:1.6' + testImplementation 'com.github.tomakehurst:wiremock-jre8:2.33.2' testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.5.5' testImplementation 'org.assertj:assertj-core:3.11.1' + testRuntimeOnly 'org.slf4j:slf4j-simple:1.7.21' } diff --git a/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java b/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java index e69e8865..5de3724b 100644 --- a/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java +++ b/metafacture-io/src/main/java/org/metafacture/io/HttpOpener.java @@ -1,5 +1,5 @@ /* - * Copyright 2013, 2014 Deutsche Nationalbibliothek + * Copyright 2013, 2022 Deutsche Nationalbibliothek et al * * Licensed under the Apache License, Version 2.0 the "License"; * you may not use this file except in compliance with the License. @@ -24,63 +24,150 @@ import org.metafacture.framework.annotations.Out; import org.metafacture.framework.helpers.DefaultObjectPipe; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; +import java.io.SequenceInputStream; +import java.net.HttpURLConnection; import java.net.URL; -import java.net.URLConnection; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; /** - * Opens a {@link URLConnection} and passes a reader to the receiver. + * Opens an {@link HttpURLConnection} and passes a reader to the receiver. * * @author Christoph Böhme * @author Jan Schnasse + * @author Jens Wille */ -@Description("Opens an HTTP resource. Supports the setting of `Accept` and `Accept-Charset` as HTTP header fields, as well as generic headers (separated by `\\n`).") +@Description("Opens an HTTP resource. Supports setting HTTP header fields `Accept`, `Accept-Charset` and `Content-Type`, as well as generic headers (separated by `\\n`). Defaults: request `method` = `GET`, request `url` = `@-` (input data), request `body` = `@-` (input data) if request method supports body and input data not already used, `Accept` header = `*/*`, `Accept-Charset` header (`encoding`) = `UTF-8`, `errorPrefix` = `ERROR: `.") @In(String.class) @Out(Reader.class) @FluxCommand("open-http") public final class HttpOpener extends DefaultObjectPipe> { - private static final Pattern HEADER_FIELD_SEPARATOR = Pattern.compile("\n"); - private static final Pattern HEADER_VALUE_SEPARATOR = Pattern.compile(":"); + public static final String ACCEPT_DEFAULT = "*/*"; + public static final String ACCEPT_HEADER = "accept"; + public static final String CONTENT_TYPE_HEADER = "content-type"; + public static final String DEFAULT_PREFIX = "ERROR: "; + public static final String ENCODING_DEFAULT = "UTF-8"; + public static final String ENCODING_HEADER = "accept-charset"; + public static final String INPUT_DESIGNATOR = "@-"; - private static final String ACCEPT_HEADER = "accept"; - private static final String ENCODING_HEADER = "accept-charset"; + public static final String DEFAULT_METHOD_NAME = "GET"; + public static final Method DEFAULT_METHOD = Method.valueOf(DEFAULT_METHOD_NAME); - private static final String ACCEPT_DEFAULT = "*/*"; - private static final String ENCODING_DEFAULT = "UTF-8"; + public static final String HEADER_FIELD_SEPARATOR = "\n"; + public static final String HEADER_VALUE_SEPARATOR = ":"; + + private static final Pattern HEADER_FIELD_SEPARATOR_PATTERN = Pattern.compile(HEADER_FIELD_SEPARATOR); + private static final Pattern HEADER_VALUE_SEPARATOR_PATTERN = Pattern.compile(HEADER_VALUE_SEPARATOR); private final Map headers = new HashMap<>(); + private Method method; + private String body; + private String errorPrefix; + private String url; + private boolean inputUsed; + + public enum Method { + + DELETE(false, true), + GET(false, true), + HEAD(false, false), + OPTIONS(false, true), + POST(true, true), + PUT(true, true), + TRACE(false, true); + + private final boolean requestHasBody; + private final boolean responseHasBody; + + Method(final boolean requestHasBody, final boolean responseHasBody) { + this.requestHasBody = requestHasBody; + this.responseHasBody = responseHasBody; + } + + /** + * Checks whether the request method accepts a request body. + * + * @return true if the request method accepts a request body + */ + public boolean getRequestHasBody() { + return requestHasBody; + } + + /** + * Checks whether the request method returns a response body. + * + * @return true if the request method returns a response body + */ + public boolean getResponseHasBody() { + return responseHasBody; + } + + } + /** * Creates an instance of {@link HttpOpener}. */ public HttpOpener() { setAccept(ACCEPT_DEFAULT); setEncoding(ENCODING_DEFAULT); + setErrorPrefix(DEFAULT_PREFIX); + setMethod(DEFAULT_METHOD); + setUrl(INPUT_DESIGNATOR); } /** - * Sets the HTTP accept header value. This is a mime-type such as text/plain - * or text/html. The default value of the accept is */* which means - * any mime-type. + * Sets the HTTP {@value ACCEPT_HEADER} header value. This is a MIME type + * such as {@code text/plain} or {@code application/json}. The default + * value for the accept header is {@value ACCEPT_DEFAULT} which means + * any MIME type. * - * @param accept mime-type to use for the HTTP accept header + * @param accept MIME type to use for the HTTP accept header */ public void setAccept(final String accept) { setHeader(ACCEPT_HEADER, accept); } /** - * Sets the preferred encoding of the HTTP response. This value is in the - * accept-charset header. Additonally, the encoding is used for reading the - * HTTP resonse if it does not specify an encoding. The default value for - * the encoding is UTF-8. + * Sets the HTTP request body. The default value for the request body is + * {@value INPUT_DESIGNATOR} if the {@link #setMethod(Method) request + * method} accepts a request body, which means it will use the {@link + * #process(String) input data} data as request body if the input has + * not already been used; otherwise, no request body will be set by + * default. + * + *

If a request body has been set, but the request method does not + * accept a body, the method may be changed to {@code POST}. + * + * @param body the request body + */ + public void setBody(final String body) { + this.body = body; + } + + /** + * Sets the HTTP {@value CONTENT_TYPE_HEADER} header value. This is a + * MIME type such as {@code text/plain} or {@code application/json}. + * + * @param contentType MIME type to use for the HTTP content-type header + */ + public void setContentType(final String contentType) { + setHeader(CONTENT_TYPE_HEADER, contentType); + } + + /** + * Sets the HTTP {@value ENCODING_HEADER} header value. This is the + * preferred encoding for the HTTP response. Additionally, the encoding + * is used for reading the HTTP response if it does not specify a content + * encoding. The default for the encoding is {@value ENCODING_DEFAULT}. * * @param encoding name of the encoding used for the accept-charset HTTP * header @@ -90,14 +177,28 @@ public void setEncoding(final String encoding) { } /** - * Sets a request property, or multiple request properties separated by - * {@code \n}. + * Sets the error prefix. The default error prefix is + * {@value DEFAULT_PREFIX}. + * + * @param errorPrefix the error prefix + */ + public void setErrorPrefix(final String errorPrefix) { + this.errorPrefix = errorPrefix; + } + + /** + * Sets a request property (header), or multiple request properties + * separated by {@value HEADER_FIELD_SEPARATOR}. Header name and value + * are separated by {@value HEADER_VALUE_SEPARATOR}. The header name is + * case-insensitive. * * @param header request property line + * + * @see #setHeader(String, String) */ public void setHeader(final String header) { - Arrays.stream(HEADER_FIELD_SEPARATOR.split(header)).forEach(h -> { - final String[] parts = HEADER_VALUE_SEPARATOR.split(h, 2); + Arrays.stream(HEADER_FIELD_SEPARATOR_PATTERN.split(header)).forEach(h -> { + final String[] parts = HEADER_VALUE_SEPARATOR_PATTERN.split(h, 2); if (parts.length == 2) { setHeader(parts[0], parts[1].trim()); } @@ -108,7 +209,7 @@ public void setHeader(final String header) { } /** - * Sets a request property. + * Sets a request property (header). The header name is case-insensitive. * * @param key request property key * @param value request property value @@ -117,21 +218,99 @@ public void setHeader(final String key, final String value) { headers.put(key.toLowerCase(), value); } + /** + * Sets the HTTP request method. The default request method is + * {@value DEFAULT_METHOD_NAME}. + * + * @param method the request method + */ + public void setMethod(final Method method) { + this.method = method; + } + + /** + * Sets the HTTP request URL. The default value for the request URL is + * {@value INPUT_DESIGNATOR}, which means it will use the {@link + * #process(String) input data} as request URL. + * + * @param url the request URL + */ + public void setUrl(final String url) { + this.url = url; + } + @Override - public void process(final String urlStr) { + public void process(final String input) { try { - final URL url = new URL(urlStr); - final URLConnection con = url.openConnection(); - headers.forEach(con::addRequestProperty); - String enc = con.getContentEncoding(); - if (enc == null) { - enc = headers.get(ENCODING_HEADER); + final String requestUrl = getInput(input, url); + final String requestBody = getInput(input, + body == null && method.getRequestHasBody() ? INPUT_DESIGNATOR : body); + + final HttpURLConnection connection = + (HttpURLConnection) new URL(requestUrl).openConnection(); + + connection.setRequestMethod(method.name()); + headers.forEach(connection::addRequestProperty); + + if (requestBody != null) { + connection.setDoOutput(true); + connection.getOutputStream().write(requestBody.getBytes()); } - getReceiver().process(new InputStreamReader(con.getInputStream(), enc)); + + final InputStream inputStream = getInputStream(connection); + final String contentEncoding = getEncoding(connection.getContentEncoding()); + + getReceiver().process(new InputStreamReader(inputStream, contentEncoding)); } catch (final IOException e) { throw new MetafactureException(e); } } + private String getInput(final String input, final String value) { + final String result; + + if (!INPUT_DESIGNATOR.equals(value)) { + result = value; + } + else if (inputUsed) { + result = null; + } + else { + inputUsed = true; + result = input; + } + + return result; + } + + private InputStream getInputStream(final HttpURLConnection connection) throws IOException { + try { + return connection.getInputStream(); + } + catch (final IOException e) { + final InputStream errorStream = connection.getErrorStream(); + if (errorStream != null) { + return getErrorStream(errorStream); + } + else { + throw e; + } + } + } + + private InputStream getErrorStream(final InputStream errorStream) { + if (errorPrefix != null) { + final InputStream errorPrefixStream = new ByteArrayInputStream(errorPrefix.getBytes()); + return new SequenceInputStream(errorPrefixStream, errorStream); + } + else { + return errorStream; + } + } + + private String getEncoding(final String contentEncoding) { + return contentEncoding != null ? contentEncoding : headers.get(ENCODING_HEADER); + } + } diff --git a/metafacture-io/src/test/java/org/metafacture/io/HttpOpenerTest.java b/metafacture-io/src/test/java/org/metafacture/io/HttpOpenerTest.java new file mode 100644 index 00000000..f86cb19c --- /dev/null +++ b/metafacture-io/src/test/java/org/metafacture/io/HttpOpenerTest.java @@ -0,0 +1,329 @@ +/* + * Copyright 2013, 2022 Deutsche Nationalbibliothek et al + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.io; + +import org.metafacture.commons.ResourceUtil; +import org.metafacture.framework.ObjectReceiver; + +import com.github.tomakehurst.wiremock.client.MappingBuilder; +import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.http.RequestMethod; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.github.tomakehurst.wiremock.matching.RequestPatternBuilder; +import com.github.tomakehurst.wiremock.matching.StringValuePattern; +import com.github.tomakehurst.wiremock.matching.UrlPattern; +import org.junit.Assert; +import org.junit.ComparisonFailure; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +import java.io.IOException; +import java.io.Reader; +import java.util.Arrays; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +/** + * Tests for class {@link HttpOpener}. + * + * @author Jens Wille + */ +public final class HttpOpenerTest { + + private static final String TEST_PATH = "/test/path"; + private static final String TEST_URL = "%s" + TEST_PATH; + + private static final String TEST_STRING = "test string"; + private static final StringValuePattern TEST_VALUE = WireMock.equalTo(TEST_STRING); + + private static final String REQUEST_BODY = "request body"; + private static final String RESPONSE_BODY = "response bödy"; // UTF-8 + + @Rule + public MockitoRule mockitoRule = MockitoJUnit.rule(); + + @Rule + public WireMockRule wireMockRule = new WireMockRule(WireMockConfiguration.wireMockConfig() + .jettyAcceptors(Runtime.getRuntime().availableProcessors()) + .dynamicPort()); + + @Mock + private ObjectReceiver receiver; + + @Captor + private ArgumentCaptor processedObject; + + @Test + public void shouldPerformGetRequestWithInputAsUrlByDefault() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.GET, (o, u) -> {}); + } + + @Test + public void shouldPerformGetRequestWithUrlParameter() throws IOException { + shouldPerformRequest(TEST_STRING, HttpOpener.Method.GET, (o, u) -> { + o.setUrl(u); + }); + } + + @Test + public void shouldPerformPostRequestWithInputAsUrl() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.POST, (o, u) -> { + o.setMethod(HttpOpener.Method.POST); + o.setBody(REQUEST_BODY); + }); + } + + @Test + public void shouldPerformPostRequestWithUrlParameter() throws IOException { + shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.POST, (o, u) -> { + o.setMethod(HttpOpener.Method.POST); + o.setUrl(u); + }); + } + + @Test + public void shouldPerformPostRequestWithBodyParameter() throws IOException { + shouldPerformRequest(TEST_STRING, HttpOpener.Method.POST, (o, u) -> { + o.setMethod(HttpOpener.Method.POST); + o.setUrl(u); + o.setBody(REQUEST_BODY); + }); + } + + @Test + public void shouldPerformPostRequestInsteadOfGetWithBodyParameter() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.POST, (o, u) -> { + o.setMethod(HttpOpener.Method.GET); + o.setBody(REQUEST_BODY); + }); + } + + @Test + public void shouldPerformPostRequestInsteadOfGetWithInputAsBodyParameter() throws IOException { + shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.POST, (o, u) -> { + o.setMethod(HttpOpener.Method.GET); + o.setUrl(u); + o.setBody("@-"); + }); + } + + @Test + public void shouldPerformGetRequestWithoutBodyWithAlreadyUsedInputAsBodyParameter() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.GET, (o, u) -> { + o.setBody("@-"); + }); + } + + @Test + public void shouldPerformPutRequestWithUrlParameter() throws IOException { + shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.PUT, (o, u) -> { + o.setMethod(HttpOpener.Method.PUT); + o.setUrl(u); + }); + } + + @Test + public void shouldPerformPutRequestWithBodyParameter() throws IOException { + shouldPerformRequest(TEST_STRING, HttpOpener.Method.PUT, (o, u) -> { + o.setMethod(HttpOpener.Method.PUT); + o.setUrl(u); + o.setBody(REQUEST_BODY); + }); + } + + @Test + public void shouldPerformDeleteRequestWithUrlParameter() throws IOException { + shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.DELETE, (o, u) -> { + o.setMethod(HttpOpener.Method.DELETE); + o.setUrl(u); + }); + } + + @Test + public void shouldPerformHeadRequestWithUrlParameter() throws IOException { + shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.HEAD, (o, u) -> { + o.setMethod(HttpOpener.Method.HEAD); + o.setUrl(u); + }); + } + + @Test + public void shouldPerformOptionsRequestWithUrlParameter() throws IOException { + shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.OPTIONS, (o, u) -> { + o.setMethod(HttpOpener.Method.OPTIONS); + o.setUrl(u); + }); + } + + @Test + public void shouldPerformTraceRequestWithUrlParameter() throws IOException { + shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.TRACE, (o, u) -> { + o.setMethod(HttpOpener.Method.TRACE); + o.setUrl(u); + }); + } + + @Test + public void shouldPerformGetRequestWithAcceptParameter() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.GET, (o, u) -> { + o.setAccept(TEST_STRING); + }, "Accept"); + } + + @Test + public void shouldPerformGetRequestWithSingleValuedHeaderParameter() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.GET, (o, u) -> { + o.setHeader("x-api-key: " + TEST_STRING); + }, "x-api-key"); + } + + @Test + public void shouldPerformGetRequestWithMultiValuedHeaderParameter() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.GET, (o, u) -> { + o.setHeader("x-api-key: " + TEST_STRING + "\nx-other-header: " + TEST_STRING); + }, "x-api-key", "x-other-header"); + } + + @Test + public void shouldPerformGetRequestWithMultipledHeaderParameters() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.GET, (o, u) -> { + o.setHeader("x-api-key: " + TEST_STRING); + o.setHeader("x-other-header: " + TEST_STRING); + }, "x-api-key", "x-other-header"); + } + + @Test + public void shouldPerformPostRequestWithContentTypeParameter() throws IOException { + shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.POST, (o, u) -> { + o.setMethod(HttpOpener.Method.POST); + o.setUrl(u); + o.setContentType(TEST_STRING); + }, "Content-Type"); + } + + @Test + public void shouldPerformPostRequestWithEncodingParameter() throws IOException { + final String encoding = "ISO-8859-1"; + final String header = "Accept-Charset"; + final StringValuePattern value = WireMock.equalTo(encoding); + + try { + shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.POST, (o, u) -> { + o.setMethod(HttpOpener.Method.POST); + o.setUrl(u); + o.setEncoding(encoding); + }, s -> s.withHeader(header, value), q -> q.withHeader(header, value), null); + } + catch (final ComparisonFailure e) { + Assert.assertEquals("expected: but was:", e.getMessage()); + } + } + + @Test + public void shouldPerformPostRequestWithEncodingParameterAndContentEncodingResponseHeader() throws IOException { + final String encoding = "ISO-8859-1"; + final String header = "Accept-Charset"; + final StringValuePattern value = WireMock.equalTo(encoding); + + shouldPerformRequest(REQUEST_BODY, HttpOpener.Method.POST, (o, u) -> { + o.setMethod(HttpOpener.Method.POST); + o.setUrl(u); + o.setEncoding(encoding); + }, + s -> s.withHeader(header, value), + q -> q.withHeader(header, value), + r -> r.withHeader("Content-Encoding", "UTF-8") + ); + } + + @Test + public void shouldPerformGetRequestWithErrorResponse() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.GET, (o, u) -> {}, + null, null, WireMock.badRequest().withBody(RESPONSE_BODY), "ERROR: " + RESPONSE_BODY); + } + + @Test + public void shouldPerformGetRequestWithErrorResponseAndErrorPrefixParameter() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.GET, (o, u) -> o.setErrorPrefix(TEST_STRING), + null, null, WireMock.badRequest().withBody(RESPONSE_BODY), TEST_STRING + RESPONSE_BODY); + } + + @Test + public void shouldPerformGetRequestWithErrorResponseAndWithoutErrorPrefixParameter() throws IOException { + shouldPerformRequest(TEST_URL, HttpOpener.Method.GET, (o, u) -> o.setErrorPrefix(null), + null, null, WireMock.badRequest().withBody(RESPONSE_BODY), RESPONSE_BODY); + } + + private void shouldPerformRequest(final String input, final HttpOpener.Method method, final BiConsumer consumer, final String... headers) throws IOException { + shouldPerformRequest(input, method, consumer, + s -> Arrays.stream(headers).forEach(h -> s.withHeader(h, TEST_VALUE)), + q -> Arrays.stream(headers).forEach(h -> q.withHeader(h, TEST_VALUE)), null); + } + + private void shouldPerformRequest(final String input, final HttpOpener.Method method, final BiConsumer consumer, final Consumer stubConsumer, final Consumer requestConsumer, final Consumer responseConsumer) throws IOException { + final ResponseDefinitionBuilder response = WireMock.ok().withBody(RESPONSE_BODY); + if (responseConsumer != null) { + responseConsumer.accept(response); + } + + shouldPerformRequest(input, method, + consumer, stubConsumer, requestConsumer, + response, method.getResponseHasBody() ? RESPONSE_BODY : ""); + } + + private void shouldPerformRequest(final String input, final HttpOpener.Method method, final BiConsumer consumer, final Consumer stubConsumer, final Consumer requestConsumer, final ResponseDefinitionBuilder response, final String responseBody) throws IOException { + final String baseUrl = wireMockRule.baseUrl(); + final String url = String.format(TEST_URL, baseUrl); + + final String methodName = method.name(); + final UrlPattern urlPattern = WireMock.urlPathEqualTo(TEST_PATH); + + final HttpOpener opener = new HttpOpener(); + opener.setReceiver(receiver); + consumer.accept(opener, url); + + final MappingBuilder stub = WireMock.request(methodName, urlPattern).willReturn(response); + if (stubConsumer != null) { + stubConsumer.accept(stub); + } + + final RequestPatternBuilder request = new RequestPatternBuilder(RequestMethod.fromString(methodName), urlPattern) + .withRequestBody(method.getRequestHasBody() ? WireMock.equalTo(REQUEST_BODY) : WireMock.absent()); + if (requestConsumer != null) { + requestConsumer.accept(request); + } + + WireMock.stubFor(stub); + + opener.process(String.format(input, baseUrl)); + opener.closeStream(); + + WireMock.verify(request); + + Mockito.verify(receiver).process(processedObject.capture()); + Assert.assertEquals(responseBody, ResourceUtil.readAll(processedObject.getValue())); + } + +}