Merge pull request #57 from renatoathaydes/dev

Next release

gradle.properties

@@ -1,8 +1,8 @@
 kotlinVersion=1.6.10
 kotestVersion=4.6.2
 kotlin.stdlib.default.dependency=false
-rawHttpCoreVersion=2.5.0
-rawHttpCliVersion=1.5.0
-rawHttpDuplexVersion=1.4.0
-rawHttpReqInEditVersion=0.4.0
-rawHttpCookiesVersion=0.3.0
+rawHttpCoreVersion=2.5.1
+rawHttpCliVersion=1.5.1
+rawHttpDuplexVersion=1.4.1
+rawHttpReqInEditVersion=0.4.1
+rawHttpCookiesVersion=0.3.1

rawhttp-cookies/build.gradle

@@ -44,6 +44,11 @@ task javadocJar(type: Jar) {
 
 /* Publishing config */
 
+java {
+    withJavadocJar()
+    withSourcesJar()
+}
+
 publishing {
     publications {
         mavenJava(MavenPublication) {

rawhttp-core/build.gradle

@@ -46,14 +46,19 @@ jar {
 
 /* Publishing config */
 
+java {
+    withJavadocJar()
+    withSourcesJar()
+}
+
 publishing {
     publications {
-        mavenJava( MavenPublication ) {
+        mavenJava(MavenPublication) {
             artifactId = 'rawhttp-core'
             from components.java
             versionMapping {
-                usage( 'java-api' ) {
-                    fromResolutionOf( 'runtimeClasspath' )
+                usage('java-api') {
+                    fromResolutionOf('runtimeClasspath')
                 }
                 usage( 'java-runtime' ) {
                     fromResolutionResult()

rawhttp-core/src/main/java/rawhttp/core/HttpMetadataParser.java

@@ -7,6 +7,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PushbackInputStream;
 import java.io.UnsupportedEncodingException;
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -267,11 +268,10 @@ private RawHttpHeaders.Builder buildHeaders(
         RawHttpHeaders.Builder builder = RawHttpHeaders.newBuilderSkippingValidation()
                 .withHeaderValuesCharset(options.getHttpHeadersOptions().getHeaderValuesCharset());
 
-        int lineNumber = 1;
+        HeaderParserState state = new HeaderParserState(new PushbackInputStream(stream));
         Map.Entry<String, String> header;
-        while ((header = parseHeaderField(stream, lineNumber, createError)) != null) {
+        while ((header = parseHeaderField(state, createError)) != null) {
             builder.with(header.getKey(), header.getValue());
-            lineNumber++;
         }
         return builder;
     }
@@ -323,26 +323,25 @@ private String parseStartLine(InputStream inputStream,
     }
 
     @Nullable
-    private Map.Entry<String, String> parseHeaderField(InputStream inputStream,
-                                                       int lineNumber,
+    private Map.Entry<String, String> parseHeaderField(HeaderParserState state,
                                                        BiFunction<String, Integer, RuntimeException> createError)
             throws IOException {
         @Nullable
-        String headerName = parseHeaderName(inputStream, lineNumber, createError);
+        String headerName = parseHeaderName(state, createError);
 
         if (headerName == null) {
             return null;
         }
 
-        String headerValue = parseHeaderValue(inputStream, lineNumber, createError);
+        String headerValue = parseHeaderValue(state, createError);
 
         return new AbstractMap.SimpleEntry<>(headerName, headerValue);
     }
 
-    private String parseHeaderName(InputStream inputStream,
-                                   int lineNumber,
+    private String parseHeaderName(HeaderParserState state,
                                    BiFunction<String, Integer, RuntimeException> createError)
             throws IOException {
+        InputStream inputStream = state.pbStream;
         int b = inputStream.read();
 
         if (b == '\r') {
@@ -369,7 +368,7 @@ private String parseHeaderName(InputStream inputStream,
                 return null; // end of headers stream
             } else {
                 inputStream.close();
-                throw createError.apply("Illegal new-line character", lineNumber);
+                throw createError.apply("Illegal new-line character", state.lineNumber);
             }
         }
         if (b < 0) {
@@ -388,30 +387,30 @@ private String parseHeaderName(InputStream inputStream,
             if (c == ':') {
                 headerName = metadataBuilder.toString();
                 if (headerName.isEmpty()) {
-                    throw createError.apply("Header name is missing", lineNumber);
+                    throw createError.apply("Header name is missing", state.lineNumber);
                 }
                 break;
             } else if (c == '\n' || c == '\r') {
-                throw createError.apply("Invalid header: missing the ':' separator", lineNumber);
+                throw createError.apply("Invalid header: missing the ':' separator", state.lineNumber);
             } else {
                 if (length > lengthLimit) {
-                    throw createError.apply("Header name is too long", lineNumber);
+                    throw createError.apply("Header name is too long", state.lineNumber);
                 }
                 if (FieldValues.isAllowedInTokens(c)) {
                     metadataBuilder.append(c);
                 } else {
-                    throw createError.apply("Illegal character in HTTP header name", lineNumber);
+                    throw createError.apply("Illegal character in HTTP header name", state.lineNumber);
                 }
             }
         } while ((b = inputStream.read()) >= 0);
 
         return metadataBuilder.toString().trim();
     }
 
-    private String parseHeaderValue(InputStream inputStream,
-                                    int lineNumber,
+    private String parseHeaderValue(HeaderParserState state,
                                     BiFunction<String, Integer, RuntimeException> createError)
             throws IOException {
+        final PushbackInputStream inputStream = state.pbStream;
         final int lengthLimit = options.getHttpHeadersOptions().getMaxHeaderValueLength();
         final boolean allowNewLineWithoutReturn = options.allowNewLineWithoutReturn();
         ByteArrayOutputStream out = new ByteArrayOutputStream(Math.min(lengthLimit, 64));
@@ -422,29 +421,53 @@ private String parseHeaderValue(InputStream inputStream,
             if (b == '\r') {
                 // expect new-line
                 int next = inputStream.read();
-                if (next < 0 || next == '\n') {
-                    break;
+                if (next < 0) break;
+                if (next == '\n') {
+                    state.lineNumber++;
+
+                    // support for multi-line headers
+                    next = inputStream.read();
+                    if (next < 0) break;
+                    boolean isMultilineHeader = (next == ' ' || next == '\t');
+                    // if multi-line header, continue as if nothing happened
+                    if (!isMultilineHeader) {
+                        // otherwise, return the byte and finish this header
+                        inputStream.unread(next);
+                        break;
+                    }
                 } else {
                     inputStream.close();
-                    throw createError.apply("Illegal character after return", lineNumber);
+                    throw createError.apply("Illegal character after return", state.lineNumber);
                 }
             } else if (b == '\n') {
                 if (!allowNewLineWithoutReturn) {
                     inputStream.close();
-                    throw createError.apply("Illegal new-line character without preceding return", lineNumber);
+                    throw createError.apply("Illegal new-line character without preceding return", state.lineNumber);
                 }
 
                 // unexpected, but let's accept new-line without returns
+                state.lineNumber++;
+
+                // support for multi-line headers
+                int next = inputStream.read();
+                if (next < 0) break;
+                boolean isMultilineHeader = (next == ' ' || next == '\t');
+                // if multi-line header, continue as if nothing happened
+                if (!isMultilineHeader) {
+                    // otherwise, return the byte and finish this header
+                    inputStream.unread(next);
+                    break;
+                }
                 break;
             } else {
                 if (length > lengthLimit) {
-                    throw createError.apply("Header value is too long", lineNumber);
+                    throw createError.apply("Header value is too long", state.lineNumber);
                 }
 
                 if (FieldValues.isAllowedInHeaderValue(b)) {
                     out.write(b);
                 } else {
-                    throw createError.apply("Illegal character in HTTP header value", lineNumber);
+                    throw createError.apply("Illegal character in HTTP header value", state.lineNumber);
                 }
             }
             length++;
@@ -563,4 +586,12 @@ private static Map.Entry<String, String> parseHostAndPort(String text) {
         return new AbstractMap.SimpleImmutableEntry<>(hostPart.toString(), portPart.toString());
     }
 
+    private static final class HeaderParserState {
+        final PushbackInputStream pbStream;
+        int lineNumber = 1;
+
+        public HeaderParserState(PushbackInputStream pbStream) {
+            this.pbStream = pbStream;
+        }
+    }
 }

rawhttp-core/src/test/kotlin/rawhttp/core/RawHttpErrorsTest.kt

@@ -54,7 +54,7 @@ class RawHttpErrorsTest {
         val examples = table(
                 headers("Request", "lineNumber", "message"),
                 row("GET / HTTP/1.1\nAccept: all\r\n", 1, "Illegal new-line character without preceding return"),
-                row("GET / HTTP/1.1\r\nAccept: all\n", 2, "Illegal new-line character without preceding return")
+                row("GET / HTTP/1.1\r\nAccept: all\r\n\n", 3, "Illegal new-line character")
         )
         forAll(examples) { request, expectedLineNumber, expectedMessage ->
             shouldThrow<InvalidHttpRequest> {

rawhttp-core/src/test/kotlin/rawhttp/core/RawHttpTest.kt

@@ -1,10 +1,12 @@
 package rawhttp.core
 
+import io.kotest.matchers.collections.shouldContainExactlyInAnyOrder
 import io.kotest.matchers.collections.shouldHaveSize
 import io.kotest.matchers.optional.bePresent
 import io.kotest.matchers.optional.shouldBePresent
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.shouldNot
+import io.kotest.matchers.string.shouldContain
 import org.junit.jupiter.api.Test
 import java.io.File
 import java.net.URI
@@ -370,6 +372,35 @@ class SimpleHttpResponseTests {
         }
     }
 
+    /**
+     * See https://www.rfc-editor.org/rfc/rfc7230#section-3.2.4
+     */
+    @Test
+    fun `Can parse deprecated multi-line headers`() {
+        SimpleHttpRequestTests::class.java.getResourceAsStream("response-headers.http")!!.use {
+            RawHttp().parseResponse(it).run {
+                startLine.statusCode shouldBe 200
+                headers.headerNames shouldContainExactlyInAnyOrder listOf(
+                    "Cache-Control",
+                    "Content-Type",
+                    "Set-Cookie",
+                    "X-Content-Type-Options",
+                    "X-Frame-Options",
+                    "X-XSS-Protection",
+                    "Strict-Transport-Security",
+                    "Content-Security-Policy",
+                    "Date",
+                    "Content-Length",
+                )
+                headers["content-length"] shouldBe listOf("95247")
+                headers.getFirst("Content-Security-Policy") shouldBePresent { csp ->
+                    // we remove the first whitespace as it marks the new beginning of a line, but keep other whitespace
+                    csp shouldContain "default-src 'self' www.google-analytics.com www.youtube.com;" +
+                            "         child-src 'self' www.youtube.com www.youtube-nocookie.com player.vimeo.com www.google.com;"
+                }
+            }
+        }
+    }
 }
 
 class CopyHttpRequestTests {

rawhttp-core/src/test/resources/rawhttp/core/response-headers.http

@@ -0,0 +1,21 @@
+HTTP/1.1 200 OK
+Cache-Control: private
+Content-Type: text/html; charset=utf-8
+Set-Cookie: PerplexCookieApproval=0; expires=Sun, 25-Aug-2024 10:40:14 GMT; path=/; secure; HttpOnly; SameSite=Lax
+X-Content-Type-Options: nosniff
+X-Frame-Options: sameorigin
+X-XSS-Protection: 1; mode=block
+Strict-Transport-Security: max-age=31536000;
+Content-Security-Policy:
+          default-src 'self' www.google-analytics.com www.youtube.com;
+          child-src 'self' www.youtube.com www.youtube-nocookie.com player.vimeo.com www.google.com;
+    frame-src 'self' www.youtube.com forms.zohopublic.eu zfrmz.eu;
+          script-src 'self' 'unsafe-inline' 'unsafe-eval' www.perplex.nl s.ytimg.com *.google-analytics.com www.googletagmanager.com www.google.com www.gstatic.com www.youtube.com player.vimeo.com www.googletagmanager.com;
+          style-src 'self' 'unsafe-inline';
+          img-src 'self' data: rating.pegi.info *.google-analytics.com *.analytics.google.com www.perplex.nl i.ytimg.com www.gravatar.com img.youtube.com i.vimeocdn.com vumbnail.com;
+          font-src 'self' data:;
+    connect-src 'self' *.google-analytics.com *.analytics.google.com;
+          form-action 'self' forms.zohopublic.eu;
+          report-uri https://perplex.report-uri.com/r/default/csp/enforce;
+Date: Thu, 25 Aug 2022 10:40:14 GMT
+Content-Length: 95247

rawhttp-duplex/build.gradle

@@ -64,14 +64,19 @@ task javadocJar(type: Jar) {
 
 /* Publishing config */
 
+java {
+    withJavadocJar()
+    withSourcesJar()
+}
+
 publishing {
     publications {
-        mavenJava( MavenPublication ) {
+        mavenJava(MavenPublication) {
             artifactId = 'rawhttp-duplex'
             from components.java
             versionMapping {
-                usage( 'java-api' ) {
-                    fromResolutionOf( 'runtimeClasspath' )
+                usage('java-api') {
+                    fromResolutionOf('runtimeClasspath')
                 }
                 usage( 'java-runtime' ) {
                     fromResolutionResult()