diff --git a/Sources/Vapor/HTTP/Headers/HTTPHeaders+ContentRange.swift b/Sources/Vapor/HTTP/Headers/HTTPHeaders+ContentRange.swift index 3130786111..493c26b082 100644 --- a/Sources/Vapor/HTTP/Headers/HTTPHeaders+ContentRange.swift +++ b/Sources/Vapor/HTTP/Headers/HTTPHeaders+ContentRange.swift @@ -5,7 +5,7 @@ extension HTTPHeaders { /// The unit in which `ContentRange`s and `Range`s are specified. This is usually `bytes`. /// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Range - public enum RangeUnit: Equatable { + public enum RangeUnit: Sendable, Equatable { case bytes case custom(value: String) @@ -21,7 +21,7 @@ extension HTTPHeaders { /// Represents the HTTP `Range` request header. /// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range - public struct Range: Equatable { + public struct Range: Sendable, Equatable { public let unit: RangeUnit public let ranges: [HTTPHeaders.Range.Value] @@ -134,7 +134,7 @@ extension HTTPHeaders.Range { /// Represents one value of the `Range` request header. /// /// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Range - public enum Value: Equatable { + public enum Value: Sendable, Equatable { ///Integer with single trailing dash, e.g. `25-` case start(value: Int) ///Integer with single leading dash, e.g. `-25` diff --git a/Sources/Vapor/HTTP/Server/HTTPServer.swift b/Sources/Vapor/HTTP/Server/HTTPServer.swift index f9a9927984..7309b7df21 100644 --- a/Sources/Vapor/HTTP/Server/HTTPServer.swift +++ b/Sources/Vapor/HTTP/Server/HTTPServer.swift @@ -119,7 +119,7 @@ public final class HTTPServer: Server, Sendable { /// Enables decompression with default configuration. public static var enabled: Self { - .enabled(limit: .ratio(10)) + .enabled(limit: .ratio(25)) } /// Enables decompression with custom configuration. @@ -168,7 +168,7 @@ public final class HTTPServer: Server, Sendable { reuseAddress: Bool = true, tcpNoDelay: Bool = true, responseCompression: CompressionConfiguration = .disabled, - requestDecompression: DecompressionConfiguration = .disabled, + requestDecompression: DecompressionConfiguration = .enabled, supportPipelining: Bool = true, supportVersions: Set? = nil, tlsConfiguration: TLSConfiguration? = nil, @@ -200,7 +200,7 @@ public final class HTTPServer: Server, Sendable { reuseAddress: Bool = true, tcpNoDelay: Bool = true, responseCompression: CompressionConfiguration = .disabled, - requestDecompression: DecompressionConfiguration = .disabled, + requestDecompression: DecompressionConfiguration = .enabled, supportPipelining: Bool = true, supportVersions: Set? = nil, tlsConfiguration: TLSConfiguration? = nil, @@ -503,6 +503,28 @@ extension ChannelPipeline { let http2 = HTTP2FramePayloadToHTTP1ServerCodec() handlers.append(http2) + /// Add response compressor if configured. + switch configuration.responseCompression.storage { + case .enabled(let initialByteBufferCapacity): + let responseCompressionHandler = HTTPResponseCompressor( + initialByteBufferCapacity: initialByteBufferCapacity + ) + handlers.append(responseCompressionHandler) + case .disabled: + break + } + + /// Add request decompressor if configured. + switch configuration.requestDecompression.storage { + case .enabled(let limit): + let requestDecompressionHandler = NIOHTTPRequestDecompressor( + limit: limit + ) + handlers.append(requestDecompressionHandler) + case .disabled: + break + } + /// Add NIO → HTTP request decoder. let serverReqDecoder = HTTPServerRequestDecoder( application: application diff --git a/Sources/Vapor/Middleware/FileMiddleware.swift b/Sources/Vapor/Middleware/FileMiddleware.swift index 9fc959bce2..d1c4b6867e 100644 --- a/Sources/Vapor/Middleware/FileMiddleware.swift +++ b/Sources/Vapor/Middleware/FileMiddleware.swift @@ -9,7 +9,8 @@ public final class FileMiddleware: Middleware { private let publicDirectory: String private let defaultFile: String? private let directoryAction: DirectoryAction - + private let advancedETagComparison: Bool + public struct BundleSetupError: Equatable, Error { /// The description of this error. @@ -22,6 +23,15 @@ public final class FileMiddleware: Middleware { public static let publicDirectoryIsNotAFolder: Self = .init(description: "Cannot find any actual folder for the given Public Directory") } + struct ETagHashes: StorageKey { + public typealias Value = [String: FileHash] + + public struct FileHash { + let lastModified: Date + let digestHex: String + } + } + /// Creates a new `FileMiddleware`. /// /// - parameters: @@ -29,10 +39,12 @@ public final class FileMiddleware: Middleware { /// - defaultFile: The name of the default file to look for and serve if a request hits any public directory. Starting with `/` implies /// an absolute path from the public directory root. If `nil`, no default files are served. /// - directoryAction: Determines the action to take when the request doesn't have a trailing slash but matches a directory. - public init(publicDirectory: String, defaultFile: String? = nil, directoryAction: DirectoryAction = .none) { + /// - advancedETagComparison: The method used when ETags are generated. If true, a byte-by-byte hash is created (and cached), otherwise a simple comparison based on the file's last modified date and size. + public init(publicDirectory: String, defaultFile: String? = nil, directoryAction: DirectoryAction = .none, advancedETagComparison: Bool = true) { self.publicDirectory = publicDirectory.addTrailingSlash() self.defaultFile = defaultFile self.directoryAction = directoryAction + self.advancedETagComparison = advancedETagComparison } public func respond(to request: Request, chainingTo next: Responder) -> EventLoopFuture { @@ -88,10 +100,9 @@ public final class FileMiddleware: Middleware { return next.respond(to: request) } } - + // stream the file - let res = request.fileio.streamFile(at: absPath) - return request.eventLoop.makeSucceededFuture(res) + return request.fileio.streamFile(at: absPath, advancedETagComparison: advancedETagComparison) } /// Creates a new `FileMiddleware` for a server contained in an Xcode Project. diff --git a/Sources/Vapor/Utilities/FileIO.swift b/Sources/Vapor/Utilities/FileIO.swift index 73c9a6fed7..7a3820989f 100644 --- a/Sources/Vapor/Utilities/FileIO.swift +++ b/Sources/Vapor/Utilities/FileIO.swift @@ -3,6 +3,7 @@ import NIOCore import NIOHTTP1 import NIOPosix import Logging +import Crypto import NIOConcurrencyHelpers extension Request { @@ -122,6 +123,7 @@ public struct FileIO: Sendable { /// - mediaType: HTTPMediaType, if not specified, will be created from file extension. /// - onCompleted: Closure to be run on completion of stream. /// - returns: A `200 OK` response containing the file stream and appropriate headers. + @available(*, deprecated, message: "Use the new `streamFile` method which returns EventLoopFuture") @preconcurrency public func streamFile( at path: String, chunkSize: Int = NonBlockingFileIO.defaultChunkSize, @@ -157,7 +159,7 @@ public struct FileIO: Sendable { // Respond with lastModified header headers.lastModified = HTTPHeaders.LastModified(value: modifiedAt) - + // Generate ETag value, "HEX value of last modified date" + "-" + "file size" let fileETag = "\"\(modifiedAt.timeIntervalSince1970)-\(fileSize)\"" headers.replaceOrAdd(name: .eTag, value: fileETag) @@ -218,10 +220,137 @@ public struct FileIO: Sendable { onCompleted(result) } }, count: byteCount, byteBufferAllocator: request.byteBufferAllocator) - + return response } + /// Generates a chunked `Response` for the specified file. This method respects values in + /// the `"ETag"` header and is capable of responding `304 Not Modified` if the file in question + /// has not been modified since last served. If `advancedETagComparison` is set to true, + /// the response will have its ETag field set to a byte-by-byte hash of the requested file. If set to false, a simple ETag consisting of the last modified date and file size + /// will be used. This method will also set the `"Content-Type"` header + /// automatically if an appropriate `MediaType` can be found for the file's suffix. + /// + /// router.get("file-stream") { req in + /// return req.fileio.streamFile(at: "/path/to/file.txt") + /// } + /// + /// - parameters: + /// - path: Path to file on the disk. + /// - chunkSize: Maximum size for the file data chunks. + /// - mediaType: HTTPMediaType, if not specified, will be created from file extension. + /// - advancedETagComparison: The method used when ETags are generated. If true, a byte-by-byte hash is created (and cached), otherwise a simple comparison based on the file's last modified date and size. + /// - onCompleted: Closure to be run on completion of stream. + /// - returns: A `200 OK` response containing the file stream and appropriate headers. + public func streamFile( + at path: String, + chunkSize: Int = NonBlockingFileIO.defaultChunkSize, + mediaType: HTTPMediaType? = nil, + advancedETagComparison: Bool, + onCompleted: @escaping @Sendable (Result) -> () = { _ in } + ) -> EventLoopFuture { + // Get file attributes for this file. + guard + let attributes = try? FileManager.default.attributesOfItem(atPath: path), + let modifiedAt = attributes[.modificationDate] as? Date, + let fileSize = (attributes[.size] as? NSNumber)?.intValue + else { + return request.eventLoop.makeSucceededFuture(Response(status: .internalServerError)) + } + + let contentRange: HTTPHeaders.Range? + if let rangeFromHeaders = request.headers.range { + if rangeFromHeaders.unit == .bytes && rangeFromHeaders.ranges.count == 1 { + contentRange = rangeFromHeaders + } else { + contentRange = nil + } + } else if request.headers.contains(name: .range) { + // Range header was supplied but could not be parsed i.e. it was invalid + request.logger.debug("Range header was provided in request but was invalid") + let response = Response(status: .badRequest) + return request.eventLoop.makeSucceededFuture(response) + } else { + contentRange = nil + } + + var eTagFuture: EventLoopFuture + + if advancedETagComparison { + eTagFuture = generateETagHash(path: path, lastModified: modifiedAt) + } else { + // Generate ETag value, "last modified date in epoch time" + "-" + "file size" + eTagFuture = request.eventLoop.makeSucceededFuture("\"\(modifiedAt.timeIntervalSince1970)-\(fileSize)\"") + } + + return eTagFuture.map { fileETag in + // Create empty headers array. + var headers: HTTPHeaders = [:] + + // Respond with lastModified header + headers.lastModified = HTTPHeaders.LastModified(value: modifiedAt) + + headers.replaceOrAdd(name: .eTag, value: fileETag) + + // Check if file has been cached already and return NotModified response if the etags match + if fileETag == request.headers.first(name: .ifNoneMatch) { + // Per RFC 9110 here: https://www.rfc-editor.org/rfc/rfc9110.html#status.304 + // and here: https://www.rfc-editor.org/rfc/rfc9110.html#name-content-encoding + // A 304 response MUST include the ETag header and a Content-Length header matching what the original resource's content length would have been were this a 200 response. + headers.replaceOrAdd(name: .contentLength, value: fileSize.description) + return Response(status: .notModified, version: .http1_1, headersNoUpdate: headers, body: .empty) + } + + // Create the HTTP response. + let response = Response(status: .ok, headers: headers) + let offset: Int64 + let byteCount: Int + if let contentRange = contentRange { + response.status = .partialContent + response.headers.add(name: .accept, value: contentRange.unit.serialize()) + if let firstRange = contentRange.ranges.first { + do { + let range = try firstRange.asResponseContentRange(limit: fileSize) + response.headers.contentRange = HTTPHeaders.ContentRange(unit: contentRange.unit, range: range) + (offset, byteCount) = try firstRange.asByteBufferBounds(withMaxSize: fileSize, logger: request.logger) + } catch { + let response = Response(status: .badRequest) + return response + } + } else { + offset = 0 + byteCount = fileSize + } + } else { + offset = 0 + byteCount = fileSize + } + // Set Content-Type header based on the media type + // Only set Content-Type if file not modified and returned above. + if + let fileExtension = path.components(separatedBy: ".").last, + let type = mediaType ?? HTTPMediaType.fileExtension(fileExtension) + { + response.headers.contentType = type + } + response.body = .init(stream: { stream in + self.read(path: path, fromOffset: offset, byteCount: byteCount, chunkSize: chunkSize) { chunk in + return stream.write(.buffer(chunk)) + }.whenComplete { result in + switch result { + case .failure(let error): + stream.write(.error(error), promise: nil) + case .success: + stream.write(.end, promise: nil) + } + onCompleted(result) + } + }, count: byteCount, byteBufferAllocator: request.byteBufferAllocator) + + return response + } + } + /// Private read method. `onRead` closure uses ByteBuffer and expects future return. /// There may be use in publicizing this in the future for reads that must be async. private func read( @@ -279,6 +408,26 @@ public struct FileIO: Sendable { } } } + + /// Generates a fresh ETag for a file or returns its currently cached one. + /// - Parameters: + /// - path: The file's path. + /// - lastModified: When the file was last modified. + /// - Returns: An `EventLoopFuture` which holds the ETag. + private func generateETagHash(path: String, lastModified: Date) -> EventLoopFuture { + if let hash = request.application.storage[FileMiddleware.ETagHashes.self]?[path], hash.lastModified == lastModified { + return request.eventLoop.makeSucceededFuture(hash.digestHex) + } else { + return collectFile(at: path).map { buffer in + let digest = SHA256.hash(data: buffer.readableBytesView) + + // update hash in dictionary + request.application.storage[FileMiddleware.ETagHashes.self]?[path] = FileMiddleware.ETagHashes.FileHash(lastModified: lastModified, digestHex: digest.hex) + + return digest.hex + } + } + } } extension HTTPHeaders.Range.Value { diff --git a/Tests/VaporTests/FileTests.swift b/Tests/VaporTests/FileTests.swift index c089588b74..0dd9345c63 100644 --- a/Tests/VaporTests/FileTests.swift +++ b/Tests/VaporTests/FileTests.swift @@ -3,17 +3,40 @@ import XCTest import Vapor import NIOCore import NIOHTTP1 +import Crypto final class FileTests: XCTestCase { func testStreamFile() throws { let app = Application(.testing) defer { app.shutdown() } + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in + do { + try result.get() + } catch { + XCTFail("File Stream should have succeeded") + } + } + } + + try app.testable(method: .running).test(.GET, "/file-stream") { res in + let test = "the quick brown fox" + XCTAssertNotNil(res.headers.first(name: .eTag)) + XCTAssertContains(res.body.string, test) + } + } + + @available(*, deprecated) + func testLegacyStreamFile() throws { + let app = Application(.testing) + defer { app.shutdown() } + app.get("file-stream") { req in return req.fileio.streamFile(at: #filePath) { result in do { try result.get() - } catch { + } catch { XCTFail("File Stream should have succeeded") } } @@ -30,8 +53,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) } var headers = HTTPHeaders() @@ -47,13 +70,13 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req -> Response in + app.get("file-stream") { req -> EventLoopFuture in var tmpPath: String repeat { tmpPath = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString).path } while (FileManager.default.fileExists(atPath: tmpPath)) - return req.fileio.streamFile(at: tmpPath) { result in + return req.fileio.streamFile(at: tmpPath, advancedETagComparison: true) { result in do { try result.get() XCTFail("File Stream should have failed") @@ -66,13 +89,59 @@ final class FileTests: XCTestCase { XCTAssertTrue(res.body.string.isEmpty) } } + + func testAdvancedETagHeaders() throws { + let app = Application(.testing) + defer { app.shutdown() } + + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in + do { + try result.get() + } catch { + XCTFail("File Stream should have succeeded") + } + } + } + + try app.testable(method: .running).test(.GET, "/file-stream") { res in + let fileData = try Data(contentsOf: URL(fileURLWithPath: #file)) + let digest = SHA256.hash(data: fileData) + let eTag = res.headers.first(name: "etag") + XCTAssertEqual(eTag, digest.hex) + } + } + + func testSimpleETagHeaders() throws { + let app = Application(.testing) + defer { app.shutdown() } + + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: false) { result in + do { + try result.get() + } catch { + XCTFail("File Stream should have succeeded") + } + } + } + + try app.testable(method: .running).test(.GET, "/file-stream") { res in + let attributes = try FileManager.default.attributesOfItem(atPath: #file) + let modifiedAt = attributes[.modificationDate] as! Date + let fileSize = (attributes[.size] as? NSNumber)!.intValue + let fileETag = "\"\(modifiedAt.timeIntervalSince1970)-\(fileSize)\"" + + XCTAssertEqual(res.headers.first(name: .eTag), fileETag) + } + } func testStreamFileContentHeaderTail() throws { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -102,8 +171,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -111,7 +180,7 @@ final class FileTests: XCTestCase { } } } - + var headerRequest = HTTPHeaders() headerRequest.range = .init(unit: .bytes, ranges: [.start(value: 20)]) try app.testable(method: .running(port: 0)).test(.GET, "/file-stream", headers: headerRequest) { res in @@ -133,8 +202,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -165,7 +234,7 @@ final class FileTests: XCTestCase { defer { app.shutdown() } app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -191,8 +260,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -217,8 +286,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -243,8 +312,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) { result in + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) { result in do { try result.get() } catch { @@ -423,8 +492,8 @@ final class FileTests: XCTestCase { let app = Application(.testing) defer { app.shutdown() } - app.get("file-stream") { req in - return req.fileio.streamFile(at: #filePath) + app.get("file-stream") { req -> EventLoopFuture in + return req.fileio.streamFile(at: #file, advancedETagComparison: true) } var headers = HTTPHeaders() diff --git a/Tests/VaporTests/ServerTests.swift b/Tests/VaporTests/ServerTests.swift index d4173031c5..bcf7a0a7d0 100644 --- a/Tests/VaporTests/ServerTests.swift +++ b/Tests/VaporTests/ServerTests.swift @@ -264,7 +264,7 @@ final class ServerTests: XCTestCase { struct Nothing: Codable {} XCTAssertNoThrow(try JSONDecoder().decode(Nothing.self, from: body)) } else { - XCTFail() + XCTFail("Missing response.body") } } @@ -277,9 +277,9 @@ final class ServerTests: XCTestCase { let smallBody = ByteBuffer(base64String: "H4sIAAAAAAAAE/NIzcnJ11Eozy/KSVEEAObG5usNAAAA")! // "Hello, world!" let bigBody = ByteBuffer(base64String: "H4sIAAAAAAAAE/NIzcnJ11HILU3OgBBJmenpqUUK5flFOSkKJRmJeQpJqWn5RamKAICcGhUqAAAA")! // "Hello, much much bigger world than before!" - // Max out at the smaller payload (.size is of compressed data) + // Max out at the smaller payload (.size is of uncompressed data) app.http.server.configuration.requestDecompression = .enabled( - limit: .size(smallBody.readableBytes) + limit: .size(smallOrigString.utf8.count) ) app.post("gzip") { $0.body.string ?? "" } @@ -314,6 +314,364 @@ final class ServerTests: XCTestCase { } } + func testHTTP1RequestDecompression() async throws { + let compressiblePayload = #"{"compressed": ["key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value"]}"# + /// To regenerate, copy the above and run `% pbpaste | gzip | base64`. To verify, run `% pbpaste | base64 -d | gzip -d` instead. + let compressedPayload = ByteBuffer(base64String: "H4sIANRAImYAA6tWSs7PLShKLS5OTVGyUohWyk6tBNJKZYk5palKOgqj/FH+KH+UP8of5RPmx9YCAMfjVAhQBgAA")! + + let app = Application(.testing) + defer { app.shutdown() } + + app.http.server.configuration.hostname = "127.0.0.1" + app.http.server.configuration.port = 0 + + app.http.server.configuration.supportVersions = [.one] + app.http.server.configuration.requestDecompression = .disabled + + /// Make sure the client doesn't keep the server open by re-using the connection. + app.http.client.configuration.maximumUsesPerConnection = 1 + + struct TestResponse: Content { + var content: ByteBuffer? + var contentLength: Int + } + + app.on(.POST, "compressed", body: .collect(maxSize: "1mb")) { request async throws in + let contentLength = request.headers.first(name: .contentLength).flatMap { Int($0) } + let contents = try await request.body.collect().get() + return TestResponse( + content: contents, + contentLength: contentLength ?? 0 + ) + } + + try app.server.start() + defer { app.server.shutdown() } + + XCTAssertNotNil(app.http.server.shared.localAddress) + guard let localAddress = app.http.server.shared.localAddress, + let port = localAddress.port else { + XCTFail("couldn't get ip/port from \(app.http.server.shared.localAddress.debugDescription)") + return + } + + let unsupportedNoncompressedResponse = try await app.client.post("http://localhost:\(port)/compressed") { request in + request.body = compressedPayload + } + + if let body = unsupportedNoncompressedResponse.body { + let decodedResponse = try JSONDecoder().decode(TestResponse.self, from: body) + XCTAssertEqual(decodedResponse.content, compressedPayload) + XCTAssertEqual(decodedResponse.contentLength, compressedPayload.readableBytes) + } else { + XCTFail("Missing unsupportedNoncompressedResponse.body") + } + + // TODO: The server should probably reject this? + let unsupportedCompressedResponse = try await app.client.post("http://localhost:\(port)/compressed") { request in + request.headers.replaceOrAdd(name: .contentEncoding, value: "gzip") + request.body = compressedPayload + } + + if let body = unsupportedCompressedResponse.body { + let decodedResponse = try JSONDecoder().decode(TestResponse.self, from: body) + XCTAssertEqual(decodedResponse.content, compressedPayload) + XCTAssertEqual(decodedResponse.contentLength, compressedPayload.readableBytes) + } else { + XCTFail("Missing unsupportedCompressedResponse.body") + } + + app.http.server.configuration.requestDecompression = .enabled(limit: .size(compressiblePayload.utf8.count)) + + let supportedUncompressedResponse = try await app.client.post("http://localhost:\(port)/compressed") { request in + request.body = compressedPayload + } + + if let body = supportedUncompressedResponse.body { + let decodedResponse = try JSONDecoder().decode(TestResponse.self, from: body) + XCTAssertEqual(decodedResponse.content, compressedPayload) + XCTAssertEqual(decodedResponse.contentLength, compressedPayload.readableBytes) + } else { + XCTFail("Missing supportedUncompressedResponse.body") + } + + let supportedCompressedResponse = try await app.client.post("http://localhost:\(port)/compressed") { request in + request.headers.replaceOrAdd(name: .contentEncoding, value: "gzip") + request.body = compressedPayload + } + + if let body = supportedCompressedResponse.body { + let decodedResponse = try JSONDecoder().decode(TestResponse.self, from: body) + XCTAssertEqual(decodedResponse.content, ByteBuffer(string: compressiblePayload)) + XCTAssertEqual(decodedResponse.contentLength, compressedPayload.readableBytes) + } else { + XCTFail("Missing supportedCompressedResponse.body") + } + } + + func testHTTP2RequestDecompression() async throws { + guard let clientCertPath = Bundle.module.url(forResource: "expired", withExtension: "crt"), + let clientKeyPath = Bundle.module.url(forResource: "expired", withExtension: "key") else { + XCTFail("Cannot load expired cert and associated key") + return + } + + let cert = try NIOSSLCertificate(file: clientCertPath.path, format: .pem) + let key = try NIOSSLPrivateKey(file: clientKeyPath.path, format: .pem) + + let compressiblePayload = #"{"compressed": ["key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value"]}"# + /// To regenerate, copy the above and run `% pbpaste | gzip | base64`. To verify, run `% pbpaste | base64 -d | gzip -d` instead. + let compressedPayload = ByteBuffer(base64String: "H4sIANRAImYAA6tWSs7PLShKLS5OTVGyUohWyk6tBNJKZYk5palKOgqj/FH+KH+UP8of5RPmx9YCAMfjVAhQBgAA")! + + let app = Application(.testing) + defer { app.shutdown() } + + app.http.server.configuration.hostname = "127.0.0.1" + app.http.server.configuration.port = 0 + + var serverConfig = TLSConfiguration.makeServerConfiguration(certificateChain: [.certificate(cert)], privateKey: .privateKey(key)) + serverConfig.certificateVerification = .noHostnameVerification + + app.http.server.configuration.tlsConfiguration = serverConfig + app.http.server.configuration.customCertificateVerifyCallback = { peerCerts, successPromise in + /// This lies and accepts the above cert, which has actually expired. + XCTAssertEqual(peerCerts, [cert]) + successPromise.succeed(.certificateVerified) + } + app.http.server.configuration.supportVersions = [.two] + app.http.server.configuration.requestDecompression = .disabled + + /// We need to disable verification on the client, because the cert we're using has expired + var clientConfig = TLSConfiguration.makeClientConfiguration() + clientConfig.certificateVerification = .none + clientConfig.certificateChain = [.certificate(cert)] + clientConfig.privateKey = .privateKey(key) + app.http.client.configuration.tlsConfiguration = clientConfig + + /// Make sure the client doesn't keep the server open by re-using the connection. + app.http.client.configuration.maximumUsesPerConnection = 1 + + struct TestResponse: Content { + var content: ByteBuffer? + var contentLength: Int + } + + app.post("compressed") { request async throws in + let contentLength = request.headers.first(name: .contentLength) + let contents = try await request.body.collect().get() + return TestResponse( + content: contents, + contentLength: contentLength.flatMap { Int($0) } ?? 0 + ) + } + + try app.server.start() + defer { app.server.shutdown() } + + XCTAssertNotNil(app.http.server.shared.localAddress) + guard let localAddress = app.http.server.shared.localAddress, + let port = localAddress.port else { + XCTFail("couldn't get ip/port from \(app.http.server.shared.localAddress.debugDescription)") + return + } + + let unsupportedNoncompressedResponse = try await app.client.post("https://localhost:\(port)/compressed") { request in + request.body = compressedPayload + } + + if let body = unsupportedNoncompressedResponse.body { + let decodedResponse = try JSONDecoder().decode(TestResponse.self, from: body) + XCTAssertEqual(decodedResponse.content, compressedPayload) + XCTAssertEqual(decodedResponse.contentLength, compressedPayload.readableBytes) + } else { + XCTFail("Missing unsupportedNoncompressedResponse.body") + } + + // TODO: The server should probably reject this? + let unsupportedCompressedResponse = try await app.client.post("https://localhost:\(port)/compressed") { request in + request.headers.replaceOrAdd(name: .contentEncoding, value: "gzip") + request.body = compressedPayload + } + + if let body = unsupportedCompressedResponse.body { + let decodedResponse = try JSONDecoder().decode(TestResponse.self, from: body) + XCTAssertEqual(decodedResponse.content, compressedPayload) + XCTAssertEqual(decodedResponse.contentLength, compressedPayload.readableBytes) + } else { + XCTFail("Missing unsupportedCompressedResponse.body") + } + + app.http.server.configuration.requestDecompression = .enabled(limit: .size(compressiblePayload.utf8.count)) + + let supportedUncompressedResponse = try await app.client.post("https://localhost:\(port)/compressed") { request in + request.body = compressedPayload + } + + if let body = supportedUncompressedResponse.body { + let decodedResponse = try JSONDecoder().decode(TestResponse.self, from: body) + XCTAssertEqual(decodedResponse.content, compressedPayload) + XCTAssertEqual(decodedResponse.contentLength, compressedPayload.readableBytes) + } else { + XCTFail("Missing supportedUncompressedResponse.body") + } + + let supportedCompressedResponse = try await app.client.post("https://localhost:\(port)/compressed") { request in + request.headers.replaceOrAdd(name: .contentEncoding, value: "gzip") + request.body = compressedPayload + } + + if let body = supportedCompressedResponse.body { + let decodedResponse = try JSONDecoder().decode(TestResponse.self, from: body) + XCTAssertEqual(decodedResponse.content, ByteBuffer(string: compressiblePayload)) + XCTAssertEqual(decodedResponse.contentLength, compressedPayload.readableBytes) + } else { + XCTFail("Missing supportedCompressedResponse.body") + } + } + + func testHTTP1ResponseDecompression() async throws { + let compressiblePayload = #"{"compressed": ["key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value"]}"# + + let app = Application(.testing) + defer { app.shutdown() } + + app.http.server.configuration.hostname = "127.0.0.1" + app.http.server.configuration.port = 0 + + app.http.server.configuration.supportVersions = [.one] + app.http.server.configuration.responseCompression = .disabled + + /// Make sure the client doesn't keep the server open by re-using the connection. + app.http.client.configuration.maximumUsesPerConnection = 1 + app.http.client.configuration.decompression = .enabled(limit: .none) + + app.get("compressed") { _ in compressiblePayload } + + try app.server.start() + defer { app.server.shutdown() } + + XCTAssertNotNil(app.http.server.shared.localAddress) + guard let localAddress = app.http.server.shared.localAddress, + let port = localAddress.port else { + XCTFail("couldn't get ip/port from \(app.http.server.shared.localAddress.debugDescription)") + return + } + + let unsupportedNoncompressedResponse = try await app.client.get("http://localhost:\(port)/compressed") { request in + request.headers.remove(name: .acceptEncoding) + } + XCTAssertNotEqual(unsupportedNoncompressedResponse.headers.first(name: .contentEncoding), "gzip") + XCTAssertEqual(unsupportedNoncompressedResponse.headers.first(name: .contentLength), "\(compressiblePayload.count)") + XCTAssertEqual(unsupportedNoncompressedResponse.body?.string, compressiblePayload) + + let unsupportedCompressedResponse = try await app.client.get("http://localhost:\(port)/compressed") { request in + request.headers.replaceOrAdd(name: .acceptEncoding, value: "gzip") + } + XCTAssertNotEqual(unsupportedCompressedResponse.headers.first(name: .contentEncoding), "gzip") + XCTAssertEqual(unsupportedCompressedResponse.headers.first(name: .contentLength), "\(compressiblePayload.count)") + XCTAssertEqual(unsupportedCompressedResponse.body?.string, compressiblePayload) + + app.http.server.configuration.responseCompression = .enabled + + let supportedUncompressedResponse = try await app.client.get("http://localhost:\(port)/compressed") { request in + request.headers.remove(name: .acceptEncoding) + } + XCTAssertNotEqual(supportedUncompressedResponse.headers.first(name: .contentEncoding), "gzip") + XCTAssertNotEqual(supportedUncompressedResponse.headers.first(name: .contentLength), "\(compressiblePayload.count)") + XCTAssertEqual(supportedUncompressedResponse.body?.string, compressiblePayload) + + let supportedCompressedResponse = try await app.client.get("http://localhost:\(port)/compressed") { request in + request.headers.replaceOrAdd(name: .acceptEncoding, value: "gzip") + } + XCTAssertEqual(supportedCompressedResponse.headers.first(name: .contentEncoding), "gzip") + XCTAssertNotEqual(supportedCompressedResponse.headers.first(name: .contentLength), "\(compressiblePayload.count)") + XCTAssertEqual(supportedCompressedResponse.body?.string, compressiblePayload) + } + + func testHTTP2ResponseDecompression() async throws { + guard let clientCertPath = Bundle.module.url(forResource: "expired", withExtension: "crt"), + let clientKeyPath = Bundle.module.url(forResource: "expired", withExtension: "key") else { + XCTFail("Cannot load expired cert and associated key") + return + } + + let cert = try NIOSSLCertificate(file: clientCertPath.path, format: .pem) + let key = try NIOSSLPrivateKey(file: clientKeyPath.path, format: .pem) + + let compressiblePayload = #"{"compressed": ["key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value", "key": "value"]}"# + + let app = Application(.testing) + defer { app.shutdown() } + + app.http.server.configuration.hostname = "127.0.0.1" + app.http.server.configuration.port = 0 + + var serverConfig = TLSConfiguration.makeServerConfiguration(certificateChain: [.certificate(cert)], privateKey: .privateKey(key)) + serverConfig.certificateVerification = .noHostnameVerification + + app.http.server.configuration.tlsConfiguration = serverConfig + app.http.server.configuration.customCertificateVerifyCallback = { peerCerts, successPromise in + /// This lies and accepts the above cert, which has actually expired. + XCTAssertEqual(peerCerts, [cert]) + successPromise.succeed(.certificateVerified) + } + app.http.server.configuration.supportVersions = [.two] + app.http.server.configuration.responseCompression = .disabled + + /// We need to disable verification on the client, because the cert we're using has expired + var clientConfig = TLSConfiguration.makeClientConfiguration() + clientConfig.certificateVerification = .none + clientConfig.certificateChain = [.certificate(cert)] + clientConfig.privateKey = .privateKey(key) + app.http.client.configuration.tlsConfiguration = clientConfig + + app.http.client.configuration.decompression = .enabled(limit: .none) + /// Make sure the client doesn't keep the server open by re-using the connection. + app.http.client.configuration.maximumUsesPerConnection = 1 + + app.get("compressed") { _ in compressiblePayload } + + try app.server.start() + defer { app.server.shutdown() } + + XCTAssertNotNil(app.http.server.shared.localAddress) + guard let localAddress = app.http.server.shared.localAddress, + let port = localAddress.port else { + XCTFail("couldn't get ip/port from \(app.http.server.shared.localAddress.debugDescription)") + return + } + + let unsupportedNoncompressedResponse = try await app.client.get("https://localhost:\(port)/compressed") { request in + request.headers.remove(name: .acceptEncoding) + } + XCTAssertNotEqual(unsupportedNoncompressedResponse.headers.first(name: .contentEncoding), "gzip") + XCTAssertEqual(unsupportedNoncompressedResponse.headers.first(name: .contentLength), "\(compressiblePayload.count)") + XCTAssertEqual(unsupportedNoncompressedResponse.body?.string, compressiblePayload) + + let unsupportedCompressedResponse = try await app.client.get("https://localhost:\(port)/compressed") { request in + request.headers.replaceOrAdd(name: .acceptEncoding, value: "gzip") + } + XCTAssertNotEqual(unsupportedCompressedResponse.headers.first(name: .contentEncoding), "gzip") + XCTAssertEqual(unsupportedCompressedResponse.headers.first(name: .contentLength), "\(compressiblePayload.count)") + XCTAssertEqual(unsupportedCompressedResponse.body?.string, compressiblePayload) + + app.http.server.configuration.responseCompression = .enabled + + let supportedUncompressedResponse = try await app.client.get("https://localhost:\(port)/compressed") { request in + request.headers.remove(name: .acceptEncoding) + } + XCTAssertNotEqual(supportedUncompressedResponse.headers.first(name: .contentEncoding), "gzip") + XCTAssertNotEqual(supportedUncompressedResponse.headers.first(name: .contentLength), "\(compressiblePayload.count)") + XCTAssertEqual(supportedUncompressedResponse.body?.string, compressiblePayload) + + let supportedCompressedResponse = try await app.client.get("https://localhost:\(port)/compressed") { request in + request.headers.replaceOrAdd(name: .acceptEncoding, value: "gzip") + } + XCTAssertEqual(supportedCompressedResponse.headers.first(name: .contentEncoding), "gzip") + XCTAssertNotEqual(supportedCompressedResponse.headers.first(name: .contentLength), "\(compressiblePayload.count)") + XCTAssertEqual(supportedCompressedResponse.body?.string, compressiblePayload) + } + func testRequestBodyStreamGetsFinalisedEvenIfClientAbandonsConnection() throws { let app = Application(.testing) app.http.server.configuration.hostname = "127.0.0.1"