From 05c49875c0fef92a00ce33f99216728ee9b255ba Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Tue, 23 Jan 2018 16:28:30 -0500 Subject: [PATCH 01/16] chttp streaming body refactor --- Sources/HTTP/Client/HTTPClient+TCP.swift | 23 +- Sources/HTTP/Client/HTTPClient.swift | 97 ++--- Sources/HTTP/Parser/BufferEmitterStream.swift | 75 ---- Sources/HTTP/Parser/CHTTPBodyStream.swift | 41 ++ Sources/HTTP/Parser/CHTTPParseResults.swift | 41 +- Sources/HTTP/Parser/CHTTPParser.swift | 292 ++++++++++---- Sources/HTTP/Parser/HTTPParser.swift | 2 +- Sources/HTTP/Parser/HTTPRequestParser.swift | 30 +- Sources/HTTP/Parser/HTTPResponseParser.swift | 117 +++--- Sources/HTTP/Server/HTTPServer.swift | 2 +- Tests/HTTPTests/HTTPClientTests.swift | 95 ++--- Tests/HTTPTests/ParserTests.swift | 368 ++++++++++-------- 12 files changed, 661 insertions(+), 522 deletions(-) delete mode 100644 Sources/HTTP/Parser/BufferEmitterStream.swift create mode 100644 Sources/HTTP/Parser/CHTTPBodyStream.swift diff --git a/Sources/HTTP/Client/HTTPClient+TCP.swift b/Sources/HTTP/Client/HTTPClient+TCP.swift index 1f6ebcb9..0947b827 100644 --- a/Sources/HTTP/Client/HTTPClient+TCP.swift +++ b/Sources/HTTP/Client/HTTPClient+TCP.swift @@ -1,12 +1,13 @@ -import Async -import TCP +//import Async +//import TCP +// +//extension HTTPClient { +// /// Create a TCP-based HTTP client. See `TCPClient`. +// public static func tcp(hostname: String, port: UInt16, on worker: Worker) throws -> HTTPClient { +// let tcpSocket = try TCPSocket(isNonBlocking: true) +// let tcpClient = try TCPClient(socket: tcpSocket) +// try tcpClient.connect(hostname: "httpbin.org", port: 80) +// return HTTPClient(stream: tcpSocket.stream(on: worker), on: worker) +// } +//} -extension HTTPClient { - /// Create a TCP-based HTTP client. See `TCPClient`. - public static func tcp(hostname: String, port: UInt16, on worker: Worker) throws -> HTTPClient { - let tcpSocket = try TCPSocket(isNonBlocking: true) - let tcpClient = try TCPClient(socket: tcpSocket) - try tcpClient.connect(hostname: "httpbin.org", port: 80) - return HTTPClient(stream: tcpSocket.stream(on: worker), on: worker) - } -} diff --git a/Sources/HTTP/Client/HTTPClient.swift b/Sources/HTTP/Client/HTTPClient.swift index 091e15f3..0314ea5c 100644 --- a/Sources/HTTP/Client/HTTPClient.swift +++ b/Sources/HTTP/Client/HTTPClient.swift @@ -1,49 +1,50 @@ -import Async -import Bits +//import Async +//import Bits +// +///// An HTTP client wrapped around TCP client +///// +///// Can handle a single `Request` at a given time. +///// +///// Multiple requests at the same time are subject to unknown behaviour +///// +///// [Learn More →](https://docs.vapor.codes/3.0/http/client/) +//public final class HTTPClient { +// /// Inverse stream, takes in responses and outputs requests +// private let queueStream: QueueStream +// +// /// Store the response map here, so it can capture +// /// the sink and source variables. +// private let responseMap: (HTTPResponse) throws -> HTTPResponse +// +// /// Creates a new Client wrapped around a `TCP.Client` +// public init(stream: Stream, on worker: Worker, maxResponseSize: Int = 10_000_000) +// where Stream: ByteStream +// { +// let queueStream = QueueStream() +// +// let serializerStream = HTTPRequestSerializer().stream(on: worker) +// let parserStream = HTTPResponseParser() +// +// stream.stream(to: parserStream) +// .stream(to: queueStream) +// .stream(to: serializerStream) +// .output(to: stream) +// +// self.responseMap = { res in +// if let onUpgrade = res.onUpgrade { +// try onUpgrade.closure(.init(stream), .init(stream), worker) +// } +// return res +// } +// +// self.queueStream = queueStream +// } +// +// /// Sends an HTTP request. +// public func send(_ request: HTTPRequest) -> Future { +// return queueStream.enqueue(request).map(to: HTTPResponse.self) { res in +// return try self.responseMap(res) +// } +// } +//} -/// An HTTP client wrapped around TCP client -/// -/// Can handle a single `Request` at a given time. -/// -/// Multiple requests at the same time are subject to unknown behaviour -/// -/// [Learn More →](https://docs.vapor.codes/3.0/http/client/) -public final class HTTPClient { - /// Inverse stream, takes in responses and outputs requests - private let queueStream: QueueStream - - /// Store the response map here, so it can capture - /// the sink and source variables. - private let responseMap: (HTTPResponse) throws -> HTTPResponse - - /// Creates a new Client wrapped around a `TCP.Client` - public init(stream: Stream, on worker: Worker, maxResponseSize: Int = 10_000_000) - where Stream: ByteStream - { - let queueStream = QueueStream() - - let serializerStream = HTTPRequestSerializer().stream(on: worker) - let parserStream = HTTPResponseParser().stream(on: worker) - - stream.stream(to: parserStream) - .stream(to: queueStream) - .stream(to: serializerStream) - .output(to: stream) - - self.responseMap = { res in - if let onUpgrade = res.onUpgrade { - try onUpgrade.closure(.init(stream), .init(stream), worker) - } - return res - } - - self.queueStream = queueStream - } - - /// Sends an HTTP request. - public func send(_ request: HTTPRequest) -> Future { - return queueStream.enqueue(request).map(to: HTTPResponse.self) { res in - return try self.responseMap(res) - } - } -} diff --git a/Sources/HTTP/Parser/BufferEmitterStream.swift b/Sources/HTTP/Parser/BufferEmitterStream.swift deleted file mode 100644 index e0e13aae..00000000 --- a/Sources/HTTP/Parser/BufferEmitterStream.swift +++ /dev/null @@ -1,75 +0,0 @@ -/// use push stream? -//import Async -//import Bits -//import Foundation -// -//internal final class ByteBufferPushStream: Async.OutputStream { -// typealias Output = ByteBuffer -// -// var downstreamDemand: UInt -// var downstream: AnyInputStream? -// var backlog = [Data]() -// var writing: Data? -// var flushedBacklog: Int -// var closed: Bool -// -// init() { -// downstreamDemand = 0 -// flushedBacklog = 0 -// closed = false -// } -// -// func eof() { -// if backlog.count == 0 { -// downstream?.close() -// } -// -// self.closed = true -// } -// -// func output(to inputStream: S) where S : Async.InputStream, Output == S.Input { -// self.downstream = AnyInputStream(inputStream) -// } -// -// func push(_ buffer: ByteBuffer) { -// flushBacklog() -// -// if downstreamDemand > 0 { -// downstreamDemand -= 1 -// downstream?.next(buffer) { -// // ignore, fixme -// } -// } else { -// backlog.append(Data(buffer: buffer)) -// } -// -// if closed && backlog.count == 0 { -// downstream?.close() -// } -// } -// -// fileprivate func flushBacklog() { -// defer { -// backlog.removeFirst(flushedBacklog) -// flushedBacklog = 0 -// -// if closed && backlog.count == 0 { -// downstream?.close() -// } -// } -// -// while backlog.count - flushedBacklog > 0, downstreamDemand > 0 { -// downstreamDemand -= 1 -// let data = backlog[flushedBacklog] -// flushedBacklog += 1 -// self.writing = data -// -// data.withByteBuffer { buffer in -// self.downstream?.next(buffer) { -// // ignore, fixme -// } -// } -// } -// } -//} - diff --git a/Sources/HTTP/Parser/CHTTPBodyStream.swift b/Sources/HTTP/Parser/CHTTPBodyStream.swift new file mode 100644 index 00000000..f4e0f704 --- /dev/null +++ b/Sources/HTTP/Parser/CHTTPBodyStream.swift @@ -0,0 +1,41 @@ +import Async +import Bits + +/// Streams ByteBuffers parsed by CHTTP during `on_message` callbacks. +final class CHTTPBodyStream: OutputStream { + /// See `OutputStream.Output` + typealias Output = ByteBuffer + + /// Creates a new `CHTTPBodyStream` + init() {} + + /// Current downstream accepting the body's byte buffers. + var downstream: AnyInputStream? + + /// Waiting output + var waiting: (ByteBuffer, Promise)? + + /// Pushes a new ByteBuffer with associated ready. + func push(_ buffer: ByteBuffer, _ ready: Promise) { + assert(waiting == nil) + if let downstream = self.downstream { + downstream.next(buffer, ready) + } else { + waiting = (buffer, ready) + } + } + + /// See `OutputStream.output(to:)` + func output(to inputStream: S) where S : InputStream, CHTTPBodyStream.Output == S.Input { + downstream = .init(inputStream) + if let (buffer, ready) = self.waiting { + self.waiting = nil + inputStream.next(buffer, ready) + } + } + + /// Closes the stream. + func close() { + downstream!.close() + } +} diff --git a/Sources/HTTP/Parser/CHTTPParseResults.swift b/Sources/HTTP/Parser/CHTTPParseResults.swift index 87832dcf..9301d437 100644 --- a/Sources/HTTP/Parser/CHTTPParseResults.swift +++ b/Sources/HTTP/Parser/CHTTPParseResults.swift @@ -14,38 +14,43 @@ import Foundation /// See the convenience methods below to see how the /// object is set and fetched from the C object. internal final class CParseResults { - // state - var headerState: HeaderState - var isComplete: Bool + /// If true, all of the headers have been sent. + var headersComplete: Bool - // message components + /// If true, the entire message has been parsed. + var messageComplete: Bool + + /// The current header parsing state (field, value, etc) + var headerState: CHTTPHeaderState + + /// The current body parsing state + var bodyState: CHTTPBodyState + + /// The HTTP method (only set for requests) + var method: http_method? + + // The HTTP version var version: HTTPVersion? + var headersIndexes: [HTTPHeaders.Index] var headersData = [UInt8]() - var currentSize: Int = 0 var maxHeaderSize: Int? - var contentLength: Int? - var headers: HTTPHeaders? - - var body: HTTPBody? - var bodyStream: PushStream - var url = [UInt8]() /// Creates a new results object - init(parser: Parser) { - self.isComplete = false + init() { + self.headersComplete = false + self.messageComplete = false self.headersIndexes = [] headersData.reserveCapacity(4096) headersIndexes.reserveCapacity(64) url.reserveCapacity(128) - self.maxHeaderSize = parser.maxHeaderSize - self.bodyStream = .init() - + self.maxHeaderSize = 100_000 self.headerState = .none + self.bodyState = .none } func addSize(_ n: Int) -> Bool { @@ -65,9 +70,9 @@ internal final class CParseResults { extension CParseResults { /// Sets the parse results object on a C parser - static func set(on parser: inout http_parser, swiftParser: Parser) -> CParseResults { + static func set(on parser: inout http_parser) -> CParseResults { let results = UnsafeMutablePointer.allocate(capacity: 1) - let new = CParseResults(parser: swiftParser) + let new = CParseResults() results.initialize(to: new) parser.data = UnsafeMutableRawPointer(results) return new diff --git a/Sources/HTTP/Parser/CHTTPParser.swift b/Sources/HTTP/Parser/CHTTPParser.swift index 061a369f..c1ad432e 100644 --- a/Sources/HTTP/Parser/CHTTPParser.swift +++ b/Sources/HTTP/Parser/CHTTPParser.swift @@ -4,79 +4,193 @@ import CHTTP import Dispatch import Foundation +/// Internal CHTTP parser protocol +internal protocol CHTTPParser: HTTPParser where Input == ByteBuffer { + /// This parser's type (request or response) + static var parserType: http_parser_type { get } + + /// If set, header data exceeding the specified size will result in an error. + var maxHeaderSize: Int? { get set } + + /// Holds the CHTTP parser's internal state. + var chttp: CHTTPParserContext { get set } + + /// Converts the CHTTP parser results and body to HTTP message. + func makeMessage(from results: CParseResults, using body: HTTPBody) throws -> Output +} + /// Possible header states -enum HeaderState { +enum CHTTPHeaderState { case none case value(HTTPHeaders.Index) case key(startIndex: Int, endIndex: Int) } +enum CHTTPMessageState { + case parsing + case streaming(Message, Future) + case waiting(Future) +} -/// Internal CHTTP parser protocol -internal protocol CHTTPParser: class, HTTPParser where Input == ByteBuffer { - static var parserType: http_parser_type { get } - var parser: http_parser { get set } - var settings: http_parser_settings { get set } - var maxHeaderSize: Int? { get set } - - var httpState: CHTTPParserState { get set } - func makeMessage(from results: CParseResults) throws -> Output +/// Possible body states +enum CHTTPBodyState { + case none + case buffer(ByteBuffer) + case stream(CHTTPBodyStream) + case readyStream(CHTTPBodyStream, Promise) } -enum CHTTPParserState { - case ready - case parsing +/// Maintains the CHTTP parser's internal state. +struct CHTTPParserContext { + /// Whether the parser is currently parsing or hasn't started yet + var isParsing: Bool + + /// Parser's message + var messageState: CHTTPMessageState + + /// The CHTTP parser's C struct + var parser: http_parser + + /// The CHTTP parer's C settings + var settings: http_parser_settings + + /// Current downstream. + var downstream: AnyInputStream? + + /// Creates a new `CHTTPParserContext` + init() { + self.parser = http_parser() + self.settings = http_parser_settings() + self.isParsing = false + self.messageState = .parsing + } +} + +/// MARK: CHTTPParser OutputStream + +extension CHTTPParser { + /// See `OutputStream.output(to:)` + public func output(to inputStream: S) where S: Async.InputStream, Self.Output == S.Input { + chttp.downstream = .init(inputStream) + } } -/// MARK: HTTPParser conformance +/// MARK: CHTTPParser InputStream extension CHTTPParser { - public func translate(input: inout TranslatingStreamInput) throws -> TranslatingStreamOutput { - guard let results = getResults() else { - throw HTTPError(identifier: "no-parser-results", reason: "An internal HTTP Parser state became invalid") + /// See `InputStream.input(_:)` + public func input(_ event: InputEvent) { + switch event { + case .close: chttp.downstream!.close() + case .error(let error): chttp.downstream!.error(error) + case .next(let input, let ready): try! handleNext(input, ready) } - - if let buffer = input.input { - /// parse the message using the C HTTP parser. - try executeParser(from: buffer) - - guard results.isComplete else { - return .insufficient() + } + + /// See `InputEvent.next` + private func handleNext(_ buffer: ByteBuffer, _ ready: Promise) throws { + guard let results = chttp.getResults() else { + throw HTTPError(identifier: "getResults", reason: "An internal HTTP Parser state became invalid") + } + + switch chttp.messageState { + case .parsing: + /// Parse the message using the CHTTP parser. + try chttp.execute(from: buffer) + + /// Check if we have received all of the messages headers + if results.headersComplete { + /// Either streaming or static will be decided + let body: HTTPBody + + /// The message is ready to move downstream, check to see + /// if we already have the HTTPBody in its entirety + if results.messageComplete { + switch results.bodyState { + case .buffer(let buffer): body = HTTPBody(Data(buffer)) + case .none: body = HTTPBody() + case .stream: fatalError("Illegal state") + case .readyStream: fatalError("Illegal state") + } + + let message = try makeMessage(from: results, using: body) + chttp.downstream!.next(message, ready) + + // the results have completed, so we are ready + // for a new request to come in + chttp.isParsing = false + CParseResults.remove(from: &chttp.parser) + } else { + // Convert body to a stream + let stream = CHTTPBodyStream() // FIX, this shouldn't backlog + switch results.bodyState { + case .buffer(let buffer): stream.push(buffer, ready) + case .none: break // push nothing + case .stream: fatalError("Illegal state") + case .readyStream: fatalError("Illegal state") + } + results.bodyState = .stream(stream) + body = HTTPBody(size: results.contentLength, stream: .init(stream)) + let message = try makeMessage(from: results, using: body) + let future = chttp.downstream!.next(message) + chttp.messageState = .streaming(message, future) + } + } else { + /// Headers not complete, request more input + ready.complete() } - + case .streaming(_, let future): + let stream: CHTTPBodyStream + + /// Close the body stream now + switch results.bodyState { + case .none: fatalError("Illegal state") + case .buffer: fatalError("Illegal state") + case .readyStream: fatalError("Illegal state") + case .stream(let s): + stream = s + // replace body state w/ new ready + results.bodyState = .readyStream(s, ready) + } + + /// Parse the message using the CHTTP parser. + try chttp.execute(from: buffer) + + if results.messageComplete { + /// Close the body stream now + stream.close() + chttp.messageState = .waiting(future) + } + case .waiting(let future): // the results have completed, so we are ready // for a new request to come in - httpState = .ready - CParseResults.remove(from: &parser) - - let message = try makeMessage(from: results) - return .sufficient(message) - } else { - input.close() - // EOF - http_parser_execute(&parser, &settings, nil, 0) - - guard results.isComplete else { - return .insufficient() + chttp.isParsing = false + CParseResults.remove(from: &chttp.parser) + chttp.messageState = .parsing + future.do { + try! self.handleNext(buffer, ready) + }.catch { error in + fatalError("\(error)") } - - let message = try makeMessage(from: results) - return .sufficient(message) } + + + // // EOF + // http_parser_execute(&parser, &settings, nil, 0) } /// Resets the parser public func reset() { - reset(Self.parserType) + chttp.reset(Self.parserType) } } /// MARK: CHTTP integration -extension CHTTPParser { +extension CHTTPParserContext { /// Parses a generic CHTTP message, filling the /// ParseResults object attached to the C praser. - internal func executeParser(from buffer: ByteBuffer) throws { + internal mutating func execute(from buffer: ByteBuffer) throws { // call the CHTTP parser let parsedCount = http_parser_execute(&parser, &settings, buffer.cPointer, buffer.count) @@ -88,9 +202,10 @@ extension CHTTPParser { } } - internal func reset(_ type: http_parser_type) { + /// Resets the parser + internal mutating func reset(_ type: http_parser_type) { http_parser_init(&parser, type) - initialize(&settings) + initialize() } } @@ -113,32 +228,31 @@ extension CParseResults { } } -extension CHTTPParser { - func getResults() -> CParseResults? { +extension CHTTPParserContext { + /// Fetches `CParseResults` from the praser. + mutating func getResults() -> CParseResults? { let results: CParseResults - - switch httpState { - case .ready: - // create a new results object and set - // a reference to it on the parser - let newResults = CParseResults.set(on: &parser, swiftParser: self) - results = newResults - httpState = .parsing - case .parsing: + if isParsing { // get the current parse results object guard let existingResults = CParseResults.get(from: &parser) else { return nil } results = existingResults + } else { + // create a new results object and set + // a reference to it on the parser + let newResults = CParseResults.set(on: &parser) + results = newResults + isParsing = true } - return results } /// Initializes the http parser settings with appropriate callbacks. - func initialize(_ settings: inout http_parser_settings) { + mutating func initialize() { // called when chunks of the url have been read settings.on_url = { parser, chunkPointer, length in + print("chttp: on_url '\(String(data: Data(chunkPointer!.makeBuffer(length: length)), encoding: .ascii)!)' (\(length)) ") guard let results = CParseResults.get(from: parser), let chunkPointer = chunkPointer @@ -161,6 +275,7 @@ extension CHTTPParser { // called when chunks of a header field have been read settings.on_header_field = { parser, chunkPointer, length in + print("chttp: on_header_field '\(String(data: Data(chunkPointer!.makeBuffer(length: length)), encoding: .ascii)!)' (\(length)) ") guard let results = CParseResults.get(from: parser), let chunkPointer = chunkPointer @@ -205,6 +320,7 @@ extension CHTTPParser { // called when chunks of a header value have been read settings.on_header_value = { parser, chunkPointer, length in + print("chttp: on_header_value '\(String(data: Data(chunkPointer!.makeBuffer(length: length)), encoding: .ascii)!)' (\(length)) ") guard let results = CParseResults.get(from: parser), let chunkPointer = chunkPointer @@ -255,10 +371,8 @@ extension CHTTPParser { // called when header parsing has completed settings.on_headers_complete = { parser in - guard - let parser = parser, - let results = CParseResults.get(from: parser) - else { + print("chttp: on_headers_complete") + guard let parser = parser, let results = CParseResults.get(from: parser) else { // signal an error return 1 } @@ -276,11 +390,12 @@ extension CHTTPParser { results.parseContentLength(index: index) let headers = HTTPHeaders(storage: results.headersData, indexes: results.headersIndexes) - - if let contentLength = results.contentLength { - results.body = HTTPBody(size: contentLength, stream: AnyOutputStream(results.bodyStream)) - } - + + /// FIXME: what was this doing? +// if let contentLength = results.contentLength { +// results.body = HTTPBody(size: contentLength, stream: AnyOutputStream(results.bodyStream)) +// } + results.headers = headers default: // no other cases need to be handled. @@ -291,41 +406,47 @@ extension CHTTPParser { let major = Int(parser.pointee.http_major) let minor = Int(parser.pointee.http_minor) results.version = HTTPVersion(major: major, minor: minor) + results.method = http_method(parser.pointee.method) + results.headersComplete = true return 0 } // called when chunks of the body have been read settings.on_body = { parser, chunk, length in - guard - let results = CParseResults.get(from: parser), - let chunk = chunk - else { + print("chttp: on_body '\(String(data: Data(chunk!.makeBuffer(length: length)), encoding: .ascii)!)' (\(length)) ") + guard let results = CParseResults.get(from: parser), let chunk = chunk else { // signal an error return 1 } - return chunk.withMemoryRebound(to: UInt8.self, capacity: length) { pointer -> Int32 in - results.bodyStream.push(ByteBuffer(start: pointer, count: length)) - - return 0 + switch results.bodyState { + case .buffer: fatalError("Unexpected buffer body state during CHTTP.on_body: \(results.bodyState)") + case .none: results.bodyState = .buffer(chunk.makeByteBuffer(length)) + case .stream: fatalError("Illegal state") + case .readyStream(let bodyStream, let ready): + bodyStream.push(chunk.makeByteBuffer(length), ready) + results.bodyState = .stream(bodyStream) // no longer ready } + + return 0 +// return chunk.withMemoryRebound(to: UInt8.self, capacity: length) { pointer -> Int32 in +// results.bodyStream.push(ByteBuffer(start: pointer, count: length)) +// +// return 0 +// } } // called when the message is finished parsing settings.on_message_complete = { parser in - guard - let parser = parser, - let results = CParseResults.get(from: parser) - else { + print("chttp: on_message_complete") + guard let parser = parser, let results = CParseResults.get(from: parser) else { // signal an error return 1 } - - results.bodyStream.close() // mark the results as complete - results.isComplete = true + results.messageComplete = true return 0 } @@ -350,6 +471,13 @@ fileprivate extension Data { } fileprivate extension UnsafePointer where Pointee == CChar { + /// Creates a Bytes array from a C pointer + fileprivate func makeByteBuffer(_ count: Int) -> ByteBuffer { + return withMemoryRebound(to: Byte.self, capacity: count) { pointer in + return ByteBuffer(start: pointer, count: count) + } + } + /// Creates a Bytes array from a C pointer fileprivate func makeBuffer(length: Int) -> UnsafeRawBufferPointer { let pointer = UnsafeBufferPointer(start: self, count: length) diff --git a/Sources/HTTP/Parser/HTTPParser.swift b/Sources/HTTP/Parser/HTTPParser.swift index ef763a80..355641b4 100644 --- a/Sources/HTTP/Parser/HTTPParser.swift +++ b/Sources/HTTP/Parser/HTTPParser.swift @@ -3,4 +3,4 @@ import Bits import Foundation /// HTTP message parser. -public protocol HTTPParser: TranslatingStream where Output: HTTPMessage {} +public protocol HTTPParser: Async.Stream where Output: HTTPMessage {} diff --git a/Sources/HTTP/Parser/HTTPRequestParser.swift b/Sources/HTTP/Parser/HTTPRequestParser.swift index 8bca2bdc..dfd6498e 100644 --- a/Sources/HTTP/Parser/HTTPRequestParser.swift +++ b/Sources/HTTP/Parser/HTTPRequestParser.swift @@ -6,44 +6,42 @@ import Foundation /// Parses requests from a readable stream. public final class HTTPRequestParser: CHTTPParser { + /// See `InputStream.Input` public typealias Input = ByteBuffer + + /// See `OutputStream.Output` public typealias Output = HTTPRequest /// See CHTTPParser.parserType static let parserType: http_parser_type = HTTP_REQUEST - // Internal variables to conform - // to the C HTTP parser protocol. - var parser: http_parser - var settings: http_parser_settings - var httpState: CHTTPParserState + /// See `CHTTPParser.chttpParserContext` + var chttp: CHTTPParserContext - /// The maxiumum possible header size - /// larger sizes will result in an error + /// See `CHTTPParser.maxHeaderSize` public var maxHeaderSize: Int? /// Creates a new Request parser. public init() { self.maxHeaderSize = 100_000 - - self.parser = http_parser() - self.settings = http_parser_settings() - self.httpState = .ready + self.chttp = .init() reset() } - func makeMessage(from results: CParseResults) throws -> HTTPRequest { + /// See `CHTTPParser.makeMessage(from:using:)` + func makeMessage(from results: CParseResults, using body: HTTPBody) throws -> HTTPRequest { // require a version to have been parsed guard let version = results.version, - let headers = results.headers + let headers = results.headers, + let cmethod = results.method else { throw HTTPError.invalidMessage() } /// switch on the C method type from the parser let method: HTTPMethod - switch http_method(parser.method) { + switch cmethod { case HTTP_DELETE: method = .delete case HTTP_GET: @@ -61,7 +59,7 @@ public final class HTTPRequestParser: CHTTPParser { /// convert the method into a string /// and use Engine's other type guard - let pointer = http_method_str(http_method(parser.method)), + let pointer = http_method_str(cmethod), let string = String(validatingUTF8: pointer) else { throw HTTPError.invalidMessage() @@ -83,7 +81,7 @@ public final class HTTPRequestParser: CHTTPParser { uri: uri, version: version, headers: headers, - body: results.body ?? HTTPBody() + body: body ) } } diff --git a/Sources/HTTP/Parser/HTTPResponseParser.swift b/Sources/HTTP/Parser/HTTPResponseParser.swift index 5f4010c6..153facc2 100644 --- a/Sources/HTTP/Parser/HTTPResponseParser.swift +++ b/Sources/HTTP/Parser/HTTPResponseParser.swift @@ -1,59 +1,60 @@ - import CHTTP -import Async -import Bits -import Foundation +// import CHTTP +//import Async +//import Bits +//import Foundation +// +///// Parses requests from a readable stream. +// public final class HTTPResponseParser: CHTTPParser { +// public typealias Input = ByteBuffer +// public typealias Output = HTTPResponse +// +// /// See CHTTPParser.parserType +// static let parserType: http_parser_type = HTTP_RESPONSE +// +// public var message: HTTPResponse? +// +// // Internal variables to conform +// // to the C HTTP parser protocol. +// var parser: http_parser +// var settings: http_parser_settings +// var httpState: CHTTPParserState +// public var messageBodyCompleted: Bool +// +// /// The maxiumum possible header size +// /// larger sizes will result in an error +// public var maxHeaderSize: Int? +// +// /// Creates a new Request parser. +// public init() { +// self.maxHeaderSize = 100_000 +// +// self.parser = http_parser() +// self.settings = http_parser_settings() +// self.httpState = .ready +// self.messageBodyCompleted = false +// reset() +// } +// +// /// See CHTTPParser.makeMessage +// func makeMessage(from results: CParseResults, using body: HTTPBody) throws -> HTTPResponse { +// // require a version to have been parsed +// guard +// let version = results.version, +// let headers = results.headers +// else { +// throw HTTPError.invalidMessage() +// } +// +// /// get response status +// let status = HTTPStatus(code: Int(parser.status_code)) +// +// // create the request +// return HTTPResponse( +// version: version, +// status: status, +// headers: headers, +// body: body +// ) +// } +//} -/// Parses requests from a readable stream. - public final class HTTPResponseParser: CHTTPParser { - public typealias Input = ByteBuffer - public typealias Output = HTTPResponse - - /// See CHTTPParser.parserType - static let parserType: http_parser_type = HTTP_RESPONSE - - public var message: HTTPResponse? - - // Internal variables to conform - // to the C HTTP parser protocol. - var parser: http_parser - var settings: http_parser_settings - var httpState: CHTTPParserState - public var messageBodyCompleted: Bool - - /// The maxiumum possible header size - /// larger sizes will result in an error - public var maxHeaderSize: Int? - - /// Creates a new Request parser. - public init() { - self.maxHeaderSize = 100_000 - - self.parser = http_parser() - self.settings = http_parser_settings() - self.httpState = .ready - self.messageBodyCompleted = false - reset() - } - - /// See CHTTPParser.makeMessage - func makeMessage(from results: CParseResults) throws -> HTTPResponse { - // require a version to have been parsed - guard - let version = results.version, - let headers = results.headers - else { - throw HTTPError.invalidMessage() - } - - /// get response status - let status = HTTPStatus(code: Int(parser.status_code)) - - // create the request - return HTTPResponse( - version: version, - status: status, - headers: headers, - body: results.body ?? HTTPBody() - ) - } -} diff --git a/Sources/HTTP/Server/HTTPServer.swift b/Sources/HTTP/Server/HTTPServer.swift index dcaa1901..485d47c5 100644 --- a/Sources/HTTP/Server/HTTPServer.swift +++ b/Sources/HTTP/Server/HTTPServer.swift @@ -26,7 +26,7 @@ public final class HTTPServer { /// set up the server stream acceptStream.drain { client in let serializerStream = HTTPResponseSerializer().stream(on: worker) - let parserStream = HTTPRequestParser().stream(on: worker) + let parserStream = HTTPRequestParser() client .stream(to: parserStream) diff --git a/Tests/HTTPTests/HTTPClientTests.swift b/Tests/HTTPTests/HTTPClientTests.swift index 250a238f..15568718 100644 --- a/Tests/HTTPTests/HTTPClientTests.swift +++ b/Tests/HTTPTests/HTTPClientTests.swift @@ -1,48 +1,49 @@ -import Async -import Bits -import HTTP -import Foundation -import TCP -import XCTest +//import Async +//import Bits +//import HTTP +//import Foundation +//import TCP +//import XCTest +// +//class HTTPClientTests: XCTestCase { +// func testTCP() throws { +// let eventLoop = try DefaultEventLoop(label: "codes.vapor.http.test.client") +// let client = try HTTPClient.tcp(hostname: "httpbin.org", port: 80, on: eventLoop) +// +// let req = HTTPRequest(method: .get, uri: "/html", headers: [.host: "httpbin.org"]) +// let res = try client.send(req).flatMap(to: Data.self) { res in +// return res.body.makeData(max: 100_000) +// }.await(on: eventLoop) +// +// XCTAssert(String(data: res, encoding: .utf8)?.contains("Moby-Dick") == true) +// XCTAssertEqual(res.count, 3741) +// } +// +// func testConnectionClose() throws { +// let eventLoop = try DefaultEventLoop(label: "codes.vapor.http.test.client") +// let client = try HTTPClient.tcp(hostname: "httpbin.org", port: 80, on: eventLoop) +// +// let req = HTTPRequest(method: .get, uri: "/status/418", headers: [.host: "httpbin.org"]) +// let res = try client.send(req).flatMap(to: Data.self) { res in +// return res.body.makeData(max: 100_000) +// }.await(on: eventLoop) +// +// XCTAssertEqual(res.count, 135) +// } +// +// func testURI() { +// var uri: URI = "http://localhost:8081/test?q=1&b=4#test" +// XCTAssertEqual(uri.scheme, "http") +// XCTAssertEqual(uri.hostname, "localhost") +// XCTAssertEqual(uri.port, 8081) +// XCTAssertEqual(uri.path, "/test") +// XCTAssertEqual(uri.query, "q=1&b=4") +// XCTAssertEqual(uri.fragment, "test") +// } +// +// static let allTests = [ +// ("testTCP", testTCP), +// ("testURI", testURI), +// ] +//} -class HTTPClientTests: XCTestCase { - func testTCP() throws { - let eventLoop = try DefaultEventLoop(label: "codes.vapor.http.test.client") - let client = try HTTPClient.tcp(hostname: "httpbin.org", port: 80, on: eventLoop) - - let req = HTTPRequest(method: .get, uri: "/html", headers: [.host: "httpbin.org"]) - let res = try client.send(req).flatMap(to: Data.self) { res in - return res.body.makeData(max: 100_000) - }.await(on: eventLoop) - - XCTAssert(String(data: res, encoding: .utf8)?.contains("Moby-Dick") == true) - XCTAssertEqual(res.count, 3741) - } - - func testConnectionClose() throws { - let eventLoop = try DefaultEventLoop(label: "codes.vapor.http.test.client") - let client = try HTTPClient.tcp(hostname: "httpbin.org", port: 80, on: eventLoop) - - let req = HTTPRequest(method: .get, uri: "/status/418", headers: [.host: "httpbin.org"]) - let res = try client.send(req).flatMap(to: Data.self) { res in - return res.body.makeData(max: 100_000) - }.await(on: eventLoop) - - XCTAssertEqual(res.count, 135) - } - - func testURI() { - var uri: URI = "http://localhost:8081/test?q=1&b=4#test" - XCTAssertEqual(uri.scheme, "http") - XCTAssertEqual(uri.hostname, "localhost") - XCTAssertEqual(uri.port, 8081) - XCTAssertEqual(uri.path, "/test") - XCTAssertEqual(uri.query, "q=1&b=4") - XCTAssertEqual(uri.fragment, "test") - } - - static let allTests = [ - ("testTCP", testTCP), - ("testURI", testURI), - ] -} diff --git a/Tests/HTTPTests/ParserTests.swift b/Tests/HTTPTests/ParserTests.swift index 06809284..f4b5def8 100644 --- a/Tests/HTTPTests/ParserTests.swift +++ b/Tests/HTTPTests/ParserTests.swift @@ -6,182 +6,220 @@ import XCTest class ParserTests : XCTestCase { let loop = try! DefaultEventLoop(label: "test") - - func testRequest() throws { - var data = """ - POST /cgi-bin/process.cgi HTTP/1.1\r - User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r - Host: www.tutorialspoint.com\r - Content-Type: text/plain\r - Content-Length: 5\r - Accept-Language: en-us\r - Accept-Encoding: gzip, deflate\r - Connection: Keep-Alive\r - \r - hello - """.data(using: .utf8) ?? Data() - let parser = HTTPRequestParser().stream(on: loop) - var message: HTTPRequest? - var completed = false - - parser.drain { _message in - message = _message - }.catch { error in - XCTFail("\(error)") - }.finally { - completed = true - } - - XCTAssertNil(message) - try parser.next(data.withByteBuffer { $0 }).await(on: loop) - parser.close() - - guard let req = message else { - XCTFail("No request parsed") - return - } + func testParserEdgeCases() throws { + let firstChunk = "GET /hello HTTP/1.1\r\nContent-Type: ".data(using: .utf8)! + let secondChunk = "text/plain\r\nContent-Length: 5\r\n\r\nwo".data(using: .utf8)! + let thirdChunk = "rl".data(using: .utf8)! + let fourthChunk = "d".data(using: .utf8)! - XCTAssertEqual(req.method, .post) - XCTAssertEqual(req.headers[.userAgent], "Mozilla/4.0 (compatible; MSIE5.01; Windows NT)") - XCTAssertEqual(req.headers[.host], "www.tutorialspoint.com") - XCTAssertEqual(req.headers[.contentType], "text/plain") - XCTAssertEqual(req.mediaType, .plainText) - XCTAssertEqual(req.headers[.contentLength], "5") - XCTAssertEqual(req.headers[.acceptLanguage], "en-us") - XCTAssertEqual(req.headers[.acceptEncoding], "gzip, deflate") - XCTAssertEqual(req.headers[.connection], "Keep-Alive") - - data = try req.body.makeData(max: 100_000).await(on: loop) - XCTAssertEqual(String(data: data, encoding: .utf8), "hello") - XCTAssert(completed) - } + let eventLoop = try DefaultEventLoop(label: "codes.vapor.engine.http.parser.test") + let parser = HTTPRequestParser() - func testResponse() throws { - var data = """ - HTTP/1.1 200 OK\r - Date: Mon, 27 Jul 2009 12:28:53 GMT\r - Server: Apache/2.2.14 (Win32)\r - Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\r - Content-Length: 7\r - Content-Type: text/html\r - Connection: Closed\r - \r - - """.data(using: .utf8) ?? Data() - - let parser = HTTPResponseParser().stream(on: loop) - var message: HTTPResponse? - var completed = false - - parser.drain { _message in - message = _message + let socket = PushStream(ByteBuffer.self) + socket.stream(to: parser).drain { message in + print("parser.drain { ... }") + print(message) + print("message.body.makeData") + message.body.makeData(max: 100).do { data in + print(data) + }.catch { error in + print("body error: \(error)") + } }.catch { error in - XCTFail("\(error)") + print("parser.catch { \(error) }") }.finally { - completed = true + print("parser.close { }") } - XCTAssertNil(message) - try parser.next(data.withByteBuffer { $0 }).await(on: loop) - parser.close() - - guard let res = message else { - XCTFail("No request parsed") - return - } - XCTAssertEqual(res.status, .ok) - XCTAssertEqual(res.headers[.date], "Mon, 27 Jul 2009 12:28:53 GMT") - XCTAssertEqual(res.headers[.server], "Apache/2.2.14 (Win32)") - XCTAssertEqual(res.headers[.lastModified], "Wed, 22 Jul 2009 19:15:56 GMT") - XCTAssertEqual(res.headers[.contentLength], "7") - XCTAssertEqual(res.headers[.contentType], "text/html") - XCTAssertEqual(res.mediaType, .html) - XCTAssertEqual(res.headers[.connection], "Closed") - - data = try res.body.makeData(max: 100_000).blockingAwait() - XCTAssertEqual(String(data: data, encoding: .utf8), "") - XCTAssert(completed) + print("(1) FIRST ---") + firstChunk.withByteBuffer(socket.push) + print("(2) SECOND ---") + secondChunk.withByteBuffer(socket.push) + print("(3) THIRD ---") + thirdChunk.withByteBuffer(socket.push) + print("(4) FOURTH ---") + fourthChunk.withByteBuffer(socket.push) + print("(c) CLOSE ---") + socket.close() } - - func testTooLargeRequest() throws { - let data = """ - POST /cgi-bin/process.cgi HTTP/1.1\r - User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r - Host: www.tutorialspoint.com\r - Content-Type: text/plain\r - Content-Length: 5\r - Accept-Language: en-us\r - Accept-Encoding: gzip, deflate\r - Connection: Keep-Alive\r - \r - hello - """.data(using: .utf8) ?? Data() - - var error = false - let p = HTTPRequestParser() - p.maxHeaderSize = data.count - 20 // body - let parser = p.stream(on: loop) - - var completed = false - - _ = parser.drain { _ in - XCTFail() - }.catch { _ in - error = true - }.finally { - completed = true - } +// +// func testRequest() throws { +// var data = """ +// POST /cgi-bin/process.cgi HTTP/1.1\r +// User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r +// Host: www.tutorialspoint.com\r +// Content-Type: text/plain\r +// Content-Length: 5\r +// Accept-Language: en-us\r +// Accept-Encoding: gzip, deflate\r +// Connection: Keep-Alive\r +// \r +// hello +// """.data(using: .utf8) ?? Data() +// +// let parser = HTTPRequestParser() +// var message: HTTPRequest? +// var completed = false +// +// parser.drain { _message in +// message = _message +// }.catch { error in +// XCTFail("\(error)") +// }.finally { +// completed = true +// } +// +// XCTAssertNil(message) +// try parser.next(data.withByteBuffer { $0 }).await(on: loop) +// parser.close() +// +// guard let req = message else { +// XCTFail("No request parsed") +// return +// } +// +// XCTAssertEqual(req.method, .post) +// XCTAssertEqual(req.headers[.userAgent], "Mozilla/4.0 (compatible; MSIE5.01; Windows NT)") +// XCTAssertEqual(req.headers[.host], "www.tutorialspoint.com") +// XCTAssertEqual(req.headers[.contentType], "text/plain") +// XCTAssertEqual(req.mediaType, .plainText) +// XCTAssertEqual(req.headers[.contentLength], "5") +// XCTAssertEqual(req.headers[.acceptLanguage], "en-us") +// XCTAssertEqual(req.headers[.acceptEncoding], "gzip, deflate") +// XCTAssertEqual(req.headers[.connection], "Keep-Alive") +// +// data = try req.body.makeData(max: 100_000).await(on: loop) +// XCTAssertEqual(String(data: data, encoding: .utf8), "hello") +// XCTAssert(completed) +// } +// +// func testResponse() throws { +// var data = """ +// HTTP/1.1 200 OK\r +// Date: Mon, 27 Jul 2009 12:28:53 GMT\r +// Server: Apache/2.2.14 (Win32)\r +// Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\r +// Content-Length: 7\r +// Content-Type: text/html\r +// Connection: Closed\r +// \r +// +// """.data(using: .utf8) ?? Data() +// +// let parser = HTTPResponseParser() +// var message: HTTPResponse? +// var completed = false +// +// parser.drain { _message in +// message = _message +// }.catch { error in +// XCTFail("\(error)") +// }.finally { +// completed = true +// } +// +// XCTAssertNil(message) +// try parser.next(data.withByteBuffer { $0 }).await(on: loop) +// parser.close() +// +// guard let res = message else { +// XCTFail("No request parsed") +// return +// } +// +// XCTAssertEqual(res.status, .ok) +// XCTAssertEqual(res.headers[.date], "Mon, 27 Jul 2009 12:28:53 GMT") +// XCTAssertEqual(res.headers[.server], "Apache/2.2.14 (Win32)") +// XCTAssertEqual(res.headers[.lastModified], "Wed, 22 Jul 2009 19:15:56 GMT") +// XCTAssertEqual(res.headers[.contentLength], "7") +// XCTAssertEqual(res.headers[.contentType], "text/html") +// XCTAssertEqual(res.mediaType, .html) +// XCTAssertEqual(res.headers[.connection], "Closed") +// +// data = try res.body.makeData(max: 100_000).blockingAwait() +// XCTAssertEqual(String(data: data, encoding: .utf8), "") +// XCTAssert(completed) +// } +// +// func testTooLargeRequest() throws { +// let data = """ +// POST /cgi-bin/process.cgi HTTP/1.1\r +// User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r +// Host: www.tutorialspoint.com\r +// Content-Type: text/plain\r +// Content-Length: 5\r +// Accept-Language: en-us\r +// Accept-Encoding: gzip, deflate\r +// Connection: Keep-Alive\r +// \r +// hello +// """.data(using: .utf8) ?? Data() +// +// var error = false +// let p = HTTPRequestParser() +// p.maxHeaderSize = data.count - 20 // body +// let parser = p +// +// var completed = false +// +// _ = parser.drain { _ in +// XCTFail() +// }.catch { _ in +// error = true +// }.finally { +// completed = true +// } +// +// try parser.next(data.withByteBuffer { $0 }).await(on: loop) +// parser.close() +// XCTAssert(error) +// XCTAssert(completed) +// } - try parser.next(data.withByteBuffer { $0 }).await(on: loop) - parser.close() - XCTAssert(error) - XCTAssert(completed) - } - - func testTooLargeResponse() throws { - let data = """ - HTTP/1.1 200 OK\r - Date: Mon, 27 Jul 2009 12:28:53 GMT\r - Server: Apache/2.2.14 (Win32)\r - Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\r - Content-Length: 7\r - Content-Type: text/html\r - Connection: Closed\r - \r - - """.data(using: .utf8) ?? Data() - - var error = false - let p = HTTPResponseParser() - p.maxHeaderSize = data.count - 20 // body - let parser = p.stream(on: loop) - - var completed = false - - _ = parser.drain { _ in - XCTFail() - }.catch { _ in - error = true - }.finally { - completed = true - } - - - try parser.next(data.withByteBuffer { $0 }).await(on: loop) - try parser.next(data.withByteBuffer { $0 }).await(on: loop) - parser.close() - XCTAssert(error) - XCTAssert(completed) - } +// func testTooLargeResponse() throws { +// let data = """ +// HTTP/1.1 200 OK\r +// Date: Mon, 27 Jul 2009 12:28:53 GMT\r +// Server: Apache/2.2.14 (Win32)\r +// Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\r +// Content-Length: 7\r +// Content-Type: text/html\r +// Connection: Closed\r +// \r +// +// """.data(using: .utf8) ?? Data() +// +// var error = false +// let p = HTTPResponseParser() +// p.maxHeaderSize = data.count - 20 // body +// let parser = p.stream(on: loop) +// +// var completed = false +// +// _ = parser.drain { _ in +// XCTFail() +// }.catch { _ in +// error = true +// }.finally { +// completed = true +// } +// +// +// try parser.next(data.withByteBuffer { $0 }).await(on: loop) +// try parser.next(data.withByteBuffer { $0 }).await(on: loop) +// parser.close() +// XCTAssert(error) +// XCTAssert(completed) +// } - static let allTests = [ - ("testRequest", testRequest), - ("testResponse", testResponse), - ("testTooLargeRequest", testTooLargeRequest), - ("testTooLargeResponse", testTooLargeResponse), - ] +// static let allTests = [ +// ("testRequest", testRequest), +// ("testResponse", testResponse), +// ("testTooLargeRequest", testTooLargeRequest), +// ("testTooLargeResponse", testTooLargeResponse), +// ] } From 715b6b882b7e2f37ce44344920d5dd35f284d9c4 Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Tue, 23 Jan 2018 17:58:12 -0500 Subject: [PATCH 02/16] protocol tester --- Sources/HTTP/Parser/CHTTPParser.swift | 17 +- Tests/HTTPTests/ParserTests.swift | 264 +++++++++++++++++++++++--- 2 files changed, 249 insertions(+), 32 deletions(-) diff --git a/Sources/HTTP/Parser/CHTTPParser.swift b/Sources/HTTP/Parser/CHTTPParser.swift index c1ad432e..17dd4b7c 100644 --- a/Sources/HTTP/Parser/CHTTPParser.swift +++ b/Sources/HTTP/Parser/CHTTPParser.swift @@ -81,7 +81,9 @@ extension CHTTPParser { /// See `InputStream.input(_:)` public func input(_ event: InputEvent) { switch event { - case .close: chttp.downstream!.close() + case .close: + chttp.close() + chttp.downstream!.close() case .error(let error): chttp.downstream!.error(error) case .next(let input, let ready): try! handleNext(input, ready) } @@ -122,10 +124,10 @@ extension CHTTPParser { CParseResults.remove(from: &chttp.parser) } else { // Convert body to a stream - let stream = CHTTPBodyStream() // FIX, this shouldn't backlog + let stream = CHTTPBodyStream() switch results.bodyState { case .buffer(let buffer): stream.push(buffer, ready) - case .none: break // push nothing + case .none: stream.push(ByteBuffer(start: nil, count: 0), ready) case .stream: fatalError("Illegal state") case .readyStream: fatalError("Illegal state") } @@ -173,10 +175,6 @@ extension CHTTPParser { fatalError("\(error)") } } - - - // // EOF - // http_parser_execute(&parser, &settings, nil, 0) } /// Resets the parser @@ -247,6 +245,11 @@ extension CHTTPParserContext { } return results } + + /// Indicates a close to the HTTP parser. + mutating func close() { + http_parser_execute(&parser, &settings, nil, 0) + } /// Initializes the http parser settings with appropriate callbacks. mutating func initialize() { diff --git a/Tests/HTTPTests/ParserTests.swift b/Tests/HTTPTests/ParserTests.swift index f4b5def8..b62fed6d 100644 --- a/Tests/HTTPTests/ParserTests.swift +++ b/Tests/HTTPTests/ParserTests.swift @@ -4,46 +4,260 @@ import Dispatch import HTTP import XCTest + +extension String { + var buffer: ByteBuffer { + return self.data(using: .utf8)!.withByteBuffer { $0 } + } +} +public final class ProtocolTester: Async.OutputStream { + /// See `OutputStream.Output` + public typealias Output = ByteBuffer + + /// Stream being tested + public var downstream: AnyInputStream? + + /// See `OutputStream.output` + public func output(to inputStream: S) where S: Async.InputStream, ProtocolTester.Output == S.Input { + downstream = .init(inputStream) + } + + private var reset: () -> () + private var fail: (String, StaticString, UInt) -> () + private var checks: [ProtocolTesterCheck] + + private struct ProtocolTesterCheck { + var minOffset: Int? + var maxOffset: Int? + var file: StaticString + var line: UInt + var checks: () throws -> () + } + + public init(onFail: @escaping (String, StaticString, UInt) -> (), reset: @escaping () -> ()) { + self.reset = reset + self.fail = onFail + checks = [] + } + + public func assert(before offset: Int, file: StaticString = #file, line: UInt = #line, callback: @escaping () throws -> ()) { + let check = ProtocolTesterCheck(minOffset: nil, maxOffset: offset, file: file, line: line, checks: callback) + checks.append(check) + } + + public func assert(after offset: Int, file: StaticString = #file, line: UInt = #line, callback: @escaping () throws -> ()) { + let check = ProtocolTesterCheck(minOffset: offset, maxOffset: nil, file: file, line: line, checks: callback) + checks.append(check) + } + + /// Runs the protocol tester w/ the supplied input + public func run(_ string: String) -> Future { + Swift.assert(downstream != nil, "ProtocolTester must be connected before running") + let buffer = string.buffer + return runMax(buffer, max: buffer.count) + } + + private func runMax(_ buffer: ByteBuffer, max: Int) -> Future { + if max > 0 { + let maxSizedChunksCount = buffer.count / max + let lastChunkSize = buffer.count % max + + var chunks: [ByteBuffer] = [] + + for i in 0.. 0 { + let lastChunk = ByteBuffer(start: buffer.baseAddress?.advanced(by: buffer.count - lastChunkSize), count: lastChunkSize) + chunks.insert(lastChunk, at: 0) + } + + reset() + return runChunks(chunks, currentOffset: 0, original: chunks).flatMap(to: Void.self) { + return self.runMax(buffer, max: max - 1) + } + } else { + downstream?.close() + return .done + } + } + + private func runChunks(_ chunks: [ByteBuffer], currentOffset: Int, original: [ByteBuffer]) -> Future { + var chunks = chunks + if let chunk = chunks.popLast() { + runChecks(offset: currentOffset, chunks: original) + return downstream!.next(chunk).flatMap(to: Void.self) { _ in + return self.runChunks(chunks, currentOffset: currentOffset + chunk.count, original: original) + } + } else { + runChecks(offset: currentOffset, chunks: original) + return .done + } + } + + private func runChecks(offset: Int, chunks: [ByteBuffer]) { + for check in checks { + var shouldRun = false + if let min = check.minOffset, offset >= min { + shouldRun = true + } + if let max = check.maxOffset, offset < max { + shouldRun = true + } + if shouldRun { + do { + try check.checks() + } catch { + var message = "Protocol test failed: \(error)" + let data = chunks.reversed().map { "[" + ProtocolTester.dataDebug(for: $0) + "]" }.joined(separator: " ") + let text = chunks.reversed().map { "[" + ProtocolTester.textDebug(for: $0) + "]" }.joined(separator: " ") + message += "\nData: \(data)" + message += "\nText: \(text)" + self.fail(message, check.file, check.line) + } + } + } + } + + static func textDebug(for buffer: ByteBuffer) -> String { + let string = String(bytes: buffer, encoding: .ascii) ?? "n/a" + return string + .replacingOccurrences(of: "\r", with: "\\r") + .replacingOccurrences(of: "\n", with: "\\n") + } + + /// See `CustomStringConvertible.description` + static func dataDebug(for buffer: ByteBuffer) -> String { + var string = "0x" + for i in 0..> 4) + let lower = Int(byte & 0b00001111) + string.append(hexMap[upper]) + string.append(hexMap[lower]) + } + return string + } + + static let hexMap = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "A", "B", "C", "D", "E", "F"] +} +extension String: Error {} + class ParserTests : XCTestCase { - let loop = try! DefaultEventLoop(label: "test") - func testParserEdgeCases() throws { - let firstChunk = "GET /hello HTTP/1.1\r\nContent-Type: ".data(using: .utf8)! - let secondChunk = "text/plain\r\nContent-Length: 5\r\n\r\nwo".data(using: .utf8)! - let thirdChunk = "rl".data(using: .utf8)! - let fourthChunk = "d".data(using: .utf8)! - let eventLoop = try DefaultEventLoop(label: "codes.vapor.engine.http.parser.test") - let parser = HTTPRequestParser() + func testParserEdgeCasesOld() throws { + // captured variables to check + var request: HTTPRequest? + var content: String? + var isClosed = false + // configure parser stream let socket = PushStream(ByteBuffer.self) - socket.stream(to: parser).drain { message in - print("parser.drain { ... }") - print(message) - print("message.body.makeData") + socket.stream(to: HTTPRequestParser()).drain { message in + request = message message.body.makeData(max: 100).do { data in - print(data) + content = String(data: data, encoding: .ascii) }.catch { error in - print("body error: \(error)") + XCTFail("body error: \(error)") } }.catch { error in - print("parser.catch { \(error) }") + XCTFail("parser error: \(error)") }.finally { - print("parser.close { }") + isClosed = true } + // pre-step + XCTAssertNil(request) + XCTAssertNil(content) + XCTAssertFalse(isClosed) + + // (1) FIRST --- + socket.push("GET /hello HTTP/1.1\r\nContent-Type: ".buffer) + XCTAssertNil(request) + XCTAssertNil(content) + XCTAssertFalse(isClosed) + + // (2) SECOND --- + socket.push("text/plain\r\nContent-Length: 5\r\n\r\nwo".buffer) + XCTAssertNotNil(request) + XCTAssertEqual(request?.uri.path, "/hello") + XCTAssertEqual(request?.method, .get) + XCTAssertNil(content) + XCTAssertFalse(isClosed) + + // (3) THIRD --- + socket.push("rl".buffer) + XCTAssertNil(content) + XCTAssertFalse(isClosed) - print("(1) FIRST ---") - firstChunk.withByteBuffer(socket.push) - print("(2) SECOND ---") - secondChunk.withByteBuffer(socket.push) - print("(3) THIRD ---") - thirdChunk.withByteBuffer(socket.push) - print("(4) FOURTH ---") - fourthChunk.withByteBuffer(socket.push) - print("(c) CLOSE ---") + // (4) FOURTH --- + socket.push("d".buffer) + XCTAssertEqual(content, "world") + XCTAssertFalse(isClosed) + + // (c) CLOSE --- socket.close() + XCTAssertTrue(isClosed) + } + + + func testParserEdgeCases() throws { + // captured variables to check + var request: HTTPRequest? + var content: String? + var isClosed = false + + // creates a protocol tester + let tester = ProtocolTester(onFail: XCTFail) { + request = nil + content = nil + isClosed = false + } + + tester.assert(before: 68) { + guard request == nil else { + throw "request was not nil" + } + } + + tester.assert(after: 68) { + guard request != nil else { + throw "request was nil" + } + } + + tester.assert(after: 73) { + guard let string = content else { + throw "content was nil" + } + + guard string == "world" else { + throw "incorrect string" + } + } + + // configure parser stream + tester.stream(to: HTTPRequestParser()).drain { message in + request = message + message.body.makeData(max: 100).do { data in + content = String(data: data, encoding: .ascii) + }.catch { error in + XCTFail("body error: \(error)") + } + }.catch { error in + XCTFail("parser error: \(error)") + }.finally { + isClosed = true + } + + try tester.run("GET /hello HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 5\r\n\r\nworld").blockingAwait() + XCTAssertTrue(isClosed) } + + // // func testRequest() throws { // var data = """ From 33eb9ca42e9659d2c4f63aada81515996f6158a1 Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Wed, 24 Jan 2018 18:56:19 -0500 Subject: [PATCH 03/16] protocol tester cleanup --- Tests/HTTPTests/HTTPParserTests.swift | 249 +++++++++++++++ Tests/HTTPTests/ParserTests.swift | 439 -------------------------- Tests/HTTPTests/ProtocolTester.swift | 171 ++++++++++ 3 files changed, 420 insertions(+), 439 deletions(-) create mode 100644 Tests/HTTPTests/HTTPParserTests.swift delete mode 100644 Tests/HTTPTests/ParserTests.swift create mode 100644 Tests/HTTPTests/ProtocolTester.swift diff --git a/Tests/HTTPTests/HTTPParserTests.swift b/Tests/HTTPTests/HTTPParserTests.swift new file mode 100644 index 00000000..b59a0c0b --- /dev/null +++ b/Tests/HTTPTests/HTTPParserTests.swift @@ -0,0 +1,249 @@ +import Async +import Bits +import Dispatch +import HTTP +import XCTest + +class HTTPParserTests: XCTestCase { + func testParserEdgeCases() throws { + // captured variables to check + var request: HTTPRequest? + var content: String? + var isClosed = false + + // creates a protocol tester + let tester = ProtocolTester( + data: "GET /hello HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 5\r\n\r\nworld", + onFail: XCTFail + ) { + request = nil + content = nil + isClosed = false + } + + tester.assert(before: "\r\n\r\n") { + guard request == nil else { + throw "request was not nil" + } + } + + tester.assert(after: "\r\n\r\n") { + guard request != nil else { + throw "request was nil" + } + } + + tester.assert(before: "world") { + guard content == nil else { + throw "content was not nil" + } + } + + tester.assert(after: "world") { + guard let string = content else { + throw "content was nil" + } + + guard string == "world" else { + throw "incorrect string" + } + } + + // configure parser stream + tester.stream(to: HTTPRequestParser()).drain { message in + request = message + message.body.makeData(max: 100).do { data in + content = String(data: data, encoding: .ascii) + }.catch { error in + XCTFail("body error: \(error)") + } + }.catch { error in + XCTFail("parser error: \(error)") + }.finally { + isClosed = true + } + + try tester.run().blockingAwait() + XCTAssertTrue(isClosed) + } + + +// +// func testRequest() throws { +// var data = """ +// POST /cgi-bin/process.cgi HTTP/1.1\r +// User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r +// Host: www.tutorialspoint.com\r +// Content-Type: text/plain\r +// Content-Length: 5\r +// Accept-Language: en-us\r +// Accept-Encoding: gzip, deflate\r +// Connection: Keep-Alive\r +// \r +// hello +// """.data(using: .utf8) ?? Data() +// +// let parser = HTTPRequestParser() +// var message: HTTPRequest? +// var completed = false +// +// parser.drain { _message in +// message = _message +// }.catch { error in +// XCTFail("\(error)") +// }.finally { +// completed = true +// } +// +// XCTAssertNil(message) +// try parser.next(data.withByteBuffer { $0 }).await(on: loop) +// parser.close() +// +// guard let req = message else { +// XCTFail("No request parsed") +// return +// } +// +// XCTAssertEqual(req.method, .post) +// XCTAssertEqual(req.headers[.userAgent], "Mozilla/4.0 (compatible; MSIE5.01; Windows NT)") +// XCTAssertEqual(req.headers[.host], "www.tutorialspoint.com") +// XCTAssertEqual(req.headers[.contentType], "text/plain") +// XCTAssertEqual(req.mediaType, .plainText) +// XCTAssertEqual(req.headers[.contentLength], "5") +// XCTAssertEqual(req.headers[.acceptLanguage], "en-us") +// XCTAssertEqual(req.headers[.acceptEncoding], "gzip, deflate") +// XCTAssertEqual(req.headers[.connection], "Keep-Alive") +// +// data = try req.body.makeData(max: 100_000).await(on: loop) +// XCTAssertEqual(String(data: data, encoding: .utf8), "hello") +// XCTAssert(completed) +// } +// +// func testResponse() throws { +// var data = """ +// HTTP/1.1 200 OK\r +// Date: Mon, 27 Jul 2009 12:28:53 GMT\r +// Server: Apache/2.2.14 (Win32)\r +// Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\r +// Content-Length: 7\r +// Content-Type: text/html\r +// Connection: Closed\r +// \r +// +// """.data(using: .utf8) ?? Data() +// +// let parser = HTTPResponseParser() +// var message: HTTPResponse? +// var completed = false +// +// parser.drain { _message in +// message = _message +// }.catch { error in +// XCTFail("\(error)") +// }.finally { +// completed = true +// } +// +// XCTAssertNil(message) +// try parser.next(data.withByteBuffer { $0 }).await(on: loop) +// parser.close() +// +// guard let res = message else { +// XCTFail("No request parsed") +// return +// } +// +// XCTAssertEqual(res.status, .ok) +// XCTAssertEqual(res.headers[.date], "Mon, 27 Jul 2009 12:28:53 GMT") +// XCTAssertEqual(res.headers[.server], "Apache/2.2.14 (Win32)") +// XCTAssertEqual(res.headers[.lastModified], "Wed, 22 Jul 2009 19:15:56 GMT") +// XCTAssertEqual(res.headers[.contentLength], "7") +// XCTAssertEqual(res.headers[.contentType], "text/html") +// XCTAssertEqual(res.mediaType, .html) +// XCTAssertEqual(res.headers[.connection], "Closed") +// +// data = try res.body.makeData(max: 100_000).blockingAwait() +// XCTAssertEqual(String(data: data, encoding: .utf8), "") +// XCTAssert(completed) +// } +// +// func testTooLargeRequest() throws { +// let data = """ +// POST /cgi-bin/process.cgi HTTP/1.1\r +// User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r +// Host: www.tutorialspoint.com\r +// Content-Type: text/plain\r +// Content-Length: 5\r +// Accept-Language: en-us\r +// Accept-Encoding: gzip, deflate\r +// Connection: Keep-Alive\r +// \r +// hello +// """.data(using: .utf8) ?? Data() +// +// var error = false +// let p = HTTPRequestParser() +// p.maxHeaderSize = data.count - 20 // body +// let parser = p +// +// var completed = false +// +// _ = parser.drain { _ in +// XCTFail() +// }.catch { _ in +// error = true +// }.finally { +// completed = true +// } +// +// try parser.next(data.withByteBuffer { $0 }).await(on: loop) +// parser.close() +// XCTAssert(error) +// XCTAssert(completed) +// } + +// func testTooLargeResponse() throws { +// let data = """ +// HTTP/1.1 200 OK\r +// Date: Mon, 27 Jul 2009 12:28:53 GMT\r +// Server: Apache/2.2.14 (Win32)\r +// Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\r +// Content-Length: 7\r +// Content-Type: text/html\r +// Connection: Closed\r +// \r +// +// """.data(using: .utf8) ?? Data() +// +// var error = false +// let p = HTTPResponseParser() +// p.maxHeaderSize = data.count - 20 // body +// let parser = p.stream(on: loop) +// +// var completed = false +// +// _ = parser.drain { _ in +// XCTFail() +// }.catch { _ in +// error = true +// }.finally { +// completed = true +// } +// +// +// try parser.next(data.withByteBuffer { $0 }).await(on: loop) +// try parser.next(data.withByteBuffer { $0 }).await(on: loop) +// parser.close() +// XCTAssert(error) +// XCTAssert(completed) +// } + +// static let allTests = [ +// ("testRequest", testRequest), +// ("testResponse", testResponse), +// ("testTooLargeRequest", testTooLargeRequest), +// ("testTooLargeResponse", testTooLargeResponse), +// ] +} + + diff --git a/Tests/HTTPTests/ParserTests.swift b/Tests/HTTPTests/ParserTests.swift deleted file mode 100644 index b62fed6d..00000000 --- a/Tests/HTTPTests/ParserTests.swift +++ /dev/null @@ -1,439 +0,0 @@ -import Async -import Bits -import Dispatch -import HTTP -import XCTest - - -extension String { - var buffer: ByteBuffer { - return self.data(using: .utf8)!.withByteBuffer { $0 } - } -} -public final class ProtocolTester: Async.OutputStream { - /// See `OutputStream.Output` - public typealias Output = ByteBuffer - - /// Stream being tested - public var downstream: AnyInputStream? - - /// See `OutputStream.output` - public func output(to inputStream: S) where S: Async.InputStream, ProtocolTester.Output == S.Input { - downstream = .init(inputStream) - } - - private var reset: () -> () - private var fail: (String, StaticString, UInt) -> () - private var checks: [ProtocolTesterCheck] - - private struct ProtocolTesterCheck { - var minOffset: Int? - var maxOffset: Int? - var file: StaticString - var line: UInt - var checks: () throws -> () - } - - public init(onFail: @escaping (String, StaticString, UInt) -> (), reset: @escaping () -> ()) { - self.reset = reset - self.fail = onFail - checks = [] - } - - public func assert(before offset: Int, file: StaticString = #file, line: UInt = #line, callback: @escaping () throws -> ()) { - let check = ProtocolTesterCheck(minOffset: nil, maxOffset: offset, file: file, line: line, checks: callback) - checks.append(check) - } - - public func assert(after offset: Int, file: StaticString = #file, line: UInt = #line, callback: @escaping () throws -> ()) { - let check = ProtocolTesterCheck(minOffset: offset, maxOffset: nil, file: file, line: line, checks: callback) - checks.append(check) - } - - /// Runs the protocol tester w/ the supplied input - public func run(_ string: String) -> Future { - Swift.assert(downstream != nil, "ProtocolTester must be connected before running") - let buffer = string.buffer - return runMax(buffer, max: buffer.count) - } - - private func runMax(_ buffer: ByteBuffer, max: Int) -> Future { - if max > 0 { - let maxSizedChunksCount = buffer.count / max - let lastChunkSize = buffer.count % max - - var chunks: [ByteBuffer] = [] - - for i in 0.. 0 { - let lastChunk = ByteBuffer(start: buffer.baseAddress?.advanced(by: buffer.count - lastChunkSize), count: lastChunkSize) - chunks.insert(lastChunk, at: 0) - } - - reset() - return runChunks(chunks, currentOffset: 0, original: chunks).flatMap(to: Void.self) { - return self.runMax(buffer, max: max - 1) - } - } else { - downstream?.close() - return .done - } - } - - private func runChunks(_ chunks: [ByteBuffer], currentOffset: Int, original: [ByteBuffer]) -> Future { - var chunks = chunks - if let chunk = chunks.popLast() { - runChecks(offset: currentOffset, chunks: original) - return downstream!.next(chunk).flatMap(to: Void.self) { _ in - return self.runChunks(chunks, currentOffset: currentOffset + chunk.count, original: original) - } - } else { - runChecks(offset: currentOffset, chunks: original) - return .done - } - } - - private func runChecks(offset: Int, chunks: [ByteBuffer]) { - for check in checks { - var shouldRun = false - if let min = check.minOffset, offset >= min { - shouldRun = true - } - if let max = check.maxOffset, offset < max { - shouldRun = true - } - if shouldRun { - do { - try check.checks() - } catch { - var message = "Protocol test failed: \(error)" - let data = chunks.reversed().map { "[" + ProtocolTester.dataDebug(for: $0) + "]" }.joined(separator: " ") - let text = chunks.reversed().map { "[" + ProtocolTester.textDebug(for: $0) + "]" }.joined(separator: " ") - message += "\nData: \(data)" - message += "\nText: \(text)" - self.fail(message, check.file, check.line) - } - } - } - } - - static func textDebug(for buffer: ByteBuffer) -> String { - let string = String(bytes: buffer, encoding: .ascii) ?? "n/a" - return string - .replacingOccurrences(of: "\r", with: "\\r") - .replacingOccurrences(of: "\n", with: "\\n") - } - - /// See `CustomStringConvertible.description` - static func dataDebug(for buffer: ByteBuffer) -> String { - var string = "0x" - for i in 0..> 4) - let lower = Int(byte & 0b00001111) - string.append(hexMap[upper]) - string.append(hexMap[lower]) - } - return string - } - - static let hexMap = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "A", "B", "C", "D", "E", "F"] -} -extension String: Error {} - -class ParserTests : XCTestCase { - - - func testParserEdgeCasesOld() throws { - // captured variables to check - var request: HTTPRequest? - var content: String? - var isClosed = false - - // configure parser stream - let socket = PushStream(ByteBuffer.self) - socket.stream(to: HTTPRequestParser()).drain { message in - request = message - message.body.makeData(max: 100).do { data in - content = String(data: data, encoding: .ascii) - }.catch { error in - XCTFail("body error: \(error)") - } - }.catch { error in - XCTFail("parser error: \(error)") - }.finally { - isClosed = true - } - - // pre-step - XCTAssertNil(request) - XCTAssertNil(content) - XCTAssertFalse(isClosed) - - // (1) FIRST --- - socket.push("GET /hello HTTP/1.1\r\nContent-Type: ".buffer) - XCTAssertNil(request) - XCTAssertNil(content) - XCTAssertFalse(isClosed) - - // (2) SECOND --- - socket.push("text/plain\r\nContent-Length: 5\r\n\r\nwo".buffer) - XCTAssertNotNil(request) - XCTAssertEqual(request?.uri.path, "/hello") - XCTAssertEqual(request?.method, .get) - XCTAssertNil(content) - XCTAssertFalse(isClosed) - - // (3) THIRD --- - socket.push("rl".buffer) - XCTAssertNil(content) - XCTAssertFalse(isClosed) - - // (4) FOURTH --- - socket.push("d".buffer) - XCTAssertEqual(content, "world") - XCTAssertFalse(isClosed) - - // (c) CLOSE --- - socket.close() - XCTAssertTrue(isClosed) - } - - - func testParserEdgeCases() throws { - // captured variables to check - var request: HTTPRequest? - var content: String? - var isClosed = false - - // creates a protocol tester - let tester = ProtocolTester(onFail: XCTFail) { - request = nil - content = nil - isClosed = false - } - - tester.assert(before: 68) { - guard request == nil else { - throw "request was not nil" - } - } - - tester.assert(after: 68) { - guard request != nil else { - throw "request was nil" - } - } - - tester.assert(after: 73) { - guard let string = content else { - throw "content was nil" - } - - guard string == "world" else { - throw "incorrect string" - } - } - - // configure parser stream - tester.stream(to: HTTPRequestParser()).drain { message in - request = message - message.body.makeData(max: 100).do { data in - content = String(data: data, encoding: .ascii) - }.catch { error in - XCTFail("body error: \(error)") - } - }.catch { error in - XCTFail("parser error: \(error)") - }.finally { - isClosed = true - } - - try tester.run("GET /hello HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 5\r\n\r\nworld").blockingAwait() - XCTAssertTrue(isClosed) - } - - -// -// func testRequest() throws { -// var data = """ -// POST /cgi-bin/process.cgi HTTP/1.1\r -// User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r -// Host: www.tutorialspoint.com\r -// Content-Type: text/plain\r -// Content-Length: 5\r -// Accept-Language: en-us\r -// Accept-Encoding: gzip, deflate\r -// Connection: Keep-Alive\r -// \r -// hello -// """.data(using: .utf8) ?? Data() -// -// let parser = HTTPRequestParser() -// var message: HTTPRequest? -// var completed = false -// -// parser.drain { _message in -// message = _message -// }.catch { error in -// XCTFail("\(error)") -// }.finally { -// completed = true -// } -// -// XCTAssertNil(message) -// try parser.next(data.withByteBuffer { $0 }).await(on: loop) -// parser.close() -// -// guard let req = message else { -// XCTFail("No request parsed") -// return -// } -// -// XCTAssertEqual(req.method, .post) -// XCTAssertEqual(req.headers[.userAgent], "Mozilla/4.0 (compatible; MSIE5.01; Windows NT)") -// XCTAssertEqual(req.headers[.host], "www.tutorialspoint.com") -// XCTAssertEqual(req.headers[.contentType], "text/plain") -// XCTAssertEqual(req.mediaType, .plainText) -// XCTAssertEqual(req.headers[.contentLength], "5") -// XCTAssertEqual(req.headers[.acceptLanguage], "en-us") -// XCTAssertEqual(req.headers[.acceptEncoding], "gzip, deflate") -// XCTAssertEqual(req.headers[.connection], "Keep-Alive") -// -// data = try req.body.makeData(max: 100_000).await(on: loop) -// XCTAssertEqual(String(data: data, encoding: .utf8), "hello") -// XCTAssert(completed) -// } -// -// func testResponse() throws { -// var data = """ -// HTTP/1.1 200 OK\r -// Date: Mon, 27 Jul 2009 12:28:53 GMT\r -// Server: Apache/2.2.14 (Win32)\r -// Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\r -// Content-Length: 7\r -// Content-Type: text/html\r -// Connection: Closed\r -// \r -// -// """.data(using: .utf8) ?? Data() -// -// let parser = HTTPResponseParser() -// var message: HTTPResponse? -// var completed = false -// -// parser.drain { _message in -// message = _message -// }.catch { error in -// XCTFail("\(error)") -// }.finally { -// completed = true -// } -// -// XCTAssertNil(message) -// try parser.next(data.withByteBuffer { $0 }).await(on: loop) -// parser.close() -// -// guard let res = message else { -// XCTFail("No request parsed") -// return -// } -// -// XCTAssertEqual(res.status, .ok) -// XCTAssertEqual(res.headers[.date], "Mon, 27 Jul 2009 12:28:53 GMT") -// XCTAssertEqual(res.headers[.server], "Apache/2.2.14 (Win32)") -// XCTAssertEqual(res.headers[.lastModified], "Wed, 22 Jul 2009 19:15:56 GMT") -// XCTAssertEqual(res.headers[.contentLength], "7") -// XCTAssertEqual(res.headers[.contentType], "text/html") -// XCTAssertEqual(res.mediaType, .html) -// XCTAssertEqual(res.headers[.connection], "Closed") -// -// data = try res.body.makeData(max: 100_000).blockingAwait() -// XCTAssertEqual(String(data: data, encoding: .utf8), "") -// XCTAssert(completed) -// } -// -// func testTooLargeRequest() throws { -// let data = """ -// POST /cgi-bin/process.cgi HTTP/1.1\r -// User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r -// Host: www.tutorialspoint.com\r -// Content-Type: text/plain\r -// Content-Length: 5\r -// Accept-Language: en-us\r -// Accept-Encoding: gzip, deflate\r -// Connection: Keep-Alive\r -// \r -// hello -// """.data(using: .utf8) ?? Data() -// -// var error = false -// let p = HTTPRequestParser() -// p.maxHeaderSize = data.count - 20 // body -// let parser = p -// -// var completed = false -// -// _ = parser.drain { _ in -// XCTFail() -// }.catch { _ in -// error = true -// }.finally { -// completed = true -// } -// -// try parser.next(data.withByteBuffer { $0 }).await(on: loop) -// parser.close() -// XCTAssert(error) -// XCTAssert(completed) -// } - -// func testTooLargeResponse() throws { -// let data = """ -// HTTP/1.1 200 OK\r -// Date: Mon, 27 Jul 2009 12:28:53 GMT\r -// Server: Apache/2.2.14 (Win32)\r -// Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\r -// Content-Length: 7\r -// Content-Type: text/html\r -// Connection: Closed\r -// \r -// -// """.data(using: .utf8) ?? Data() -// -// var error = false -// let p = HTTPResponseParser() -// p.maxHeaderSize = data.count - 20 // body -// let parser = p.stream(on: loop) -// -// var completed = false -// -// _ = parser.drain { _ in -// XCTFail() -// }.catch { _ in -// error = true -// }.finally { -// completed = true -// } -// -// -// try parser.next(data.withByteBuffer { $0 }).await(on: loop) -// try parser.next(data.withByteBuffer { $0 }).await(on: loop) -// parser.close() -// XCTAssert(error) -// XCTAssert(completed) -// } - -// static let allTests = [ -// ("testRequest", testRequest), -// ("testResponse", testResponse), -// ("testTooLargeRequest", testTooLargeRequest), -// ("testTooLargeResponse", testTooLargeResponse), -// ] -} - - diff --git a/Tests/HTTPTests/ProtocolTester.swift b/Tests/HTTPTests/ProtocolTester.swift new file mode 100644 index 00000000..dfb16a5a --- /dev/null +++ b/Tests/HTTPTests/ProtocolTester.swift @@ -0,0 +1,171 @@ +import Async +import Bits + +public final class ProtocolTester: Async.OutputStream { + /// See `OutputStream.Output` + public typealias Output = ByteBuffer + + /// Stream being tested + public var downstream: AnyInputStream? + + /// See `OutputStream.output` + public func output(to inputStream: S) where S: Async.InputStream, ProtocolTester.Output == S.Input { + downstream = .init(inputStream) + } + + /// Callback to indicate test is restarting + private var reset: () -> () + + /// Callback to indicate a test failure + private var fail: (String, StaticString, UInt) -> () + + /// The added checks + private var checks: [ProtocolTesterCheck] + + /// The test data + private var data: String + + /// Creates a new `ProtocolTester` + public init(data: String, onFail: @escaping (String, StaticString, UInt) -> (), reset: @escaping () -> ()) { + self.reset = reset + self.fail = onFail + self.data = data + checks = [] + } + + /// Adds a "before" offset assertion to the tester. + public func assert(before substring: String, file: StaticString = #file, line: UInt = #line, callback: @escaping () throws -> ()) { + let check = ProtocolTesterCheck(minOffset: nil, maxOffset: data.offset(of: substring), file: file, line: line, checks: callback) + checks.append(check) + } + + /// Adds an "after" offset assertion to the tester. + public func assert(after substring: String, file: StaticString = #file, line: UInt = #line, callback: @escaping () throws -> ()) { + let check = ProtocolTesterCheck(minOffset: data.offset(of: substring), maxOffset: nil, file: file, line: line, checks: callback) + checks.append(check) + } + + /// Runs the protocol tester w/ the supplied input + public func run() -> Future { + Swift.assert(downstream != nil, "ProtocolTester must be connected before running") + let buffer = data.buffer + return runMax(buffer, max: buffer.count) + } + + /// Recurisvely runs tests, splitting the supplied buffer until max == 0 + private func runMax(_ buffer: ByteBuffer, max: Int) -> Future { + if max > 0 { + let maxSizedChunksCount = buffer.count / max + let lastChunkSize = buffer.count % max + + var chunks: [ByteBuffer] = [] + + for i in 0.. 0 { + let lastChunk = ByteBuffer(start: buffer.baseAddress?.advanced(by: buffer.count - lastChunkSize), count: lastChunkSize) + chunks.insert(lastChunk, at: 0) + } + + reset() + return runChunks(chunks, currentOffset: 0).flatMap(to: Void.self) { + return self.runMax(buffer, max: max - 1) + } + } else { + downstream?.close() + return .done + } + } + + /// Recursively passes each chunk to downstream until chunks.count == 0 + private func runChunks(_ chunks: [ByteBuffer], currentOffset: Int) -> Future { + var chunks = chunks + if let chunk = chunks.popLast() { + runChecks(offset: currentOffset, chunks: chunks) + return downstream!.next(chunk).flatMap(to: Void.self) { _ in + return self.runChunks(chunks, currentOffset: currentOffset + chunk.count) + } + } else { + runChecks(offset: currentOffset, chunks: chunks) + return .done + } + } + + /// Runs checks for the supplied offset. + private func runChecks(offset: Int, chunks: [ByteBuffer]) { + for check in checks { + var shouldRun = false + if let min = check.minOffset, offset >= min { + shouldRun = true + } + if let max = check.maxOffset, offset < max { + shouldRun = true + } + if shouldRun { + do { + try check.checks() + } catch { + var message = "Protocol test failed: \(error)" + let data = chunks.reversed().map { "[" + ProtocolTester.dataDebug(for: $0) + "]" }.joined(separator: " ") + let text = chunks.reversed().map { "[" + ProtocolTester.textDebug(for: $0) + "]" }.joined(separator: " ") + message += "\nData: \(data)" + message += "\nText: \(text)" + self.fail(message, check.file, check.line) + } + } + } + } + + /// Creates TEXT formatted debug string for a ByteBuffer + private static func textDebug(for buffer: ByteBuffer) -> String { + let string = String(bytes: buffer, encoding: .ascii) ?? "n/a" + return string + .replacingOccurrences(of: "\r", with: "\\r") + .replacingOccurrences(of: "\n", with: "\\n") + } + + /// Create HEX formatted debug string for a ByteBuffer + private static func dataDebug(for buffer: ByteBuffer) -> String { + var string = "0x" + for i in 0..> 4) + let lower = Int(byte & 0b00001111) + string.append(hexMap[upper]) + string.append(hexMap[lower]) + } + return string + } + + /// HEX map. + private static let hexMap = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "A", "B", "C", "D", "E", "F"] +} + +/// A stored protocol tester check. +private struct ProtocolTesterCheck { + var minOffset: Int? + var maxOffset: Int? + var file: StaticString + var line: UInt + var checks: () throws -> () +} + +extension String { + /// Byte buffer representation + /// Note: String must be static or a reference held for the duration of this buffer's use + var buffer: ByteBuffer { + return self.data(using: .utf8)!.withByteBuffer { $0 } + } +} + +extension String { + /// Returns int offset of the supplied string, crashing if it doesn't exist + fileprivate func offset(of string: String) -> Int { + return range(of: string)!.upperBound.encodedOffset + } +} + +extension String: Error {} From 4a6167c14bc4b91c73862f7fa0e8082e09c9cad4 Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Thu, 25 Jan 2018 13:18:21 -0500 Subject: [PATCH 04/16] chttp header copy optimizations --- Sources/HTTP/Message/HTTPHeaders.swift | 8 +- Sources/HTTP/Parser/CHTTPParseResults.swift | 97 ---- Sources/HTTP/Parser/CHTTPParser.swift | 461 ++--------------- Sources/HTTP/Parser/CHTTPParserContext.swift | 491 +++++++++++++++++++ Sources/HTTP/Parser/HTTPRequestParser.swift | 23 +- Tests/HTTPTests/HTTPParserTests.swift | 57 ++- Tests/HTTPTests/ProtocolTester.swift | 7 + 7 files changed, 613 insertions(+), 531 deletions(-) delete mode 100644 Sources/HTTP/Parser/CHTTPParseResults.swift create mode 100644 Sources/HTTP/Parser/CHTTPParserContext.swift diff --git a/Sources/HTTP/Message/HTTPHeaders.swift b/Sources/HTTP/Message/HTTPHeaders.swift index 59d712fa..c47ea4a9 100644 --- a/Sources/HTTP/Message/HTTPHeaders.swift +++ b/Sources/HTTP/Message/HTTPHeaders.swift @@ -14,13 +14,9 @@ import Bits /// /// [Learn More →](https://docs.vapor.codes/3.0/http/headers/) public struct HTTPHeaders: Codable { - struct Index: CustomStringConvertible, CustomDebugStringConvertible { + struct Index: CustomStringConvertible { var description: String { - return "" - } - - var debugDescription: String { - return "" + return "[\(nameStartIndex)..<\(nameEndIndex):\(valueStartIndex)..<\(valueEndIndex)]" } var nameStartIndex: Int diff --git a/Sources/HTTP/Parser/CHTTPParseResults.swift b/Sources/HTTP/Parser/CHTTPParseResults.swift deleted file mode 100644 index 9301d437..00000000 --- a/Sources/HTTP/Parser/CHTTPParseResults.swift +++ /dev/null @@ -1,97 +0,0 @@ -import Async -import Bits -import CHTTP -import Dispatch -import Foundation - -/// The parse results object helps get around -/// the issue of not being able to capture context -/// with C closures. -/// -/// All C closures must be sent some object that -/// this parse results object can be retreived from. -/// -/// See the convenience methods below to see how the -/// object is set and fetched from the C object. -internal final class CParseResults { - /// If true, all of the headers have been sent. - var headersComplete: Bool - - /// If true, the entire message has been parsed. - var messageComplete: Bool - - /// The current header parsing state (field, value, etc) - var headerState: CHTTPHeaderState - - /// The current body parsing state - var bodyState: CHTTPBodyState - - /// The HTTP method (only set for requests) - var method: http_method? - - // The HTTP version - var version: HTTPVersion? - - var headersIndexes: [HTTPHeaders.Index] - var headersData = [UInt8]() - var currentSize: Int = 0 - var maxHeaderSize: Int? - var contentLength: Int? - var headers: HTTPHeaders? - var url = [UInt8]() - - /// Creates a new results object - init() { - self.headersComplete = false - self.messageComplete = false - self.headersIndexes = [] - headersData.reserveCapacity(4096) - headersIndexes.reserveCapacity(64) - url.reserveCapacity(128) - self.maxHeaderSize = 100_000 - self.headerState = .none - self.bodyState = .none - } - - func addSize(_ n: Int) -> Bool { - if let maxHeaderSize = maxHeaderSize { - guard currentSize + n <= maxHeaderSize else { - return false - } - - self.currentSize += n - } - - return true - } -} - -// MARK: Convenience - -extension CParseResults { - /// Sets the parse results object on a C parser - static func set(on parser: inout http_parser) -> CParseResults { - let results = UnsafeMutablePointer.allocate(capacity: 1) - let new = CParseResults() - results.initialize(to: new) - parser.data = UnsafeMutableRawPointer(results) - return new - } - - static func remove(from parser: inout http_parser) { - if let results = parser.data { - let pointer = results.assumingMemoryBound(to: CParseResults.self) - pointer.deinitialize() - pointer.deallocate(capacity: 1) - } - } - - /// Fetches the parse results object from the C parser - static func get(from parser: UnsafePointer?) -> CParseResults? { - return parser? - .pointee - .data - .assumingMemoryBound(to: CParseResults.self) - .pointee - } -} diff --git a/Sources/HTTP/Parser/CHTTPParser.swift b/Sources/HTTP/Parser/CHTTPParser.swift index 17dd4b7c..134c2a75 100644 --- a/Sources/HTTP/Parser/CHTTPParser.swift +++ b/Sources/HTTP/Parser/CHTTPParser.swift @@ -6,64 +6,14 @@ import Foundation /// Internal CHTTP parser protocol internal protocol CHTTPParser: HTTPParser where Input == ByteBuffer { - /// This parser's type (request or response) - static var parserType: http_parser_type { get } - - /// If set, header data exceeding the specified size will result in an error. - var maxHeaderSize: Int? { get set } + /// Current downstream. + var downstream: AnyInputStream? { get set } /// Holds the CHTTP parser's internal state. - var chttp: CHTTPParserContext { get set } + var chttp: CHTTPParserContext { get set } /// Converts the CHTTP parser results and body to HTTP message. - func makeMessage(from results: CParseResults, using body: HTTPBody) throws -> Output -} - -/// Possible header states -enum CHTTPHeaderState { - case none - case value(HTTPHeaders.Index) - case key(startIndex: Int, endIndex: Int) -} - -enum CHTTPMessageState { - case parsing - case streaming(Message, Future) - case waiting(Future) -} - -/// Possible body states -enum CHTTPBodyState { - case none - case buffer(ByteBuffer) - case stream(CHTTPBodyStream) - case readyStream(CHTTPBodyStream, Promise) -} - -/// Maintains the CHTTP parser's internal state. -struct CHTTPParserContext { - /// Whether the parser is currently parsing or hasn't started yet - var isParsing: Bool - - /// Parser's message - var messageState: CHTTPMessageState - - /// The CHTTP parser's C struct - var parser: http_parser - - /// The CHTTP parer's C settings - var settings: http_parser_settings - - /// Current downstream. - var downstream: AnyInputStream? - - /// Creates a new `CHTTPParserContext` - init() { - self.parser = http_parser() - self.settings = http_parser_settings() - self.isParsing = false - self.messageState = .parsing - } + func makeMessage(using body: HTTPBody) throws -> Output } /// MARK: CHTTPParser OutputStream @@ -71,7 +21,7 @@ struct CHTTPParserContext { extension CHTTPParser { /// See `OutputStream.output(to:)` public func output(to inputStream: S) where S: Async.InputStream, Self.Output == S.Input { - chttp.downstream = .init(inputStream) + downstream = .init(inputStream) } } @@ -80,419 +30,104 @@ extension CHTTPParser { extension CHTTPParser { /// See `InputStream.input(_:)` public func input(_ event: InputEvent) { + guard let downstream = self.downstream else { + fatalError("Unexpected `nil` downstream on CHTTPParser.input(close)") + } + switch event { case .close: chttp.close() - chttp.downstream!.close() - case .error(let error): chttp.downstream!.error(error) - case .next(let input, let ready): try! handleNext(input, ready) + downstream.close() + case .error(let error): + downstream.error(error) + case .next(let input, let ready): + do { + try handleNext(input, ready, downstream) + } catch { + downstream.error(error) + } } } /// See `InputEvent.next` - private func handleNext(_ buffer: ByteBuffer, _ ready: Promise) throws { - guard let results = chttp.getResults() else { - throw HTTPError(identifier: "getResults", reason: "An internal HTTP Parser state became invalid") - } - - switch chttp.messageState { + private func handleNext(_ buffer: ByteBuffer, _ ready: Promise, _ downstream: AnyInputStream) throws { + switch chttp.state { case .parsing: /// Parse the message using the CHTTP parser. try chttp.execute(from: buffer) + /// Copies raw header data from the buffer + chttp.copyHeaders(from: buffer) + /// Check if we have received all of the messages headers - if results.headersComplete { + if chttp.headersComplete { /// Either streaming or static will be decided let body: HTTPBody /// The message is ready to move downstream, check to see /// if we already have the HTTPBody in its entirety - if results.messageComplete { - switch results.bodyState { + if chttp.messageComplete { + switch chttp.bodyState { case .buffer(let buffer): body = HTTPBody(Data(buffer)) case .none: body = HTTPBody() case .stream: fatalError("Illegal state") case .readyStream: fatalError("Illegal state") } - let message = try makeMessage(from: results, using: body) - chttp.downstream!.next(message, ready) - - // the results have completed, so we are ready - // for a new request to come in - chttp.isParsing = false - CParseResults.remove(from: &chttp.parser) + print(chttp.headersIndexes) + let message = try makeMessage(using: body) + downstream.next(message, ready) + chttp.reset() } else { // Convert body to a stream let stream = CHTTPBodyStream() - switch results.bodyState { + switch chttp.bodyState { case .buffer(let buffer): stream.push(buffer, ready) case .none: stream.push(ByteBuffer(start: nil, count: 0), ready) case .stream: fatalError("Illegal state") case .readyStream: fatalError("Illegal state") } - results.bodyState = .stream(stream) - body = HTTPBody(size: results.contentLength, stream: .init(stream)) - let message = try makeMessage(from: results, using: body) - let future = chttp.downstream!.next(message) - chttp.messageState = .streaming(message, future) + chttp.bodyState = .stream(stream) + body = HTTPBody(size: chttp.contentLength, stream: .init(stream)) + print(chttp.headersIndexes) + let message = try makeMessage(using: body) + let nextMessageFuture = downstream.next(message) + chttp.state = .streaming(nextMessageFuture) } } else { /// Headers not complete, request more input ready.complete() } - case .streaming(_, let future): + case .streaming(let nextMessageFuture): let stream: CHTTPBodyStream /// Close the body stream now - switch results.bodyState { + switch chttp.bodyState { case .none: fatalError("Illegal state") case .buffer: fatalError("Illegal state") case .readyStream: fatalError("Illegal state") case .stream(let s): stream = s // replace body state w/ new ready - results.bodyState = .readyStream(s, ready) + chttp.bodyState = .readyStream(s, ready) } /// Parse the message using the CHTTP parser. try chttp.execute(from: buffer) - if results.messageComplete { + if chttp.messageComplete { /// Close the body stream now stream.close() - chttp.messageState = .waiting(future) + chttp.state = .streamingClosed(nextMessageFuture) } - case .waiting(let future): - // the results have completed, so we are ready - // for a new request to come in - chttp.isParsing = false - CParseResults.remove(from: &chttp.parser) - chttp.messageState = .parsing - future.do { - try! self.handleNext(buffer, ready) + case .streamingClosed(let nextMessageFuture): + chttp.reset() + nextMessageFuture.map(to: Void.self) { + return try self.handleNext(buffer, ready, downstream) }.catch { error in - fatalError("\(error)") - } - } - } - - /// Resets the parser - public func reset() { - chttp.reset(Self.parserType) - } -} - -/// MARK: CHTTP integration - -extension CHTTPParserContext { - /// Parses a generic CHTTP message, filling the - /// ParseResults object attached to the C praser. - internal mutating func execute(from buffer: ByteBuffer) throws { - // call the CHTTP parser - let parsedCount = http_parser_execute(&parser, &settings, buffer.cPointer, buffer.count) - - // if the parsed count does not equal the bytes passed - // to the parser, it is signaling an error - // - 1 to allow room for filtering a possibly final \r\n which I observed the parser does - guard parsedCount >= buffer.count - 2, parsedCount <= buffer.count else { - throw HTTPError.invalidMessage() - } - } - - /// Resets the parser - internal mutating func reset(_ type: http_parser_type) { - http_parser_init(&parser, type) - initialize() - } -} - -extension CParseResults { - func parseContentLength(index: HTTPHeaders.Index) { - if self.contentLength == nil { - let namePointer = UnsafePointer(self.headersData).advanced(by: index.nameStartIndex) - let nameLength = index.nameEndIndex - index.nameStartIndex - let nameBuffer = ByteBuffer(start: namePointer, count: nameLength) - - if lowercasedContentLength.caseInsensitiveEquals(to: nameBuffer) { - let pointer = UnsafePointer(self.headersData).advanced(by: index.valueStartIndex) - let length = index.valueEndIndex - index.valueStartIndex - - pointer.withMemoryRebound(to: Int8.self, capacity: length) { pointer in - self.contentLength = numericCast(strtol(pointer, nil, 10)) - } - } - } - } -} - -extension CHTTPParserContext { - /// Fetches `CParseResults` from the praser. - mutating func getResults() -> CParseResults? { - let results: CParseResults - if isParsing { - // get the current parse results object - guard let existingResults = CParseResults.get(from: &parser) else { - return nil - } - results = existingResults - } else { - // create a new results object and set - // a reference to it on the parser - let newResults = CParseResults.set(on: &parser) - results = newResults - isParsing = true - } - return results - } - - /// Indicates a close to the HTTP parser. - mutating func close() { - http_parser_execute(&parser, &settings, nil, 0) - } - - /// Initializes the http parser settings with appropriate callbacks. - mutating func initialize() { - // called when chunks of the url have been read - settings.on_url = { parser, chunkPointer, length in - print("chttp: on_url '\(String(data: Data(chunkPointer!.makeBuffer(length: length)), encoding: .ascii)!)' (\(length)) ") - guard - let results = CParseResults.get(from: parser), - let chunkPointer = chunkPointer - else { - // signal an error - return 1 - } - - guard results.addSize(length) else { - return 1 - } - - // append the url bytes to the results - chunkPointer.withMemoryRebound(to: UInt8.self, capacity: length) { chunkPointer in - results.url.append(contentsOf: ByteBuffer(start: chunkPointer, count: length)) - } - - return 0 - } - - // called when chunks of a header field have been read - settings.on_header_field = { parser, chunkPointer, length in - print("chttp: on_header_field '\(String(data: Data(chunkPointer!.makeBuffer(length: length)), encoding: .ascii)!)' (\(length)) ") - guard - let results = CParseResults.get(from: parser), - let chunkPointer = chunkPointer - else { - // signal an error - return 1 - } - - guard results.addSize(length + 4) else { // + ": \r\n" - return 1 - } - - // check current header parsing state - switch results.headerState { - case .none: - // nothing is being parsed, start a new key - results.headerState = .key(startIndex: results.headersData.count, endIndex: results.headersData.count + length) - case .value(let index): - // there was previously a value being parsed. - // it is now finished. - - results.headersIndexes.append(index) - - results.headersData.append(.carriageReturn) - results.headersData.append(.newLine) - - results.parseContentLength(index: index) - - // start a new key - results.headerState = .key(startIndex: results.headersData.count, endIndex: results.headersData.count + length) - case .key(let start, let end): - // there is a key currently being parsed. - results.headerState = .key(startIndex: start, endIndex: end + length) - } - - chunkPointer.withMemoryRebound(to: UInt8.self, capacity: length) { chunkPointer in - results.headersData.append(contentsOf: ByteBuffer(start: chunkPointer, count: length)) - } - - return 0 - } - - // called when chunks of a header value have been read - settings.on_header_value = { parser, chunkPointer, length in - print("chttp: on_header_value '\(String(data: Data(chunkPointer!.makeBuffer(length: length)), encoding: .ascii)!)' (\(length)) ") - guard - let results = CParseResults.get(from: parser), - let chunkPointer = chunkPointer - else { - // signal an error - return 1 - } - - guard results.addSize(length + 2) else { // + "\r\n" - return 1 - } - - // check the current header parsing state - switch results.headerState { - case .none: - // nothing has been parsed, so this - // value is useless. - // (this should never be reached) - results.headerState = .none - case .value(var index): - // there was previously a value being parsed. - // add the new bytes to it. - index.nameEndIndex += length - results.headerState = .value(index) - case .key(let key): - // there was previously a key being parsed. - // it is now finished. - results.headersData.append(contentsOf: headerSeparator) - - // Set a dummy hashvalue - let index = HTTPHeaders.Index( - nameStartIndex: key.startIndex, - nameEndIndex: key.endIndex, - valueStartIndex: results.headersData.count, - valueEndIndex: results.headersData.count + length, - invalidated: false - ) - - results.headerState = .value(index) - } - - chunkPointer.withMemoryRebound(to: UInt8.self, capacity: length) { chunkPointer in - results.headersData.append(contentsOf: ByteBuffer(start: chunkPointer, count: length)) - } - - return 0 - } - - // called when header parsing has completed - settings.on_headers_complete = { parser in - print("chttp: on_headers_complete") - guard let parser = parser, let results = CParseResults.get(from: parser) else { - // signal an error - return 1 - } - - // check the current header parsing state - switch results.headerState { - case .value(let index): - // there was previously a value being parsed. - // it should be added to the headers dict. - - results.headersIndexes.append(index) - results.headersData.append(.carriageReturn) - results.headersData.append(.newLine) - - results.parseContentLength(index: index) - - let headers = HTTPHeaders(storage: results.headersData, indexes: results.headersIndexes) - - /// FIXME: what was this doing? -// if let contentLength = results.contentLength { -// results.body = HTTPBody(size: contentLength, stream: AnyOutputStream(results.bodyStream)) -// } - - results.headers = headers - default: - // no other cases need to be handled. - break - } - - // parse version - let major = Int(parser.pointee.http_major) - let minor = Int(parser.pointee.http_minor) - results.version = HTTPVersion(major: major, minor: minor) - results.method = http_method(parser.pointee.method) - results.headersComplete = true - - return 0 - } - - // called when chunks of the body have been read - settings.on_body = { parser, chunk, length in - print("chttp: on_body '\(String(data: Data(chunk!.makeBuffer(length: length)), encoding: .ascii)!)' (\(length)) ") - guard let results = CParseResults.get(from: parser), let chunk = chunk else { - // signal an error - return 1 - } - - switch results.bodyState { - case .buffer: fatalError("Unexpected buffer body state during CHTTP.on_body: \(results.bodyState)") - case .none: results.bodyState = .buffer(chunk.makeByteBuffer(length)) - case .stream: fatalError("Illegal state") - case .readyStream(let bodyStream, let ready): - bodyStream.push(chunk.makeByteBuffer(length), ready) - results.bodyState = .stream(bodyStream) // no longer ready - } - - return 0 -// return chunk.withMemoryRebound(to: UInt8.self, capacity: length) { pointer -> Int32 in -// results.bodyStream.push(ByteBuffer(start: pointer, count: length)) -// -// return 0 -// } - } - - // called when the message is finished parsing - settings.on_message_complete = { parser in - print("chttp: on_message_complete") - guard let parser = parser, let results = CParseResults.get(from: parser) else { - // signal an error - return 1 + downstream.error(error) + ready.complete() } - - // mark the results as complete - results.messageComplete = true - - return 0 - } - } -} - -// MARK: Utilities - -extension UnsafeBufferPointer where Element == Byte { - fileprivate var cPointer: UnsafePointer { - return baseAddress.unsafelyUnwrapped.withMemoryRebound(to: CChar.self, capacity: count) { $0 } - } -} - -fileprivate let headerSeparator: [UInt8] = [.colon, .space] -fileprivate let lowercasedContentLength = HTTPHeaders.Name.contentLength.lowercased - -fileprivate extension Data { - fileprivate var cPointer: UnsafePointer { - return withUnsafeBytes { $0 } - } -} - -fileprivate extension UnsafePointer where Pointee == CChar { - /// Creates a Bytes array from a C pointer - fileprivate func makeByteBuffer(_ count: Int) -> ByteBuffer { - return withMemoryRebound(to: Byte.self, capacity: count) { pointer in - return ByteBuffer(start: pointer, count: count) - } - } - - /// Creates a Bytes array from a C pointer - fileprivate func makeBuffer(length: Int) -> UnsafeRawBufferPointer { - let pointer = UnsafeBufferPointer(start: self, count: length) - - guard let base = pointer.baseAddress else { - return UnsafeRawBufferPointer(start: nil, count: 0) - } - - return base.withMemoryRebound(to: UInt8.self, capacity: length) { pointer in - return UnsafeRawBufferPointer(start: pointer, count: length) } } } - - diff --git a/Sources/HTTP/Parser/CHTTPParserContext.swift b/Sources/HTTP/Parser/CHTTPParserContext.swift new file mode 100644 index 00000000..0f1707ac --- /dev/null +++ b/Sources/HTTP/Parser/CHTTPParserContext.swift @@ -0,0 +1,491 @@ +import Async +import Bits +import CHTTP + +/// Maintains the CHTTP parser's internal state. +internal final class CHTTPParserContext { + /// Parser's message + var state: CHTTPParserState + + /// The CHTTP parser's C struct + fileprivate var parser: http_parser + + /// The CHTTP parer's C settings + fileprivate var settings: http_parser_settings + + /// If true, the start line has been parsed. + var startLineComplete: Bool + + /// If true, all of the headers have been sent. + var headersComplete: Bool + + /// If true, the entire message has been parsed. + var messageComplete: Bool + + /// The current header parsing state (field, value, etc) + var headerState: CHTTPHeaderState + + /// The current body parsing state + var bodyState: CHTTPBodyState + + /// The HTTP method (only set for requests) + var method: http_method? + + // The HTTP version + var version: HTTPVersion? + + var maxStartLineAndHeadersSize: Int + var contentLength: Int? + + var headers: HTTPHeaders? + var headersIndexes: [HTTPHeaders.Index] + + private var currentURLSize: Int + private var currentHeadersSize: Int + + var headerStart: UnsafePointer? + var headerStartOffset: Int + var bodyStart: UnsafePointer? + + var headersData: [UInt8] + var urlData: [UInt8] + + /// Creates a new `CHTTPParserContext` + init(_ type: http_parser_type) { + self.parser = http_parser() + self.settings = http_parser_settings() + self.state = .parsing + self.startLineComplete = false + self.headersComplete = false + self.messageComplete = false + self.currentHeadersSize = 0 + self.currentURLSize = 0 + self.maxStartLineAndHeadersSize = 100_000 + self.headerState = .none + self.bodyState = .none + self.headersIndexes = [] + urlData = [] + headersData = [] + headerStart = nil + headerStartOffset = 0 + headersIndexes.reserveCapacity(64) + set(on: &self.parser) + http_parser_init(&parser, type) + initialize() + } +} + +/// Current parser message state. +enum CHTTPParserState { + /// Currently parsing an HTTP message, not yet streaming. + case parsing + /// We are currently streaming HTTP body to a message. + /// The future contained is when downstream will be ready for a new HTTP message. + case streaming(Future) + /// Previously streaming an HTTP body, now finished. + /// The future contained is when downstream will be ready for a new HTTP message. + case streamingClosed(Future) +} + +/// Possible header states +enum CHTTPHeaderState { + case none + case value(HTTPHeaders.Index) + case key(startIndex: Int, endIndex: Int) +} + + +/// Possible body states +enum CHTTPBodyState { + case none + case buffer(ByteBuffer) + case stream(CHTTPBodyStream) + case readyStream(CHTTPBodyStream, Promise) +} + + +/// MARK: Internal + +extension CHTTPParserContext { + /// Parses a generic CHTTP message, filling the + /// ParseResults object attached to the C praser. + internal func execute(from buffer: ByteBuffer) throws { + // call the CHTTP parser + let parsedCount = http_parser_execute(&parser, &settings, buffer.cPointer, buffer.count) + + // if the parsed count does not equal the bytes passed + // to the parser, it is signaling an error + // - 1 to allow room for filtering a possibly final \r\n which I observed the parser does + guard parsedCount >= buffer.count - 2, parsedCount <= buffer.count else { + throw HTTPError.invalidMessage() + } + } + + /// Resets the parser context, preparing it for a new message. + internal func reset() { + print(" RESET") + startLineComplete = false + headersComplete = false + messageComplete = false + headerState = .none + bodyState = .none + state = .parsing + urlData = [] + headersData = [] + currentURLSize = 0 + currentHeadersSize = 0 + headersIndexes = [] + headersIndexes.reserveCapacity(64) + headers = nil + method = nil + headerStart = nil + headerStartOffset = 0 + bodyStart = nil + } + + internal func copyHeaders(from buffer: ByteBuffer) { + print("copy headers") + guard startLineComplete else { + return + } + + /// start is known header start or buffer start + let start: UnsafePointer + if let headerStart = self.headerStart { + start = headerStart + } else { + start = buffer.cPointer + } + + /// end is known body start or buffer end + let end: UnsafePointer + if let bodyStart = self.bodyStart { + // end of headers is the body start + end = bodyStart + } else { + // body hasn't started yet + // get the end of this buffer as *char + end = buffer.cPointer.advanced(by: buffer.count) + } + + let headerSize = start.distance(to: end) + // append the length of the headers in this buffer to the header start offset + headerStartOffset += start.distance(to: end) + let buffer = ByteBuffer(start: start.withMemoryRebound(to: Byte.self, capacity: headerSize) { $0 }, count: headerSize) + headersData.append(contentsOf: buffer) + headerStart = nil + + if headersComplete { + print(" set headers") + headers = HTTPHeaders(storage: headersData, indexes: headersIndexes) + } + } + + /// Indicates a close to the HTTP parser. + internal func close() { + http_parser_execute(&parser, &settings, nil, 0) + CHTTPParserContext.remove(from: &self.parser) + } +} + +/// MARK: C-Baton Access + +extension CHTTPParserContext { + /// Sets the parse results object on a C parser + fileprivate func set(on parser: inout http_parser) { + let results = UnsafeMutablePointer.allocate(capacity: 1) + results.initialize(to: self) + parser.data = UnsafeMutableRawPointer(results) + } + + fileprivate static func remove(from parser: inout http_parser) { + if let results = parser.data { + let pointer = results.assumingMemoryBound(to: CHTTPParserContext.self) + pointer.deinitialize() + pointer.deallocate(capacity: 1) + } + } + + /// Fetches the parse results object from the C parser + fileprivate static func get(from parser: UnsafePointer?) -> CHTTPParserContext? { + return parser? + .pointee + .data + .assumingMemoryBound(to: CHTTPParserContext.self) + .pointee + } +} + +/// Private methods + +extension CHTTPParserContext { + /// Returns true if adding the supplied length to the current + /// size is still within maximum size boundaries. + fileprivate func isUnderMaxSize() -> Bool { + guard (currentURLSize + currentHeadersSize) <= maxStartLineAndHeadersSize else { + return false + } + return true + } + + /// Initializes the http parser settings with appropriate callbacks. + fileprivate func initialize() { + // called when chunks of the url have been read + settings.on_url = { parser, chunk, count in + guard let results = CHTTPParserContext.get(from: parser), let chunk = chunk else { + // signal an error + return 1 + } + + // increase url count + results.currentURLSize += count + + // verify we are within max size limits + guard results.isUnderMaxSize() else { + // signal an error + return 1 + } + + /// FIXME: optimize url append + // append the url bytes to the results + chunk.withMemoryRebound(to: Byte.self, capacity: count) { chunkPointer in + let buffer = ByteBuffer(start: chunkPointer, count: count) + results.urlData.append(contentsOf: buffer) + } + + // return success + return 0 + } + + // called when chunks of a header field have been read + settings.on_header_field = { parser, chunk, count in + guard let results = CHTTPParserContext.get(from: parser), let chunk = chunk else { + // signal an error + return 1 + } + results.startLineComplete = true + + let start: UnsafePointer + if let existing = results.headerStart { + start = existing + } else { + results.headerStart = chunk + start = chunk + } + print("on_header_field") + + + // check current header parsing state + switch results.headerState { + case .none: + let distance = start.distance(to: chunk) + results.headerStartOffset + // nothing is being parsed, start a new key + results.headerState = .key(startIndex: distance, endIndex: distance + count) + case .value(let index): + let distance = start.distance(to: chunk) + results.headerStartOffset + // there was previously a value being parsed. + // it is now finished. + results.headersIndexes.append(index) + // start a new key + results.headerState = .key(startIndex: distance, endIndex: distance + count) + case .key(let start, let end): + // there is a key currently being parsed, extend the count index + results.headerState = .key(startIndex: start, endIndex: end + count) + } + + // verify total size has not exceeded max + results.currentHeadersSize += count + // verify we are within max size limits + guard results.isUnderMaxSize() else { + return 1 + } + + return 0 + } + + // called when chunks of a header value have been read + settings.on_header_value = { parser, chunk, count in + guard let results = CHTTPParserContext.get(from: parser), let chunk = chunk else { + // signal an error + return 1 + } + print("on_header_value") + + let start: UnsafePointer + if let existing = results.headerStart { + start = existing + } else { + results.headerStart = chunk + start = chunk + } + + // increase headers size + results.currentHeadersSize += count + + // verify we are within max size limits + guard results.isUnderMaxSize() else { + return 1 + } + + // check the current header parsing state + switch results.headerState { + case .none: fatalError("Illegal header state `none` during `on_header_value`") + case .value(var index): + // there was previously a value being parsed. + // add the new bytes to it. + index.valueEndIndex += count + results.headerState = .value(index) + case .key(let key): + // there was previously a value being parsed. + // it is now finished. + // results.headersData.append(contentsOf: headerSeparator) + + let distance = start.distance(to: chunk) + results.headerStartOffset + + // create a full HTTP headers index + let index = HTTPHeaders.Index( + nameStartIndex: key.startIndex, + nameEndIndex: key.endIndex, + valueStartIndex: distance, + valueEndIndex: distance + count, + invalidated: false + ) + results.headerState = .value(index) + } + return 0 + } + + // called when header parsing has completed + settings.on_headers_complete = { parser in + guard let parser = parser, let results = CHTTPParserContext.get(from: parser) else { + // signal an error + return 1 + } + print("on_headers_complete") + + // check the current header parsing state + switch results.headerState { + case .value(let index): + // there was previously a value being parsed. + // it is now finished. + results.headersIndexes.append(index) + + // let headers = HTTPHeaders(storage: results.headersData, indexes: results.headersIndexes) + + /// FIXME: what was this doing? + // if let contentLength = results.contentLength { + // results.body = HTTPBody(size: contentLength, stream: AnyOutputStream(results.bodyStream)) + // } + + // results.headers = headers + case .key: fatalError("Unexpected header state .key during on_headers_complete") + case .none: fatalError("Unexpected header state .none during on_headers_complete") + } + + // parse version + let major = Int(parser.pointee.http_major) + let minor = Int(parser.pointee.http_minor) + results.version = HTTPVersion(major: major, minor: minor) + results.method = http_method(parser.pointee.method) + results.headersComplete = true + + return 0 + } + + // called when chunks of the body have been read + settings.on_body = { parser, chunk, length in + guard let results = CHTTPParserContext.get(from: parser), let chunk = chunk else { + // signal an error + return 1 + } + print("on_body") + results.bodyStart = chunk + + switch results.bodyState { + case .buffer: fatalError("Unexpected buffer body state during CHTTP.on_body: \(results.bodyState)") + case .none: results.bodyState = .buffer(chunk.makeByteBuffer(length)) + case .stream: fatalError("Illegal state") + case .readyStream(let bodyStream, let ready): + bodyStream.push(chunk.makeByteBuffer(length), ready) + results.bodyState = .stream(bodyStream) // no longer ready + } + + return 0 + } + + // called when the message is finished parsing + settings.on_message_complete = { parser in + guard let parser = parser, let results = CHTTPParserContext.get(from: parser) else { + // signal an error + return 1 + } + print("on_message_complete") + + // mark the results as complete + results.messageComplete = true + + return 0 + } + } +} + +// MARK: Utilities + +extension UnsafeBufferPointer where Element == Byte { + fileprivate var cPointer: UnsafePointer { + return baseAddress.unsafelyUnwrapped.withMemoryRebound(to: CChar.self, capacity: count) { $0 } + } +} + +fileprivate let headerSeparator: [UInt8] = [.colon, .space] +fileprivate let lowercasedContentLength = HTTPHeaders.Name.contentLength.lowercased + +//fileprivate extension Data { +// fileprivate var cPointer: UnsafePointer { +// return withUnsafeBytes { $0 } +// } +//} + +fileprivate extension UnsafePointer where Pointee == CChar { + /// Creates a Bytes array from a C pointer + fileprivate func makeByteBuffer(_ count: Int) -> ByteBuffer { + return withMemoryRebound(to: Byte.self, capacity: count) { pointer in + return ByteBuffer(start: pointer, count: count) + } + } + + /// Creates a Bytes array from a C pointer + fileprivate func makeBuffer(length: Int) -> UnsafeRawBufferPointer { + let pointer = UnsafeBufferPointer(start: self, count: length) + + guard let base = pointer.baseAddress else { + return UnsafeRawBufferPointer(start: nil, count: 0) + } + + return base.withMemoryRebound(to: UInt8.self, capacity: length) { pointer in + return UnsafeRawBufferPointer(start: pointer, count: length) + } + } +} + + +//extension CHTTPParserContext { +// fileprivate func parseContentLength(index: HTTPHeaders.Index) { +// if self.contentLength == nil { +// let namePointer = UnsafePointer(self.headersData).advanced(by: index.nameStartIndex) +// let nameLength = index.nameEndIndex - index.nameStartIndex +// let nameBuffer = ByteBuffer(start: namePointer, count: nameLength) +// +// if lowercasedContentLength.caseInsensitiveEquals(to: nameBuffer) { +// let pointer = UnsafePointer(self.headersData).advanced(by: index.valueStartIndex) +// let length = index.valueEndIndex - index.valueStartIndex +// +// pointer.withMemoryRebound(to: Int8.self, capacity: length) { pointer in +// self.contentLength = numericCast(strtol(pointer, nil, 10)) +// } +// } +// } +// } +//} +// + diff --git a/Sources/HTTP/Parser/HTTPRequestParser.swift b/Sources/HTTP/Parser/HTTPRequestParser.swift index dfd6498e..e9be4b2b 100644 --- a/Sources/HTTP/Parser/HTTPRequestParser.swift +++ b/Sources/HTTP/Parser/HTTPRequestParser.swift @@ -12,29 +12,24 @@ public final class HTTPRequestParser: CHTTPParser { /// See `OutputStream.Output` public typealias Output = HTTPRequest - /// See CHTTPParser.parserType - static let parserType: http_parser_type = HTTP_REQUEST - /// See `CHTTPParser.chttpParserContext` - var chttp: CHTTPParserContext + var chttp: CHTTPParserContext - /// See `CHTTPParser.maxHeaderSize` - public var maxHeaderSize: Int? + /// Current downstream accepting parsed messages. + var downstream: AnyInputStream? /// Creates a new Request parser. public init() { - self.maxHeaderSize = 100_000 - self.chttp = .init() - reset() + self.chttp = .init(HTTP_REQUEST) } /// See `CHTTPParser.makeMessage(from:using:)` - func makeMessage(from results: CParseResults, using body: HTTPBody) throws -> HTTPRequest { + func makeMessage(using body: HTTPBody) throws -> HTTPRequest { // require a version to have been parsed guard - let version = results.version, - let headers = results.headers, - let cmethod = results.method + let version = chttp.version, + let headers = chttp.headers, + let cmethod = chttp.method else { throw HTTPError.invalidMessage() } @@ -68,7 +63,7 @@ public final class HTTPRequestParser: CHTTPParser { } // parse the uri from the url bytes. - var uri = URI(buffer: results.url) + var uri = URI(buffer: chttp.urlData) // if there is no scheme, use http by default if uri.scheme?.isEmpty == true { diff --git a/Tests/HTTPTests/HTTPParserTests.swift b/Tests/HTTPTests/HTTPParserTests.swift index b59a0c0b..93128d06 100644 --- a/Tests/HTTPTests/HTTPParserTests.swift +++ b/Tests/HTTPTests/HTTPParserTests.swift @@ -5,6 +5,53 @@ import HTTP import XCTest class HTTPParserTests: XCTestCase { + func testParserEdgeCasesOld() throws { + let firstChunk = "GET /hello HTTP/1.1\r\nContent-Type: ".data(using: .utf8)! + let secondChunk = "text/plain\r\nContent-Length: 5\r\n\r\nwo".data(using: .utf8)! + let thirdChunk = "rl".data(using: .utf8)! + let fourthChunk = "d".data(using: .utf8)! + + let parser = HTTPRequestParser() + + let socket = PushStream(ByteBuffer.self) + socket.stream(to: parser).drain { message in + print("parser.drain { ... }") + print(message) + print("message.body.makeData") + message.body.makeData(max: 100).do { data in + print(data) + }.catch { error in + print("body error: \(error)") + } + }.catch { error in + print("parser.catch { \(error) }") + }.finally { + print("parser.close { }") + } + + + print("(1) FIRST ---") + firstChunk.withByteBuffer(socket.push) + print("(2) SECOND ---") + secondChunk.withByteBuffer(socket.push) + print("(3) THIRD ---") + thirdChunk.withByteBuffer(socket.push) + print("(4) FOURTH ---") + fourthChunk.withByteBuffer(socket.push) + + print("(1) FIRST ---") + firstChunk.withByteBuffer(socket.push) + print("(2) SECOND ---") + secondChunk.withByteBuffer(socket.push) + print("(3) THIRD ---") + thirdChunk.withByteBuffer(socket.push) + print("(4) FOURTH ---") + fourthChunk.withByteBuffer(socket.push) + + print("(c) CLOSE ---") + socket.close() + } + func testParserEdgeCases() throws { // captured variables to check var request: HTTPRequest? @@ -28,9 +75,17 @@ class HTTPParserTests: XCTestCase { } tester.assert(after: "\r\n\r\n") { - guard request != nil else { + guard let req = request else { throw "request was nil" } + + guard req.headers[.contentType] == "text/plain" else { + throw "incorrect content type" + } + + guard req.headers[.contentLength] == "5" else { + throw "incorrect content length" + } } tester.assert(before: "world") { diff --git a/Tests/HTTPTests/ProtocolTester.swift b/Tests/HTTPTests/ProtocolTester.swift index dfb16a5a..264f6190 100644 --- a/Tests/HTTPTests/ProtocolTester.swift +++ b/Tests/HTTPTests/ProtocolTester.swift @@ -70,6 +70,12 @@ public final class ProtocolTester: Async.OutputStream { chunks.insert(lastChunk, at: 0) } + print("=== TEST: \(max)") + let data = chunks.reversed().map { "[" + ProtocolTester.dataDebug(for: $0) + "]" }.joined(separator: " ") + let text = chunks.reversed().map { "[" + ProtocolTester.textDebug(for: $0) + "]" }.joined(separator: " ") + print("Data: \(data)") + print("Text: \(text)") + reset() return runChunks(chunks, currentOffset: 0).flatMap(to: Void.self) { return self.runMax(buffer, max: max - 1) @@ -82,6 +88,7 @@ public final class ProtocolTester: Async.OutputStream { /// Recursively passes each chunk to downstream until chunks.count == 0 private func runChunks(_ chunks: [ByteBuffer], currentOffset: Int) -> Future { + print("--- run \(chunks.count)") var chunks = chunks if let chunk = chunks.popLast() { runChecks(offset: currentOffset, chunks: chunks) From 7c02193ab32edbaefc4aed311180b982e25ef49b Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Thu, 25 Jan 2018 14:05:39 -0500 Subject: [PATCH 05/16] parser context cleanup --- Sources/HTTP/Parser/CHTTPParser.swift | 2 +- Sources/HTTP/Parser/CHTTPParserContext.swift | 135 ++++++++++++------- 2 files changed, 89 insertions(+), 48 deletions(-) diff --git a/Sources/HTTP/Parser/CHTTPParser.swift b/Sources/HTTP/Parser/CHTTPParser.swift index 134c2a75..a16e94b1 100644 --- a/Sources/HTTP/Parser/CHTTPParser.swift +++ b/Sources/HTTP/Parser/CHTTPParser.swift @@ -88,7 +88,7 @@ extension CHTTPParser { case .readyStream: fatalError("Illegal state") } chttp.bodyState = .stream(stream) - body = HTTPBody(size: chttp.contentLength, stream: .init(stream)) + body = HTTPBody(size: nil, stream: .init(stream)) print(chttp.headersIndexes) let message = try makeMessage(using: body) let nextMessageFuture = downstream.next(message) diff --git a/Sources/HTTP/Parser/CHTTPParserContext.swift b/Sources/HTTP/Parser/CHTTPParserContext.swift index 0f1707ac..a1361464 100644 --- a/Sources/HTTP/Parser/CHTTPParserContext.swift +++ b/Sources/HTTP/Parser/CHTTPParserContext.swift @@ -4,15 +4,6 @@ import CHTTP /// Maintains the CHTTP parser's internal state. internal final class CHTTPParserContext { - /// Parser's message - var state: CHTTPParserState - - /// The CHTTP parser's C struct - fileprivate var parser: http_parser - - /// The CHTTP parer's C settings - fileprivate var settings: http_parser_settings - /// If true, the start line has been parsed. var startLineComplete: Bool @@ -22,52 +13,96 @@ internal final class CHTTPParserContext { /// If true, the entire message has been parsed. var messageComplete: Bool + + /// Parser's message + var state: CHTTPParserState + /// The current header parsing state (field, value, etc) var headerState: CHTTPHeaderState /// The current body parsing state var bodyState: CHTTPBodyState - /// The HTTP method (only set for requests) + + /// The parsed HTTP method var method: http_method? - // The HTTP version + /// The parsed HTTP version var version: HTTPVersion? - var maxStartLineAndHeadersSize: Int - var contentLength: Int? - + /// The parsed HTTP headers. var headers: HTTPHeaders? + + + /// Raw headers data + var headersData: [UInt8] + + /// Parsed indexes into the header data var headersIndexes: [HTTPHeaders.Index] + + /// Raw URL data + var urlData: [UInt8] + + + /// Maximum allowed size of the start line + headers data (not including some start line componenets and white space) + private var maxStartLineAndHeadersSize: Int + + /// Current URL size in start line (excluding other start line components) private var currentURLSize: Int + + /// Current size of header data exclusing whitespace private var currentHeadersSize: Int - var headerStart: UnsafePointer? - var headerStartOffset: Int - var bodyStart: UnsafePointer? - var headersData: [UInt8] - var urlData: [UInt8] + /// Pointer to the last start location of headers. + /// If not set, there have been no header start events yet. + private var headerStart: UnsafePointer? + + /// Current header start offset from previous run(s) of the parser + private var headerStartOffset: Int + + /// Pointer to the last start location of the body. + /// If not set, there have been no body start events yet. + private var bodyStart: UnsafePointer? + + + /// The CHTTP parser's C struct + fileprivate var parser: http_parser + + /// The CHTTP parer's C settings + fileprivate var settings: http_parser_settings /// Creates a new `CHTTPParserContext` init(_ type: http_parser_type) { - self.parser = http_parser() - self.settings = http_parser_settings() - self.state = .parsing self.startLineComplete = false self.headersComplete = false self.messageComplete = false - self.currentHeadersSize = 0 - self.currentURLSize = 0 - self.maxStartLineAndHeadersSize = 100_000 + + self.state = .parsing self.headerState = .none self.bodyState = .none + + self.method = nil + self.version = nil + self.headers = nil + + self.headersData = [] self.headersIndexes = [] - urlData = [] - headersData = [] - headerStart = nil - headerStartOffset = 0 + + self.urlData = [] + + self.maxStartLineAndHeadersSize = 100_000 + self.currentURLSize = 0 + self.currentHeadersSize = 0 + + self.headerStart = nil + self.headerStartOffset = 0 + self.bodyStart = nil + + self.parser = http_parser() + self.settings = http_parser_settings() + headersIndexes.reserveCapacity(64) set(on: &self.parser) http_parser_init(&parser, type) @@ -123,26 +158,32 @@ extension CHTTPParserContext { /// Resets the parser context, preparing it for a new message. internal func reset() { - print(" RESET") - startLineComplete = false - headersComplete = false - messageComplete = false - headerState = .none - bodyState = .none - state = .parsing - urlData = [] - headersData = [] - currentURLSize = 0 - currentHeadersSize = 0 - headersIndexes = [] - headersIndexes.reserveCapacity(64) - headers = nil - method = nil - headerStart = nil - headerStartOffset = 0 - bodyStart = nil + self.startLineComplete = false + self.headersComplete = false + self.messageComplete = false + + self.state = .parsing + self.headerState = .none + self.bodyState = .none + + self.method = nil + self.version = nil + self.headers = nil + + self.headersData = [] + self.headersIndexes = [] + + self.urlData = [] + + self.currentURLSize = 0 + self.currentHeadersSize = 0 + + self.headerStart = nil + self.headerStartOffset = 0 + self.bodyStart = nil } + /// Copies raw header data from the buffer into `headersData` internal func copyHeaders(from buffer: ByteBuffer) { print("copy headers") guard startLineComplete else { From 5195fbcee4eab6a766a55f3bfe3b6a329b8e5aa9 Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Thu, 25 Jan 2018 14:14:52 -0500 Subject: [PATCH 06/16] additional comments --- Sources/HTTP/Parser/CHTTPParserContext.swift | 117 ++++++------------- 1 file changed, 38 insertions(+), 79 deletions(-) diff --git a/Sources/HTTP/Parser/CHTTPParserContext.swift b/Sources/HTTP/Parser/CHTTPParserContext.swift index a1361464..48adc501 100644 --- a/Sources/HTTP/Parser/CHTTPParserContext.swift +++ b/Sources/HTTP/Parser/CHTTPParserContext.swift @@ -185,8 +185,9 @@ extension CHTTPParserContext { /// Copies raw header data from the buffer into `headersData` internal func copyHeaders(from buffer: ByteBuffer) { - print("copy headers") guard startLineComplete else { + /// we should not copy headers until the start line is complete + /// (there will be no `headerStart` pointer, and buffer start contains non-header data) return } @@ -194,6 +195,7 @@ extension CHTTPParserContext { let start: UnsafePointer if let headerStart = self.headerStart { start = headerStart + self.headerStart = nil } else { start = buffer.cPointer } @@ -209,15 +211,22 @@ extension CHTTPParserContext { end = buffer.cPointer.advanced(by: buffer.count) } - let headerSize = start.distance(to: end) + /// current distance from start to end + let distance = start.distance(to: end) + // append the length of the headers in this buffer to the header start offset - headerStartOffset += start.distance(to: end) - let buffer = ByteBuffer(start: start.withMemoryRebound(to: Byte.self, capacity: headerSize) { $0 }, count: headerSize) + headerStartOffset += distance + + /// create buffer view of current header data and append it + let buffer = ByteBuffer( + start: start.withMemoryRebound(to: Byte.self, capacity: distance) { $0 }, + count: distance + ) headersData.append(contentsOf: buffer) - headerStart = nil + /// if this buffer copy is happening after headers complete indication, + /// set the headers struct for later retreival if headersComplete { - print(" set headers") headers = HTTPHeaders(storage: headersData, indexes: headersIndexes) } } @@ -229,16 +238,18 @@ extension CHTTPParserContext { } } -/// MARK: C-Baton Access +/// MARK: C Baton Access extension CHTTPParserContext { - /// Sets the parse results object on a C parser + /// Sets C pointer for this context on the http_parser's data. + /// Use `CHTTPParserContext.get(from:)` to fetch back. fileprivate func set(on parser: inout http_parser) { let results = UnsafeMutablePointer.allocate(capacity: 1) results.initialize(to: self) parser.data = UnsafeMutableRawPointer(results) } + /// Removes C pointer from http_parser data fileprivate static func remove(from parser: inout http_parser) { if let results = parser.data { let pointer = results.assumingMemoryBound(to: CHTTPParserContext.self) @@ -247,7 +258,7 @@ extension CHTTPParserContext { } } - /// Fetches the parse results object from the C parser + /// Fetches the parse results object from the C http_parser data fileprivate static func get(from parser: UnsafePointer?) -> CHTTPParserContext? { return parser? .pointee @@ -304,8 +315,12 @@ extension CHTTPParserContext { // signal an error return 1 } + + /// Header fields are the first indication that the start-line has completed. results.startLineComplete = true + /// Get headerStart pointer. If nil, then there has not + /// been a header event yet. let start: UnsafePointer if let existing = results.headerStart { start = existing @@ -315,7 +330,6 @@ extension CHTTPParserContext { } print("on_header_field") - // check current header parsing state switch results.headerState { case .none: @@ -336,6 +350,7 @@ extension CHTTPParserContext { // verify total size has not exceeded max results.currentHeadersSize += count + // verify we are within max size limits guard results.isUnderMaxSize() else { return 1 @@ -352,6 +367,8 @@ extension CHTTPParserContext { } print("on_header_value") + /// Get headerStart pointer. If nil, then there has not + /// been a header event yet. let start: UnsafePointer if let existing = results.headerStart { start = existing @@ -370,17 +387,13 @@ extension CHTTPParserContext { // check the current header parsing state switch results.headerState { - case .none: fatalError("Illegal header state `none` during `on_header_value`") + case .none: fatalError("Unexpected headerState (.key) during chttp.on_header_value") case .value(var index): // there was previously a value being parsed. // add the new bytes to it. index.valueEndIndex += count results.headerState = .value(index) case .key(let key): - // there was previously a value being parsed. - // it is now finished. - // results.headersData.append(contentsOf: headerSeparator) - let distance = start.distance(to: chunk) + results.headerStartOffset // create a full HTTP headers index @@ -406,24 +419,13 @@ extension CHTTPParserContext { // check the current header parsing state switch results.headerState { - case .value(let index): - // there was previously a value being parsed. - // it is now finished. - results.headersIndexes.append(index) - - // let headers = HTTPHeaders(storage: results.headersData, indexes: results.headersIndexes) - - /// FIXME: what was this doing? - // if let contentLength = results.contentLength { - // results.body = HTTPBody(size: contentLength, stream: AnyOutputStream(results.bodyStream)) - // } - - // results.headers = headers - case .key: fatalError("Unexpected header state .key during on_headers_complete") - case .none: fatalError("Unexpected header state .none during on_headers_complete") + case .value(let index): results.headersIndexes.append(index) + case .key: fatalError("Unexpected headerState (.key) during chttp.on_headers_complete") + case .none: fatalError("Unexpected headerState (.none) during chttp.on_headers_complete") } - // parse version + /// if headers are complete, so is the start line. + /// parse all start-line information now let major = Int(parser.pointee.http_major) let minor = Int(parser.pointee.http_minor) results.version = HTTPVersion(major: major, minor: minor) @@ -443,9 +445,9 @@ extension CHTTPParserContext { results.bodyStart = chunk switch results.bodyState { - case .buffer: fatalError("Unexpected buffer body state during CHTTP.on_body: \(results.bodyState)") + case .buffer: fatalError("Unexpected bodyState (.buffer) during chttp.on_body.") case .none: results.bodyState = .buffer(chunk.makeByteBuffer(length)) - case .stream: fatalError("Illegal state") + case .stream: fatalError("Unexpected bodyState (.stream) during chttp.on_body.") case .readyStream(let bodyStream, let ready): bodyStream.push(chunk.makeByteBuffer(length), ready) results.bodyState = .stream(bodyStream) // no longer ready @@ -472,61 +474,18 @@ extension CHTTPParserContext { // MARK: Utilities -extension UnsafeBufferPointer where Element == Byte { +extension UnsafeBufferPointer where Element == Byte /* ByteBuffer */ { + /// Creates a C pointer from a Byte Buffer fileprivate var cPointer: UnsafePointer { return baseAddress.unsafelyUnwrapped.withMemoryRebound(to: CChar.self, capacity: count) { $0 } } } -fileprivate let headerSeparator: [UInt8] = [.colon, .space] -fileprivate let lowercasedContentLength = HTTPHeaders.Name.contentLength.lowercased - -//fileprivate extension Data { -// fileprivate var cPointer: UnsafePointer { -// return withUnsafeBytes { $0 } -// } -//} - fileprivate extension UnsafePointer where Pointee == CChar { - /// Creates a Bytes array from a C pointer + /// Creates a Bytes Buffer from a C pointer. fileprivate func makeByteBuffer(_ count: Int) -> ByteBuffer { return withMemoryRebound(to: Byte.self, capacity: count) { pointer in return ByteBuffer(start: pointer, count: count) } } - - /// Creates a Bytes array from a C pointer - fileprivate func makeBuffer(length: Int) -> UnsafeRawBufferPointer { - let pointer = UnsafeBufferPointer(start: self, count: length) - - guard let base = pointer.baseAddress else { - return UnsafeRawBufferPointer(start: nil, count: 0) - } - - return base.withMemoryRebound(to: UInt8.self, capacity: length) { pointer in - return UnsafeRawBufferPointer(start: pointer, count: length) - } - } } - - -//extension CHTTPParserContext { -// fileprivate func parseContentLength(index: HTTPHeaders.Index) { -// if self.contentLength == nil { -// let namePointer = UnsafePointer(self.headersData).advanced(by: index.nameStartIndex) -// let nameLength = index.nameEndIndex - index.nameStartIndex -// let nameBuffer = ByteBuffer(start: namePointer, count: nameLength) -// -// if lowercasedContentLength.caseInsensitiveEquals(to: nameBuffer) { -// let pointer = UnsafePointer(self.headersData).advanced(by: index.valueStartIndex) -// let length = index.valueEndIndex - index.valueStartIndex -// -// pointer.withMemoryRebound(to: Int8.self, capacity: length) { pointer in -// self.contentLength = numericCast(strtol(pointer, nil, 10)) -// } -// } -// } -// } -//} -// - From 5a262130e6868c205f52b22069188db2af3cc262 Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Thu, 25 Jan 2018 14:23:32 -0500 Subject: [PATCH 07/16] updates' --- Sources/HTTP/Parser/CHTTPParser.swift | 3 --- Sources/HTTP/Parser/CHTTPParserContext.swift | 10 +++++----- Tests/HTTPTests/HTTPParserTests.swift | 2 +- Tests/HTTPTests/HTTPServerTests.swift | 6 +++--- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/Sources/HTTP/Parser/CHTTPParser.swift b/Sources/HTTP/Parser/CHTTPParser.swift index a16e94b1..41a755b7 100644 --- a/Sources/HTTP/Parser/CHTTPParser.swift +++ b/Sources/HTTP/Parser/CHTTPParser.swift @@ -73,8 +73,6 @@ extension CHTTPParser { case .stream: fatalError("Illegal state") case .readyStream: fatalError("Illegal state") } - - print(chttp.headersIndexes) let message = try makeMessage(using: body) downstream.next(message, ready) chttp.reset() @@ -89,7 +87,6 @@ extension CHTTPParser { } chttp.bodyState = .stream(stream) body = HTTPBody(size: nil, stream: .init(stream)) - print(chttp.headersIndexes) let message = try makeMessage(using: body) let nextMessageFuture = downstream.next(message) chttp.state = .streaming(nextMessageFuture) diff --git a/Sources/HTTP/Parser/CHTTPParserContext.swift b/Sources/HTTP/Parser/CHTTPParserContext.swift index 48adc501..9c0198f7 100644 --- a/Sources/HTTP/Parser/CHTTPParserContext.swift +++ b/Sources/HTTP/Parser/CHTTPParserContext.swift @@ -328,7 +328,7 @@ extension CHTTPParserContext { results.headerStart = chunk start = chunk } - print("on_header_field") + //print("on_header_field") // check current header parsing state switch results.headerState { @@ -365,7 +365,7 @@ extension CHTTPParserContext { // signal an error return 1 } - print("on_header_value") + //print("on_header_value") /// Get headerStart pointer. If nil, then there has not /// been a header event yet. @@ -415,7 +415,7 @@ extension CHTTPParserContext { // signal an error return 1 } - print("on_headers_complete") + //print("on_headers_complete") // check the current header parsing state switch results.headerState { @@ -441,7 +441,7 @@ extension CHTTPParserContext { // signal an error return 1 } - print("on_body") + //print("on_body") results.bodyStart = chunk switch results.bodyState { @@ -462,7 +462,7 @@ extension CHTTPParserContext { // signal an error return 1 } - print("on_message_complete") + //print("on_message_complete") // mark the results as complete results.messageComplete = true diff --git a/Tests/HTTPTests/HTTPParserTests.swift b/Tests/HTTPTests/HTTPParserTests.swift index 93128d06..f0103ccc 100644 --- a/Tests/HTTPTests/HTTPParserTests.swift +++ b/Tests/HTTPTests/HTTPParserTests.swift @@ -60,7 +60,7 @@ class HTTPParserTests: XCTestCase { // creates a protocol tester let tester = ProtocolTester( - data: "GET /hello HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 5\r\n\r\nworld", + data: "GET /hello HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 5\r\n\r\nworld", onFail: XCTFail ) { request = nil diff --git a/Tests/HTTPTests/HTTPServerTests.swift b/Tests/HTTPTests/HTTPServerTests.swift index 8655db86..67107481 100644 --- a/Tests/HTTPTests/HTTPServerTests.swift +++ b/Tests/HTTPTests/HTTPServerTests.swift @@ -28,9 +28,9 @@ class HTTPServerTests: XCTestCase { } } -// let group = DispatchGroup() -// group.enter() -// group.wait() + let group = DispatchGroup() + group.enter() + group.wait() let exp = expectation(description: "all requests complete") var num = 1024 From 73b58a1aa4ca45223c6069d75cddbf0ec80ed981 Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Thu, 25 Jan 2018 17:16:46 -0500 Subject: [PATCH 08/16] fix tests --- Sources/HTTP/Client/HTTPClient+TCP.swift | 24 +- Sources/HTTP/Client/HTTPClient.swift | 98 +++---- Sources/HTTP/Message/HTTPBody.swift | 246 +++++++++--------- Sources/HTTP/Parser/CHTTPParser.swift | 6 +- Sources/HTTP/Parser/CHTTPParserContext.swift | 66 ++--- Sources/HTTP/Parser/HTTPRequestParser.swift | 6 +- Sources/HTTP/Parser/HTTPResponseParser.swift | 99 +++---- .../Serializer/HTTPRequestSerializer.swift | 3 +- .../Serializer/HTTPResponseSerializer.swift | 3 +- Sources/HTTP/Serializer/HTTPSerializer.swift | 2 +- Sources/WebSocket/WebSocket.swift | 6 +- Tests/HTTPTests/HTTPClientTests.swift | 96 +++---- Tests/HTTPTests/HTTPParserTests.swift | 231 +--------------- .../HTTPTests/HTTPSerializerStreamTests.swift | 138 ---------- Tests/HTTPTests/HTTPSerializerTests.swift | 82 ++++++ Tests/HTTPTests/HTTPServerTests.swift | 6 +- Tests/HTTPTests/ProtocolTester.swift | 7 - 17 files changed, 412 insertions(+), 707 deletions(-) delete mode 100644 Tests/HTTPTests/HTTPSerializerStreamTests.swift create mode 100644 Tests/HTTPTests/HTTPSerializerTests.swift diff --git a/Sources/HTTP/Client/HTTPClient+TCP.swift b/Sources/HTTP/Client/HTTPClient+TCP.swift index 0947b827..ce760e02 100644 --- a/Sources/HTTP/Client/HTTPClient+TCP.swift +++ b/Sources/HTTP/Client/HTTPClient+TCP.swift @@ -1,13 +1,13 @@ -//import Async -//import TCP -// -//extension HTTPClient { -// /// Create a TCP-based HTTP client. See `TCPClient`. -// public static func tcp(hostname: String, port: UInt16, on worker: Worker) throws -> HTTPClient { -// let tcpSocket = try TCPSocket(isNonBlocking: true) -// let tcpClient = try TCPClient(socket: tcpSocket) -// try tcpClient.connect(hostname: "httpbin.org", port: 80) -// return HTTPClient(stream: tcpSocket.stream(on: worker), on: worker) -// } -//} +import Async +import TCP + +extension HTTPClient { + /// Create a TCP-based HTTP client. See `TCPClient`. + public static func tcp(hostname: String, port: UInt16, on worker: Worker) throws -> HTTPClient { + let tcpSocket = try TCPSocket(isNonBlocking: true) + let tcpClient = try TCPClient(socket: tcpSocket) + try tcpClient.connect(hostname: "httpbin.org", port: 80) + return HTTPClient(stream: tcpSocket.stream(on: worker), on: worker) + } +} diff --git a/Sources/HTTP/Client/HTTPClient.swift b/Sources/HTTP/Client/HTTPClient.swift index 0314ea5c..0bd24ecf 100644 --- a/Sources/HTTP/Client/HTTPClient.swift +++ b/Sources/HTTP/Client/HTTPClient.swift @@ -1,50 +1,50 @@ -//import Async -//import Bits -// -///// An HTTP client wrapped around TCP client -///// -///// Can handle a single `Request` at a given time. -///// -///// Multiple requests at the same time are subject to unknown behaviour -///// -///// [Learn More →](https://docs.vapor.codes/3.0/http/client/) -//public final class HTTPClient { -// /// Inverse stream, takes in responses and outputs requests -// private let queueStream: QueueStream -// -// /// Store the response map here, so it can capture -// /// the sink and source variables. -// private let responseMap: (HTTPResponse) throws -> HTTPResponse -// -// /// Creates a new Client wrapped around a `TCP.Client` -// public init(stream: Stream, on worker: Worker, maxResponseSize: Int = 10_000_000) -// where Stream: ByteStream -// { -// let queueStream = QueueStream() -// -// let serializerStream = HTTPRequestSerializer().stream(on: worker) -// let parserStream = HTTPResponseParser() -// -// stream.stream(to: parserStream) -// .stream(to: queueStream) -// .stream(to: serializerStream) -// .output(to: stream) -// -// self.responseMap = { res in -// if let onUpgrade = res.onUpgrade { -// try onUpgrade.closure(.init(stream), .init(stream), worker) -// } -// return res -// } -// -// self.queueStream = queueStream -// } -// -// /// Sends an HTTP request. -// public func send(_ request: HTTPRequest) -> Future { -// return queueStream.enqueue(request).map(to: HTTPResponse.self) { res in -// return try self.responseMap(res) -// } -// } -//} +import Async +import Bits + +/// An HTTP client wrapped around TCP client +/// +/// Can handle a single `Request` at a given time. +/// +/// Multiple requests at the same time are subject to unknown behaviour +/// +/// [Learn More →](https://docs.vapor.codes/3.0/http/client/) +public final class HTTPClient { + /// Inverse stream, takes in responses and outputs requests + private let queueStream: QueueStream + + /// Store the response map here, so it can capture + /// the sink and source variables. + private let responseMap: (HTTPResponse) throws -> HTTPResponse + + /// Creates a new Client wrapped around a `TCP.Client` + public init(stream: Stream, on worker: Worker, maxResponseSize: Int = 10_000_000) + where Stream: ByteStream + { + let queueStream = QueueStream() + + let serializerStream = HTTPRequestSerializer().stream(on: worker) + let parserStream = HTTPResponseParser() + + stream.stream(to: parserStream) + .stream(to: queueStream) + .stream(to: serializerStream) + .output(to: stream) + + self.responseMap = { res in + if let onUpgrade = res.onUpgrade { + try onUpgrade.closure(.init(stream), .init(stream), worker) + } + return res + } + + self.queueStream = queueStream + } + + /// Sends an HTTP request. + public func send(_ request: HTTPRequest) -> Future { + return queueStream.enqueue(request).map(to: HTTPResponse.self) { res in + return try self.responseMap(res) + } + } +} diff --git a/Sources/HTTP/Message/HTTPBody.swift b/Sources/HTTP/Message/HTTPBody.swift index 1ddd9651..ca18ce78 100644 --- a/Sources/HTTP/Message/HTTPBody.swift +++ b/Sources/HTTP/Message/HTTPBody.swift @@ -10,79 +10,14 @@ import TCP /// /// [Learn More →](https://docs.vapor.codes/3.0/http/body/) public struct HTTPBody: Codable { - /// The internal storage medium. - /// - /// NOTE: This is an implementation detail - enum Storage: Codable { - case none - case data(Data) - case staticString(StaticString) - case dispatchData(DispatchData) - case string(String) - case chunkedOutputStream(OutputChunkedStreamClosure) - case binaryOutputStream(size: Int?, stream: AnyOutputStream) - - func encode(to encoder: Encoder) throws { - switch self { - case .none: return - case .data(let data): - try data.encode(to: encoder) - case .dispatchData(let data): - try Data(data).encode(to: encoder) - case .staticString(let string): - try Data(bytes: string.utf8Start, count: string.utf8CodeUnitCount).encode(to: encoder) - case .string(let string): - try string.encode(to: encoder) - case .chunkedOutputStream(_): - /// FIXME: properly encode stream - return - case .binaryOutputStream(_): - /// FIXME: properly encode stream - return - } - } - - init(from decoder: Decoder) throws { - self = .data(try Data(from: decoder)) - } - - /// The size of this buffer - var count: Int { - switch self { - case .data(let data): return data.count - case .dispatchData(let data): return data.count - case .staticString(let staticString): return staticString.utf8CodeUnitCount - case .string(let string): return string.utf8.count - case .chunkedOutputStream, .none: - /// FIXME: convert to data then return count? - return 0 - case .binaryOutputStream(let size, _): - return size ?? 0 - } - } - - /// Accesses the bytes of this data - func withUnsafeBytes(_ run: ((BytesPointer) throws -> (Return))) throws -> Return { - switch self { - case .data(let data): - return try data.withUnsafeBytes(run) - case .dispatchData(let data): - return try data.withUnsafeBytes(body: run) - case .staticString(let staticString): - return try run(staticString.utf8Start) - case .string(let string): - return try string.withCString { pointer in - return try pointer.withMemoryRebound(to: UInt8.self, capacity: self.count, run) - } - case .none, .chunkedOutputStream(_), .binaryOutputStream(_): - throw HTTPError(identifier: "invalid-stream-acccess", reason: "A BodyStream was being accessed as a sequential byte buffer, which is impossible.") - } - } - } - /// The underlying storage type - var storage: Storage - + var storage: HTTPBodyStorage + + /// Internal HTTPBody init with underlying storage type. + internal init(storage: HTTPBodyStorage) { + self.storage = storage + } + /// Creates an empty body public init() { storage = .none @@ -108,23 +43,19 @@ public struct HTTPBody: Codable { self.storage = .string(string) } - /// Output the body stream to the chunk encoding stream - /// When supplied in this closure - typealias OutputChunkedStreamClosure = (HTTPChunkEncodingStream) -> (HTTPChunkEncodingStream) - /// A chunked body stream public init(chunked stream: S) where S: Async.OutputStream, S.Output == ByteBuffer { self.storage = .chunkedOutputStream(stream.stream) } /// A chunked body stream - public init(size: Int?, stream: AnyOutputStream) { - self.storage = .binaryOutputStream(size: size, stream: stream) + public init(stream: AnyOutputStream, count: @escaping () -> Int?) { + self.storage = .binaryOutputStream(count: count, stream: stream) } /// Decodes a body from from a Decoder public init(from decoder: Decoder) throws { - self.storage = try Storage(from: decoder) + self.storage = try HTTPBodyStorage(from: decoder) } /// Executes a closure with a pointer to the start of the data @@ -136,49 +67,11 @@ public struct HTTPBody: Codable { /// Get body data. public func makeData(max: Int) -> Future { - switch storage { - case .none: - return Future(Data()) - case .data(let data): - return Future(data) - case .dispatchData(let dispatch): - return Future(Data(dispatch)) - case .staticString(let string): - return Future(Data(bytes: string.utf8Start, count: string.utf8CodeUnitCount)) - case .string(let string): - return Future(Data(string.utf8)) - case .chunkedOutputStream(_): - return Future(error: HTTPError(identifier: "chunked-output-stream", reason: "Cannot convert a chunked output stream to a `Data` buffer")) - case .binaryOutputStream(let size, let stream): - let promise = Promise() - var data = Data() - - if let size = size { - data.reserveCapacity(size) - } - - let drain = DrainStream(ByteBuffer.self, onInput: { buffer in - if let size = size { - guard data.count + buffer.count <= size else { - throw HTTPError(identifier: "body-size", reason: "The body was larger than the request.") - } - } - - guard data.count + buffer.count <= max else { - throw HTTPError(identifier: "body-size", reason: "The body was larger than the limit") - } - - data.append(Data(buffer: buffer)) - }, onError: promise.fail, onClose: { - promise.complete(data) - }) - stream.output(to: drain) - return promise.future - } + return storage.makeData(max: max) } /// The size of the data buffer - public var count: Int { + public var count: Int? { return self.storage.count } } @@ -199,3 +92,118 @@ extension String: HTTPBodyRepresentable { } } + +/// Output the body stream to the chunk encoding stream +/// When supplied in this closure +typealias OutputChunkedStreamClosure = (HTTPChunkEncodingStream) -> (HTTPChunkEncodingStream) + +/// The internal storage medium. +/// +/// NOTE: This is an implementation detail +enum HTTPBodyStorage: Codable { + case none + case buffer(ByteBuffer) + case data(Data) + case staticString(StaticString) + case dispatchData(DispatchData) + case string(String) + case chunkedOutputStream(OutputChunkedStreamClosure) + case binaryOutputStream(count: () -> Int?, stream: AnyOutputStream) + + /// See `Encodable.encode(to:)` + func encode(to encoder: Encoder) throws { + switch self { + case .none: return + case .data(let data): try data.encode(to: encoder) + case .buffer(let buffer): try Data(buffer).encode(to: encoder) + case .dispatchData(let data): try Data(data).encode(to: encoder) + case .staticString(let string): try Data(bytes: string.utf8Start, count: string.utf8CodeUnitCount).encode(to: encoder) + case .string(let string): try string.encode(to: encoder) + case .chunkedOutputStream, .binaryOutputStream: + throw HTTPError( + identifier: "streamingBody", + reason: "A BodyStream cannot be encoded with `encode(to:)`." + ) + } + } + + /// See `Decodable.init(from:)` + init(from decoder: Decoder) throws { + self = .data(try Data(from: decoder)) + } + + /// The size of the HTTP body's data. + /// `nil` of the body is a non-determinate stream. + var count: Int? { + switch self { + case .data(let data): return data.count + case .dispatchData(let data): return data.count + case .staticString(let staticString): return staticString.utf8CodeUnitCount + case .string(let string): return string.utf8.count + case .buffer(let buffer): return buffer.count + case .chunkedOutputStream, .none: return nil + case .binaryOutputStream(let size, _): return size() + } + } + + /// Accesses the bytes of this data + func withUnsafeBytes(_ run: ((BytesPointer) throws -> (Return))) throws -> Return { + switch self { + case .data(let data): + return try data.withUnsafeBytes(run) + case .dispatchData(let data): + return try data.withUnsafeBytes(body: run) + case .staticString(let staticString): + return try run(staticString.utf8Start) + case .string(let string): + return try string.withCString { pointer in + return try pointer.withMemoryRebound(to: UInt8.self, capacity: string.utf8.count, run) + } + case .buffer(let buffer): + return try run(buffer.baseAddress!) + case .none, .chunkedOutputStream(_), .binaryOutputStream(_): + throw HTTPError( + identifier: "streamingBody", + reason: "A BodyStream was being accessed as a sequential byte buffer, which is impossible." + ) + } + } + + func makeData(max: Int) -> Future { + switch self { + case .none: return Future(Data()) + case .buffer(let buffer): return Future(Data(buffer)) + case .data(let data): return Future(data) + case .dispatchData(let dispatch): return Future(Data(dispatch)) + case .staticString(let string): return Future(Data(bytes: string.utf8Start, count: string.utf8CodeUnitCount)) + case .string(let string): return Future(Data(string.utf8)) + case .chunkedOutputStream(_): + let error = HTTPError( + identifier: "chunkedBody", + reason: "Cannot use `makeData(max:)` on a chunk-encoded body." + ) + return Future(error: error) + case .binaryOutputStream(let size, let stream): + let promise = Promise() + let size = size() ?? 0 + var data = Data() + data.reserveCapacity(size) + + let drain = DrainStream(ByteBuffer.self, onInput: { buffer in + guard data.count + buffer.count <= size else { + throw HTTPError(identifier: "bodySize", reason: "The body was larger than the request.") + } + + guard data.count + buffer.count <= max else { + throw HTTPError(identifier: "bodySize", reason: "The body was larger than the limit") + } + + data.append(Data(buffer: buffer)) + }, onError: promise.fail, onClose: { + promise.complete(data) + }) + stream.output(to: drain) + return promise.future + } + } + } diff --git a/Sources/HTTP/Parser/CHTTPParser.swift b/Sources/HTTP/Parser/CHTTPParser.swift index 41a755b7..3e77e778 100644 --- a/Sources/HTTP/Parser/CHTTPParser.swift +++ b/Sources/HTTP/Parser/CHTTPParser.swift @@ -68,7 +68,7 @@ extension CHTTPParser { /// if we already have the HTTPBody in its entirety if chttp.messageComplete { switch chttp.bodyState { - case .buffer(let buffer): body = HTTPBody(Data(buffer)) + case .buffer(let buffer): body = HTTPBody(storage: .buffer(buffer)) case .none: body = HTTPBody() case .stream: fatalError("Illegal state") case .readyStream: fatalError("Illegal state") @@ -86,7 +86,9 @@ extension CHTTPParser { case .readyStream: fatalError("Illegal state") } chttp.bodyState = .stream(stream) - body = HTTPBody(size: nil, stream: .init(stream)) + body = HTTPBody(stream: .init(stream)) { + return self.chttp.headers?[.contentLength].flatMap(Int.init) + } let message = try makeMessage(using: body) let nextMessageFuture = downstream.next(message) chttp.state = .streaming(nextMessageFuture) diff --git a/Sources/HTTP/Parser/CHTTPParserContext.swift b/Sources/HTTP/Parser/CHTTPParserContext.swift index 9c0198f7..ec73da89 100644 --- a/Sources/HTTP/Parser/CHTTPParserContext.swift +++ b/Sources/HTTP/Parser/CHTTPParserContext.swift @@ -27,6 +27,9 @@ internal final class CHTTPParserContext { /// The parsed HTTP method var method: http_method? + /// The parsed HTTP status + var statusCode: Int? + /// The parsed HTTP version var version: HTTPVersion? @@ -238,36 +241,6 @@ extension CHTTPParserContext { } } -/// MARK: C Baton Access - -extension CHTTPParserContext { - /// Sets C pointer for this context on the http_parser's data. - /// Use `CHTTPParserContext.get(from:)` to fetch back. - fileprivate func set(on parser: inout http_parser) { - let results = UnsafeMutablePointer.allocate(capacity: 1) - results.initialize(to: self) - parser.data = UnsafeMutableRawPointer(results) - } - - /// Removes C pointer from http_parser data - fileprivate static func remove(from parser: inout http_parser) { - if let results = parser.data { - let pointer = results.assumingMemoryBound(to: CHTTPParserContext.self) - pointer.deinitialize() - pointer.deallocate(capacity: 1) - } - } - - /// Fetches the parse results object from the C http_parser data - fileprivate static func get(from parser: UnsafePointer?) -> CHTTPParserContext? { - return parser? - .pointee - .data - .assumingMemoryBound(to: CHTTPParserContext.self) - .pointee - } -} - /// Private methods extension CHTTPParserContext { @@ -430,6 +403,7 @@ extension CHTTPParserContext { let minor = Int(parser.pointee.http_minor) results.version = HTTPVersion(major: major, minor: minor) results.method = http_method(parser.pointee.method) + results.statusCode = Int(parser.pointee.status_code) results.headersComplete = true return 0 @@ -472,6 +446,38 @@ extension CHTTPParserContext { } } + + +/// MARK: C Baton Access + +extension CHTTPParserContext { + /// Sets C pointer for this context on the http_parser's data. + /// Use `CHTTPParserContext.get(from:)` to fetch back. + fileprivate func set(on parser: inout http_parser) { + let results = UnsafeMutablePointer.allocate(capacity: 1) + results.initialize(to: self) + parser.data = UnsafeMutableRawPointer(results) + } + + /// Removes C pointer from http_parser data + fileprivate static func remove(from parser: inout http_parser) { + if let results = parser.data { + let pointer = results.assumingMemoryBound(to: CHTTPParserContext.self) + pointer.deinitialize() + pointer.deallocate(capacity: 1) + } + } + + /// Fetches the parse results object from the C http_parser data + fileprivate static func get(from parser: UnsafePointer?) -> CHTTPParserContext? { + return parser? + .pointee + .data + .assumingMemoryBound(to: CHTTPParserContext.self) + .pointee + } +} + // MARK: Utilities extension UnsafeBufferPointer where Element == Byte /* ByteBuffer */ { diff --git a/Sources/HTTP/Parser/HTTPRequestParser.swift b/Sources/HTTP/Parser/HTTPRequestParser.swift index e9be4b2b..56bf0a08 100644 --- a/Sources/HTTP/Parser/HTTPRequestParser.swift +++ b/Sources/HTTP/Parser/HTTPRequestParser.swift @@ -26,11 +26,7 @@ public final class HTTPRequestParser: CHTTPParser { /// See `CHTTPParser.makeMessage(from:using:)` func makeMessage(using body: HTTPBody) throws -> HTTPRequest { // require a version to have been parsed - guard - let version = chttp.version, - let headers = chttp.headers, - let cmethod = chttp.method - else { + guard let version = chttp.version, let headers = chttp.headers, let cmethod = chttp.method else { throw HTTPError.invalidMessage() } diff --git a/Sources/HTTP/Parser/HTTPResponseParser.swift b/Sources/HTTP/Parser/HTTPResponseParser.swift index 153facc2..59636ce5 100644 --- a/Sources/HTTP/Parser/HTTPResponseParser.swift +++ b/Sources/HTTP/Parser/HTTPResponseParser.swift @@ -1,60 +1,41 @@ -// import CHTTP -//import Async -//import Bits -//import Foundation -// -///// Parses requests from a readable stream. -// public final class HTTPResponseParser: CHTTPParser { -// public typealias Input = ByteBuffer -// public typealias Output = HTTPResponse -// -// /// See CHTTPParser.parserType -// static let parserType: http_parser_type = HTTP_RESPONSE -// -// public var message: HTTPResponse? -// -// // Internal variables to conform -// // to the C HTTP parser protocol. -// var parser: http_parser -// var settings: http_parser_settings -// var httpState: CHTTPParserState -// public var messageBodyCompleted: Bool -// -// /// The maxiumum possible header size -// /// larger sizes will result in an error -// public var maxHeaderSize: Int? -// -// /// Creates a new Request parser. -// public init() { -// self.maxHeaderSize = 100_000 -// -// self.parser = http_parser() -// self.settings = http_parser_settings() -// self.httpState = .ready -// self.messageBodyCompleted = false -// reset() -// } -// -// /// See CHTTPParser.makeMessage -// func makeMessage(from results: CParseResults, using body: HTTPBody) throws -> HTTPResponse { -// // require a version to have been parsed -// guard -// let version = results.version, -// let headers = results.headers -// else { -// throw HTTPError.invalidMessage() -// } -// -// /// get response status -// let status = HTTPStatus(code: Int(parser.status_code)) -// -// // create the request -// return HTTPResponse( -// version: version, -// status: status, -// headers: headers, -// body: body -// ) -// } -//} + import CHTTP +import Async +import Bits +import Foundation + +/// Parses requests from a readable stream. + public final class HTTPResponseParser: CHTTPParser { + /// See `InputStream.Input` + public typealias Input = ByteBuffer + + /// See `OutputStream.Output` + public typealias Output = HTTPResponse + + /// See `CHTTPParser.chttpParserContext` + var chttp: CHTTPParserContext + + /// Current downstream accepting parsed messages. + var downstream: AnyInputStream? + + /// Creates a new Request parser. + public init() { + self.chttp = .init(HTTP_RESPONSE) + } + + /// See CHTTPParser.makeMessage + func makeMessage(using body: HTTPBody) throws -> HTTPResponse { + // require a version to have been parsed + guard let version = chttp.version, let headers = chttp.headers, let statusCode = chttp.statusCode else { + throw HTTPError.invalidMessage() + } + + // create the request + return HTTPResponse( + version: version, + status: HTTPStatus(code: statusCode), + headers: headers, + body: body + ) + } +} diff --git a/Sources/HTTP/Serializer/HTTPRequestSerializer.swift b/Sources/HTTP/Serializer/HTTPRequestSerializer.swift index d9ff7087..e02faa21 100644 --- a/Sources/HTTP/Serializer/HTTPRequestSerializer.swift +++ b/Sources/HTTP/Serializer/HTTPRequestSerializer.swift @@ -38,7 +38,8 @@ public final class HTTPRequestSerializer: _HTTPSerializer { if case .chunkedOutputStream = message.body.storage { headers[.transferEncoding] = "chunked" } else { - headers.appendValue(message.body.count.bytes(reserving: 6), forName: .contentLength) + let count = message.body.count ?? 0 + headers.appendValue(count.bytes(reserving: 6), forName: .contentLength) } self.headersData = headers.storage diff --git a/Sources/HTTP/Serializer/HTTPResponseSerializer.swift b/Sources/HTTP/Serializer/HTTPResponseSerializer.swift index bd190222..549e5d47 100644 --- a/Sources/HTTP/Serializer/HTTPResponseSerializer.swift +++ b/Sources/HTTP/Serializer/HTTPResponseSerializer.swift @@ -41,7 +41,8 @@ public final class HTTPResponseSerializer: _HTTPSerializer { if case .chunkedOutputStream = message.body.storage { headers[.transferEncoding] = "chunked" } else { - headers.appendValue(message.body.count.description, forName: .contentLength) + let count = message.body.count ?? 0 + headers.appendValue(count.bytes(reserving: 6), forName: .contentLength) } self.headersData = headers.storage diff --git a/Sources/HTTP/Serializer/HTTPSerializer.swift b/Sources/HTTP/Serializer/HTTPSerializer.swift index c0f68ac4..ec3e918d 100644 --- a/Sources/HTTP/Serializer/HTTPSerializer.swift +++ b/Sources/HTTP/Serializer/HTTPSerializer.swift @@ -137,7 +137,7 @@ extension _HTTPSerializer { state: .streaming(stream) ) default: - bufferSize = body.count + bufferSize = body.count ?? 0 writeSize = min(outputSize, bufferSize - offset) try body.withUnsafeBytes { pointer in diff --git a/Sources/WebSocket/WebSocket.swift b/Sources/WebSocket/WebSocket.swift index 472e73d3..2d60fd65 100644 --- a/Sources/WebSocket/WebSocket.swift +++ b/Sources/WebSocket/WebSocket.swift @@ -104,11 +104,7 @@ public final class WebSocket { let serializer = HTTPRequestSerializer().stream(on: self.worker) let serializerStream = PushStream() - let responseParser = HTTPResponseParser() - responseParser.maxHeaderSize = 50_000 - - let parser = responseParser.stream(on: self.worker) - + let parser = HTTPResponseParser() serializerStream.stream(to: serializer).output(to: self.sink) let drain = DrainStream(onInput: { response in diff --git a/Tests/HTTPTests/HTTPClientTests.swift b/Tests/HTTPTests/HTTPClientTests.swift index 15568718..16f8366b 100644 --- a/Tests/HTTPTests/HTTPClientTests.swift +++ b/Tests/HTTPTests/HTTPClientTests.swift @@ -1,49 +1,49 @@ -//import Async -//import Bits -//import HTTP -//import Foundation -//import TCP -//import XCTest -// -//class HTTPClientTests: XCTestCase { -// func testTCP() throws { -// let eventLoop = try DefaultEventLoop(label: "codes.vapor.http.test.client") -// let client = try HTTPClient.tcp(hostname: "httpbin.org", port: 80, on: eventLoop) -// -// let req = HTTPRequest(method: .get, uri: "/html", headers: [.host: "httpbin.org"]) -// let res = try client.send(req).flatMap(to: Data.self) { res in -// return res.body.makeData(max: 100_000) -// }.await(on: eventLoop) -// -// XCTAssert(String(data: res, encoding: .utf8)?.contains("Moby-Dick") == true) -// XCTAssertEqual(res.count, 3741) -// } -// -// func testConnectionClose() throws { -// let eventLoop = try DefaultEventLoop(label: "codes.vapor.http.test.client") -// let client = try HTTPClient.tcp(hostname: "httpbin.org", port: 80, on: eventLoop) -// -// let req = HTTPRequest(method: .get, uri: "/status/418", headers: [.host: "httpbin.org"]) -// let res = try client.send(req).flatMap(to: Data.self) { res in -// return res.body.makeData(max: 100_000) -// }.await(on: eventLoop) -// -// XCTAssertEqual(res.count, 135) -// } -// -// func testURI() { -// var uri: URI = "http://localhost:8081/test?q=1&b=4#test" -// XCTAssertEqual(uri.scheme, "http") -// XCTAssertEqual(uri.hostname, "localhost") -// XCTAssertEqual(uri.port, 8081) -// XCTAssertEqual(uri.path, "/test") -// XCTAssertEqual(uri.query, "q=1&b=4") -// XCTAssertEqual(uri.fragment, "test") -// } -// -// static let allTests = [ -// ("testTCP", testTCP), -// ("testURI", testURI), -// ] -//} +import Async +import Bits +import HTTP +import Foundation +import TCP +import XCTest + +class HTTPClientTests: XCTestCase { + func testTCP() throws { + let eventLoop = try DefaultEventLoop(label: "codes.vapor.http.test.client") + let client = try HTTPClient.tcp(hostname: "httpbin.org", port: 80, on: eventLoop) + + let req = HTTPRequest(method: .get, uri: "/html", headers: [.host: "httpbin.org"]) + let res = try client.send(req).flatMap(to: Data.self) { res in + return res.body.makeData(max: 100_000) + }.await(on: eventLoop) + + XCTAssert(String(data: res, encoding: .utf8)?.contains("Moby-Dick") == true) + XCTAssertEqual(res.count, 3741) + } + + func testConnectionClose() throws { + let eventLoop = try DefaultEventLoop(label: "codes.vapor.http.test.client") + let client = try HTTPClient.tcp(hostname: "httpbin.org", port: 80, on: eventLoop) + + let req = HTTPRequest(method: .get, uri: "/status/418", headers: [.host: "httpbin.org"]) + let res = try client.send(req).flatMap(to: Data.self) { res in + return res.body.makeData(max: 100_000) + }.await(on: eventLoop) + + XCTAssertEqual(res.count, 135) + } + + func testURI() { + var uri: URI = "http://localhost:8081/test?q=1&b=4#test" + XCTAssertEqual(uri.scheme, "http") + XCTAssertEqual(uri.hostname, "localhost") + XCTAssertEqual(uri.port, 8081) + XCTAssertEqual(uri.path, "/test") + XCTAssertEqual(uri.query, "q=1&b=4") + XCTAssertEqual(uri.fragment, "test") + } + + static let allTests = [ + ("testTCP", testTCP), + ("testURI", testURI), + ] +} diff --git a/Tests/HTTPTests/HTTPParserTests.swift b/Tests/HTTPTests/HTTPParserTests.swift index f0103ccc..5ed21cca 100644 --- a/Tests/HTTPTests/HTTPParserTests.swift +++ b/Tests/HTTPTests/HTTPParserTests.swift @@ -5,54 +5,7 @@ import HTTP import XCTest class HTTPParserTests: XCTestCase { - func testParserEdgeCasesOld() throws { - let firstChunk = "GET /hello HTTP/1.1\r\nContent-Type: ".data(using: .utf8)! - let secondChunk = "text/plain\r\nContent-Length: 5\r\n\r\nwo".data(using: .utf8)! - let thirdChunk = "rl".data(using: .utf8)! - let fourthChunk = "d".data(using: .utf8)! - - let parser = HTTPRequestParser() - - let socket = PushStream(ByteBuffer.self) - socket.stream(to: parser).drain { message in - print("parser.drain { ... }") - print(message) - print("message.body.makeData") - message.body.makeData(max: 100).do { data in - print(data) - }.catch { error in - print("body error: \(error)") - } - }.catch { error in - print("parser.catch { \(error) }") - }.finally { - print("parser.close { }") - } - - - print("(1) FIRST ---") - firstChunk.withByteBuffer(socket.push) - print("(2) SECOND ---") - secondChunk.withByteBuffer(socket.push) - print("(3) THIRD ---") - thirdChunk.withByteBuffer(socket.push) - print("(4) FOURTH ---") - fourthChunk.withByteBuffer(socket.push) - - print("(1) FIRST ---") - firstChunk.withByteBuffer(socket.push) - print("(2) SECOND ---") - secondChunk.withByteBuffer(socket.push) - print("(3) THIRD ---") - thirdChunk.withByteBuffer(socket.push) - print("(4) FOURTH ---") - fourthChunk.withByteBuffer(socket.push) - - print("(c) CLOSE ---") - socket.close() - } - - func testParserEdgeCases() throws { + func testRequest() throws { // captured variables to check var request: HTTPRequest? var content: String? @@ -122,183 +75,7 @@ class HTTPParserTests: XCTestCase { XCTAssertTrue(isClosed) } - -// -// func testRequest() throws { -// var data = """ -// POST /cgi-bin/process.cgi HTTP/1.1\r -// User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r -// Host: www.tutorialspoint.com\r -// Content-Type: text/plain\r -// Content-Length: 5\r -// Accept-Language: en-us\r -// Accept-Encoding: gzip, deflate\r -// Connection: Keep-Alive\r -// \r -// hello -// """.data(using: .utf8) ?? Data() -// -// let parser = HTTPRequestParser() -// var message: HTTPRequest? -// var completed = false -// -// parser.drain { _message in -// message = _message -// }.catch { error in -// XCTFail("\(error)") -// }.finally { -// completed = true -// } -// -// XCTAssertNil(message) -// try parser.next(data.withByteBuffer { $0 }).await(on: loop) -// parser.close() -// -// guard let req = message else { -// XCTFail("No request parsed") -// return -// } -// -// XCTAssertEqual(req.method, .post) -// XCTAssertEqual(req.headers[.userAgent], "Mozilla/4.0 (compatible; MSIE5.01; Windows NT)") -// XCTAssertEqual(req.headers[.host], "www.tutorialspoint.com") -// XCTAssertEqual(req.headers[.contentType], "text/plain") -// XCTAssertEqual(req.mediaType, .plainText) -// XCTAssertEqual(req.headers[.contentLength], "5") -// XCTAssertEqual(req.headers[.acceptLanguage], "en-us") -// XCTAssertEqual(req.headers[.acceptEncoding], "gzip, deflate") -// XCTAssertEqual(req.headers[.connection], "Keep-Alive") -// -// data = try req.body.makeData(max: 100_000).await(on: loop) -// XCTAssertEqual(String(data: data, encoding: .utf8), "hello") -// XCTAssert(completed) -// } -// -// func testResponse() throws { -// var data = """ -// HTTP/1.1 200 OK\r -// Date: Mon, 27 Jul 2009 12:28:53 GMT\r -// Server: Apache/2.2.14 (Win32)\r -// Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\r -// Content-Length: 7\r -// Content-Type: text/html\r -// Connection: Closed\r -// \r -// -// """.data(using: .utf8) ?? Data() -// -// let parser = HTTPResponseParser() -// var message: HTTPResponse? -// var completed = false -// -// parser.drain { _message in -// message = _message -// }.catch { error in -// XCTFail("\(error)") -// }.finally { -// completed = true -// } -// -// XCTAssertNil(message) -// try parser.next(data.withByteBuffer { $0 }).await(on: loop) -// parser.close() -// -// guard let res = message else { -// XCTFail("No request parsed") -// return -// } -// -// XCTAssertEqual(res.status, .ok) -// XCTAssertEqual(res.headers[.date], "Mon, 27 Jul 2009 12:28:53 GMT") -// XCTAssertEqual(res.headers[.server], "Apache/2.2.14 (Win32)") -// XCTAssertEqual(res.headers[.lastModified], "Wed, 22 Jul 2009 19:15:56 GMT") -// XCTAssertEqual(res.headers[.contentLength], "7") -// XCTAssertEqual(res.headers[.contentType], "text/html") -// XCTAssertEqual(res.mediaType, .html) -// XCTAssertEqual(res.headers[.connection], "Closed") -// -// data = try res.body.makeData(max: 100_000).blockingAwait() -// XCTAssertEqual(String(data: data, encoding: .utf8), "") -// XCTAssert(completed) -// } -// -// func testTooLargeRequest() throws { -// let data = """ -// POST /cgi-bin/process.cgi HTTP/1.1\r -// User-Agent: Mozilla/4.0 (compatible; MSIE5.01; Windows NT)\r -// Host: www.tutorialspoint.com\r -// Content-Type: text/plain\r -// Content-Length: 5\r -// Accept-Language: en-us\r -// Accept-Encoding: gzip, deflate\r -// Connection: Keep-Alive\r -// \r -// hello -// """.data(using: .utf8) ?? Data() -// -// var error = false -// let p = HTTPRequestParser() -// p.maxHeaderSize = data.count - 20 // body -// let parser = p -// -// var completed = false -// -// _ = parser.drain { _ in -// XCTFail() -// }.catch { _ in -// error = true -// }.finally { -// completed = true -// } -// -// try parser.next(data.withByteBuffer { $0 }).await(on: loop) -// parser.close() -// XCTAssert(error) -// XCTAssert(completed) -// } - -// func testTooLargeResponse() throws { -// let data = """ -// HTTP/1.1 200 OK\r -// Date: Mon, 27 Jul 2009 12:28:53 GMT\r -// Server: Apache/2.2.14 (Win32)\r -// Last-Modified: Wed, 22 Jul 2009 19:15:56 GMT\r -// Content-Length: 7\r -// Content-Type: text/html\r -// Connection: Closed\r -// \r -// -// """.data(using: .utf8) ?? Data() -// -// var error = false -// let p = HTTPResponseParser() -// p.maxHeaderSize = data.count - 20 // body -// let parser = p.stream(on: loop) -// -// var completed = false -// -// _ = parser.drain { _ in -// XCTFail() -// }.catch { _ in -// error = true -// }.finally { -// completed = true -// } -// -// -// try parser.next(data.withByteBuffer { $0 }).await(on: loop) -// try parser.next(data.withByteBuffer { $0 }).await(on: loop) -// parser.close() -// XCTAssert(error) -// XCTAssert(completed) -// } - -// static let allTests = [ -// ("testRequest", testRequest), -// ("testResponse", testResponse), -// ("testTooLargeRequest", testTooLargeRequest), -// ("testTooLargeResponse", testTooLargeResponse), -// ] + static let allTests = [ + ("testRequest", testRequest), + ] } - - diff --git a/Tests/HTTPTests/HTTPSerializerStreamTests.swift b/Tests/HTTPTests/HTTPSerializerStreamTests.swift deleted file mode 100644 index ad1c7670..00000000 --- a/Tests/HTTPTests/HTTPSerializerStreamTests.swift +++ /dev/null @@ -1,138 +0,0 @@ -import Async -import Bits -import HTTP -import Foundation -import XCTest - -class HTTPSerializerStreamTests: XCTestCase { - let loop = try! DefaultEventLoop(label: "test") - - func testResponse() throws { - /// output and output request for later in test - var output: [ByteBuffer] = [] - - /// setup the mock app - let mockApp = PushStream(HTTPResponse.self) - mockApp.stream( - to: HTTPResponseSerializer().stream(on: loop) - ).drain { buffer in - output.append(buffer) - }.catch { err in - XCTFail("\(err)") - }.finally { - // closed - } - - /// sanity check - XCTAssertEqual(output.count, 0) - - /// emit response - let body = "" - let response = try HTTPResponse( - status: .ok, - body: body - ) - XCTAssertEqual(output.count, 0) - mockApp.push(response) - - /// there should only be one buffer since we - /// called `.drain(1)`. this buffer should contain - /// the entire response - XCTAssertEqual(output.count, 1) - XCTAssertEqual(output.first?.count, 45) - } - - func testResponseStreamingBody() throws { - /// output and output request for later in test - var output: [Data] = [] - var closed = false - - /// setup the mock app - let mockApp = PushStream(HTTPResponse.self) - mockApp.stream( - to: HTTPResponseSerializer().stream(on: loop) - ).drain { buffer in - output.append(Data(buffer)) - }.catch { err in - XCTFail("\(err)") - }.finally { - closed = true - } - - /// sanity check - XCTAssertEqual(output.count, 0) - - /// create a streaming body - let bodyEmitter = PushStream(ByteBuffer.self) - - /// emit response - let response = HTTPResponse( - status: .ok, - body: HTTPBody(chunked: bodyEmitter) - ) - mockApp.push(response) - - /// there should only be one buffer since we - /// called `.drain(1)`. this buffer should contain - /// the entire response sans body - if output.count == 1 { - let message = String(bytes: output[0], encoding: .utf8) - XCTAssertEqual(message, "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n") - } else { - XCTFail("Invalid output count: \(output.count) != 1") - } - /// the count should still be one, we are - /// waiting on the body now - XCTAssertEqual(output.count, 1) - - /// Request and emit additional output - let a = "hello".data(using: .utf8)! - a.withByteBuffer(bodyEmitter.push) - if output.count == 2 { - let message = String(data: output[1], encoding: .utf8) - XCTAssertEqual(message, "5\r\nhello\r\n") - } else { - XCTFail("Invalid output count: \(output.count) != 2") - } - - /// Request and emit additional output - let b = "test".data(using: .utf8)! - b.withByteBuffer(bodyEmitter.push) - if output.count == 3 { - let message = String(data: output[2], encoding: .utf8) - XCTAssertEqual(message, "4\r\ntest\r\n") - } else { - XCTFail("Invalid output count: \(output.count) != 3") - } - - XCTAssertEqual(output.count, 3) - bodyEmitter.close() - if output.count == 4 { - let message = String(data: output[3], encoding: .utf8) - XCTAssertEqual(message, "0\r\n\r\n") - } else { - XCTFail("Invalid output count: \(output.count) != 4") - } - /// parsing stream should remain open, just ready for another message - XCTAssertTrue(!closed) - - /// emit response 2 - let response2 = try HTTPResponse( - status: .ok, - body: "hello" - ) - mockApp.push(response2) - if output.count == 5 { - let message = String(data: output[4], encoding: .utf8) - XCTAssertEqual(message, "HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello") - } else { - XCTFail("Invalid output count: \(output.count) != 5") - } - } - - static let allTests = [ - ("testResponse", testResponse), - ("testResponseStreamingBody", testResponseStreamingBody), - ] -} - diff --git a/Tests/HTTPTests/HTTPSerializerTests.swift b/Tests/HTTPTests/HTTPSerializerTests.swift new file mode 100644 index 00000000..baf8d656 --- /dev/null +++ b/Tests/HTTPTests/HTTPSerializerTests.swift @@ -0,0 +1,82 @@ +import Async +import Bits +import HTTP +import Foundation +import XCTest + +class HTTPSerializerTests: XCTestCase { + func testResponse() throws { + // captured variables to check + var response: HTTPResponse? + var content: String? + var isClosed = false + + // creates a protocol tester + let tester = ProtocolTester( + data: "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 5\r\n\r\nworld", + onFail: XCTFail + ) { + response = nil + content = nil + isClosed = false + } + + tester.assert(before: "\r\n\r\n") { + guard response == nil else { + throw "response was not nil" + } + } + + tester.assert(after: "\r\n\r\n") { + guard let res = response else { + throw "response was nil" + } + + guard res.headers[.contentType] == "text/plain" else { + throw "incorrect content type" + } + + guard res.headers[.contentLength] == "5" else { + throw "incorrect content length" + } + } + + tester.assert(before: "world") { + guard content == nil else { + throw "content was not nil" + } + } + + tester.assert(after: "world") { + guard let string = content else { + throw "content was nil" + } + + guard string == "world" else { + throw "incorrect string" + } + } + + // configure parser stream + tester.stream(to: HTTPResponseParser()).drain { message in + response = message + message.body.makeData(max: 100).do { data in + content = String(data: data, encoding: .ascii) + }.catch { error in + XCTFail("body error: \(error)") + } + }.catch { error in + XCTFail("parser error: \(error)") + }.finally { + isClosed = true + } + + try tester.run().blockingAwait() + XCTAssertTrue(isClosed) + } + + static let allTests = [ + ("testResponse", testResponse), + ] +} + diff --git a/Tests/HTTPTests/HTTPServerTests.swift b/Tests/HTTPTests/HTTPServerTests.swift index 67107481..8655db86 100644 --- a/Tests/HTTPTests/HTTPServerTests.swift +++ b/Tests/HTTPTests/HTTPServerTests.swift @@ -28,9 +28,9 @@ class HTTPServerTests: XCTestCase { } } - let group = DispatchGroup() - group.enter() - group.wait() +// let group = DispatchGroup() +// group.enter() +// group.wait() let exp = expectation(description: "all requests complete") var num = 1024 diff --git a/Tests/HTTPTests/ProtocolTester.swift b/Tests/HTTPTests/ProtocolTester.swift index 264f6190..dfb16a5a 100644 --- a/Tests/HTTPTests/ProtocolTester.swift +++ b/Tests/HTTPTests/ProtocolTester.swift @@ -70,12 +70,6 @@ public final class ProtocolTester: Async.OutputStream { chunks.insert(lastChunk, at: 0) } - print("=== TEST: \(max)") - let data = chunks.reversed().map { "[" + ProtocolTester.dataDebug(for: $0) + "]" }.joined(separator: " ") - let text = chunks.reversed().map { "[" + ProtocolTester.textDebug(for: $0) + "]" }.joined(separator: " ") - print("Data: \(data)") - print("Text: \(text)") - reset() return runChunks(chunks, currentOffset: 0).flatMap(to: Void.self) { return self.runMax(buffer, max: max - 1) @@ -88,7 +82,6 @@ public final class ProtocolTester: Async.OutputStream { /// Recursively passes each chunk to downstream until chunks.count == 0 private func runChunks(_ chunks: [ByteBuffer], currentOffset: Int) -> Future { - print("--- run \(chunks.count)") var chunks = chunks if let chunk = chunks.popLast() { runChecks(offset: currentOffset, chunks: chunks) From 7420b2acc8b698bf995c4b30c5c55737c2b38293 Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Thu, 25 Jan 2018 21:55:47 -0500 Subject: [PATCH 09/16] http headers refactor --- Package.swift | 1 + Sources/HTTP/Client/HTTPClient.swift | 2 +- Sources/HTTP/Content/MediaType.swift | 6 +- Sources/HTTP/Message/HTTPBody.swift | 30 +- Sources/HTTP/Message/HTTPHeaderIndex.swift | 37 + Sources/HTTP/Message/HTTPHeaderName.swift | 430 +++++++++++ Sources/HTTP/Message/HTTPHeaderStorage.swift | 224 ++++++ Sources/HTTP/Message/HTTPHeaders.swift | 696 ++---------------- Sources/HTTP/Message/HTTPMessage.swift | 13 + Sources/HTTP/Parser/CHTTPBodyStream.swift | 4 +- Sources/HTTP/Parser/CHTTPParser.swift | 7 +- Sources/HTTP/Parser/CHTTPParserContext.swift | 12 +- Sources/HTTP/Response/HTTPResponse.swift | 5 +- .../Serializer/HTTPChunkEncodingStream.swift | 5 +- .../Serializer/HTTPRequestSerializer.swift | 132 ++-- .../Serializer/HTTPResponseSerializer.swift | 75 +- Sources/HTTP/Serializer/HTTPSerializer.swift | 273 ++++--- Sources/HTTP/Server/HTTPResponder.swift | 88 ++- Sources/HTTP/Server/HTTPServer.swift | 19 +- Sources/Performance/main.swift | 129 +--- Tests/HTTPTests/HTTPServerTests.swift | 6 +- 21 files changed, 1131 insertions(+), 1063 deletions(-) create mode 100644 Sources/HTTP/Message/HTTPHeaderIndex.swift create mode 100644 Sources/HTTP/Message/HTTPHeaderName.swift create mode 100644 Sources/HTTP/Message/HTTPHeaderStorage.swift diff --git a/Package.swift b/Package.swift index 9df25e11..251b55b4 100644 --- a/Package.swift +++ b/Package.swift @@ -36,6 +36,7 @@ let package = Package( .testTarget(name: "FormURLEncodedTests", dependencies: ["FormURLEncoded"]), .target(name: "HTTP", dependencies: ["CHTTP", "TCP"]), .testTarget(name: "HTTPTests", dependencies: ["HTTP"]), + .target(name: "Performance", dependencies: ["HTTP", "TCP"]), // .target(name: "HTTP2", dependencies: ["HTTP", "TLS", "Pufferfish"]), // .testTarget(name: "HTTP2Tests", dependencies: ["HTTP2"]), .target(name: "Multipart", dependencies: ["Debugging", "HTTP"]), diff --git a/Sources/HTTP/Client/HTTPClient.swift b/Sources/HTTP/Client/HTTPClient.swift index 0bd24ecf..bd209250 100644 --- a/Sources/HTTP/Client/HTTPClient.swift +++ b/Sources/HTTP/Client/HTTPClient.swift @@ -22,7 +22,7 @@ public final class HTTPClient { { let queueStream = QueueStream() - let serializerStream = HTTPRequestSerializer().stream(on: worker) + let serializerStream = HTTPRequestSerializer() let parserStream = HTTPResponseParser() stream.stream(to: parserStream) diff --git a/Sources/HTTP/Content/MediaType.swift b/Sources/HTTP/Content/MediaType.swift index 562f31d9..218ad93f 100644 --- a/Sources/HTTP/Content/MediaType.swift +++ b/Sources/HTTP/Content/MediaType.swift @@ -197,11 +197,7 @@ extension HTTPMessage { return MediaType(string: contentType) } set { - if let newValue = newValue { - headers.appendValue(newValue.bytes, forName: .contentType) - } else { - headers.removeValues(forName: .contentType) - } + headers[.contentType] = newValue?.description // FIXME: performance } } } diff --git a/Sources/HTTP/Message/HTTPBody.swift b/Sources/HTTP/Message/HTTPBody.swift index ca18ce78..29d9656f 100644 --- a/Sources/HTTP/Message/HTTPBody.swift +++ b/Sources/HTTP/Message/HTTPBody.swift @@ -61,7 +61,7 @@ public struct HTTPBody: Codable { /// Executes a closure with a pointer to the start of the data /// /// Can be used to read data from this buffer until the `count`. - public func withUnsafeBytes(_ run: ((BytesPointer) throws -> (Return))) throws -> Return { + public func withUnsafeBytes(_ run: (ByteBuffer) throws -> Return) throws -> Return { return try self.storage.withUnsafeBytes(run) } @@ -141,27 +141,37 @@ enum HTTPBodyStorage: Codable { case .staticString(let staticString): return staticString.utf8CodeUnitCount case .string(let string): return string.utf8.count case .buffer(let buffer): return buffer.count - case .chunkedOutputStream, .none: return nil + case .none: return 0 + case .chunkedOutputStream: return nil case .binaryOutputStream(let size, _): return size() } } /// Accesses the bytes of this data - func withUnsafeBytes(_ run: ((BytesPointer) throws -> (Return))) throws -> Return { + func withUnsafeBytes(_ run: (ByteBuffer) throws -> Return) throws -> Return { switch self { case .data(let data): - return try data.withUnsafeBytes(run) + return try data.withByteBuffer(run) case .dispatchData(let data): - return try data.withUnsafeBytes(body: run) + let data = Data(data) + return try data.withByteBuffer(run) case .staticString(let staticString): - return try run(staticString.utf8Start) + return staticString.withUTF8Buffer { buffer in + return try! run(buffer) // FIXME: throwing + } case .string(let string): - return try string.withCString { pointer in - return try pointer.withMemoryRebound(to: UInt8.self, capacity: string.utf8.count, run) + let buffer = string.withCString { pointer in + return ByteBuffer( + start: pointer.withMemoryRebound(to: UInt8.self, capacity: string.utf8.count) { $0 }, + count: string.utf8.count + ) } + return try run(buffer) case .buffer(let buffer): - return try run(buffer.baseAddress!) - case .none, .chunkedOutputStream(_), .binaryOutputStream(_): + return try run(buffer) + case .none: + return try run(ByteBuffer(start: nil, count: 0)) + case .chunkedOutputStream(_), .binaryOutputStream(_): throw HTTPError( identifier: "streamingBody", reason: "A BodyStream was being accessed as a sequential byte buffer, which is impossible." diff --git a/Sources/HTTP/Message/HTTPHeaderIndex.swift b/Sources/HTTP/Message/HTTPHeaderIndex.swift new file mode 100644 index 00000000..59331e9f --- /dev/null +++ b/Sources/HTTP/Message/HTTPHeaderIndex.swift @@ -0,0 +1,37 @@ +/// The location of a `Name: Value` pair in the raw HTTP headers data. +struct HTTPHeaderIndex { + /// Start offset of the header's name field. + var nameStartIndex: Int + /// End offset of the header's name field + var nameEndIndex: Int + + /// Start offset of the header's value field. + var valueStartIndex: Int + /// End offset of the header's value field. + var valueEndIndex: Int +} + +extension HTTPHeaderIndex { + /// The lowest index of this header. + var startIndex: Int { + return nameStartIndex + } + + /// The highest index of this header. + var endIndex: Int { + return valueEndIndex + 2 // include trailing \r\n + } + + /// The length of this header. + var size: Int { + return endIndex - startIndex + } +} + +extension HTTPHeaderIndex: CustomStringConvertible { + /// See `CustomStringConvertible.description` + var description: String { + return "[\(nameStartIndex)..<\(nameEndIndex):\(valueStartIndex)..<\(valueEndIndex)]" + } + +} diff --git a/Sources/HTTP/Message/HTTPHeaderName.swift b/Sources/HTTP/Message/HTTPHeaderName.swift new file mode 100644 index 00000000..67bfed0d --- /dev/null +++ b/Sources/HTTP/Message/HTTPHeaderName.swift @@ -0,0 +1,430 @@ +/// Type used for the name of a HTTP header in the `HTTPHeaders` storage. +public struct HTTPHeaderName: Codable, Hashable, ExpressibleByStringLiteral, CustomStringConvertible { + /// See `Hashable.hashValue` + public var hashValue: Int { + return lowercased.djb2 + } + + /// Original header bytes + internal let original: [UInt8] + + /// Lowercased-ASCII version of the header. + internal let lowercased: [UInt8] + + /// Create a HTTP header name with the provided String. + public init(_ name: String) { + original = Array(name.utf8) + lowercased = original.lowercasedASCIIString() + } + + /// Create a HTTP header name with the provided String. + internal init(data: [UInt8]) { + original = data + lowercased = data.lowercasedASCIIString() + } + + /// See `ExpressibleByStringLiteral.init(stringLiteral:)` + public init(stringLiteral: String) { + self.init(stringLiteral) + } + + /// See `ExpressibleByStringLiteral.init(unicodeScalarLiteral:)` + public init(unicodeScalarLiteral: String) { + self.init(unicodeScalarLiteral) + } + + /// See `ExpressibleByStringLiteral.init(extendedGraphemeClusterLiteral:)` + public init(extendedGraphemeClusterLiteral: String) { + self.init(extendedGraphemeClusterLiteral) + } + + /// See `CustomStringConvertible.description` + public var description: String { + return String(bytes: original, encoding: .utf8) ?? "" + } + + /// See `Equatable.==` + public static func ==(lhs: Name, rhs: Name) -> Bool { + return lhs.lowercased == rhs.lowercased + } + + // https://www.iana.org/assignments/message-headers/message-headers.xhtml + // Permanent Message Header Field Names + + /// A-IM header. + public static let aIM = HTTPHeaderName("A-IM") + /// Accept header. + public static let accept = HTTPHeaderName("Accept") + /// Accept-Additions header. + public static let acceptAdditions = HTTPHeaderName("Accept-Additions") + /// Accept-Charset header. + public static let acceptCharset = HTTPHeaderName("Accept-Charset") + /// Accept-Datetime header. + public static let acceptDatetime = HTTPHeaderName("Accept-Datetime") + /// Accept-Encoding header. + public static let acceptEncoding = HTTPHeaderName("Accept-Encoding") + /// Accept-Features header. + public static let acceptFeatures = HTTPHeaderName("Accept-Features") + /// Accept-Language header. + public static let acceptLanguage = HTTPHeaderName("Accept-Language") + /// Accept-Patch header. + public static let acceptPatch = HTTPHeaderName("Accept-Patch") + /// Accept-Post header. + public static let acceptPost = HTTPHeaderName("Accept-Post") + /// Accept-Ranges header. + public static let acceptRanges = HTTPHeaderName("Accept-Ranges") + /// Accept-Age header. + public static let age = HTTPHeaderName("Age") + /// Accept-Allow header. + public static let allow = HTTPHeaderName("Allow") + /// ALPN header. + public static let alpn = HTTPHeaderName("ALPN") + /// Alt-Svc header. + public static let altSvc = HTTPHeaderName("Alt-Svc") + /// Alt-Used header. + public static let altUsed = HTTPHeaderName("Alt-Used") + /// Alternatives header. + public static let alternates = HTTPHeaderName("Alternates") + /// Apply-To-Redirect-Ref header. + public static let applyToRedirectRef = HTTPHeaderName("Apply-To-Redirect-Ref") + /// Authentication-Control header. + public static let authenticationControl = HTTPHeaderName("Authentication-Control") + /// Authentication-Info header. + public static let authenticationInfo = HTTPHeaderName("Authentication-Info") + /// Authorization header. + public static let authorization = HTTPHeaderName("Authorization") + /// C-Ext header. + public static let cExt = HTTPHeaderName("C-Ext") + /// C-Man header. + public static let cMan = HTTPHeaderName("C-Man") + /// C-Opt header. + public static let cOpt = HTTPHeaderName("C-Opt") + /// C-PEP header. + public static let cPEP = HTTPHeaderName("C-PEP") + /// C-PEP-Indo header. + public static let cPEPInfo = HTTPHeaderName("C-PEP-Info") + /// Cache-Control header. + public static let cacheControl = HTTPHeaderName("Cache-Control") + /// CalDav-Timezones header. + public static let calDAVTimezones = HTTPHeaderName("CalDAV-Timezones") + /// Close header. + public static let close = HTTPHeaderName("Close") + /// Connection header. + public static let connection = HTTPHeaderName("Connection") + /// Content-Base. + public static let contentBase = HTTPHeaderName("Content-Base") + /// Content-Disposition header. + public static let contentDisposition = HTTPHeaderName("Content-Disposition") + /// Content-Encoding header. + public static let contentEncoding = HTTPHeaderName("Content-Encoding") + /// Content-ID header. + public static let contentID = HTTPHeaderName("Content-ID") + /// Content-Language header. + public static let contentLanguage = HTTPHeaderName("Content-Language") + /// Content-Length header. + public static let contentLength = HTTPHeaderName("Content-Length") + /// Content-Location header. + public static let contentLocation = HTTPHeaderName("Content-Location") + /// Content-MD5 header. + public static let contentMD5 = HTTPHeaderName("Content-MD5") + /// Content-Range header. + public static let contentRange = HTTPHeaderName("Content-Range") + /// Content-Script-Type header. + public static let contentScriptType = HTTPHeaderName("Content-Script-Type") + /// Content-Style-Type header. + public static let contentStyleType = HTTPHeaderName("Content-Style-Type") + /// Content-Type header. + public static let contentType = HTTPHeaderName("Content-Type") + /// Content-Version header. + public static let contentVersion = HTTPHeaderName("Content-Version") + /// Content-Cookie header. + public static let cookie = HTTPHeaderName("Cookie") + /// Content-Cookie2 header. + public static let cookie2 = HTTPHeaderName("Cookie2") + /// DASL header. + public static let dasl = HTTPHeaderName("DASL") + /// DASV header. + public static let dav = HTTPHeaderName("DAV") + /// Date header. + public static let date = HTTPHeaderName("Date") + /// Default-Style header. + public static let defaultStyle = HTTPHeaderName("Default-Style") + /// Delta-Base header. + public static let deltaBase = HTTPHeaderName("Delta-Base") + /// Depth header. + public static let depth = HTTPHeaderName("Depth") + /// Derived-From header. + public static let derivedFrom = HTTPHeaderName("Derived-From") + /// Destination header. + public static let destination = HTTPHeaderName("Destination") + /// Differential-ID header. + public static let differentialID = HTTPHeaderName("Differential-ID") + /// Digest header. + public static let digest = HTTPHeaderName("Digest") + /// ETag header. + public static let eTag = HTTPHeaderName("ETag") + /// Expect header. + public static let expect = HTTPHeaderName("Expect") + /// Expires header. + public static let expires = HTTPHeaderName("Expires") + /// Ext header. + public static let ext = HTTPHeaderName("Ext") + /// Forwarded header. + public static let forwarded = HTTPHeaderName("Forwarded") + /// From header. + public static let from = HTTPHeaderName("From") + /// GetProfile header. + public static let getProfile = HTTPHeaderName("GetProfile") + /// Hobareg header. + public static let hobareg = HTTPHeaderName("Hobareg") + /// Host header. + public static let host = HTTPHeaderName("Host") + /// HTTP2-Settings header. + public static let http2Settings = HTTPHeaderName("HTTP2-Settings") + /// IM header. + public static let im = HTTPHeaderName("IM") + /// If header. + public static let `if` = HTTPHeaderName("If") + /// If-Match header. + public static let ifMatch = HTTPHeaderName("If-Match") + /// If-Modified-Since header. + public static let ifModifiedSince = HTTPHeaderName("If-Modified-Since") + /// If-None-Match header. + public static let ifNoneMatch = HTTPHeaderName("If-None-Match") + /// If-Range header. + public static let ifRange = HTTPHeaderName("If-Range") + /// If-Schedule-Tag-Match header. + public static let ifScheduleTagMatch = HTTPHeaderName("If-Schedule-Tag-Match") + /// If-Unmodified-Since header. + public static let ifUnmodifiedSince = HTTPHeaderName("If-Unmodified-Since") + /// Keep-Alive header. + public static let keepAlive = HTTPHeaderName("Keep-Alive") + /// Label header. + public static let label = HTTPHeaderName("Label") + /// Last-Modified header. + public static let lastModified = HTTPHeaderName("Last-Modified") + /// Link header. + public static let link = HTTPHeaderName("Link") + /// Location header. + public static let location = HTTPHeaderName("Location") + /// Lock-Token header. + public static let lockToken = HTTPHeaderName("Lock-Token") + /// Man header. + public static let man = HTTPHeaderName("Man") + /// Max-Forwards header. + public static let maxForwards = HTTPHeaderName("Max-Forwards") + /// Memento-Date header. + public static let mementoDatetime = HTTPHeaderName("Memento-Datetime") + /// Meter header. + public static let meter = HTTPHeaderName("Meter") + /// MIME-Version header. + public static let mimeVersion = HTTPHeaderName("MIME-Version") + /// Negotiate header. + public static let negotiate = HTTPHeaderName("Negotiate") + /// Opt header. + public static let opt = HTTPHeaderName("Opt") + /// Optional-WWW-Authenticate header. + public static let optionalWWWAuthenticate = HTTPHeaderName("Optional-WWW-Authenticate") + /// Ordering-Type header. + public static let orderingType = HTTPHeaderName("Ordering-Type") + /// Origin header. + public static let origin = HTTPHeaderName("Origin") + /// Overwrite header. + public static let overwrite = HTTPHeaderName("Overwrite") + /// P3P header. + public static let p3p = HTTPHeaderName("P3P") + /// PEP header. + public static let pep = HTTPHeaderName("PEP") + /// PICS-Label header. + public static let picsLabel = HTTPHeaderName("PICS-Label") + /// Pep-Info header. + public static let pepInfo = HTTPHeaderName("Pep-Info") + /// Position header. + public static let position = HTTPHeaderName("Position") + /// Pragma header. + public static let pragma = HTTPHeaderName("Pragma") + /// Prefer header. + public static let prefer = HTTPHeaderName("Prefer") + /// Preference-Applied header. + public static let preferenceApplied = HTTPHeaderName("Preference-Applied") + /// ProfileObject header. + public static let profileObject = HTTPHeaderName("ProfileObject") + /// Protocol header. + public static let `protocol` = HTTPHeaderName("Protocol") + /// Protocol-Info header. + public static let protocolInfo = HTTPHeaderName("Protocol-Info") + /// Protocol-Query header. + public static let protocolQuery = HTTPHeaderName("Protocol-Query") + /// Protocol-Request header. + public static let protocolRequest = HTTPHeaderName("Protocol-Request") + /// Proxy-Authenticate header. + public static let proxyAuthenticate = HTTPHeaderName("Proxy-Authenticate") + /// Proxy-Authentication-Info header. + public static let proxyAuthenticationInfo = HTTPHeaderName("Proxy-Authentication-Info") + /// Proxy-Authorization header. + public static let proxyAuthorization = HTTPHeaderName("Proxy-Authorization") + /// Proxy-Features header. + public static let proxyFeatures = HTTPHeaderName("Proxy-Features") + /// Proxy-Instruction header. + public static let proxyInstruction = HTTPHeaderName("Proxy-Instruction") + /// Public header. + public static let `public` = HTTPHeaderName("Public") + /// Public-Key-Pins header. + public static let publicKeyPins = HTTPHeaderName("Public-Key-Pins") + /// Public-Key-Pins-Report-Only header. + public static let publicKeyPinsReportOnly = HTTPHeaderName("Public-Key-Pins-Report-Only") + /// Range header. + public static let range = HTTPHeaderName("Range") + /// Redirect-Ref header. + public static let redirectRef = HTTPHeaderName("Redirect-Ref") + /// Referer header. + public static let referer = HTTPHeaderName("Referer") + /// Retry-After header. + public static let retryAfter = HTTPHeaderName("Retry-After") + /// Safe header. + public static let safe = HTTPHeaderName("Safe") + /// Schedule-Reply header. + public static let scheduleReply = HTTPHeaderName("Schedule-Reply") + /// Schedule-Tag header. + public static let scheduleTag = HTTPHeaderName("Schedule-Tag") + /// Sec-WebSocket-Accept header. + public static let secWebSocketAccept = HTTPHeaderName("Sec-WebSocket-Accept") + /// Sec-WebSocket-Extensions header. + public static let secWebSocketExtensions = HTTPHeaderName("Sec-WebSocket-Extensions") + /// Sec-WebSocket-Key header. + public static let secWebSocketKey = HTTPHeaderName("Sec-WebSocket-Key") + /// Sec-WebSocket-Protocol header. + public static let secWebSocketProtocol = HTTPHeaderName("Sec-WebSocket-Protocol") + /// Sec-WebSocket-Version header. + public static let secWebSocketVersion = HTTPHeaderName("Sec-WebSocket-Version") + /// Security-Scheme header. + public static let securityScheme = HTTPHeaderName("Security-Scheme") + /// Server header. + public static let server = HTTPHeaderName("Server") + /// Set-Cookie header. + public static let setCookie = HTTPHeaderName("Set-Cookie") + /// Set-Cookie2 header. + public static let setCookie2 = HTTPHeaderName("Set-Cookie2") + /// SetProfile header. + public static let setProfile = HTTPHeaderName("SetProfile") + /// SLUG header. + public static let slug = HTTPHeaderName("SLUG") + /// SoapAction header. + public static let soapAction = HTTPHeaderName("SoapAction") + /// Status-URI header. + public static let statusURI = HTTPHeaderName("Status-URI") + /// Strict-Transport-Security header. + public static let strictTransportSecurity = HTTPHeaderName("Strict-Transport-Security") + /// Surrogate-Capability header. + public static let surrogateCapability = HTTPHeaderName("Surrogate-Capability") + /// Surrogate-Control header. + public static let surrogateControl = HTTPHeaderName("Surrogate-Control") + /// TCN header. + public static let tcn = HTTPHeaderName("TCN") + /// TE header. + public static let te = HTTPHeaderName("TE") + /// Timeout header. + public static let timeout = HTTPHeaderName("Timeout") + /// Topic header. + public static let topic = HTTPHeaderName("Topic") + /// Trailer header. + public static let trailer = HTTPHeaderName("Trailer") + /// Transfer-Encoding header. + public static let transferEncoding = HTTPHeaderName("Transfer-Encoding") + /// TTL header. + public static let ttl = HTTPHeaderName("TTL") + /// Urgency header. + public static let urgency = HTTPHeaderName("Urgency") + /// URI header. + public static let uri = HTTPHeaderName("URI") + /// Upgrade header. + public static let upgrade = HTTPHeaderName("Upgrade") + /// User-Agent header. + public static let userAgent = HTTPHeaderName("User-Agent") + /// Variant-Vary header. + public static let variantVary = HTTPHeaderName("Variant-Vary") + /// Vary header. + public static let vary = HTTPHeaderName("Vary") + /// Via header. + public static let via = HTTPHeaderName("Via") + /// WWW-Authenticate header. + public static let wwwAuthenticate = HTTPHeaderName("WWW-Authenticate") + /// Want-Digest header. + public static let wantDigest = HTTPHeaderName("Want-Digest") + /// Warning header. + public static let warning = HTTPHeaderName("Warning") + /// X-Frame-Options header. + public static let xFrameOptions = HTTPHeaderName("X-Frame-Options") + + // https://www.iana.org/assignments/message-headers/message-headers.xhtml + // Provisional Message Header Field Names + /// Access-Control header. + public static let accessControl = HTTPHeaderName("Access-Control") + /// Access-Control-Allow-Credentials header. + public static let accessControlAllowCredentials = HTTPHeaderName("Access-Control-Allow-Credentials") + /// Access-Control-Allow-Headers header. + public static let accessControlAllowHeaders = HTTPHeaderName("Access-Control-Allow-Headers") + /// Access-Control-Allow-Methods header. + public static let accessControlAllowMethods = HTTPHeaderName("Access-Control-Allow-Methods") + /// Access-Control-Allow-Origin header. + public static let accessControlAllowOrigin = HTTPHeaderName("Access-Control-Allow-Origin") + /// Access-Control-Max-Age header. + public static let accessControlMaxAge = HTTPHeaderName("Access-Control-Max-Age") + /// Access-Control-Request-Method header. + public static let accessControlRequestMethod = HTTPHeaderName("Access-Control-Request-Method") + /// Access-Control-Request-Headers header. + public static let accessControlRequestHeaders = HTTPHeaderName("Access-Control-Request-Headers") + /// Compliance header. + public static let compliance = HTTPHeaderName("Compliance") + /// Content-Transfer-Encoding header. + public static let contentTransferEncoding = HTTPHeaderName("Content-Transfer-Encoding") + /// Cost header. + public static let cost = HTTPHeaderName("Cost") + /// EDIINT-Features header. + public static let ediintFeatures = HTTPHeaderName("EDIINT-Features") + /// Message-ID header. + public static let messageID = HTTPHeaderName("Message-ID") + /// Method-Check header. + public static let methodCheck = HTTPHeaderName("Method-Check") + /// Method-Check-Expires header. + public static let methodCheckExpires = HTTPHeaderName("Method-Check-Expires") + /// Non-Compliance header. + public static let nonCompliance = HTTPHeaderName("Non-Compliance") + /// Optional header. + public static let optional = HTTPHeaderName("Optional") + /// Referer-Root header. + public static let refererRoot = HTTPHeaderName("Referer-Root") + /// Resolution-Hint header. + public static let resolutionHint = HTTPHeaderName("Resolution-Hint") + /// Resolver-Location header. + public static let resolverLocation = HTTPHeaderName("Resolver-Location") + /// SubOK header. + public static let subOK = HTTPHeaderName("SubOK") + /// Subst header. + public static let subst = HTTPHeaderName("Subst") + /// Title header. + public static let title = HTTPHeaderName("Title") + /// UA-Color header. + public static let uaColor = HTTPHeaderName("UA-Color") + /// UA-Media header. + public static let uaMedia = HTTPHeaderName("UA-Media") + /// UA-Pixels header. + public static let uaPixels = HTTPHeaderName("UA-Pixels") + /// UA-Resolution header. + public static let uaResolution = HTTPHeaderName("UA-Resolution") + /// UA-Windowpixels header. + public static let uaWindowpixels = HTTPHeaderName("UA-Windowpixels") + /// Version header. + public static let version = HTTPHeaderName("Version") + /// X-Device-Accept header. + public static let xDeviceAccept = HTTPHeaderName("X-Device-Accept") + /// X-Device-Accept-Charset header. + public static let xDeviceAcceptCharset = HTTPHeaderName("X-Device-Accept-Charset") + /// X-Device-Accept-Encoding header. + public static let xDeviceAcceptEncoding = HTTPHeaderName("X-Device-Accept-Encoding") + /// X-Device-Accept-Language header. + public static let xDeviceAcceptLanguage = HTTPHeaderName("X-Device-Accept-Language") + /// X-Device-User-Agent header. + public static let xDeviceUserAgent = HTTPHeaderName("X-Device-User-Agent") +} + diff --git a/Sources/HTTP/Message/HTTPHeaderStorage.swift b/Sources/HTTP/Message/HTTPHeaderStorage.swift new file mode 100644 index 00000000..5070294d --- /dev/null +++ b/Sources/HTTP/Message/HTTPHeaderStorage.swift @@ -0,0 +1,224 @@ +import Bits +import COperatingSystem + +/// COW storage for HTTP headers. +final class HTTPHeaderStorage { + /// Valid view into the HTTPHeader's internal buffer. + private var view: ByteBuffer + + /// The HTTPHeader's internal storage. + private var buffer: MutableByteBuffer + + /// The HTTPHeader's known indexes into the storage. + private var indexes: [HTTPHeaderIndex?] + + /// Creates a new, empty `HTTPHeaders`. + public init() { + let storageSize = 1024 + let buffer = MutableByteBuffer(start: .allocate(capacity: storageSize), count: storageSize) + memcpy(buffer.baseAddress, defaultHeaders.baseAddress!, defaultHeadersSize) + self.buffer = buffer + self.view = ByteBuffer(start: buffer.baseAddress, count: defaultHeadersSize) + self.indexes = [] + } + + /// Create a new `HTTPHeaders` with explicit storage and indexes. + internal init(bytes: Bytes, indexes: [HTTPHeaderIndex]) { + let storageSize = view.count + let buffer = MutableByteBuffer(start: .allocate(capacity: storageSize), count: storageSize) + memcpy(buffer.baseAddress, bytes, storageSize) + self.buffer = buffer + self.view = ByteBuffer(start: buffer.baseAddress, count: storageSize) + self.indexes = indexes + } + + + /// Removes all headers with this name + internal func removeValues(for name: HTTPHeaderName) { + /// loop over all indexes + for i in 0.. 0 { + /// there is valid data after this header that we must relocate + let destination = buffer.start.advanced(by: index.startIndex) // deleted header's start + let source = buffer.start.advanced(by: index.endIndex) // deleted header's end + memcpy(destination, source, displacedBytes) + } else { + // no headers after this, simply shorten the valid buffer + self.view = ByteBuffer(start: buffer.start, count: view.count - index.size) + } + } + } + } + + /// Appends the supplied string to the header data. + /// Note: This will naively append data, not deleting existing values. Use in + /// conjunction with `removeValues(for:)` for that behavior. + internal func appendValue(_ value: String, for name: HTTPHeaderName) { + let value = value.buffer + + /// create the new header index + let index = HTTPHeaderIndex( + nameStartIndex: view.count, + nameEndIndex: view.count + name.lowercased.count, + valueStartIndex: view.count + name.lowercased.count + 2, + valueEndIndex: view.count + name.lowercased.count + 2 + value.count + ) + indexes.append(index) + + /// if value is bigger than internal buffer, increase size + if index.size > buffer.count - view.count { + increaseBufferSize(by: value.count * 2) // multiply by 2 to potentially reduce realloc calls + } + + // + memcpy(buffer.start.advanced(by: index.nameStartIndex), name.lowercased, name.lowercased.count) + // `: ` + memcpy(buffer.start.advanced(by: index.nameEndIndex), headerSeparator.start, headerSeparatorSize) + // + memcpy(buffer.start.advanced(by: index.valueStartIndex), value.start, value.count) + // `\r\n` + memcpy(buffer.start.advanced(by: index.valueEndIndex), headerEnding.start, headerEndingSize) + + view = ByteBuffer(start: buffer.start, count: view.count + index.endIndex) + } + + /// Fetches the String value for a given header index. + /// Use `indexes(for:)` to fetch indexes for a given header name. + internal func value(for header: HTTPHeaderIndex) -> String? { + return String(bytes: view[header.valueStartIndex.. String? { + return String(bytes: view[header.nameStartIndex.. [HTTPHeaderIndex] { + return indexes.flatMap { $0 } + } + + /// Scans the boundary of the value associated with a name + internal func indexes(for name: HTTPHeaderName) -> [HTTPHeaderIndex] { + var valueRanges: [HTTPHeaderIndex] = [] + + for index in indexes { + guard let index = index else { + continue + } + + if headerAt(index, matchesName: name) { + valueRanges.append(index) + } + } + + return valueRanges + } + + + /// Returns true if the header at the supplied index matches a given name. + internal func headerAt(_ index: HTTPHeaderIndex, matchesName name: HTTPHeaderName) -> Bool { + let nameSize = index.nameEndIndex - index.nameStartIndex + guard name.lowercased.count == nameSize else { + return false + } + + let nameData: ByteBuffer = name.lowercased.withUnsafeBufferPointer { $0 } + + for i in 0..= .A && headerByte <= .Z && headerByte &+ asciiCasingOffset == nameByte { + continue + } + + return false + } + + return true + } + + /// An internal API that blindly adds a header without checking for doubles + internal func withByteBuffer(_ closure: (ByteBuffer) -> T) -> T { + /// need room for trailing `\r\n` + if view.count + 2 > buffer.count { + increaseBufferSize(by: 2) + } + let sub = ByteBuffer(start: view.start, count: view.count + 2) + memcpy(buffer.start.advanced(by: view.count), headerEnding.start, headerEndingSize) + return closure(sub) + } + + /// Increases the internal buffer size by the supplied count. + internal func increaseBufferSize(by count: Int) { + let newSize = buffer.count + count + let pointer: MutableBytesPointer = realloc(UnsafeMutableRawPointer(buffer.start), newSize) + .assumingMemoryBound(to: Byte.self) + buffer = MutableByteBuffer(start: pointer, count: newSize) + } +} + +extension HTTPHeaderStorage: CustomStringConvertible { + /// See `CustomStringConvertible.description` + public var description: String { + return String(bytes: view, encoding: .ascii) ?? "n/a" + } +} + +/// MARK: Utility + +extension UnsafeMutableBufferPointer { + var start: UnsafeMutablePointer { + return baseAddress! + } +} + +extension UnsafeBufferPointer { + var start: UnsafePointer { + return baseAddress! + } +} + +extension String { + var buffer: ByteBuffer { + let count = utf8.count + return withCString { cPointer in + return ByteBuffer(start: cPointer.withMemoryRebound(to: Byte.self, capacity: count) { $0 }, count: count) + } + } +} + +/// MARK: Static Data + +private let headerSeparatorStaticString: StaticString = ": " +private let headerSeparator: ByteBuffer = headerSeparatorStaticString.withUTF8Buffer { $0 } +private let headerSeparatorSize: Int = headerSeparator.count + +private let headerEndingStaticString: StaticString = "\r\n" +private let headerEnding: ByteBuffer = headerEndingStaticString.withUTF8Buffer { $0 } +private let headerEndingSize: Int = headerEnding.count + +private let defaultHeadersStaticString: StaticString = "Content-Length: 0\r\n\r\n" +private let defaultHeaders: ByteBuffer = defaultHeadersStaticString.withUTF8Buffer { $0 } +private let defaultHeadersSize: Int = defaultHeaders.count diff --git a/Sources/HTTP/Message/HTTPHeaders.swift b/Sources/HTTP/Message/HTTPHeaders.swift index c47ea4a9..169310b2 100644 --- a/Sources/HTTP/Message/HTTPHeaders.swift +++ b/Sources/HTTP/Message/HTTPHeaders.swift @@ -14,43 +14,27 @@ import Bits /// /// [Learn More →](https://docs.vapor.codes/3.0/http/headers/) public struct HTTPHeaders: Codable { - struct Index: CustomStringConvertible { - var description: String { - return "[\(nameStartIndex)..<\(nameEndIndex):\(valueStartIndex)..<\(valueEndIndex)]" - } - - var nameStartIndex: Int - var nameEndIndex: Int - var valueStartIndex: Int - var valueEndIndex: Int - var invalidated: Bool - } - - var headerIndexes = [HTTPHeaders.Index]() - var storage: [UInt8] - - public func withByteBuffer(_ run: (ByteBuffer) throws -> T) rethrows -> T { - return try storage.withUnsafeBufferPointer(run) - } - + /// The HTTPHeader's raw data storage + internal var storage: HTTPHeaderStorage + + /// Creates a new, empty `HTTPHeaders`. public init() { - self.storage = [UInt8]() - self.storage.reserveCapacity(4096) - self.headerIndexes.reserveCapacity(50) + self.storage = .init() } - - internal init(storage: [UInt8], indexes: [HTTPHeaders.Index]) { + + /// Create a new `HTTPHeaders` with explicit storage and indexes. + internal init(storage: HTTPHeaderStorage) { self.storage = storage - self.storage.reserveCapacity(storage.count + 1024) - self.headerIndexes = indexes } - + + /// See `Encodable.encode(to:)` public func encode(to encoder: Encoder) throws { try Array(self).encode(to: encoder) } - + + /// See `Decodable.init(from:)` public init(from decoder: Decoder) throws { - let headers = try [Name: String](from: decoder) + let headers = try [HTTPHeaderName: String](from: decoder) self.init() @@ -62,167 +46,82 @@ public struct HTTPHeaders: Codable { /// Accesses the (first) value associated with the `Name` if any /// /// [Learn More →](https://docs.vapor.codes/3.0/http/headers/#accessing-headers) - public subscript(name: Name) -> String? { + public subscript(name: HTTPHeaderName) -> String? { get { switch name { - case Name.setCookie: // Exception, see note in [RFC7230, section 3.2.2] + case HTTPHeaderName.setCookie: // Exception, see note in [RFC7230, section 3.2.2] return self[valuesFor: .setCookie].first default: let values = self[valuesFor: name] - if values.isEmpty { return nil } - return values.joined(separator: ",") } } set { - guard let newValue = newValue else { - self.removeValues(forName: name) - return - } - - let indexes = self.indexes(forName: name) - - if indexes.count == 1, - indexes[0].valueEndIndex - indexes[0].valueStartIndex == newValue.utf8.count - { - storage.withUnsafeMutableBufferPointer { (buffer: inout MutableByteBuffer) in - _ = memcpy( - buffer.baseAddress!.advanced(by: indexes[0].valueStartIndex), - newValue, - indexes[0].valueEndIndex - indexes[0].valueStartIndex - ) - } - } else { - removeValues(forName: name) - - appendValue(newValue, forName: name) + storage.removeValues(for: name) + if let value = newValue { + storage.appendValue(value, for: name) } } } - /// Removes all headers with this name - public mutating func removeValues(forName name: Name) { - // Reverse iteration to keep the index positions intact - for i in 0.. [String] { + get { + return storage.indexes(for: name).flatMap { storage.value(for: $0) } } - } - - /// Reserves X bytes of extra capacity in advance - public mutating func reserveAdditionalCapacity(bytes: Int) { - storage.reserveCapacity(storage.count + bytes) - } - - func clean() -> Data { - var data = Data() - data.reserveCapacity(self.storage.count) - - var start: Int? = nil - - for index in headerIndexes { - if index.invalidated { - if let start = start { - data.append(contentsOf: self.storage[start.. String? { + public subscript(name: HTTPHeaderName, attribute: String) -> String? { get { guard let header = self[name] else { return nil } guard let range = header.range(of: "\(attribute)=") else { return nil } - + let remainder = header[range.upperBound...] - + var string: String - + if let end = remainder.index(of: ";") { string = String(remainder[remainder.startIndex.. 1 { string.removeFirst() string.removeLast() } - + return string } } - - /// Accesses all values associated with the `Name` - public subscript(valuesFor name: Name) -> [String] { - get { - return self.indexes(forName: name).flatMap { index in - return String(bytes: storage[index.valueStartIndex.. HTTPHeaders { return lhs } + +/// MARK: Literal Conformances + extension HTTPHeaders : ExpressibleByDictionaryLiteral { /// Creates HTTP headers. - public init(dictionaryLiteral: (Name, String)...) { - storage = [UInt8]() - - // 64 bytes per key-value pair shouldn't be too far off - storage.reserveCapacity(dictionaryLiteral.count * 64) + public init(dictionaryLiteral: (HTTPHeaderName, String)...) { + self.init() for (name, value) in dictionaryLiteral { - appendValue(value, forName: name) + storage.appendValue(value, for: name) } } } extension HTTPHeaders { /// Used instead of HTTPHeaders to save CPU on dictionary construction - /// :nodoc: public struct Literal : ExpressibleByDictionaryLiteral { - let fields: [(name: Name, value: String)] + let fields: [(name: HTTPHeaderName, value: String)] - public init(dictionaryLiteral: (Name, String)...) { + public init(dictionaryLiteral: (HTTPHeaderName, String)...) { fields = dictionaryLiteral } } @@ -265,51 +163,7 @@ extension HTTPHeaders { /// Appends a header to the headers public mutating func append(_ literal: HTTPHeaders.Literal) { for (name, value) in literal.fields { - appendValue(value, forName: name) - } - } - - /// Scans the boundary of the value associated with a name - fileprivate func indexes(forName name: Name) -> [Index] { - var valueRanges = [Index]() - - for index in self.headerIndexes { - if equal(index: index, name: name) { - valueRanges.append(index) - } - } - - return valueRanges - } - - fileprivate func equal(index: Index, name: Name) -> Bool { - let indexSize = index.nameEndIndex - index.nameStartIndex - - guard name.lowercased.count == indexSize else { - return false - } - - return storage.withUnsafeBufferPointer { lhsBuffer in - return name.lowercased.withUnsafeBufferPointer { (rhsBuffer: ByteBuffer) in - let lhs = lhsBuffer.baseAddress! - let rhs = rhsBuffer.baseAddress! - - for i in 0..= .A && byte <= .Z && byte &+ asciiCasingOffset == rhs[i] { - continue - } - - return false - } - - return true - } + storage.appendValue(value, for: name) } } @@ -321,451 +175,21 @@ extension HTTPHeaders { } } +/// MARK: Sequence Conformance + extension HTTPHeaders: Sequence { /// Iterates over all headers - public func makeIterator() -> AnyIterator<(name: Name, value: String)> { + public func makeIterator() -> AnyIterator<(name: HTTPHeaderName, value: String)> { let storage = self.storage - var indexIterator = self.headerIndexes.makeIterator() - + var indexIterator = storage.validIndexes().makeIterator() return AnyIterator { - guard let index = indexIterator.next() else { + guard let header = indexIterator.next() else { return nil } - let name = storage[index.nameStartIndex.. Bool { - return lhs.lowercased == rhs.lowercased - } - - // https://www.iana.org/assignments/message-headers/message-headers.xhtml - // Permanent Message Header Field Names - - /// A-IM header. - public static let aIM = Name("A-IM") - /// Accept header. - public static let accept = Name("Accept") - /// Accept-Additions header. - public static let acceptAdditions = Name("Accept-Additions") - /// Accept-Charset header. - public static let acceptCharset = Name("Accept-Charset") - /// Accept-Datetime header. - public static let acceptDatetime = Name("Accept-Datetime") - /// Accept-Encoding header. - public static let acceptEncoding = Name("Accept-Encoding") - /// Accept-Features header. - public static let acceptFeatures = Name("Accept-Features") - /// Accept-Language header. - public static let acceptLanguage = Name("Accept-Language") - /// Accept-Patch header. - public static let acceptPatch = Name("Accept-Patch") - /// Accept-Post header. - public static let acceptPost = Name("Accept-Post") - /// Accept-Ranges header. - public static let acceptRanges = Name("Accept-Ranges") - /// Accept-Age header. - public static let age = Name("Age") - /// Accept-Allow header. - public static let allow = Name("Allow") - /// ALPN header. - public static let alpn = Name("ALPN") - /// Alt-Svc header. - public static let altSvc = Name("Alt-Svc") - /// Alt-Used header. - public static let altUsed = Name("Alt-Used") - /// Alternatives header. - public static let alternates = Name("Alternates") - /// Apply-To-Redirect-Ref header. - public static let applyToRedirectRef = Name("Apply-To-Redirect-Ref") - /// Authentication-Control header. - public static let authenticationControl = Name("Authentication-Control") - /// Authentication-Info header. - public static let authenticationInfo = Name("Authentication-Info") - /// Authorization header. - public static let authorization = Name("Authorization") - /// C-Ext header. - public static let cExt = Name("C-Ext") - /// C-Man header. - public static let cMan = Name("C-Man") - /// C-Opt header. - public static let cOpt = Name("C-Opt") - /// C-PEP header. - public static let cPEP = Name("C-PEP") - /// C-PEP-Indo header. - public static let cPEPInfo = Name("C-PEP-Info") - /// Cache-Control header. - public static let cacheControl = Name("Cache-Control") - /// CalDav-Timezones header. - public static let calDAVTimezones = Name("CalDAV-Timezones") - /// Close header. - public static let close = Name("Close") - /// Connection header. - public static let connection = Name("Connection") - /// Content-Base. - public static let contentBase = Name("Content-Base") - /// Content-Disposition header. - public static let contentDisposition = Name("Content-Disposition") - /// Content-Encoding header. - public static let contentEncoding = Name("Content-Encoding") - /// Content-ID header. - public static let contentID = Name("Content-ID") - /// Content-Language header. - public static let contentLanguage = Name("Content-Language") - /// Content-Length header. - public static let contentLength = Name("Content-Length") - /// Content-Location header. - public static let contentLocation = Name("Content-Location") - /// Content-MD5 header. - public static let contentMD5 = Name("Content-MD5") - /// Content-Range header. - public static let contentRange = Name("Content-Range") - /// Content-Script-Type header. - public static let contentScriptType = Name("Content-Script-Type") - /// Content-Style-Type header. - public static let contentStyleType = Name("Content-Style-Type") - /// Content-Type header. - public static let contentType = Name("Content-Type") - /// Content-Version header. - public static let contentVersion = Name("Content-Version") - /// Content-Cookie header. - public static let cookie = Name("Cookie") - /// Content-Cookie2 header. - public static let cookie2 = Name("Cookie2") - /// DASL header. - public static let dasl = Name("DASL") - /// DASV header. - public static let dav = Name("DAV") - /// Date header. - public static let date = Name("Date") - /// Default-Style header. - public static let defaultStyle = Name("Default-Style") - /// Delta-Base header. - public static let deltaBase = Name("Delta-Base") - /// Depth header. - public static let depth = Name("Depth") - /// Derived-From header. - public static let derivedFrom = Name("Derived-From") - /// Destination header. - public static let destination = Name("Destination") - /// Differential-ID header. - public static let differentialID = Name("Differential-ID") - /// Digest header. - public static let digest = Name("Digest") - /// ETag header. - public static let eTag = Name("ETag") - /// Expect header. - public static let expect = Name("Expect") - /// Expires header. - public static let expires = Name("Expires") - /// Ext header. - public static let ext = Name("Ext") - /// Forwarded header. - public static let forwarded = Name("Forwarded") - /// From header. - public static let from = Name("From") - /// GetProfile header. - public static let getProfile = Name("GetProfile") - /// Hobareg header. - public static let hobareg = Name("Hobareg") - /// Host header. - public static let host = Name("Host") - /// HTTP2-Settings header. - public static let http2Settings = Name("HTTP2-Settings") - /// IM header. - public static let im = Name("IM") - /// If header. - public static let `if` = Name("If") - /// If-Match header. - public static let ifMatch = Name("If-Match") - /// If-Modified-Since header. - public static let ifModifiedSince = Name("If-Modified-Since") - /// If-None-Match header. - public static let ifNoneMatch = Name("If-None-Match") - /// If-Range header. - public static let ifRange = Name("If-Range") - /// If-Schedule-Tag-Match header. - public static let ifScheduleTagMatch = Name("If-Schedule-Tag-Match") - /// If-Unmodified-Since header. - public static let ifUnmodifiedSince = Name("If-Unmodified-Since") - /// Keep-Alive header. - public static let keepAlive = Name("Keep-Alive") - /// Label header. - public static let label = Name("Label") - /// Last-Modified header. - public static let lastModified = Name("Last-Modified") - /// Link header. - public static let link = Name("Link") - /// Location header. - public static let location = Name("Location") - /// Lock-Token header. - public static let lockToken = Name("Lock-Token") - /// Man header. - public static let man = Name("Man") - /// Max-Forwards header. - public static let maxForwards = Name("Max-Forwards") - /// Memento-Date header. - public static let mementoDatetime = Name("Memento-Datetime") - /// Meter header. - public static let meter = Name("Meter") - /// MIME-Version header. - public static let mimeVersion = Name("MIME-Version") - /// Negotiate header. - public static let negotiate = Name("Negotiate") - /// Opt header. - public static let opt = Name("Opt") - /// Optional-WWW-Authenticate header. - public static let optionalWWWAuthenticate = Name("Optional-WWW-Authenticate") - /// Ordering-Type header. - public static let orderingType = Name("Ordering-Type") - /// Origin header. - public static let origin = Name("Origin") - /// Overwrite header. - public static let overwrite = Name("Overwrite") - /// P3P header. - public static let p3p = Name("P3P") - /// PEP header. - public static let pep = Name("PEP") - /// PICS-Label header. - public static let picsLabel = Name("PICS-Label") - /// Pep-Info header. - public static let pepInfo = Name("Pep-Info") - /// Position header. - public static let position = Name("Position") - /// Pragma header. - public static let pragma = Name("Pragma") - /// Prefer header. - public static let prefer = Name("Prefer") - /// Preference-Applied header. - public static let preferenceApplied = Name("Preference-Applied") - /// ProfileObject header. - public static let profileObject = Name("ProfileObject") - /// Protocol header. - public static let `protocol` = Name("Protocol") - /// Protocol-Info header. - public static let protocolInfo = Name("Protocol-Info") - /// Protocol-Query header. - public static let protocolQuery = Name("Protocol-Query") - /// Protocol-Request header. - public static let protocolRequest = Name("Protocol-Request") - /// Proxy-Authenticate header. - public static let proxyAuthenticate = Name("Proxy-Authenticate") - /// Proxy-Authentication-Info header. - public static let proxyAuthenticationInfo = Name("Proxy-Authentication-Info") - /// Proxy-Authorization header. - public static let proxyAuthorization = Name("Proxy-Authorization") - /// Proxy-Features header. - public static let proxyFeatures = Name("Proxy-Features") - /// Proxy-Instruction header. - public static let proxyInstruction = Name("Proxy-Instruction") - /// Public header. - public static let `public` = Name("Public") - /// Public-Key-Pins header. - public static let publicKeyPins = Name("Public-Key-Pins") - /// Public-Key-Pins-Report-Only header. - public static let publicKeyPinsReportOnly = Name("Public-Key-Pins-Report-Only") - /// Range header. - public static let range = Name("Range") - /// Redirect-Ref header. - public static let redirectRef = Name("Redirect-Ref") - /// Referer header. - public static let referer = Name("Referer") - /// Retry-After header. - public static let retryAfter = Name("Retry-After") - /// Safe header. - public static let safe = Name("Safe") - /// Schedule-Reply header. - public static let scheduleReply = Name("Schedule-Reply") - /// Schedule-Tag header. - public static let scheduleTag = Name("Schedule-Tag") - /// Sec-WebSocket-Accept header. - public static let secWebSocketAccept = Name("Sec-WebSocket-Accept") - /// Sec-WebSocket-Extensions header. - public static let secWebSocketExtensions = Name("Sec-WebSocket-Extensions") - /// Sec-WebSocket-Key header. - public static let secWebSocketKey = Name("Sec-WebSocket-Key") - /// Sec-WebSocket-Protocol header. - public static let secWebSocketProtocol = Name("Sec-WebSocket-Protocol") - /// Sec-WebSocket-Version header. - public static let secWebSocketVersion = Name("Sec-WebSocket-Version") - /// Security-Scheme header. - public static let securityScheme = Name("Security-Scheme") - /// Server header. - public static let server = Name("Server") - /// Set-Cookie header. - public static let setCookie = Name("Set-Cookie") - /// Set-Cookie2 header. - public static let setCookie2 = Name("Set-Cookie2") - /// SetProfile header. - public static let setProfile = Name("SetProfile") - /// SLUG header. - public static let slug = Name("SLUG") - /// SoapAction header. - public static let soapAction = Name("SoapAction") - /// Status-URI header. - public static let statusURI = Name("Status-URI") - /// Strict-Transport-Security header. - public static let strictTransportSecurity = Name("Strict-Transport-Security") - /// Surrogate-Capability header. - public static let surrogateCapability = Name("Surrogate-Capability") - /// Surrogate-Control header. - public static let surrogateControl = Name("Surrogate-Control") - /// TCN header. - public static let tcn = Name("TCN") - /// TE header. - public static let te = Name("TE") - /// Timeout header. - public static let timeout = Name("Timeout") - /// Topic header. - public static let topic = Name("Topic") - /// Trailer header. - public static let trailer = Name("Trailer") - /// Transfer-Encoding header. - public static let transferEncoding = Name("Transfer-Encoding") - /// TTL header. - public static let ttl = Name("TTL") - /// Urgency header. - public static let urgency = Name("Urgency") - /// URI header. - public static let uri = Name("URI") - /// Upgrade header. - public static let upgrade = Name("Upgrade") - /// User-Agent header. - public static let userAgent = Name("User-Agent") - /// Variant-Vary header. - public static let variantVary = Name("Variant-Vary") - /// Vary header. - public static let vary = Name("Vary") - /// Via header. - public static let via = Name("Via") - /// WWW-Authenticate header. - public static let wwwAuthenticate = Name("WWW-Authenticate") - /// Want-Digest header. - public static let wantDigest = Name("Want-Digest") - /// Warning header. - public static let warning = Name("Warning") - /// X-Frame-Options header. - public static let xFrameOptions = Name("X-Frame-Options") - - // https://www.iana.org/assignments/message-headers/message-headers.xhtml - // Provisional Message Header Field Names - /// Access-Control header. - public static let accessControl = Name("Access-Control") - /// Access-Control-Allow-Credentials header. - public static let accessControlAllowCredentials = Name("Access-Control-Allow-Credentials") - /// Access-Control-Allow-Headers header. - public static let accessControlAllowHeaders = Name("Access-Control-Allow-Headers") - /// Access-Control-Allow-Methods header. - public static let accessControlAllowMethods = Name("Access-Control-Allow-Methods") - /// Access-Control-Allow-Origin header. - public static let accessControlAllowOrigin = Name("Access-Control-Allow-Origin") - /// Access-Control-Max-Age header. - public static let accessControlMaxAge = Name("Access-Control-Max-Age") - /// Access-Control-Request-Method header. - public static let accessControlRequestMethod = Name("Access-Control-Request-Method") - /// Access-Control-Request-Headers header. - public static let accessControlRequestHeaders = Name("Access-Control-Request-Headers") - /// Compliance header. - public static let compliance = Name("Compliance") - /// Content-Transfer-Encoding header. - public static let contentTransferEncoding = Name("Content-Transfer-Encoding") - /// Cost header. - public static let cost = Name("Cost") - /// EDIINT-Features header. - public static let ediintFeatures = Name("EDIINT-Features") - /// Message-ID header. - public static let messageID = Name("Message-ID") - /// Method-Check header. - public static let methodCheck = Name("Method-Check") - /// Method-Check-Expires header. - public static let methodCheckExpires = Name("Method-Check-Expires") - /// Non-Compliance header. - public static let nonCompliance = Name("Non-Compliance") - /// Optional header. - public static let optional = Name("Optional") - /// Referer-Root header. - public static let refererRoot = Name("Referer-Root") - /// Resolution-Hint header. - public static let resolutionHint = Name("Resolution-Hint") - /// Resolver-Location header. - public static let resolverLocation = Name("Resolver-Location") - /// SubOK header. - public static let subOK = Name("SubOK") - /// Subst header. - public static let subst = Name("Subst") - /// Title header. - public static let title = Name("Title") - /// UA-Color header. - public static let uaColor = Name("UA-Color") - /// UA-Media header. - public static let uaMedia = Name("UA-Media") - /// UA-Pixels header. - public static let uaPixels = Name("UA-Pixels") - /// UA-Resolution header. - public static let uaResolution = Name("UA-Resolution") - /// UA-Windowpixels header. - public static let uaWindowpixels = Name("UA-Windowpixels") - /// Version header. - public static let version = Name("Version") - /// X-Device-Accept header. - public static let xDeviceAccept = Name("X-Device-Accept") - /// X-Device-Accept-Charset header. - public static let xDeviceAcceptCharset = Name("X-Device-Accept-Charset") - /// X-Device-Accept-Encoding header. - public static let xDeviceAcceptEncoding = Name("X-Device-Accept-Encoding") - /// X-Device-Accept-Language header. - public static let xDeviceAcceptLanguage = Name("X-Device-Accept-Language") - /// X-Device-User-Agent header. - public static let xDeviceUserAgent = Name("X-Device-User-Agent") - } -} - diff --git a/Sources/HTTP/Message/HTTPMessage.swift b/Sources/HTTP/Message/HTTPMessage.swift index 684d223b..4e3ed718 100644 --- a/Sources/HTTP/Message/HTTPMessage.swift +++ b/Sources/HTTP/Message/HTTPMessage.swift @@ -71,6 +71,19 @@ public struct HTTPOnUpgrade: Codable { } } +extension HTTPMessage { + /// Sets Content-Length / Transfer-Encoding headers. + internal mutating func updateBodyHeaders() { + if let count = body.count { + if count > 0 { + headers[.contentLength] = count.description + } + } else { + headers[.transferEncoding] = "chunked" + } + } +} + // MARK: Debug string extension HTTPMessage { diff --git a/Sources/HTTP/Parser/CHTTPBodyStream.swift b/Sources/HTTP/Parser/CHTTPBodyStream.swift index f4e0f704..8f84ce12 100644 --- a/Sources/HTTP/Parser/CHTTPBodyStream.swift +++ b/Sources/HTTP/Parser/CHTTPBodyStream.swift @@ -19,7 +19,7 @@ final class CHTTPBodyStream: OutputStream { func push(_ buffer: ByteBuffer, _ ready: Promise) { assert(waiting == nil) if let downstream = self.downstream { - downstream.next(buffer, ready) + downstream.input(.next(buffer, ready)) } else { waiting = (buffer, ready) } @@ -30,7 +30,7 @@ final class CHTTPBodyStream: OutputStream { downstream = .init(inputStream) if let (buffer, ready) = self.waiting { self.waiting = nil - inputStream.next(buffer, ready) + inputStream.input(.next(buffer, ready)) } } diff --git a/Sources/HTTP/Parser/CHTTPParser.swift b/Sources/HTTP/Parser/CHTTPParser.swift index 3e77e778..4f3de286 100644 --- a/Sources/HTTP/Parser/CHTTPParser.swift +++ b/Sources/HTTP/Parser/CHTTPParser.swift @@ -74,7 +74,7 @@ extension CHTTPParser { case .readyStream: fatalError("Illegal state") } let message = try makeMessage(using: body) - downstream.next(message, ready) + downstream.input(.next(message, ready)) chttp.reset() } else { // Convert body to a stream @@ -90,8 +90,9 @@ extension CHTTPParser { return self.chttp.headers?[.contentLength].flatMap(Int.init) } let message = try makeMessage(using: body) - let nextMessageFuture = downstream.next(message) - chttp.state = .streaming(nextMessageFuture) + let nextMessagePromise = Promise(Void.self) + downstream.input(.next(message, nextMessagePromise)) + chttp.state = .streaming(nextMessagePromise.future) } } else { /// Headers not complete, request more input diff --git a/Sources/HTTP/Parser/CHTTPParserContext.swift b/Sources/HTTP/Parser/CHTTPParserContext.swift index ec73da89..a53e3108 100644 --- a/Sources/HTTP/Parser/CHTTPParserContext.swift +++ b/Sources/HTTP/Parser/CHTTPParserContext.swift @@ -41,7 +41,7 @@ internal final class CHTTPParserContext { var headersData: [UInt8] /// Parsed indexes into the header data - var headersIndexes: [HTTPHeaders.Index] + var headersIndexes: [HTTPHeaderIndex] /// Raw URL data @@ -128,7 +128,7 @@ enum CHTTPParserState { /// Possible header states enum CHTTPHeaderState { case none - case value(HTTPHeaders.Index) + case value(HTTPHeaderIndex) case key(startIndex: Int, endIndex: Int) } @@ -230,7 +230,8 @@ extension CHTTPParserContext { /// if this buffer copy is happening after headers complete indication, /// set the headers struct for later retreival if headersComplete { - headers = HTTPHeaders(storage: headersData, indexes: headersIndexes) + let storage = HTTPHeaderStorage(bytes: headersData, indexes: headersIndexes) + headers = HTTPHeaders(storage: storage) } } @@ -370,12 +371,11 @@ extension CHTTPParserContext { let distance = start.distance(to: chunk) + results.headerStartOffset // create a full HTTP headers index - let index = HTTPHeaders.Index( + let index = HTTPHeaderIndex( nameStartIndex: key.startIndex, nameEndIndex: key.endIndex, valueStartIndex: distance, - valueEndIndex: distance + count, - invalidated: false + valueEndIndex: distance + count ) results.headerState = .value(index) } diff --git a/Sources/HTTP/Response/HTTPResponse.swift b/Sources/HTTP/Response/HTTPResponse.swift index b7fb70eb..1d610a2c 100644 --- a/Sources/HTTP/Response/HTTPResponse.swift +++ b/Sources/HTTP/Response/HTTPResponse.swift @@ -38,7 +38,9 @@ public struct HTTPResponse: HTTPMessage { /// See Message.body /// /// [Learn More →](https://docs.vapor.codes/3.0/http/body/) - public var body: HTTPBody + public var body: HTTPBody { + didSet { updateBodyHeaders() } + } /// See Message.onUpgrade public var onUpgrade: HTTPOnUpgrade? @@ -54,6 +56,7 @@ public struct HTTPResponse: HTTPMessage { self.status = status self.headers = headers self.body = body + updateBodyHeaders() } } diff --git a/Sources/HTTP/Serializer/HTTPChunkEncodingStream.swift b/Sources/HTTP/Serializer/HTTPChunkEncodingStream.swift index d505a40f..f1a6491e 100644 --- a/Sources/HTTP/Serializer/HTTPChunkEncodingStream.swift +++ b/Sources/HTTP/Serializer/HTTPChunkEncodingStream.swift @@ -48,12 +48,13 @@ final class HTTPChunkEncodingStream: Async.Stream { // FIXME: Improve performance let hexNumber = String(input.count, radix: 16, uppercase: true).data(using: .utf8)! self.chunk = hexNumber + crlf + Data(input) + crlf - downstream!.next(self.chunk!.withByteBuffer { $0 }, done) + downstream!.input(.next(self.chunk!.withByteBuffer { $0 }, done)) case .error(let error): downstream?.error(error) case .close: isClosed = true - _ = downstream?.next(eof.withByteBuffer { $0 }) + let promise = Promise(Void.self) + downstream?.input(.next(eof.withByteBuffer { $0 }, promise)) downstream?.close() } } diff --git a/Sources/HTTP/Serializer/HTTPRequestSerializer.swift b/Sources/HTTP/Serializer/HTTPRequestSerializer.swift index e02faa21..ed1cebd0 100644 --- a/Sources/HTTP/Serializer/HTTPRequestSerializer.swift +++ b/Sources/HTTP/Serializer/HTTPRequestSerializer.swift @@ -3,88 +3,70 @@ import Bits import Dispatch import Foundation -/// Converts requests to DispatchData. -public final class HTTPRequestSerializer: _HTTPSerializer { - public typealias SerializationState = HTTPSerializerState +/// https://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers +let maxStartLineSize = 2048 + +/// Serializing stream, converts HTTP Request to ByteBuffer. +public final class HTTPRequestSerializer: HTTPSerializer { + /// See `InputStream.Input` public typealias Input = HTTPRequest + + /// See `OutputStream.Output` public typealias Output = ByteBuffer - - /// Serialized message - var firstLine: [UInt8]? - - /// Headers - var headersData: [UInt8]? - /// Static body data - var body: HTTPBody? - - public let state: ByteSerializerState - let buffer: MutableByteBuffer - - /// Create a new HTTPResponseSerializer - public init(bufferSize: Int = 2048) { - self.state = .init() - - let pointer = MutableBytesPointer.allocate(capacity: bufferSize) - self.buffer = MutableByteBuffer(start: pointer, count: bufferSize) + /// See `HTTPSerializer.downstream` + public var downstream: AnyInputStream? + + /// See `HTTPSerializer.context` + public var context: HTTPSerializerContext + + /// Holds start lines for serialization + private let startLineBuffer: MutableByteBuffer + + /// Creates a new `HTTPRequestSerializer` + init() { + context = .init() + let pointer = MutableBytesPointer.allocate(capacity: maxStartLineSize) + startLineBuffer = .init(start: pointer, count: maxStartLineSize) } - - /// Set up the variables for Message serialization - public func setMessage(to message: HTTPRequest) { - var headers = message.headers - - headers[.contentLength] = nil - - if case .chunkedOutputStream = message.body.storage { - headers[.transferEncoding] = "chunked" - } else { - let count = message.body.count ?? 0 - headers.appendValue(count.bytes(reserving: 6), forName: .contentLength) + + /// See `HTTPSerializer.serializeStartLine(for:)` + public func serializeStartLine(for message: HTTPRequest) -> ByteBuffer { + guard startLineBuffer.count > + /// GET /foo HTTP/1.1\r\n + message.method.bytes.count + 1 + message.uri.pathBytes.count + 1 + http1newLineBuffer.count + else { + fatalError("Start line too large for buffer") } - - self.headersData = headers.storage - - self.firstLine = message.firstLine - - self.body = message.body - } - - deinit { - self.buffer.baseAddress?.deallocate(capacity: self.buffer.count) + + var address = startLineBuffer.baseAddress!.advanced(by: 0) + /// FIXME: static string? + let methodBytes: ByteBuffer = message.method.bytes.withUnsafeBufferPointer { $0 } + memcpy(address, methodBytes.baseAddress!, methodBytes.count) + + address = address.advanced(by: methodBytes.count) + address.pointee = .space + + address = address.advanced(by: 1) + let pathCount = message.uri.path.utf8.count + let pathBytes = message.uri.path.withCString { $0 } + memcpy(address, pathBytes, pathCount) + + address = address.advanced(by: pathCount) + memcpy(address, http1newLineBuffer.baseAddress!, http1newLineBuffer.count) + address = address.advanced(by: http1newLineBuffer.count) + + return ByteBuffer( + start: startLineBuffer.baseAddress, + count: startLineBuffer.baseAddress!.distance(to: address) + ) } -} -fileprivate extension HTTPRequest { - var firstLine: [UInt8] { - var firstLine = self.method.bytes - firstLine.reserveCapacity(self.headers.storage.count + 256) - - firstLine.append(.space) - - if self.uri.pathBytes.first != .forwardSlash { - firstLine.append(.forwardSlash) - } - - firstLine.append(contentsOf: self.uri.pathBytes) - - if let query = self.uri.query { - firstLine.append(.questionMark) - firstLine.append(contentsOf: query.utf8) - } - - if let fragment = self.uri.fragment { - firstLine.append(.numberSign) - firstLine.append(contentsOf: fragment.utf8) - } - - firstLine.append(contentsOf: http1newLine) - - return firstLine + deinit { + startLineBuffer.baseAddress?.deinitialize() + startLineBuffer.baseAddress?.deallocate(capacity: maxStartLineSize) } } -fileprivate let crlf = Data([ - .carriageReturn, - .newLine -]) -fileprivate let http1newLine = [UInt8](" HTTP/1.1\r\n".utf8) +fileprivate let http1newLine: StaticString = " HTTP/1.1\r\n" +fileprivate let http1newLineBuffer = http1newLine.withUTF8Buffer { $0 } diff --git a/Sources/HTTP/Serializer/HTTPResponseSerializer.swift b/Sources/HTTP/Serializer/HTTPResponseSerializer.swift index 549e5d47..d34e5871 100644 --- a/Sources/HTTP/Serializer/HTTPResponseSerializer.swift +++ b/Sources/HTTP/Serializer/HTTPResponseSerializer.swift @@ -5,72 +5,31 @@ import Dispatch import Foundation /// Converts responses to Data. -public final class HTTPResponseSerializer: _HTTPSerializer { - let buffer: MutableByteBuffer - - public var state: ByteSerializerState - - /// See HTTPSerializer.Message +public final class HTTPResponseSerializer: HTTPSerializer { + /// See `InputStream.Input` public typealias Input = HTTPResponse - public typealias SerializationState = HTTPSerializerState + + /// See `OutputStream.Output` public typealias Output = ByteBuffer - /// Serialized message - var firstLine: [UInt8]? - - /// Headers - var headersData: [UInt8]? + /// See `HTTPSerializer.downstream` + public var downstream: AnyInputStream? - /// Body data - var body: HTTPBody? + /// See `HTTPSerializer.context` + public var context: HTTPSerializerContext /// Create a new HTTPResponseSerializer - public init(bufferSize: Int = 2048) { - self.state = .init() - - let pointer = MutableBytesPointer.allocate(capacity: bufferSize) - self.buffer = MutableByteBuffer(start: pointer, count: bufferSize) - } - - /// Set up the variables for Message serialization - public func setMessage(to message: HTTPResponse) { - var headers = message.headers - - headers[.contentLength] = nil - - if case .chunkedOutputStream = message.body.storage { - headers[.transferEncoding] = "chunked" - } else { - let count = message.body.count ?? 0 - headers.appendValue(count.bytes(reserving: 6), forName: .contentLength) - } - - self.headersData = headers.storage - - self.firstLine = message.firstLine - - self.body = message.body - } - - deinit { - self.buffer.baseAddress?.deallocate(capacity: self.buffer.count) + public init() { + context = .init() } -} -fileprivate extension HTTPResponse { - var firstLine: [UInt8] { - // First line - var http1Line = http1Prefix - http1Line.reserveCapacity(128) - - http1Line.append(contentsOf: self.status.code.bytes(reserving: 3)) - http1Line.append(.space) - http1Line.append(contentsOf: self.status.messageBytes) - http1Line.append(contentsOf: crlf) - return http1Line + /// See `HTTPSerializer.serializeStartLine(for:)` + public func serializeStartLine(for message: HTTPResponse) -> ByteBuffer { + switch message.status { + case .ok: return okStartLine.withUTF8Buffer { $0 } + default: fatalError() + } } } -private let http1Prefix = [UInt8]("HTTP/1.1 ".utf8) -private let crlf = [UInt8]("\r\n".utf8) -private let headerKeyValueSeparator = [UInt8](": ".utf8) +private let okStartLine: StaticString = "HTTP/1.1 200 OK\r\n" diff --git a/Sources/HTTP/Serializer/HTTPSerializer.swift b/Sources/HTTP/Serializer/HTTPSerializer.swift index ec3e918d..223bcad1 100644 --- a/Sources/HTTP/Serializer/HTTPSerializer.swift +++ b/Sources/HTTP/Serializer/HTTPSerializer.swift @@ -4,164 +4,159 @@ import Dispatch import Foundation /// A helper for Request and Response serializer that keeps state -public enum HTTPSerializerState { - case noMessage - case firstLine(offset: Int) - case headers(offset: Int) - case crlf(offset: Int) - case staticBody(offset: Int) - case streaming(AnyOutputStream) - - mutating func next() { - switch self { - case .firstLine: self = .headers(offset: 0) - case .headers: self = .crlf(offset: 0) - case .crlf: self = .staticBody(offset: 0) - default: self = .noMessage - } +public indirect enum HTTPSerializerState { + case startLine + case headers + case body + case done + case continueBuffer(ByteBuffer, nextState: HTTPSerializerState) +} + +public final class HTTPSerializerContext { + var state: HTTPSerializerState + + private let buffer: MutableByteBuffer + private var bufferOffset: Int + + func drain() -> ByteBuffer { + defer { bufferOffset = 0 } + return ByteBuffer(start: buffer.baseAddress, count: bufferOffset) } - - mutating func advance(_ n: Int) { - switch self { - case .firstLine(let offset): self = .firstLine(offset: offset + n) - case .headers(let offset): self = .headers(offset: offset + n) - case .crlf(let offset): self = .crlf(offset: offset + n) - case .staticBody(let offset): self = .staticBody(offset: offset + n) - default: self = .noMessage - } + + init() { + let bufferSize: Int = 2048 + bufferOffset = 0 + let pointer = MutableBytesPointer.allocate(capacity: bufferSize) + self.buffer = MutableByteBuffer(start: pointer, count: bufferSize) + self.state = .startLine } - - var ready: Bool { - if case .noMessage = self { - return true + + func append(_ data: ByteBuffer) -> ByteBuffer? { + let writeSize = min(data.count, buffer.count - bufferOffset) + memcpy(buffer.baseAddress!.advanced(by: bufferOffset), data.baseAddress!, writeSize) + bufferOffset += writeSize + guard writeSize >= data.count else { + return ByteBuffer(start: data.baseAddress!.advanced(by: writeSize), count: data.count - writeSize) } - - return false + return nil + } + + deinit { + buffer.baseAddress?.deinitialize() + buffer.baseAddress?.deallocate(capacity: buffer.count) } } /// Internal Swift HTTP serializer protocol. -public protocol HTTPSerializer: class, ByteSerializer where Input: HTTPMessage { - func setMessage(to message: Input) +public protocol HTTPSerializer: Async.Stream where Input: HTTPMessage, Output == ByteBuffer { + var context: HTTPSerializerContext { get } + var downstream: AnyInputStream? { get set } + func serializeStartLine(for message: Input) -> ByteBuffer } -internal protocol _HTTPSerializer: HTTPSerializer where SerializationState == HTTPSerializerState { - /// Serialized message - var firstLine: [UInt8]? { get } - - /// Headers - var headersData: [UInt8]? { get } +extension HTTPSerializer { + public func input(_ event: InputEvent) { + guard let downstream = self.downstream else { + fatalError() + } + switch event { + case .close: downstream.close() + case .error(let e): downstream.error(e) + case .next(let input, let ready): + try! serialize(input, downstream, ready) + } + } - /// Body data - var body: HTTPBody? { get } - - var buffer: MutableByteBuffer { get } -} + public func output(to inputStream: S) where S: Async.InputStream, HTTPRequestSerializer.Output == S.Input { + downstream = .init(inputStream) + } -extension _HTTPSerializer { - public func serialize(_ input: Input, state previousState: SerializationState?) throws -> ByteSerializerResult { - var bufferSize: Int - var writeOffset = 0 - - var state: SerializationState - - if let previousState = previousState { - state = previousState - } else { - self.setMessage(to: input) - state = .firstLine(offset: 0) - } - - while !state.ready { - let _offset: Int - let writeSize: Int - let outputSize = buffer.count - writeOffset - - switch state { - case .noMessage: - throw HTTPError(identifier: "no-message", reason: "Serialization requested without a message") - case .firstLine(let offset): - _offset = offset - guard let firstLine = self.firstLine else { - throw HTTPError(identifier: "invalid-state", reason: "Missing first line metadata") - } - - bufferSize = firstLine.count - writeSize = min(outputSize, bufferSize - offset) - - firstLine.withUnsafeBytes { pointer in - _ = memcpy(buffer.baseAddress!.advanced(by: writeOffset), pointer.baseAddress!.advanced(by: offset), writeSize) - } - case .headers(let offset): - _offset = offset - guard let headersData = self.headersData else { - throw HTTPError(identifier: "invalid-state", reason: "Missing header state") - } - - bufferSize = headersData.count - writeSize = min(outputSize, bufferSize - offset) - - headersData.withUnsafeBufferPointer { headerBuffer in - _ = memcpy(buffer.baseAddress!.advanced(by: writeOffset), headerBuffer.baseAddress!.advanced(by: offset), writeSize) - } - case .crlf(let offset): - _offset = offset - bufferSize = 2 - writeSize = min(outputSize, bufferSize - offset) - - crlf.withUnsafeBufferPointer { crlfBuffer in - _ = memcpy(buffer.baseAddress!.advanced(by: writeOffset), crlfBuffer.baseAddress!.advanced(by: offset), writeSize) + fileprivate func serialize(_ message: Input, _ downstream: AnyInputStream, _ nextMessage: Promise) throws { + switch context.state { + case .startLine: + if let remaining = context.append(serializeStartLine(for: message)) { + context.state = .continueBuffer(remaining, nextState: .headers) + write(message, downstream, nextMessage) + } else { + context.state = .headers + try serialize(message, downstream, nextMessage) + } + case .headers: + let buffer = message.headers.storage.withByteBuffer { $0 } + if let remaining = context.append(buffer) { + context.state = .continueBuffer(remaining, nextState: .body) + write(message, downstream, nextMessage) + } else { + context.state = .body + try serialize(message, downstream, nextMessage) + } + case .body: + let byteBuffer: ByteBuffer? + + switch message.body.storage { + case .data(let data): + byteBuffer = ByteBuffer(start: data.withUnsafeBytes { $0 }, count: data.count) + case .dispatchData(let data): + byteBuffer = ByteBuffer(start: data.withUnsafeBytes { $0 }, count: data.count) + case .staticString(let staticString): + byteBuffer = ByteBuffer(start: staticString.utf8Start, count: staticString.utf8CodeUnitCount) + case .string(let string): + let bytePointer = string.withCString { pointer in + return pointer.withMemoryRebound(to: UInt8.self, capacity: string.utf8.count) { $0 } } - case .staticBody(let offset): - _offset = offset - - guard let body = self.body else { - state.next() - continue + byteBuffer = ByteBuffer(start: bytePointer, count: string.utf8.count) + case .buffer(let buffer): + byteBuffer = buffer + case .none: + byteBuffer = nil + case .chunkedOutputStream(_), .binaryOutputStream(_): + byteBuffer = nil + } + + if let buffer = byteBuffer { + if let remaining = context.append(buffer) { + context.state = .continueBuffer(remaining, nextState: .done) + write(message, downstream, nextMessage) + } else { + context.state = .done + write(message, downstream, nextMessage) } - - switch body.storage { + } else { + switch message.body.storage { case .none: - return .complete(ByteBuffer(start: buffer.baseAddress, count: writeOffset)) - case .chunkedOutputStream(let streamBuilder): - let stream = HTTPChunkEncodingStream() - let result = AnyOutputStream(streamBuilder(stream)) - - return .incomplete( - ByteBuffer(start: buffer.baseAddress, count: writeOffset), - state: .streaming(result) - ) - case .binaryOutputStream(_, let stream): - return .incomplete( - ByteBuffer(start: buffer.baseAddress, count: writeOffset), - state: .streaming(stream) - ) - default: - bufferSize = body.count ?? 0 - writeSize = min(outputSize, bufferSize - offset) - - try body.withUnsafeBytes { pointer in - _ = memcpy(buffer.baseAddress!.advanced(by: writeOffset), pointer.advanced(by: offset), writeSize) - } + context.state = .done + write(message, downstream, nextMessage) + default: fatalError() } - case .streaming(let stream): - state.next() - return .awaiting(stream, state: nil) } - - writeOffset += writeSize - - if _offset + writeSize < bufferSize { - state.advance(writeSize) - return .incomplete(ByteBuffer(start: buffer.baseAddress, count: writeOffset), state: state) + case .continueBuffer(let remainingStartLine, let then): + if let remaining = context.append(remainingStartLine) { + context.state = .continueBuffer(remaining, nextState: then) + write(message, downstream, nextMessage) } else { - state.next() + context.state = then + try serialize(message, downstream, nextMessage) } + case .done: + context.state = .startLine + nextMessage.complete() } - - return .complete(ByteBuffer(start: buffer.baseAddress, count: writeOffset)) } -} -fileprivate let crlf: [UInt8] = [.carriageReturn, .newLine] + fileprivate func write(_ message: Input, _ downstream: AnyInputStream, _ nextMessage: Promise) { + let promise = Promise(Void.self) + downstream.input(.next(context.drain(), promise)) + promise.future.addAwaiter { result in + switch result { + case .error(let error): downstream.error(error) + case .expectation: + do { + try self.serialize(message, downstream, nextMessage) + } catch { + downstream.error(error) + } + } + } + } +} diff --git a/Sources/HTTP/Server/HTTPResponder.swift b/Sources/HTTP/Server/HTTPResponder.swift index 19fdff65..1d18a77f 100644 --- a/Sources/HTTP/Server/HTTPResponder.swift +++ b/Sources/HTTP/Server/HTTPResponder.swift @@ -1,4 +1,5 @@ import Async +import Bits /// Converts HTTPRequests to future HTTPResponses on the supplied event loop. public protocol HTTPResponder { @@ -7,10 +8,89 @@ public protocol HTTPResponder { } extension HTTPResponder { - /// Converts an HTTPResponder to an HTTPRequest -> HTTPResponse stream. - public func stream(on worker: Worker) -> MapStream { - return MapStream { req in - return try self.respond(to: req, on: worker) + public func stream( + upgradingTo byteStream: AnyStream, + on worker: Worker + ) -> HTTPResponderStream { + return .init(responder: self, byteStream: byteStream, on: worker) + } +} + +public final class HTTPResponderStream: Stream where Responder: HTTPResponder { + /// See `InputStream.Input` + public typealias Input = HTTPRequest + + /// See `OutputStream.Output` + public typealias Output = HTTPResponse + + /// Current downstream accepting our responses + private var downstream: AnyInputStream? + + /// The responder powering this stream. + private let responder: Responder + + /// Byte stream to use for on upgrade. + private let byteStream: AnyStream + + /// Current event loop + private let eventLoop: EventLoop + + /// Creates a new `HTTPResponderStream` + internal init(responder: Responder, byteStream: AnyStream, on worker: Worker) { + self.responder = responder + self.byteStream = byteStream + self.eventLoop = worker.eventLoop + } + + /// See `InputStream.input(_:)` + public func input(_ event: InputEvent) { + guard let downstream = self.downstream else { + fatalError("Unexpected nil downstream during HTTPResponderStream.input") + } + + switch event { + case .close: downstream.close() + case .error(let error): downstream.error(error) + case .next(let input, let nextRequest): + do { + let byteStream = self.byteStream + let eventLoop = self.eventLoop + + try responder.respond(to: input, on: eventLoop).addAwaiter { res in + switch res { + case .error(let error): downstream.error(error) + case .expectation(let res): + if let onUpgrade = res.onUpgrade { + let receivedResponse = Promise(Void.self) + downstream.input(.next(res, receivedResponse)) + receivedResponse.future.addAwaiter { rec in + do { + switch rec { + case .error(let error): downstream.error(error) + case .expectation: + try onUpgrade.closure( + byteStream.outputStream, + byteStream.inputStream, + eventLoop + ) + } + } catch { + downstream.error(error) + } + } + } else { + downstream.input(.next(res, nextRequest)) + } + } + } + } catch { + downstream.error(error) + } } } + + /// See `OutputStream.output(to:)` + public func output(to inputStream: S) where S : InputStream, HTTPResponderStream.Output == S.Input { + downstream = .init(inputStream) + } } diff --git a/Sources/HTTP/Server/HTTPServer.swift b/Sources/HTTP/Server/HTTPServer.swift index 485d47c5..d7d2715a 100644 --- a/Sources/HTTP/Server/HTTPServer.swift +++ b/Sources/HTTP/Server/HTTPServer.swift @@ -20,28 +20,17 @@ public final class HTTPServer { public var onError: ErrorHandler? /// Create a new HTTP server with the supplied accept stream. - public init(acceptStream: AcceptStream, worker: Worker, responder: HTTPResponder) - where AcceptStream: OutputStream, AcceptStream.Output: ByteStream + public init(acceptStream: AcceptStream, worker: Worker, responder: Responder) + where AcceptStream: OutputStream, AcceptStream.Output: ByteStream, Responder: HTTPResponder { /// set up the server stream acceptStream.drain { client in - let serializerStream = HTTPResponseSerializer().stream(on: worker) + let serializerStream = HTTPResponseSerializer() let parserStream = HTTPRequestParser() client .stream(to: parserStream) - .stream(to: responder.stream(on: worker).stream()) - .map(to: HTTPResponse.self) { response in - /// map the responder adding http upgrade support - if let onUpgrade = response.onUpgrade { - do { - try onUpgrade.closure(.init(client), .init(client), worker) - } catch { - self.onError?(error) - } - } - return response - } + .stream(to: responder.stream(upgradingTo: .init(client), on: worker)) .stream(to: serializerStream) .output(to: client) }.catch { err in diff --git a/Sources/Performance/main.swift b/Sources/Performance/main.swift index 4dee341f..8e4a6b47 100644 --- a/Sources/Performance/main.swift +++ b/Sources/Performance/main.swift @@ -1,114 +1,37 @@ -import Core -import Dispatch -import Foundation +import Async import HTTP import TCP +import Foundation -extension String: Swift.Error { } - -struct User: Codable { - var name: String - var age: Int -} - -extension User: ContentCodable { - static func decodeContent(from message: Message) throws -> User? { - guard message.mediaType == .json else { - throw "only json supported" - } - - return try JSONDecoder().decode(User.self, from: message.body.data) - } - - func encodeContent(to message: Message) throws { - message.mediaType = .json - message.body = try Body(JSONEncoder().encode(self)) - } - -} - - -struct Application: Responder { - func respond(to req: Request) throws -> Future { - // let user = User(name: "Vapor", age: 2) - // print(String(cString: __dispatch_queue_get_label(nil), encoding: .utf8)) - // try! res.content(user) - let p = Promise() - let res = try Response(status: .ok, body: "hi") - p.complete(res) - return p.future - } -} - - -// MARK: Client -do { - final class RequestEmitter: Core.OutputStream { - typealias Output = Request - var outputStream: OutputHandler? - var errorStream: ErrorHandler? - - init() {} - - func emit(_ request: Request) { - outputStream?(request) - } - } - - let emitter = RequestEmitter() - let serializer = RequestSerializer() - let parser = ResponseParser() +let tcpSocket = try TCPSocket(isNonBlocking: true) +let tcpServer = try TCPServer(socket: tcpSocket) - let socket = try TCP.Socket() - try socket.connect(hostname: "google.com", port: 80) - let client = TCP.Client(socket: socket, queue: .global()) +let hostname = "localhost" +let port: UInt16 = 8123 +try tcpServer.start(hostname: hostname, port: port, backlog: 128) - emitter.stream(to: serializer) - .stream(to: client) - .stream(to: parser) - .drain { response in - print(String(data: response.body.data, encoding: .utf8)!) - } - emitter.errorStream = { error in - print(error) +struct EchoResponder: HTTPResponder { + func respond(to req: HTTPRequest, on Worker: Worker) throws -> Future { + return Future(.init(body: req.body)) } - client.start() - - - let request = try Request(method: .get, uri: URI(path: "/"), body: "hello") - request.headers[.host] = "google.com" - request.headers[.userAgent] = "vapor/engine" - - emitter.emit(request) } -// MARK: Server -do { - let app = Application() - let server = try TCP.Server() - - server.drain { client in - let parser = HTTP.RequestParser() - let serializer = HTTP.ResponseSerializer() - - client.stream(to: parser) - .stream(to: app.makeStream(on: client.queue)) - .stream(to: serializer) - .drain(into: client) - - client.start() - } - - server.errorStream = { error in - debugPrint(error) +let workerCount = ProcessInfo.processInfo.activeProcessorCount +for i in 1...workerCount { + let loop = try DefaultEventLoop(label: "codes.vapor.engine.performance.\(i)") + let serverStream = tcpServer.stream(on: loop) + + _ = HTTPServer( + acceptStream: serverStream.map(to: SocketStream.self) { $0.socket.stream(on: loop) }, + worker: loop, + responder: EchoResponder() + ) + + print("Starting worker #\(i) on \(hostname):\(port)") + if i == workerCount { + loop.runLoop() + } else { + Thread.async { loop.runLoop() } } - - try server.start(port: 8080) - print("Server started...") } - - -let group = DispatchGroup() -group.enter() -group.wait() diff --git a/Tests/HTTPTests/HTTPServerTests.swift b/Tests/HTTPTests/HTTPServerTests.swift index 8655db86..67107481 100644 --- a/Tests/HTTPTests/HTTPServerTests.swift +++ b/Tests/HTTPTests/HTTPServerTests.swift @@ -28,9 +28,9 @@ class HTTPServerTests: XCTestCase { } } -// let group = DispatchGroup() -// group.enter() -// group.wait() + let group = DispatchGroup() + group.enter() + group.wait() let exp = expectation(description: "all requests complete") var num = 1024 From 656a5041e7782c402b773ee7940ab2e22d1d1ada Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Thu, 25 Jan 2018 22:23:49 -0500 Subject: [PATCH 10/16] parser header re-use --- Sources/HTTP/Message/HTTPHeaderName.swift | 2 +- Sources/HTTP/Message/HTTPHeaderStorage.swift | 65 ++++++++++++++++++-- Sources/HTTP/Message/HTTPHeaders.swift | 45 ++++++++------ Sources/HTTP/Parser/CHTTPParserContext.swift | 12 ++-- Tests/HTTPTests/HTTPServerTests.swift | 6 +- 5 files changed, 98 insertions(+), 32 deletions(-) diff --git a/Sources/HTTP/Message/HTTPHeaderName.swift b/Sources/HTTP/Message/HTTPHeaderName.swift index 67bfed0d..719acff6 100644 --- a/Sources/HTTP/Message/HTTPHeaderName.swift +++ b/Sources/HTTP/Message/HTTPHeaderName.swift @@ -44,7 +44,7 @@ public struct HTTPHeaderName: Codable, Hashable, ExpressibleByStringLiteral, Cus } /// See `Equatable.==` - public static func ==(lhs: Name, rhs: Name) -> Bool { + public static func ==(lhs: HTTPHeaderName, rhs: HTTPHeaderName) -> Bool { return lhs.lowercased == rhs.lowercased } diff --git a/Sources/HTTP/Message/HTTPHeaderStorage.swift b/Sources/HTTP/Message/HTTPHeaderStorage.swift index 5070294d..4b0c84af 100644 --- a/Sources/HTTP/Message/HTTPHeaderStorage.swift +++ b/Sources/HTTP/Message/HTTPHeaderStorage.swift @@ -12,9 +12,9 @@ final class HTTPHeaderStorage { /// The HTTPHeader's known indexes into the storage. private var indexes: [HTTPHeaderIndex?] - /// Creates a new, empty `HTTPHeaders`. + /// Creates a new `HTTPHeaders` with default content. public init() { - let storageSize = 1024 + let storageSize = 64 let buffer = MutableByteBuffer(start: .allocate(capacity: storageSize), count: storageSize) memcpy(buffer.baseAddress, defaultHeaders.baseAddress!, defaultHeadersSize) self.buffer = buffer @@ -22,9 +22,17 @@ final class HTTPHeaderStorage { self.indexes = [] } + /// Internal init for truly empty header storage. + internal init(reserving: Int) { + let buffer = MutableByteBuffer(start: .allocate(capacity: reserving), count: reserving) + self.buffer = buffer + self.view = ByteBuffer(start: buffer.baseAddress, count: 0) + self.indexes = [] + } + /// Create a new `HTTPHeaders` with explicit storage and indexes. internal init(bytes: Bytes, indexes: [HTTPHeaderIndex]) { - let storageSize = view.count + let storageSize = bytes.count let buffer = MutableByteBuffer(start: .allocate(capacity: storageSize), count: storageSize) memcpy(buffer.baseAddress, bytes, storageSize) self.buffer = buffer @@ -33,6 +41,28 @@ final class HTTPHeaderStorage { } + /// Create a new `HTTPHeaders` with explicit storage and indexes. + private init(view: ByteBuffer, buffer: MutableByteBuffer, indexes: [HTTPHeaderIndex?]) { + self.view = view + self.buffer = buffer + self.indexes = indexes + } + + /// Creates a new, identical copy of the header storage. + internal func copy() -> HTTPHeaderStorage { + print("copy") + let newBuffer = MutableByteBuffer( + start: MutableBytesPointer.allocate(capacity: buffer.count), + count: buffer.count + ) + memcpy(buffer.start, newBuffer.start, buffer.count) + return HTTPHeaderStorage( + view: ByteBuffer(start: newBuffer.start, count: view.count), + buffer: newBuffer, + indexes: indexes + ) + } + /// Removes all headers with this name internal func removeValues(for name: HTTPHeaderName) { /// loop over all indexes @@ -177,6 +207,33 @@ final class HTTPHeaderStorage { .assumingMemoryBound(to: Byte.self) buffer = MutableByteBuffer(start: pointer, count: newSize) } + + /// Manually appends a byte buffer to the header storage. + internal func manualAppend(_ bytes: ByteBuffer) { + let count = (bytes.count + view.count) - buffer.count + if count > 0 { + // not enough room + increaseBufferSize(by: count) + } + memcpy(buffer.start.advanced(by: view.count), bytes.start, bytes.count) + view = ByteBuffer(start: buffer.start, count: view.count + bytes.count) + } + + /// Resets the internal view, ignoring any added data. + internal func manualReset() { + self.indexes = [] + self.view = ByteBuffer(start: buffer.start, count: 0) + } + + /// Manually set indexes. + internal func manualIndexes(_ indexes: [HTTPHeaderIndex]) { + self.indexes = indexes + } + + deinit { + buffer.start.deinitialize() + buffer.start.deallocate(capacity: buffer.count) + } } extension HTTPHeaderStorage: CustomStringConvertible { @@ -219,6 +276,6 @@ private let headerEndingStaticString: StaticString = "\r\n" private let headerEnding: ByteBuffer = headerEndingStaticString.withUTF8Buffer { $0 } private let headerEndingSize: Int = headerEnding.count -private let defaultHeadersStaticString: StaticString = "Content-Length: 0\r\n\r\n" +private let defaultHeadersStaticString: StaticString = "Content-Length: 0\r\n" private let defaultHeaders: ByteBuffer = defaultHeadersStaticString.withUTF8Buffer { $0 } private let defaultHeadersSize: Int = defaultHeaders.count diff --git a/Sources/HTTP/Message/HTTPHeaders.swift b/Sources/HTTP/Message/HTTPHeaders.swift index 169310b2..d2337f03 100644 --- a/Sources/HTTP/Message/HTTPHeaders.swift +++ b/Sources/HTTP/Message/HTTPHeaders.swift @@ -15,6 +15,8 @@ import Bits /// [Learn More →](https://docs.vapor.codes/3.0/http/headers/) public struct HTTPHeaders: Codable { /// The HTTPHeader's raw data storage + /// Note: For COW to work properly, this must only be + /// accessed from the public methods on this struct. internal var storage: HTTPHeaderStorage /// Creates a new, empty `HTTPHeaders`. @@ -43,6 +45,28 @@ public struct HTTPHeaders: Codable { } } + /// Accesses all values associated with the `Name` + public subscript(valuesFor name: HTTPHeaderName) -> [String] { + get { + return storage.indexes(for: name).flatMap { storage.value(for: $0) } + } + set { + if !isKnownUniquelyReferenced(&storage) { + /// this storage is being referenced from two places + /// copy now to ensure COW behavior + storage = storage.copy() + } + storage.removeValues(for: name) + for value in newValue { + storage.appendValue(value, for: name) + } + } + } +} + +/// MARK: Convenience + +extension HTTPHeaders { /// Accesses the (first) value associated with the `Name` if any /// /// [Learn More →](https://docs.vapor.codes/3.0/http/headers/#accessing-headers) @@ -60,30 +84,15 @@ public struct HTTPHeaders: Codable { } } set { - storage.removeValues(for: name) if let value = newValue { - storage.appendValue(value, for: name) - } - } - } - - /// Accesses all values associated with the `Name` - public subscript(valuesFor name: HTTPHeaderName) -> [String] { - get { - return storage.indexes(for: name).flatMap { storage.value(for: $0) } - } - set { - storage.removeValues(for: name) - for value in newValue { - storage.appendValue(value, for: name) + self[valuesFor: name] = [value] + } else { + self[valuesFor: name] = [] } } } -} -/// MARK: Convenience -extension HTTPHeaders { /// https://tools.ietf.org/html/rfc2616#section-3.6 /// /// "Parameters are in the form of attribute/value pairs." diff --git a/Sources/HTTP/Parser/CHTTPParserContext.swift b/Sources/HTTP/Parser/CHTTPParserContext.swift index a53e3108..e087232e 100644 --- a/Sources/HTTP/Parser/CHTTPParserContext.swift +++ b/Sources/HTTP/Parser/CHTTPParserContext.swift @@ -38,7 +38,7 @@ internal final class CHTTPParserContext { /// Raw headers data - var headersData: [UInt8] + var headersData: HTTPHeaderStorage /// Parsed indexes into the header data var headersIndexes: [HTTPHeaderIndex] @@ -90,7 +90,7 @@ internal final class CHTTPParserContext { self.version = nil self.headers = nil - self.headersData = [] + self.headersData = .init(reserving: 64) self.headersIndexes = [] self.urlData = [] @@ -173,7 +173,7 @@ extension CHTTPParserContext { self.version = nil self.headers = nil - self.headersData = [] + self.headersData.manualReset() self.headersIndexes = [] self.urlData = [] @@ -225,13 +225,13 @@ extension CHTTPParserContext { start: start.withMemoryRebound(to: Byte.self, capacity: distance) { $0 }, count: distance ) - headersData.append(contentsOf: buffer) + headersData.manualAppend(buffer) /// if this buffer copy is happening after headers complete indication, /// set the headers struct for later retreival if headersComplete { - let storage = HTTPHeaderStorage(bytes: headersData, indexes: headersIndexes) - headers = HTTPHeaders(storage: storage) + headersData.manualIndexes(headersIndexes) + headers = HTTPHeaders(storage: headersData) } } diff --git a/Tests/HTTPTests/HTTPServerTests.swift b/Tests/HTTPTests/HTTPServerTests.swift index 67107481..8655db86 100644 --- a/Tests/HTTPTests/HTTPServerTests.swift +++ b/Tests/HTTPTests/HTTPServerTests.swift @@ -28,9 +28,9 @@ class HTTPServerTests: XCTestCase { } } - let group = DispatchGroup() - group.enter() - group.wait() +// let group = DispatchGroup() +// group.enter() +// group.wait() let exp = expectation(description: "all requests complete") var num = 1024 From 82c2561ebe58f6aa8535102cb34f1ddaf07cdf92 Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Thu, 25 Jan 2018 23:04:29 -0500 Subject: [PATCH 11/16] http tests passing --- Sources/HTTP/Message/HTTPHeaderStorage.swift | 41 +++++++++++-------- Sources/HTTP/Message/HTTPHeaders.swift | 7 +++- Sources/HTTP/Parser/CHTTPParserContext.swift | 2 +- .../Serializer/HTTPRequestSerializer.swift | 2 +- Sources/Multipart/Parser.swift | 2 +- Sources/WebSocket/WebSocket.swift | 2 +- Tests/HTTPTests/HTTPHeaderTests.swift | 32 +++++++++++++++ Tests/HTTPTests/HTTPParserTests.swift | 6 ++- 8 files changed, 69 insertions(+), 25 deletions(-) create mode 100644 Tests/HTTPTests/HTTPHeaderTests.swift diff --git a/Sources/HTTP/Message/HTTPHeaderStorage.swift b/Sources/HTTP/Message/HTTPHeaderStorage.swift index 4b0c84af..5fbd0e94 100644 --- a/Sources/HTTP/Message/HTTPHeaderStorage.swift +++ b/Sources/HTTP/Message/HTTPHeaderStorage.swift @@ -13,13 +13,12 @@ final class HTTPHeaderStorage { private var indexes: [HTTPHeaderIndex?] /// Creates a new `HTTPHeaders` with default content. - public init() { + public static func `default`() -> HTTPHeaderStorage { let storageSize = 64 let buffer = MutableByteBuffer(start: .allocate(capacity: storageSize), count: storageSize) memcpy(buffer.baseAddress, defaultHeaders.baseAddress!, defaultHeadersSize) - self.buffer = buffer - self.view = ByteBuffer(start: buffer.baseAddress, count: defaultHeadersSize) - self.indexes = [] + let view = ByteBuffer(start: buffer.baseAddress, count: defaultHeadersSize) + return HTTPHeaderStorage(view: view, buffer: buffer, indexes: [defaultHeaderIndex]) } /// Internal init for truly empty header storage. @@ -50,12 +49,12 @@ final class HTTPHeaderStorage { /// Creates a new, identical copy of the header storage. internal func copy() -> HTTPHeaderStorage { - print("copy") + print("🐄 COPY") let newBuffer = MutableByteBuffer( start: MutableBytesPointer.allocate(capacity: buffer.count), count: buffer.count ) - memcpy(buffer.start, newBuffer.start, buffer.count) + memcpy(newBuffer.start, buffer.start, buffer.count) return HTTPHeaderStorage( view: ByteBuffer(start: newBuffer.start, count: view.count), buffer: newBuffer, @@ -78,7 +77,7 @@ final class HTTPHeaderStorage { /// calculate how much valid storage data is placed /// after this header - let displacedBytes = view.count - index.startIndex + let displacedBytes = view.count - index.endIndex if displacedBytes > 0 { /// there is valid data after this header that we must relocate @@ -87,8 +86,8 @@ final class HTTPHeaderStorage { memcpy(destination, source, displacedBytes) } else { // no headers after this, simply shorten the valid buffer - self.view = ByteBuffer(start: buffer.start, count: view.count - index.size) } + self.view = ByteBuffer(start: buffer.start, count: view.count - index.size) } } } @@ -97,14 +96,14 @@ final class HTTPHeaderStorage { /// Note: This will naively append data, not deleting existing values. Use in /// conjunction with `removeValues(for:)` for that behavior. internal func appendValue(_ value: String, for name: HTTPHeaderName) { - let value = value.buffer + let valueCount = value.utf8.count /// create the new header index let index = HTTPHeaderIndex( nameStartIndex: view.count, - nameEndIndex: view.count + name.lowercased.count, - valueStartIndex: view.count + name.lowercased.count + 2, - valueEndIndex: view.count + name.lowercased.count + 2 + value.count + nameEndIndex: view.count + name.original.count, + valueStartIndex: view.count + name.original.count + 2, + valueEndIndex: view.count + name.original.count + 2 + valueCount ) indexes.append(index) @@ -114,15 +113,17 @@ final class HTTPHeaderStorage { } // - memcpy(buffer.start.advanced(by: index.nameStartIndex), name.lowercased, name.lowercased.count) + memcpy(buffer.start.advanced(by: index.nameStartIndex), name.original, name.original.count) // `: ` memcpy(buffer.start.advanced(by: index.nameEndIndex), headerSeparator.start, headerSeparatorSize) // - memcpy(buffer.start.advanced(by: index.valueStartIndex), value.start, value.count) + _ = value.withByteBuffer { valueBuffer in + memcpy(buffer.start.advanced(by: index.valueStartIndex), valueBuffer.start, valueCount) + } // `\r\n` memcpy(buffer.start.advanced(by: index.valueEndIndex), headerEnding.start, headerEndingSize) - view = ByteBuffer(start: buffer.start, count: view.count + index.endIndex) + view = ByteBuffer(start: buffer.start, count: view.count + index.size) } /// Fetches the String value for a given header index. @@ -167,10 +168,11 @@ final class HTTPHeaderStorage { return false } + let headerData = ByteBuffer(start: view.start.advanced(by: index.startIndex), count: index.size) let nameData: ByteBuffer = name.lowercased.withUnsafeBufferPointer { $0 } for i in 0..(_ closure: (ByteBuffer) -> T) -> T { let count = utf8.count return withCString { cPointer in - return ByteBuffer(start: cPointer.withMemoryRebound(to: Byte.self, capacity: count) { $0 }, count: count) + return cPointer.withMemoryRebound(to: Byte.self, capacity: count) { + return closure(ByteBuffer(start: $0, count: count)) + } } } } @@ -279,3 +283,4 @@ private let headerEndingSize: Int = headerEnding.count private let defaultHeadersStaticString: StaticString = "Content-Length: 0\r\n" private let defaultHeaders: ByteBuffer = defaultHeadersStaticString.withUTF8Buffer { $0 } private let defaultHeadersSize: Int = defaultHeaders.count +private let defaultHeaderIndex = HTTPHeaderIndex(nameStartIndex: 0, nameEndIndex: 14, valueStartIndex: 16, valueEndIndex: 17) diff --git a/Sources/HTTP/Message/HTTPHeaders.swift b/Sources/HTTP/Message/HTTPHeaders.swift index d2337f03..15223a5e 100644 --- a/Sources/HTTP/Message/HTTPHeaders.swift +++ b/Sources/HTTP/Message/HTTPHeaders.swift @@ -21,7 +21,7 @@ public struct HTTPHeaders: Codable { /// Creates a new, empty `HTTPHeaders`. public init() { - self.storage = .init() + self.storage = .default() } /// Create a new `HTTPHeaders` with explicit storage and indexes. @@ -62,6 +62,11 @@ public struct HTTPHeaders: Codable { } } } + + /// An internal API that blindly adds a header without checking for doubles + public func withByteBuffer(_ closure: (ByteBuffer) -> T) -> T { + return storage.withByteBuffer(closure) + } } /// MARK: Convenience diff --git a/Sources/HTTP/Parser/CHTTPParserContext.swift b/Sources/HTTP/Parser/CHTTPParserContext.swift index e087232e..5ca4abe7 100644 --- a/Sources/HTTP/Parser/CHTTPParserContext.swift +++ b/Sources/HTTP/Parser/CHTTPParserContext.swift @@ -173,7 +173,7 @@ extension CHTTPParserContext { self.version = nil self.headers = nil - self.headersData.manualReset() + self.headersData = .init(reserving: 64) // .manualReset() self.headersIndexes = [] self.urlData = [] diff --git a/Sources/HTTP/Serializer/HTTPRequestSerializer.swift b/Sources/HTTP/Serializer/HTTPRequestSerializer.swift index ed1cebd0..ac04675b 100644 --- a/Sources/HTTP/Serializer/HTTPRequestSerializer.swift +++ b/Sources/HTTP/Serializer/HTTPRequestSerializer.swift @@ -24,7 +24,7 @@ public final class HTTPRequestSerializer: HTTPSerializer { private let startLineBuffer: MutableByteBuffer /// Creates a new `HTTPRequestSerializer` - init() { + public init() { context = .init() let pointer = MutableBytesPointer.allocate(capacity: maxStartLineSize) startLineBuffer = .init(start: pointer, count: maxStartLineSize) diff --git a/Sources/Multipart/Parser.swift b/Sources/Multipart/Parser.swift index c849d3ae..03463f70 100644 --- a/Sources/Multipart/Parser.swift +++ b/Sources/Multipart/Parser.swift @@ -118,7 +118,7 @@ public final class MultipartParser { throw MultipartError(identifier: "multipart:invalid-header-value", reason: "Invalid multipart header value string encoding") } - headers[HTTPHeaders.Name(key)] = value + headers[HTTPHeaderName(key)] = value } return headers diff --git a/Sources/WebSocket/WebSocket.swift b/Sources/WebSocket/WebSocket.swift index 2d60fd65..b79ddfe3 100644 --- a/Sources/WebSocket/WebSocket.swift +++ b/Sources/WebSocket/WebSocket.swift @@ -101,7 +101,7 @@ public final class WebSocket { let id = OSRandom().data(count: 16).base64EncodedString() // Creates an HTTP client for the handshake - let serializer = HTTPRequestSerializer().stream(on: self.worker) + let serializer = HTTPRequestSerializer() let serializerStream = PushStream() let parser = HTTPResponseParser() diff --git a/Tests/HTTPTests/HTTPHeaderTests.swift b/Tests/HTTPTests/HTTPHeaderTests.swift new file mode 100644 index 00000000..93d739eb --- /dev/null +++ b/Tests/HTTPTests/HTTPHeaderTests.swift @@ -0,0 +1,32 @@ +import Async +import Bits +import HTTP +import Foundation +import TCP +import XCTest + +class HTTPHeaderTests: XCTestCase { + func testHeaders() throws { + var headers = HTTPHeaders() + XCTAssertEqual(headers.description, "Content-Length: 0\r\n") + headers[.contentType] = "text/plain" + XCTAssertEqual(headers.description, "Content-Length: 0\r\nContent-Type: text/plain\r\n") + headers[.contentType] = "text/plain" + XCTAssertEqual(headers.description, "Content-Length: 0\r\nContent-Type: text/plain\r\n") + headers[.contentType] = nil + XCTAssertEqual(headers.description, "Content-Length: 0\r\n") + headers[.contentType] = nil + headers[.contentType] = nil + headers[.contentType] = nil + XCTAssertEqual(headers.description, "Content-Length: 0\r\n") + let hugeString = String(repeating: "hi", count: 25_000) + headers[HTTPHeaderName("foo")] = hugeString + XCTAssertEqual(headers.description, "Content-Length: 0\r\nfoo: \(hugeString)\r\n") + headers[HTTPHeaderName("FOO")] = nil + XCTAssertEqual(headers.description, "Content-Length: 0\r\n") + } + + static let allTests = [ + ("testHeaders", testHeaders) + ] +} diff --git a/Tests/HTTPTests/HTTPParserTests.swift b/Tests/HTTPTests/HTTPParserTests.swift index 5ed21cca..11e67b74 100644 --- a/Tests/HTTPTests/HTTPParserTests.swift +++ b/Tests/HTTPTests/HTTPParserTests.swift @@ -32,8 +32,10 @@ class HTTPParserTests: XCTestCase { throw "request was nil" } - guard req.headers[.contentType] == "text/plain" else { - throw "incorrect content type" + let contentType = req.headers[.contentType] + guard contentType == "text/plain" else { + print(req.headers) + throw "incorrect content type: \(contentType ?? "nil")" } guard req.headers[.contentLength] == "5" else { From 91d9553fa3f1beb2da0b9dc8189ec5643b50a4de Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Thu, 25 Jan 2018 23:20:49 -0500 Subject: [PATCH 12/16] updates --- Sources/HTTP/Parser/CHTTPParserContext.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Sources/HTTP/Parser/CHTTPParserContext.swift b/Sources/HTTP/Parser/CHTTPParserContext.swift index 5ca4abe7..72e1b7e6 100644 --- a/Sources/HTTP/Parser/CHTTPParserContext.swift +++ b/Sources/HTTP/Parser/CHTTPParserContext.swift @@ -173,7 +173,8 @@ extension CHTTPParserContext { self.version = nil self.headers = nil - self.headersData = .init(reserving: 64) // .manualReset() + //self.headersData = .init(reserving: 64) + self.headersData.manualReset() // can we get this to work w/ COW? self.headersIndexes = [] self.urlData = [] From f287034e19f4cb26bc7516e4d21ad88d03076e9c Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Fri, 26 Jan 2018 20:14:01 -0500 Subject: [PATCH 13/16] chttp refactor cleanup --- Sources/HTTP/Message/HTTPBody.swift | 34 +++++++++++- Sources/HTTP/Message/HTTPHeaderStorage.swift | 57 +++++++++----------- Sources/HTTP/Message/HTTPHeaders.swift | 9 +++- Sources/HTTP/Message/HTTPMessage.swift | 24 ++++++--- Sources/HTTP/Parser/CHTTPParserContext.swift | 49 +++++++++-------- Sources/HTTP/Request/HTTPRequest.swift | 5 +- 6 files changed, 112 insertions(+), 66 deletions(-) diff --git a/Sources/HTTP/Message/HTTPBody.swift b/Sources/HTTP/Message/HTTPBody.swift index 29d9656f..59d10a77 100644 --- a/Sources/HTTP/Message/HTTPBody.swift +++ b/Sources/HTTP/Message/HTTPBody.swift @@ -1,4 +1,4 @@ - import Async +import Async import Foundation import Dispatch import Bits @@ -92,6 +92,38 @@ extension String: HTTPBodyRepresentable { } } +extension HTTPBody: CustomStringConvertible { + /// See `CustomStringConvertible.description + public var description: String { + switch storage { + case .binaryOutputStream: return " (use `debugPrint(_:)` to consume)" + case .chunkedOutputStream: return " (use `debugPrint(_:)` to consume)" + case .data, .buffer, .dispatchData, .none, .staticString, .string: return debugDescription + } + } + } + +extension HTTPBody: CustomDebugStringConvertible { + /// See `CustomDebugStringConvertible.debugDescription` + public var debugDescription: String { + switch storage { + case .buffer(let buffer): return String(bytes: buffer, encoding: .ascii) ?? "n/a" + case .data(let data): return String(data: data, encoding: .ascii) ?? "n/a" + case .dispatchData(let data): return String(data: Data(data), encoding: .ascii) ?? "n/a" + case .none: return "" + case .staticString(let string): return string.description + case .string(let string): return string + case .chunkedOutputStream, .binaryOutputStream: + do { + let data = try makeData(max: 2048).blockingAwait(timeout: .seconds(5)) + let string = String(data: data, encoding: .ascii) ?? "Error: Decoding body data as ASCII failed." + return " " + string + } catch { + return "Error collecting body data: \(error)" + } + } + } + } /// Output the body stream to the chunk encoding stream /// When supplied in this closure diff --git a/Sources/HTTP/Message/HTTPHeaderStorage.swift b/Sources/HTTP/Message/HTTPHeaderStorage.swift index 5fbd0e94..99eaecd9 100644 --- a/Sources/HTTP/Message/HTTPHeaderStorage.swift +++ b/Sources/HTTP/Message/HTTPHeaderStorage.swift @@ -22,11 +22,12 @@ final class HTTPHeaderStorage { } /// Internal init for truly empty header storage. - internal init(reserving: Int) { - let buffer = MutableByteBuffer(start: .allocate(capacity: reserving), count: reserving) + internal init(copying bytes: ByteBuffer, with indexes: [HTTPHeaderIndex]) { + let buffer = MutableByteBuffer(start: .allocate(capacity: bytes.count), count: bytes.count) self.buffer = buffer - self.view = ByteBuffer(start: buffer.baseAddress, count: 0) - self.indexes = [] + memcpy(buffer.start, bytes.start, bytes.count) + self.view = ByteBuffer(start: buffer.baseAddress, count: bytes.count) + self.indexes = indexes } /// Create a new `HTTPHeaders` with explicit storage and indexes. @@ -39,7 +40,6 @@ final class HTTPHeaderStorage { self.indexes = indexes } - /// Create a new `HTTPHeaders` with explicit storage and indexes. private init(view: ByteBuffer, buffer: MutableByteBuffer, indexes: [HTTPHeaderIndex?]) { self.view = view @@ -49,7 +49,6 @@ final class HTTPHeaderStorage { /// Creates a new, identical copy of the header storage. internal func copy() -> HTTPHeaderStorage { - print("🐄 COPY") let newBuffer = MutableByteBuffer( start: MutableBytesPointer.allocate(capacity: buffer.count), count: buffer.count @@ -204,32 +203,7 @@ final class HTTPHeaderStorage { /// Increases the internal buffer size by the supplied count. internal func increaseBufferSize(by count: Int) { - let newSize = buffer.count + count - let pointer: MutableBytesPointer = realloc(UnsafeMutableRawPointer(buffer.start), newSize) - .assumingMemoryBound(to: Byte.self) - buffer = MutableByteBuffer(start: pointer, count: newSize) - } - - /// Manually appends a byte buffer to the header storage. - internal func manualAppend(_ bytes: ByteBuffer) { - let count = (bytes.count + view.count) - buffer.count - if count > 0 { - // not enough room - increaseBufferSize(by: count) - } - memcpy(buffer.start.advanced(by: view.count), bytes.start, bytes.count) - view = ByteBuffer(start: buffer.start, count: view.count + bytes.count) - } - - /// Resets the internal view, ignoring any added data. - internal func manualReset() { - self.indexes = [] - self.view = ByteBuffer(start: buffer.start, count: 0) - } - - /// Manually set indexes. - internal func manualIndexes(_ indexes: [HTTPHeaderIndex]) { - self.indexes = indexes + buffer = buffer.increaseBufferSize(by: count) } deinit { @@ -239,8 +213,15 @@ final class HTTPHeaderStorage { } extension HTTPHeaderStorage: CustomStringConvertible { - /// See `CustomStringConvertible.description` + /// See `CustomStringConvertible.description public var description: String { + return debugDescription + } +} + +extension HTTPHeaderStorage: CustomDebugStringConvertible { + /// See `CustomDebugStringConvertible.debugDescription` + public var debugDescription: String { return String(bytes: view, encoding: .ascii) ?? "n/a" } } @@ -270,6 +251,16 @@ extension String { } } +extension UnsafeMutableBufferPointer { + /// Increases the mutable buffer size by the supplied count. + internal mutating func increaseBufferSize(by count: Int) -> UnsafeMutableBufferPointer { + let newSize = self.count + count + let pointer = realloc(UnsafeMutableRawPointer(start), newSize * MemoryLayout.size) + .assumingMemoryBound(to: Element.self) + return .init(start: pointer, count: newSize) + } +} + /// MARK: Static Data private let headerSeparatorStaticString: StaticString = ": " diff --git a/Sources/HTTP/Message/HTTPHeaders.swift b/Sources/HTTP/Message/HTTPHeaders.swift index 15223a5e..74f49601 100644 --- a/Sources/HTTP/Message/HTTPHeaders.swift +++ b/Sources/HTTP/Message/HTTPHeaders.swift @@ -133,12 +133,19 @@ extension HTTPHeaders { /// MARK: Utility extension HTTPHeaders: CustomStringConvertible { - /// See `CustomStringConvertible.description` + /// See `CustomStringConvertible.description public var description: String { return storage.description } } +extension HTTPHeaders: CustomDebugStringConvertible { + /// See `CustomStringConvertible.description` + public var debugDescription: String { + return storage.debugDescription + } +} + /// Joins two headers, overwriting the data in `lhs` with `rhs`' equivalent for duplicated public func +(lhs: HTTPHeaders, rhs: HTTPHeaders) -> HTTPHeaders { var lhs = lhs diff --git a/Sources/HTTP/Message/HTTPMessage.swift b/Sources/HTTP/Message/HTTPMessage.swift index 4e3ed718..d5baba32 100644 --- a/Sources/HTTP/Message/HTTPMessage.swift +++ b/Sources/HTTP/Message/HTTPMessage.swift @@ -30,7 +30,7 @@ import TCP /// to add your own stored properties to requests and responses /// that can be accessed simply by importing the module that /// adds them. This is how much of Vapor's functionality is created. -public protocol HTTPMessage: Codable, CustomDebugStringConvertible { +public protocol HTTPMessage: Codable, CustomStringConvertible, CustomDebugStringConvertible { /// The HTTP version of this message. var version: HTTPVersion { get set } /// The HTTP headers. @@ -87,15 +87,23 @@ extension HTTPMessage { // MARK: Debug string extension HTTPMessage { - /// A debug description for this HTTP message. - public var debugDescription: String { + /// See `CustomStringConvertible.description + public var description: String { var desc: [String] = [] + desc.append("(\(Self.self))") + desc.append(headers.description) + desc.append(body.description) + return desc.joined(separator: "\n") + } +} - desc.append("HTTP.\(Self.self)") - for header in headers { - desc.append("\(header.name): \(header.value)") - } - +extension HTTPMessage { + /// See `CustomDebugStringConvertible.debugDescription` + public var debugDescription: String { + var desc: [String] = [] + desc.append("(\(Self.self))") + desc.append(headers.debugDescription) + desc.append(body.debugDescription) return desc.joined(separator: "\n") } } diff --git a/Sources/HTTP/Parser/CHTTPParserContext.swift b/Sources/HTTP/Parser/CHTTPParserContext.swift index 72e1b7e6..244beaae 100644 --- a/Sources/HTTP/Parser/CHTTPParserContext.swift +++ b/Sources/HTTP/Parser/CHTTPParserContext.swift @@ -38,7 +38,10 @@ internal final class CHTTPParserContext { /// Raw headers data - var headersData: HTTPHeaderStorage + var headersData: MutableByteBuffer + + /// Current header start offset from previous run(s) of the parser + var headersDataSize: Int /// Parsed indexes into the header data var headersIndexes: [HTTPHeaderIndex] @@ -62,9 +65,6 @@ internal final class CHTTPParserContext { /// If not set, there have been no header start events yet. private var headerStart: UnsafePointer? - /// Current header start offset from previous run(s) of the parser - private var headerStartOffset: Int - /// Pointer to the last start location of the body. /// If not set, there have been no body start events yet. private var bodyStart: UnsafePointer? @@ -90,7 +90,8 @@ internal final class CHTTPParserContext { self.version = nil self.headers = nil - self.headersData = .init(reserving: 64) + self.headersData = .init(start: .allocate(capacity: 64), count: 64) + self.headersDataSize = 0 self.headersIndexes = [] self.urlData = [] @@ -100,7 +101,6 @@ internal final class CHTTPParserContext { self.currentHeadersSize = 0 self.headerStart = nil - self.headerStartOffset = 0 self.bodyStart = nil self.parser = http_parser() @@ -111,6 +111,11 @@ internal final class CHTTPParserContext { http_parser_init(&parser, type) initialize() } + + deinit { + headersData.start.deinitialize() + headersData.start.deallocate(capacity: headersData.count) + } } /// Current parser message state. @@ -173,8 +178,7 @@ extension CHTTPParserContext { self.version = nil self.headers = nil - //self.headersData = .init(reserving: 64) - self.headersData.manualReset() // can we get this to work w/ COW? + self.headersDataSize = 0 self.headersIndexes = [] self.urlData = [] @@ -183,7 +187,6 @@ extension CHTTPParserContext { self.currentHeadersSize = 0 self.headerStart = nil - self.headerStartOffset = 0 self.bodyStart = nil } @@ -218,21 +221,23 @@ extension CHTTPParserContext { /// current distance from start to end let distance = start.distance(to: end) - // append the length of the headers in this buffer to the header start offset - headerStartOffset += distance + let overflow = (headersDataSize + distance) - headersData.count + if overflow > 0 { + headersData = headersData.increaseBufferSize(by: overflow) + } - /// create buffer view of current header data and append it - let buffer = ByteBuffer( - start: start.withMemoryRebound(to: Byte.self, capacity: distance) { $0 }, - count: distance - ) - headersData.manualAppend(buffer) + // append the length of the headers in this buffer to the header start offset + memcpy(headersData.start.advanced(by: headersDataSize), start, distance) + headersDataSize += distance /// if this buffer copy is happening after headers complete indication, /// set the headers struct for later retreival if headersComplete { - headersData.manualIndexes(headersIndexes) - headers = HTTPHeaders(storage: headersData) + let storage = HTTPHeaderStorage( + copying: ByteBuffer(start: headersData.start, count: headersDataSize), + with: headersIndexes + ) + headers = HTTPHeaders(storage: storage) } } @@ -308,11 +313,11 @@ extension CHTTPParserContext { // check current header parsing state switch results.headerState { case .none: - let distance = start.distance(to: chunk) + results.headerStartOffset + let distance = start.distance(to: chunk) + results.headersDataSize // nothing is being parsed, start a new key results.headerState = .key(startIndex: distance, endIndex: distance + count) case .value(let index): - let distance = start.distance(to: chunk) + results.headerStartOffset + let distance = start.distance(to: chunk) + results.headersDataSize // there was previously a value being parsed. // it is now finished. results.headersIndexes.append(index) @@ -369,7 +374,7 @@ extension CHTTPParserContext { index.valueEndIndex += count results.headerState = .value(index) case .key(let key): - let distance = start.distance(to: chunk) + results.headerStartOffset + let distance = start.distance(to: chunk) + results.headersDataSize // create a full HTTP headers index let index = HTTPHeaderIndex( diff --git a/Sources/HTTP/Request/HTTPRequest.swift b/Sources/HTTP/Request/HTTPRequest.swift index 96935d09..081e46b0 100644 --- a/Sources/HTTP/Request/HTTPRequest.swift +++ b/Sources/HTTP/Request/HTTPRequest.swift @@ -44,7 +44,9 @@ public struct HTTPRequest: HTTPMessage { /// See `Message.body` /// /// [Learn More →](https://docs.vapor.codes/3.0/http/body/) - public var body: HTTPBody + public var body: HTTPBody { + didSet { updateBodyHeaders() } + } /// See Message.onUpgrade public var onUpgrade: HTTPOnUpgrade? @@ -62,6 +64,7 @@ public struct HTTPRequest: HTTPMessage { self.version = version self.headers = headers self.body = body + updateBodyHeaders() } } From 69ead250ed52bdc9fedca95edc153fd3b438b312 Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Fri, 26 Jan 2018 20:23:00 -0500 Subject: [PATCH 14/16] remove unnecessary public --- Sources/HTTP/Message/HTTPHeaderStorage.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/HTTP/Message/HTTPHeaderStorage.swift b/Sources/HTTP/Message/HTTPHeaderStorage.swift index 99eaecd9..72156fb8 100644 --- a/Sources/HTTP/Message/HTTPHeaderStorage.swift +++ b/Sources/HTTP/Message/HTTPHeaderStorage.swift @@ -13,7 +13,7 @@ final class HTTPHeaderStorage { private var indexes: [HTTPHeaderIndex?] /// Creates a new `HTTPHeaders` with default content. - public static func `default`() -> HTTPHeaderStorage { + static func `default`() -> HTTPHeaderStorage { let storageSize = 64 let buffer = MutableByteBuffer(start: .allocate(capacity: storageSize), count: storageSize) memcpy(buffer.baseAddress, defaultHeaders.baseAddress!, defaultHeadersSize) From 1fe28832b6a1925bb78f5c9172b6135132ca1c06 Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Mon, 29 Jan 2018 14:12:54 -0500 Subject: [PATCH 15/16] tests passing --- Sources/HTTP/Message/HTTPHeaders.swift | 5 ++ Sources/HTTP/Parser/HTTPResponseParser.swift | 63 ++++++++++++++++++- .../Serializer/HTTPResponseSerializer.swift | 56 ++++++++++++++++- Sources/Multipart/Parser.swift | 4 +- Sources/Multipart/Serializer.swift | 1 - Sources/WebSocket/Upgrade.swift | 2 +- Tests/MultipartTests/MultipartTests.swift | 5 +- Tests/WebSocketTests/WebSocketTests.swift | 4 +- 8 files changed, 130 insertions(+), 10 deletions(-) diff --git a/Sources/HTTP/Message/HTTPHeaders.swift b/Sources/HTTP/Message/HTTPHeaders.swift index 74f49601..1ae708e1 100644 --- a/Sources/HTTP/Message/HTTPHeaders.swift +++ b/Sources/HTTP/Message/HTTPHeaders.swift @@ -24,6 +24,11 @@ public struct HTTPHeaders: Codable { self.storage = .default() } + /// Creates an empty HTTPHeaders (no Content-Length 0) + public static func empty() -> HTTPHeaders { + return HTTPHeaders(storage: HTTPHeaderStorage(bytes: [], indexes: [])) + } + /// Create a new `HTTPHeaders` with explicit storage and indexes. internal init(storage: HTTPHeaderStorage) { self.storage = storage diff --git a/Sources/HTTP/Parser/HTTPResponseParser.swift b/Sources/HTTP/Parser/HTTPResponseParser.swift index 59636ce5..e01b6eb5 100644 --- a/Sources/HTTP/Parser/HTTPResponseParser.swift +++ b/Sources/HTTP/Parser/HTTPResponseParser.swift @@ -29,10 +29,71 @@ import Foundation throw HTTPError.invalidMessage() } + let status: HTTPStatus + switch statusCode { + case 101: status = .upgrade + case 200: status = .ok + case 201: status = .created + case 202: status = .accepted + case 204: status = .noContent + case 205: status = .resetContent + case 206: status = .partialContent + case 300: status = .multipleChoices + case 301: status = .movedPermanently + case 302: status = .found + case 303: status = .seeOther + case 304: status = .notModified + case 305: status = .useProxy + case 306: status = .switchProxy + case 307: status = .temporaryRedirect + case 308: status = .permanentRedirect + case 400: status = .badRequest + case 401: status = .unauthorized + case 403: status = .forbidden + case 404: status = .notFound + case 405: status = .methodNotAllowed + case 406: status = .notAcceptable + case 407: status = .proxyAuthenticationRequired + case 408: status = .requestTimeout + case 409: status = .conflict + case 410: status = .gone + case 411: status = .lengthRequired + case 412: status = .preconditionFailed + case 413: status = .requestEntityTooLarge + case 414: status = .requestURITooLong + case 415: status = .unsupportedMediaType + case 416: status = .requestedRangeNotSatisfiable + case 417: status = .expectationFailed + case 418: status = .imATeapot + case 419: status = .authenticationTimeout + case 420: status = .enhanceYourCalm + case 421: status = .misdirectedRequest + case 422: status = .unprocessableEntity + case 423: status = .locked + case 424: status = .failedDependency + case 426: status = .upgradeRequired + case 428: status = .preconditionRequired + case 429: status = .tooManyRequests + case 431: status = .requestHeaderFieldsTooLarge + case 451: status = .unavailableForLegalReasons + case 500: status = .internalServerError + case 501: status = .notImplemented + case 502: status = .badGateway + case 503: status = .serviceUnavailable + case 504: status = .gatewayTimeout + case 505: status = .httpVersionNotSupported + case 506: status = .variantAlsoNegotiates + case 507: status = .insufficientStorage + case 508: status = .loopDetected + case 510: status = .notExtended + case 511: status = .networkAuthenticationRequired + default: status = HTTPStatus(code: statusCode) + } + // create the request return HTTPResponse( version: version, - status: HTTPStatus(code: statusCode), + status: status, headers: headers, body: body ) diff --git a/Sources/HTTP/Serializer/HTTPResponseSerializer.swift b/Sources/HTTP/Serializer/HTTPResponseSerializer.swift index d34e5871..3702ff24 100644 --- a/Sources/HTTP/Serializer/HTTPResponseSerializer.swift +++ b/Sources/HTTP/Serializer/HTTPResponseSerializer.swift @@ -4,6 +4,8 @@ import Bits import Dispatch import Foundation +private let startLineBufferSize = 1024 + /// Converts responses to Data. public final class HTTPResponseSerializer: HTTPSerializer { /// See `InputStream.Input` @@ -17,6 +19,9 @@ public final class HTTPResponseSerializer: HTTPSerializer { /// See `HTTPSerializer.context` public var context: HTTPSerializerContext + + /// Start line buffer for non-precoded start lines. + private var startLineBuffer: MutableByteBuffer? /// Create a new HTTPResponseSerializer public init() { @@ -27,9 +32,58 @@ public final class HTTPResponseSerializer: HTTPSerializer { public func serializeStartLine(for message: HTTPResponse) -> ByteBuffer { switch message.status { case .ok: return okStartLine.withUTF8Buffer { $0 } - default: fatalError() + case .notFound: return notFoundStartLine.withUTF8Buffer { $0 } + case .internalServerError: return internalServerErrorStartLine.withUTF8Buffer { $0 } + default: + let buffer: MutableByteBuffer + if let existing = self.startLineBuffer { + buffer = existing + } else { + let new = MutableByteBuffer(start: .allocate(capacity: startLineBufferSize), count: startLineBufferSize) + buffer = new + } + + // `HTTP/1.1 ` + var pos = buffer.start.advanced(by: 0) + memcpy(pos, version.withUTF8Buffer { $0 }.start, version.utf8CodeUnitCount) + + // `200` + pos = pos.advanced(by: version.utf8CodeUnitCount) + let codeBytes = message.status.code.bytes() + memcpy(pos, codeBytes, codeBytes.count) + + // ` ` + pos = pos.advanced(by: codeBytes.count) + pos[0] = .space + + // `OK` + pos = pos.advanced(by: 1) + let messageBytes = message.status.messageBytes + memcpy(pos, messageBytes, messageBytes.count) + + // `\r\n` + pos = pos.advanced(by: messageBytes.count) + pos[0] = .carriageReturn + pos[1] = .newLine + pos = pos.advanced(by: 2) + + // view + let view = ByteBuffer(start: buffer.start, count: buffer.start.distance(to: pos)) + print(String(bytes: view, encoding: .ascii)) + return view + } + } + + deinit { + if let buffer = startLineBuffer { + buffer.baseAddress!.deinitialize() + buffer.baseAddress?.deallocate(capacity: buffer.count) } } } +private let version: StaticString = "HTTP/1.1 " + private let okStartLine: StaticString = "HTTP/1.1 200 OK\r\n" +private let notFoundStartLine: StaticString = "HTTP/1.1 404 Not Found\r\n" +private let internalServerErrorStartLine: StaticString = "HTTP/1.1 500 Internal Server Error\r\n" diff --git a/Sources/Multipart/Parser.swift b/Sources/Multipart/Parser.swift index 03463f70..0a3654c4 100644 --- a/Sources/Multipart/Parser.swift +++ b/Sources/Multipart/Parser.swift @@ -92,7 +92,7 @@ public final class MultipartParser { /// Reads the headers at the current position fileprivate func readHeaders() throws -> HTTPHeaders { - var headers = HTTPHeaders() + var headers = HTTPHeaders.empty() // headers headerScan: while position < data.count, try carriageReturnNewLine() { @@ -117,7 +117,7 @@ public final class MultipartParser { guard let value = try scanStringUntil(.carriageReturn) else { throw MultipartError(identifier: "multipart:invalid-header-value", reason: "Invalid multipart header value string encoding") } - + headers[HTTPHeaderName(key)] = value } diff --git a/Sources/Multipart/Serializer.swift b/Sources/Multipart/Serializer.swift index 4e77e740..b2cb4947 100644 --- a/Sources/Multipart/Serializer.swift +++ b/Sources/Multipart/Serializer.swift @@ -31,7 +31,6 @@ public final class MultipartSerializer { body.append(buffer) } - body.append(contentsOf: [.carriageReturn, .newLine]) body.append(part.data) body.append(contentsOf: [.carriageReturn, .newLine]) } diff --git a/Sources/WebSocket/Upgrade.swift b/Sources/WebSocket/Upgrade.swift index b7f08ac3..039478d4 100644 --- a/Sources/WebSocket/Upgrade.swift +++ b/Sources/WebSocket/Upgrade.swift @@ -28,7 +28,7 @@ extension WebSocket { try settings.apply(on: request) let headers = try buildWebSocketHeaders(for: request) - var response = HTTPResponse(status: 101, headers: headers) + var response = HTTPResponse(status: .upgrade, headers: headers) try settings.apply(on: &response, request: request) diff --git a/Tests/MultipartTests/MultipartTests.swift b/Tests/MultipartTests/MultipartTests.swift index ff19f50a..0da33012 100644 --- a/Tests/MultipartTests/MultipartTests.swift +++ b/Tests/MultipartTests/MultipartTests.swift @@ -47,8 +47,9 @@ class MultipartTests: XCTestCase { XCTAssertEqual(try form.getString(named: "test"), "eqw-dd-sa----123;1[234") XCTAssertEqual(try form.getFile(named: "named").data, Data(named.utf8)) XCTAssertEqual(try form.getFile(named: "multinamed[]").data, Data(multinamed.utf8)) - - XCTAssertEqual(MultipartSerializer(form: form).serialize(), data) + + let a = String(data: MultipartSerializer(form: form).serialize(), encoding: .ascii) + XCTAssertEqual(a, string) } func testMultifile() throws { diff --git a/Tests/WebSocketTests/WebSocketTests.swift b/Tests/WebSocketTests/WebSocketTests.swift index 863a16ec..44edd67a 100644 --- a/Tests/WebSocketTests/WebSocketTests.swift +++ b/Tests/WebSocketTests/WebSocketTests.swift @@ -38,11 +38,11 @@ final class WebSocketTests : XCTestCase { let read = try client.socket.read(max: 512) let string = String(data: read, encoding: .utf8) XCTAssertEqual(string, """ - HTTP/1.1 101 \r + HTTP/1.1 101 Upgrade\r + Content-Length: 0\r Upgrade: websocket\r Connection: Upgrade\r Sec-WebSocket-Accept: U5ZWHrbsu7snP3DY1Q5P3e8AkOk=\r - Content-Length: 0\r \r """) From ad9e2089b41236c4ce7ed50fd036a7a0ba3bc25f Mon Sep 17 00:00:00 2001 From: Tanner Nelson Date: Mon, 29 Jan 2018 14:50:15 -0500 Subject: [PATCH 16/16] linux fixes --- Sources/HTTP/Message/HTTPHeaderStorage.swift | 4 +-- .../Serializer/HTTPResponseSerializer.swift | 1 - Tests/HTTPTests/ProtocolTester.swift | 27 ++++++++----------- Tests/LinuxMain.swift | 3 ++- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/Sources/HTTP/Message/HTTPHeaderStorage.swift b/Sources/HTTP/Message/HTTPHeaderStorage.swift index 72156fb8..dd36cf32 100644 --- a/Sources/HTTP/Message/HTTPHeaderStorage.swift +++ b/Sources/HTTP/Message/HTTPHeaderStorage.swift @@ -16,7 +16,7 @@ final class HTTPHeaderStorage { static func `default`() -> HTTPHeaderStorage { let storageSize = 64 let buffer = MutableByteBuffer(start: .allocate(capacity: storageSize), count: storageSize) - memcpy(buffer.baseAddress, defaultHeaders.baseAddress!, defaultHeadersSize) + memcpy(buffer.start, defaultHeaders.start, defaultHeadersSize) let view = ByteBuffer(start: buffer.baseAddress, count: defaultHeadersSize) return HTTPHeaderStorage(view: view, buffer: buffer, indexes: [defaultHeaderIndex]) } @@ -34,7 +34,7 @@ final class HTTPHeaderStorage { internal init(bytes: Bytes, indexes: [HTTPHeaderIndex]) { let storageSize = bytes.count let buffer = MutableByteBuffer(start: .allocate(capacity: storageSize), count: storageSize) - memcpy(buffer.baseAddress, bytes, storageSize) + memcpy(buffer.start, bytes, storageSize) self.buffer = buffer self.view = ByteBuffer(start: buffer.baseAddress, count: storageSize) self.indexes = indexes diff --git a/Sources/HTTP/Serializer/HTTPResponseSerializer.swift b/Sources/HTTP/Serializer/HTTPResponseSerializer.swift index 3702ff24..7fda1da3 100644 --- a/Sources/HTTP/Serializer/HTTPResponseSerializer.swift +++ b/Sources/HTTP/Serializer/HTTPResponseSerializer.swift @@ -69,7 +69,6 @@ public final class HTTPResponseSerializer: HTTPSerializer { // view let view = ByteBuffer(start: buffer.start, count: buffer.start.distance(to: pos)) - print(String(bytes: view, encoding: .ascii)) return view } } diff --git a/Tests/HTTPTests/ProtocolTester.swift b/Tests/HTTPTests/ProtocolTester.swift index dfb16a5a..3bb39b86 100644 --- a/Tests/HTTPTests/ProtocolTester.swift +++ b/Tests/HTTPTests/ProtocolTester.swift @@ -22,34 +22,37 @@ public final class ProtocolTester: Async.OutputStream { /// The added checks private var checks: [ProtocolTesterCheck] + /// The original string data + private let original: String + /// The test data - private var data: String + private var data: Bytes /// Creates a new `ProtocolTester` public init(data: String, onFail: @escaping (String, StaticString, UInt) -> (), reset: @escaping () -> ()) { self.reset = reset self.fail = onFail - self.data = data + self.original = data + self.data = Bytes(data.utf8) checks = [] } /// Adds a "before" offset assertion to the tester. public func assert(before substring: String, file: StaticString = #file, line: UInt = #line, callback: @escaping () throws -> ()) { - let check = ProtocolTesterCheck(minOffset: nil, maxOffset: data.offset(of: substring), file: file, line: line, checks: callback) + let check = ProtocolTesterCheck(minOffset: nil, maxOffset: original.offset(of: substring), file: file, line: line, checks: callback) checks.append(check) } /// Adds an "after" offset assertion to the tester. public func assert(after substring: String, file: StaticString = #file, line: UInt = #line, callback: @escaping () throws -> ()) { - let check = ProtocolTesterCheck(minOffset: data.offset(of: substring), maxOffset: nil, file: file, line: line, checks: callback) + let check = ProtocolTesterCheck(minOffset: original.offset(of: substring), maxOffset: nil, file: file, line: line, checks: callback) checks.append(check) } /// Runs the protocol tester w/ the supplied input public func run() -> Future { Swift.assert(downstream != nil, "ProtocolTester must be connected before running") - let buffer = data.buffer - return runMax(buffer, max: buffer.count) + return runMax(ByteBuffer(start: &data, count: data.count), max: data.count) } /// Recurisvely runs tests, splitting the supplied buffer until max == 0 @@ -69,7 +72,7 @@ public final class ProtocolTester: Async.OutputStream { let lastChunk = ByteBuffer(start: buffer.baseAddress?.advanced(by: buffer.count - lastChunkSize), count: lastChunkSize) chunks.insert(lastChunk, at: 0) } - + reset() return runChunks(chunks, currentOffset: 0).flatMap(to: Void.self) { return self.runMax(buffer, max: max - 1) @@ -141,7 +144,7 @@ public final class ProtocolTester: Async.OutputStream { } /// HEX map. - private static let hexMap = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "0", "A", "B", "C", "D", "E", "F"] + private static let hexMap = ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "A", "B", "C", "D", "E", "F", "0"] } /// A stored protocol tester check. @@ -153,14 +156,6 @@ private struct ProtocolTesterCheck { var checks: () throws -> () } -extension String { - /// Byte buffer representation - /// Note: String must be static or a reference held for the duration of this buffer's use - var buffer: ByteBuffer { - return self.data(using: .utf8)!.withByteBuffer { $0 } - } -} - extension String { /// Returns int offset of the supplied string, crashing if it doesn't exist fileprivate func offset(of string: String) -> Int { diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift index 08b69b7b..b95cbc67 100644 --- a/Tests/LinuxMain.swift +++ b/Tests/LinuxMain.swift @@ -10,7 +10,8 @@ XCTMain([ // MARK: HTTP testCase(HTTPClientTests.allTests), testCase(HTTPServerTests.allTests), - testCase(HTTPSerializerStreamTests.allTests), + testCase(HTTPParserTests.allTests), + testCase(HTTPSerializerTests.allTests), testCase(UtilityTests.allTests), testCase(FormURLEncodedCodableTests.allTests),