From 3f6ac39f5253fb9c4c07744cbfefb654bea911ca Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Wed, 9 Mar 2022 16:07:41 +0100 Subject: [PATCH 01/13] Add ControlPlaneResponseDecoder --- .../ControlPlaneRequest.swift | 45 +- .../ControlPlaneResponseDecoder.swift | 502 ++++++++++++++++++ .../LambdaRuntimeError.swift | 66 +++ .../ControlPlaneResponseDecoderTests.swift | 76 +++ scripts/soundness.sh | 2 +- 5 files changed, 677 insertions(+), 14 deletions(-) create mode 100644 Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift create mode 100644 Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift create mode 100644 Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift diff --git a/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift b/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift index 14c5f2a7..3b3ba272 100644 --- a/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift +++ b/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift @@ -2,7 +2,7 @@ // // This source file is part of the SwiftAWSLambdaRuntime open source project // -// Copyright (c) 2017-2021 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Copyright (c) 2021-2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors // Licensed under Apache License v2.0 // // See LICENSE.txt for license information @@ -29,12 +29,27 @@ enum ControlPlaneResponse: Hashable { } struct Invocation: Hashable { - let requestID: String - let deadlineInMillisSinceEpoch: Int64 - let invokedFunctionARN: String - let traceID: String - let clientContext: String? - let cognitoIdentity: String? + var requestID: String + var deadlineInMillisSinceEpoch: Int64 + var invokedFunctionARN: String + var traceID: String + var clientContext: String? + var cognitoIdentity: String? + + init(requestID: String, + deadlineInMillisSinceEpoch: Int64, + invokedFunctionARN: String, + traceID: String, + clientContext: String?, + cognitoIdentity: String? + ) { + self.requestID = requestID + self.deadlineInMillisSinceEpoch = deadlineInMillisSinceEpoch + self.invokedFunctionARN = invokedFunctionARN + self.traceID = traceID + self.clientContext = clientContext + self.cognitoIdentity = cognitoIdentity + } init(headers: HTTPHeaders) throws { guard let requestID = headers.first(name: AmazonHeaders.requestID), !requestID.isEmpty else { @@ -51,12 +66,16 @@ struct Invocation: Hashable { throw Lambda.RuntimeError.invocationMissingHeader(AmazonHeaders.invokedFunctionARN) } - self.requestID = requestID - self.deadlineInMillisSinceEpoch = unixTimeInMilliseconds - self.invokedFunctionARN = invokedFunctionARN - self.traceID = headers.first(name: AmazonHeaders.traceID) ?? "Root=\(AmazonHeaders.generateXRayTraceID());Sampled=0" - self.clientContext = headers["Lambda-Runtime-Client-Context"].first - self.cognitoIdentity = headers["Lambda-Runtime-Cognito-Identity"].first + let traceID = headers.first(name: AmazonHeaders.traceID) ?? "Root=\(AmazonHeaders.generateXRayTraceID());Sampled=0" + + self.init( + requestID: requestID, + deadlineInMillisSinceEpoch: unixTimeInMilliseconds, + invokedFunctionARN: invokedFunctionARN, + traceID: traceID, + clientContext: headers["Lambda-Runtime-Client-Context"].first, + cognitoIdentity: headers["Lambda-Runtime-Cognito-Identity"].first + ) } } diff --git a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift new file mode 100644 index 00000000..b2f6fcc8 --- /dev/null +++ b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift @@ -0,0 +1,502 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftAWSLambdaRuntime open source project +// +// Copyright (c) 2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore +#if canImport(Darwin) +import Darwin +#else +import Glibc +#endif + +struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { + typealias InboundOut = ControlPlaneResponse + + private enum State { + case waitingForNewResponse + case parsingHead(PartialHead) + case waitingForBody(PartialHead) + case receivingBody(PartialHead, ByteBuffer) + } + + private var state: State + + init() { + self.state = .waitingForNewResponse + } + + mutating func decode(buffer: inout ByteBuffer) throws -> ControlPlaneResponse? { + switch self.state { + case .waitingForNewResponse: + guard case .decoded(let head) = try self.decodeResponseHead(from: &buffer) else { + return nil + } + + guard case .decoded(let body) = try self.decodeBody(from: &buffer) else { + return nil + } + + return try self.decodeResponse(head: head, body: body) + + case .parsingHead: + guard case .decoded(let head) = try self.decodeHeaderLines(from: &buffer) else { + return nil + } + + guard case .decoded(let body) = try self.decodeBody(from: &buffer) else { + return nil + } + + return try self.decodeResponse(head: head, body: body) + + case .waitingForBody(let head), .receivingBody(let head, _): + guard case .decoded(let body) = try self.decodeBody(from: &buffer) else { + return nil + } + + return try self.decodeResponse(head: head, body: body) + } + } + + mutating func decodeLast(buffer: inout ByteBuffer, seenEOF: Bool) throws -> ControlPlaneResponse? { + try self.decode(buffer: &buffer) + } + + // MARK: - Private Methods - + + private enum DecodeResult { + case needMoreData + case decoded(T) + } + + private mutating func decodeResponseHead(from buffer: inout ByteBuffer) throws -> DecodeResult { + guard case .decoded = try self.decodeResponseStatusLine(from: &buffer) else { + return .needMoreData + } + + return try self.decodeHeaderLines(from: &buffer) + } + + private mutating func decodeResponseStatusLine(from buffer: inout ByteBuffer) throws -> DecodeResult { + guard case .waitingForNewResponse = self.state else { + preconditionFailure("Invalid state: \(self.state)") + } + + guard case .decoded(var lineBuffer) = try self.decodeCRLFTerminatedLine(from: &buffer) else { + return .needMoreData + } + + let statusCode = try self.decodeStatusLine(from: &lineBuffer) + self.state = .parsingHead(.init(statusCode: statusCode)) + return .decoded(statusCode) + } + + private mutating func decodeHeaderLines(from buffer: inout ByteBuffer) throws -> DecodeResult { + guard case .parsingHead(var head) = self.state else { + preconditionFailure("Invalid state: \(self.state)") + } + + while true { + guard case .decoded(var nextLine) = try self.decodeCRLFTerminatedLine(from: &buffer) else { + self.state = .parsingHead(head) + return .needMoreData + } + + switch try self.decodeHeaderLine(from: &nextLine) { + case .headerEnd: + self.state = .waitingForBody(head) + return .decoded(head) + + case .contentLength(let length): + head.contentLength = length // TODO: This can crash + + case .contentType: + break // switch + + case .requestID(let requestID): + head.requestID = requestID + + case .traceID(let traceID): + head.traceID = traceID + + case .functionARN(let arn): + head.invokedFunctionARN = arn + + case .cognitoIdentity(let cognitoIdentity): + head.cognitoIdentity = cognitoIdentity + + case .deadlineMS(let deadline): + head.deadlineInMillisSinceEpoch = deadline + + case .weDontCare: + break // switch + } + } + } + + enum BodyEncoding { + case chunked + case plain(length: Int) + case none + } + + private mutating func decodeBody(from buffer: inout ByteBuffer) throws -> DecodeResult { + switch self.state { + case .waitingForBody(let partialHead): + switch partialHead.contentLength { + case .none: + return .decoded(nil) + case .some(let length): + if let slice = buffer.readSlice(length: length) { + self.state = .waitingForNewResponse + return .decoded(slice) + } + return .needMoreData + } + + case .waitingForNewResponse, .parsingHead, .receivingBody: + preconditionFailure("Invalid state: \(self.state)") + } + } + + private mutating func decodeResponse(head: PartialHead, body: ByteBuffer?) throws -> ControlPlaneResponse { + switch head.statusCode { + case 200: + guard let body = body else { + preconditionFailure("TODO: implement") + } + return .next(try Invocation(head: head), body) + case 202: + return .accepted + case 400..<600: + preconditionFailure("TODO: implement") + + default: + throw LambdaRuntimeError.unexpectedStatusCode + } + } + + mutating func decodeStatusLine(from buffer: inout ByteBuffer) throws -> Int { + guard buffer.readableBytes >= 11 else { + throw LambdaRuntimeError.responseHeadInvalidStatusLine + } + + let cmp = buffer.readableBytesView.withUnsafeBytes { ptr in + memcmp("HTTP/1.1 ", ptr.baseAddress, 8) == 0 ? true : false + } + buffer.moveReaderIndex(forwardBy: 9) + + guard cmp else { + throw LambdaRuntimeError.responseHeadInvalidStatusLine + } + + let statusAsString = buffer.readString(length: 3)! + guard let status = Int(statusAsString) else { + throw LambdaRuntimeError.responseHeadInvalidStatusLine + } + + return status + } + + private mutating func decodeCRLFTerminatedLine(from buffer: inout ByteBuffer) throws -> DecodeResult { + guard let crIndex = buffer.readableBytesView.firstIndex(of: UInt8(ascii: "\r")) else { + if buffer.readableBytes > 256 { + throw LambdaRuntimeError.responseHeadMoreThan256BytesBeforeCRLF + } + return .needMoreData + } + let lfIndex = buffer.readableBytesView.index(after: crIndex) + guard lfIndex < buffer.readableBytesView.endIndex else { + // the buffer is split exactly after the \r and \n. Let's wait for more data + return .needMoreData + } + + guard buffer.readableBytesView[lfIndex] == UInt8(ascii: "\n") else { + throw LambdaRuntimeError.responseHeadInvalidHeader + } + + let slice = buffer.readSlice(length: crIndex - buffer.readerIndex)! + buffer.moveReaderIndex(forwardBy: 2) // move over \r\n + return .decoded(slice) + } + + private enum HeaderLineContent: Equatable { + case traceID(String) + case contentType + case contentLength(Int) + case cognitoIdentity(String) + case deadlineMS(Int) + case functionARN(String) + case requestID(LambdaRequestID) + + case weDontCare + case headerEnd + } + + private mutating func decodeHeaderLine(from buffer: inout ByteBuffer) throws -> HeaderLineContent { + guard let colonIndex = buffer.readableBytesView.firstIndex(of: UInt8(ascii: ":")) else { + if buffer.readableBytes == 0 { + return .headerEnd + } + throw LambdaRuntimeError.responseHeadHeaderMissingColon + } + + // based on colonIndex we can already make some good guesses... + // 4: Date + // 12: Content-Type + // 14: Content-Length + // 17: Transfer-Encoding + // 23: Lambda-Runtime-Trace-Id + // 26: Lambda-Runtime-Deadline-Ms + // 29: Lambda-Runtime-Aws-Request-Id + // Lambda-Runtime-Client-Context + // 31: Lambda-Runtime-Cognito-Identity + // 35: Lambda-Runtime-Invoked-Function-Arn + + switch colonIndex { + case 4: + if buffer.readHeaderName("date") { + return .weDontCare + } + + case 12: + if buffer.readHeaderName("content-type") { + return .weDontCare + } + + case 14: + if buffer.readHeaderName("content-length") { + buffer.moveReaderIndex(forwardBy: 1) // move forward for colon + try self.decodeOptionalWhiteSpaceBeforeFieldValue(from: &buffer) + guard let length = buffer.readIntegerFromHeader() else { + throw LambdaRuntimeError.responseHeadInvalidDeadlineValue + } + return .contentLength(length) + } + + case 17: + if buffer.readHeaderName("transfer-encoding") { + buffer.moveReaderIndex(forwardBy: 1) // move forward for colon + try self.decodeOptionalWhiteSpaceBeforeFieldValue(from: &buffer) + guard let length = buffer.readIntegerFromHeader() else { + throw LambdaRuntimeError.responseHeadInvalidDeadlineValue + } + return .contentLength(length) + } + + case 23: + if buffer.readHeaderName("lambda-runtime-trace-id") { + buffer.moveReaderIndex(forwardBy: 1) + guard let string = try self.decodeHeaderValue(from: &buffer) else { + throw LambdaRuntimeError.responseHeadInvalidTraceIDValue + } + return .traceID(string) + } + + case 26: + if buffer.readHeaderName("lambda-runtime-deadline-ms") { + buffer.moveReaderIndex(forwardBy: 1) // move forward for colon + try self.decodeOptionalWhiteSpaceBeforeFieldValue(from: &buffer) + guard let deadline = buffer.readIntegerFromHeader() else { + throw LambdaRuntimeError.responseHeadInvalidContentLengthValue + } + return .deadlineMS(deadline) + } + + case 29: + if buffer.readHeaderName("lambda-runtime-aws-request-id") { + buffer.moveReaderIndex(forwardBy: 1) // move forward for colon + try self.decodeOptionalWhiteSpaceBeforeFieldValue(from: &buffer) + guard let requestID = buffer.readRequestID() else { + throw LambdaRuntimeError.responseHeadInvalidRequestIDValue + } + return .requestID(requestID) + } + if buffer.readHeaderName("lambda-runtime-client-context") { + return .weDontCare + } + + case 31: + if buffer.readHeaderName("lambda-runtime-cognito-identity") { + return .weDontCare + } + + case 35: + if buffer.readHeaderName("lambda-runtime-invoked-function-arn") { + buffer.moveReaderIndex(forwardBy: 1) + guard let string = try self.decodeHeaderValue(from: &buffer) else { + throw LambdaRuntimeError.responseHeadInvalidTraceIDValue + } + return .functionARN(string) + } + + default: + return .weDontCare + } + + return .weDontCare + } + + @discardableResult + mutating func decodeOptionalWhiteSpaceBeforeFieldValue(from buffer: inout ByteBuffer) throws -> Int { + let startIndex = buffer.readerIndex + guard let index = buffer.readableBytesView.firstIndex(where: { ($0 != UInt8(ascii: " ") && $0 != UInt8(ascii: "\t")) }) else { + throw LambdaRuntimeError.responseHeadHeaderMissingFieldValue + } + buffer.moveReaderIndex(to: index) + return index - startIndex + } + + private func decodeHeaderValue(from buffer: inout ByteBuffer) throws -> String? { + func isNotOptionalWhiteSpace(_ val: UInt8) -> Bool { + val != UInt8(ascii: " ") && val != UInt8(ascii: "\t") + } + + guard let firstCharacterIndex = buffer.readableBytesView.firstIndex(where: isNotOptionalWhiteSpace), + let lastCharacterIndex = buffer.readableBytesView.lastIndex(where: isNotOptionalWhiteSpace) + else { + throw LambdaRuntimeError.responseHeadHeaderMissingFieldValue + } + + let string = buffer.getString(at: firstCharacterIndex, length: lastCharacterIndex + 1 - firstCharacterIndex) + buffer.moveReaderIndex(to: buffer.writerIndex) + return string + } + +} + +extension ControlPlaneResponseDecoder { + + fileprivate struct PartialHead { + var statusCode: Int + var contentLength: Int? + + var requestID: LambdaRequestID? + var deadlineInMillisSinceEpoch: Int? + var invokedFunctionARN: String? + var traceID: String? + var clientContext: String? + var cognitoIdentity: String? + + init(statusCode: Int) { + self.statusCode = statusCode + self.contentLength = nil + + self.requestID = nil + self.deadlineInMillisSinceEpoch = nil + self.invokedFunctionARN = nil + self.traceID = nil + self.clientContext = nil + self.cognitoIdentity = nil + } + } +} + +extension ByteBuffer { + fileprivate mutating func readHeaderName(_ name: String) -> Bool { + let result = self.withUnsafeReadableBytes { inputBuffer in + name.utf8.withContiguousStorageIfAvailable { nameBuffer -> Bool in + assert(inputBuffer.count >= nameBuffer.count) + + for idx in 0 ..< nameBuffer.count { + // let's hope this gets vectorised ;) + if inputBuffer[idx] & 0xdf != nameBuffer[idx] & 0xdf { + return false + } + } + return true + } + }! + + if result { + self.moveReaderIndex(forwardBy: name.utf8.count) + return true + } + + return false + } + + mutating func readIntegerFromHeader() -> Int? { + guard let ascii = self.readInteger(as: UInt8.self), UInt8(ascii: "0") <= ascii && ascii <= UInt8(ascii: "9") else { + return nil + } + var value = Int(ascii - UInt8(ascii: "0")) + loop: while let ascii = self.readInteger(as: UInt8.self) { + switch ascii { + case UInt8(ascii: "0")...UInt8(ascii: "9"): + value = value * 10 + value += Int(ascii - UInt8(ascii: "0")) + + case UInt8(ascii: " "), UInt8(ascii: "\t"): + // verify that all following characters are also whitespace + guard self.readableBytesView.allSatisfy({ $0 == UInt8(ascii: " ") || $0 == UInt8(ascii: "\t") }) else { + return nil + } + return value + + default: + return nil + } + } + + return value + } + +// mutating func validateHeaderValue(_ value: String) -> Bool { +// func isNotOptionalWhiteSpace(_ val: UInt8) -> Bool { +// val != UInt8(ascii: " ") && val != UInt8(ascii: "\t") +// } +// +// guard let firstCharacterIndex = self.readableBytesView.firstIndex(where: isNotOptionalWhiteSpace), +// let lastCharacterIndex = self.readableBytesView.lastIndex(where: isNotOptionalWhiteSpace) +// else { +// return false +// } +// +// self.com +// } + + mutating func readOptionalWhiteSpace() { + + } +} + +extension Invocation { + + fileprivate init(head: ControlPlaneResponseDecoder.PartialHead) throws { + guard let requestID = head.requestID else { + throw LambdaRuntimeError.invocationHeadMissingRequestID + } + + guard let deadlineInMillisSinceEpoch = head.deadlineInMillisSinceEpoch else { + throw LambdaRuntimeError.invocationHeadMissingDeadlineInMillisSinceEpoch + } + + guard let invokedFunctionARN = head.invokedFunctionARN else { + throw LambdaRuntimeError.invocationHeadMissingFunctionARN + } + + guard let traceID = head.traceID else { + throw LambdaRuntimeError.invocationHeadMissingTraceID + } + + self = Invocation( + requestID: requestID.lowercased, + deadlineInMillisSinceEpoch: Int64(deadlineInMillisSinceEpoch), + invokedFunctionARN: invokedFunctionARN, + traceID: traceID, + clientContext: head.clientContext, + cognitoIdentity: head.cognitoIdentity + ) + } +} diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift b/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift new file mode 100644 index 00000000..10ee1931 --- /dev/null +++ b/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift @@ -0,0 +1,66 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftAWSLambdaRuntime open source project +// +// Copyright (c) 2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +struct LambdaRuntimeError: Error, Hashable { + enum Base: Hashable { + case unsolicitedResponse + case unexpectedStatusCode + + case responseHeadInvalidStatusLine + case responseHeadMissingContentLengthOrTransferEncodingChunked + case responseHeadMoreThan256BytesBeforeCRLF + case responseHeadHeaderMissingColon + case responseHeadHeaderMissingFieldValue + case responseHeadInvalidHeader + case responseHeadInvalidContentLengthValue + case responseHeadInvalidRequestIDValue + case responseHeadInvalidTraceIDValue + case responseHeadInvalidDeadlineValue + + case invocationHeadMissingRequestID + case invocationHeadMissingDeadlineInMillisSinceEpoch + case invocationHeadMissingFunctionARN + case invocationHeadMissingTraceID + + case controlPlaneErrorResponse(ErrorResponse) + } + + private let base: Base + + private init(_ base: Base) { + self.base = base + } + + static var unsolicitedResponse = LambdaRuntimeError(.unsolicitedResponse) + static var unexpectedStatusCode = LambdaRuntimeError(.unexpectedStatusCode) + static var responseHeadInvalidStatusLine = LambdaRuntimeError(.responseHeadInvalidStatusLine) + static var responseHeadMissingContentLengthOrTransferEncodingChunked = + LambdaRuntimeError(.responseHeadMissingContentLengthOrTransferEncodingChunked) + static var responseHeadMoreThan256BytesBeforeCRLF = LambdaRuntimeError(.responseHeadMoreThan256BytesBeforeCRLF) + static var responseHeadHeaderMissingColon = LambdaRuntimeError(.responseHeadHeaderMissingColon) + static var responseHeadHeaderMissingFieldValue = LambdaRuntimeError(.responseHeadHeaderMissingFieldValue) + static var responseHeadInvalidHeader = LambdaRuntimeError(.responseHeadInvalidHeader) + static var responseHeadInvalidContentLengthValue = LambdaRuntimeError(.responseHeadInvalidContentLengthValue) + static var responseHeadInvalidRequestIDValue = LambdaRuntimeError(.responseHeadInvalidRequestIDValue) + static var responseHeadInvalidTraceIDValue = LambdaRuntimeError(.responseHeadInvalidTraceIDValue) + static var responseHeadInvalidDeadlineValue = LambdaRuntimeError(.responseHeadInvalidDeadlineValue) + static var invocationHeadMissingRequestID = LambdaRuntimeError(.invocationHeadMissingRequestID) + static var invocationHeadMissingDeadlineInMillisSinceEpoch = LambdaRuntimeError(.invocationHeadMissingDeadlineInMillisSinceEpoch) + static var invocationHeadMissingFunctionARN = LambdaRuntimeError(.invocationHeadMissingFunctionARN) + static var invocationHeadMissingTraceID = LambdaRuntimeError(.invocationHeadMissingTraceID) + + static func controlPlaneErrorResponse(_ response: ErrorResponse) -> Self { + LambdaRuntimeError(.controlPlaneErrorResponse(response)) + } +} diff --git a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift new file mode 100644 index 00000000..b3d9be0e --- /dev/null +++ b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift @@ -0,0 +1,76 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftAWSLambdaRuntime open source project +// +// Copyright (c) 2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@testable import AWSLambdaRuntimeCore +import NIOCore +import XCTest +import NIOTestUtils + +final class ControlPlaneResponseDecoderTests: XCTestCase { + + func testNextAndAcceptedResponse() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 200 OK\r\n\ + Content-Type: application/json\r\n\ + Lambda-Runtime-Aws-Request-Id: 9028dc49-a01b-4b44-8ffe-4912e9dabbbd\r\n\ + Lambda-Runtime-Deadline-Ms: 1638392696671\r\n\ + Lambda-Runtime-Invoked-Function-Arn: arn:aws:lambda:eu-central-1:079477498937:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\r\n\ + Lambda-Runtime-Trace-Id: Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0\r\n\ + Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ + Content-Length: 49\r\n\ + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + let invocation = Invocation( + requestID: "9028dc49-a01b-4b44-8ffe-4912e9dabbbd", + deadlineInMillisSinceEpoch: 1638392696671, + invokedFunctionARN: "arn:aws:lambda:eu-central-1:079477498937:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x", + traceID: "Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0", + clientContext: nil, + cognitoIdentity: nil + ) + let next: ControlPlaneResponse = .next(invocation, ByteBuffer(string: #"{"name":"Fabian","key2":"value2","key3":"value3"}"#)) + + let acceptedResponse = ByteBuffer(string: """ + HTTP/1.1 202 Accepted\r\n\ + Content-Type: application/json\r\n\ + Date: Sun, 05 Dec 2021 11:53:40 GMT\r\n\ + Content-Length: 16\r\n\ + \r\n\ + {"status":"OK"}\n + """ + ) + + let pairs: [(ByteBuffer, [ControlPlaneResponse])] = [ + (nextResponse, [next]), + (acceptedResponse, [.accepted]), + (nextResponse + acceptedResponse, [next, .accepted]) + ] + + XCTAssertNoThrow(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: pairs, + decoderFactory: { ControlPlaneResponseDecoder() } + )) + } +} + +extension ByteBuffer { + static func + (lhs: Self, rhs: Self) -> ByteBuffer { + var new = lhs + var rhs = rhs + new.writeBuffer(&rhs) + return new + } +} diff --git a/scripts/soundness.sh b/scripts/soundness.sh index d9145903..603ab19a 100755 --- a/scripts/soundness.sh +++ b/scripts/soundness.sh @@ -19,7 +19,7 @@ here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" function replace_acceptable_years() { # this needs to replace all acceptable forms with 'YEARS' - sed -e 's/2017-2018/YEARS/' -e 's/2017-2020/YEARS/' -e 's/2017-2021/YEARS/' -e 's/2020-2021/YEARS/' -e 's/2019/YEARS/' -e 's/2020/YEARS/' -e 's/2021/YEARS/' -e 's/2022/YEARS/' + sed -e 's/2017-2018/YEARS/' -e 's/2017-2020/YEARS/' -e 's/2017-2021/YEARS/' -e 's/2020-2021/YEARS/' -e 's/2021-2022/YEARS/' -e 's/2019/YEARS/' -e 's/2020/YEARS/' -e 's/2021/YEARS/' -e 's/2022/YEARS/' } printf "=> Checking for unacceptable language... " From 59d3f08e0b74e2ef926456aed2c636512b870eed Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Mon, 3 Jan 2022 10:06:31 +0100 Subject: [PATCH 02/13] code format --- .../ControlPlaneRequest.swift | 5 +- .../ControlPlaneResponseDecoder.swift | 173 +++++++++--------- .../LambdaRuntimeError.swift | 14 +- .../ControlPlaneResponseDecoderTests.swift | 13 +- 4 files changed, 99 insertions(+), 106 deletions(-) diff --git a/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift b/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift index 3b3ba272..a0123467 100644 --- a/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift +++ b/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift @@ -35,14 +35,13 @@ struct Invocation: Hashable { var traceID: String var clientContext: String? var cognitoIdentity: String? - + init(requestID: String, deadlineInMillisSinceEpoch: Int64, invokedFunctionARN: String, traceID: String, clientContext: String?, - cognitoIdentity: String? - ) { + cognitoIdentity: String?) { self.requestID = requestID self.deadlineInMillisSinceEpoch = deadlineInMillisSinceEpoch self.invokedFunctionARN = invokedFunctionARN diff --git a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift index b2f6fcc8..65d27d09 100644 --- a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift +++ b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift @@ -21,135 +21,135 @@ import Glibc struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { typealias InboundOut = ControlPlaneResponse - + private enum State { case waitingForNewResponse case parsingHead(PartialHead) case waitingForBody(PartialHead) case receivingBody(PartialHead, ByteBuffer) } - + private var state: State - + init() { self.state = .waitingForNewResponse } - + mutating func decode(buffer: inout ByteBuffer) throws -> ControlPlaneResponse? { switch self.state { case .waitingForNewResponse: guard case .decoded(let head) = try self.decodeResponseHead(from: &buffer) else { return nil } - + guard case .decoded(let body) = try self.decodeBody(from: &buffer) else { return nil } - + return try self.decodeResponse(head: head, body: body) - + case .parsingHead: guard case .decoded(let head) = try self.decodeHeaderLines(from: &buffer) else { return nil } - + guard case .decoded(let body) = try self.decodeBody(from: &buffer) else { return nil } - + return try self.decodeResponse(head: head, body: body) - + case .waitingForBody(let head), .receivingBody(let head, _): guard case .decoded(let body) = try self.decodeBody(from: &buffer) else { return nil } - + return try self.decodeResponse(head: head, body: body) } } - + mutating func decodeLast(buffer: inout ByteBuffer, seenEOF: Bool) throws -> ControlPlaneResponse? { try self.decode(buffer: &buffer) } - + // MARK: - Private Methods - - + private enum DecodeResult { case needMoreData case decoded(T) } - + private mutating func decodeResponseHead(from buffer: inout ByteBuffer) throws -> DecodeResult { guard case .decoded = try self.decodeResponseStatusLine(from: &buffer) else { return .needMoreData } - + return try self.decodeHeaderLines(from: &buffer) } - + private mutating func decodeResponseStatusLine(from buffer: inout ByteBuffer) throws -> DecodeResult { guard case .waitingForNewResponse = self.state else { preconditionFailure("Invalid state: \(self.state)") } - + guard case .decoded(var lineBuffer) = try self.decodeCRLFTerminatedLine(from: &buffer) else { return .needMoreData } - + let statusCode = try self.decodeStatusLine(from: &lineBuffer) self.state = .parsingHead(.init(statusCode: statusCode)) return .decoded(statusCode) } - + private mutating func decodeHeaderLines(from buffer: inout ByteBuffer) throws -> DecodeResult { guard case .parsingHead(var head) = self.state else { preconditionFailure("Invalid state: \(self.state)") } - + while true { guard case .decoded(var nextLine) = try self.decodeCRLFTerminatedLine(from: &buffer) else { self.state = .parsingHead(head) return .needMoreData } - + switch try self.decodeHeaderLine(from: &nextLine) { case .headerEnd: self.state = .waitingForBody(head) return .decoded(head) - + case .contentLength(let length): head.contentLength = length // TODO: This can crash - + case .contentType: break // switch - + case .requestID(let requestID): head.requestID = requestID - + case .traceID(let traceID): head.traceID = traceID - + case .functionARN(let arn): head.invokedFunctionARN = arn - + case .cognitoIdentity(let cognitoIdentity): head.cognitoIdentity = cognitoIdentity - + case .deadlineMS(let deadline): head.deadlineInMillisSinceEpoch = deadline - + case .weDontCare: break // switch } } } - + enum BodyEncoding { case chunked case plain(length: Int) case none } - + private mutating func decodeBody(from buffer: inout ByteBuffer) throws -> DecodeResult { switch self.state { case .waitingForBody(let partialHead): @@ -163,12 +163,12 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { } return .needMoreData } - + case .waitingForNewResponse, .parsingHead, .receivingBody: preconditionFailure("Invalid state: \(self.state)") } } - + private mutating func decodeResponse(head: PartialHead, body: ByteBuffer?) throws -> ControlPlaneResponse { switch head.statusCode { case 200: @@ -178,36 +178,36 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { return .next(try Invocation(head: head), body) case 202: return .accepted - case 400..<600: + case 400 ..< 600: preconditionFailure("TODO: implement") - + default: throw LambdaRuntimeError.unexpectedStatusCode } } - + mutating func decodeStatusLine(from buffer: inout ByteBuffer) throws -> Int { guard buffer.readableBytes >= 11 else { throw LambdaRuntimeError.responseHeadInvalidStatusLine } - + let cmp = buffer.readableBytesView.withUnsafeBytes { ptr in memcmp("HTTP/1.1 ", ptr.baseAddress, 8) == 0 ? true : false } buffer.moveReaderIndex(forwardBy: 9) - + guard cmp else { throw LambdaRuntimeError.responseHeadInvalidStatusLine } - + let statusAsString = buffer.readString(length: 3)! guard let status = Int(statusAsString) else { throw LambdaRuntimeError.responseHeadInvalidStatusLine } - + return status } - + private mutating func decodeCRLFTerminatedLine(from buffer: inout ByteBuffer) throws -> DecodeResult { guard let crIndex = buffer.readableBytesView.firstIndex(of: UInt8(ascii: "\r")) else { if buffer.readableBytes > 256 { @@ -220,16 +220,16 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { // the buffer is split exactly after the \r and \n. Let's wait for more data return .needMoreData } - + guard buffer.readableBytesView[lfIndex] == UInt8(ascii: "\n") else { throw LambdaRuntimeError.responseHeadInvalidHeader } - + let slice = buffer.readSlice(length: crIndex - buffer.readerIndex)! buffer.moveReaderIndex(forwardBy: 2) // move over \r\n return .decoded(slice) } - + private enum HeaderLineContent: Equatable { case traceID(String) case contentType @@ -238,7 +238,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { case deadlineMS(Int) case functionARN(String) case requestID(LambdaRequestID) - + case weDontCare case headerEnd } @@ -250,7 +250,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { } throw LambdaRuntimeError.responseHeadHeaderMissingColon } - + // based on colonIndex we can already make some good guesses... // 4: Date // 12: Content-Type @@ -262,18 +262,18 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { // Lambda-Runtime-Client-Context // 31: Lambda-Runtime-Cognito-Identity // 35: Lambda-Runtime-Invoked-Function-Arn - + switch colonIndex { case 4: if buffer.readHeaderName("date") { return .weDontCare } - + case 12: if buffer.readHeaderName("content-type") { return .weDontCare } - + case 14: if buffer.readHeaderName("content-length") { buffer.moveReaderIndex(forwardBy: 1) // move forward for colon @@ -283,7 +283,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { } return .contentLength(length) } - + case 17: if buffer.readHeaderName("transfer-encoding") { buffer.moveReaderIndex(forwardBy: 1) // move forward for colon @@ -293,7 +293,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { } return .contentLength(length) } - + case 23: if buffer.readHeaderName("lambda-runtime-trace-id") { buffer.moveReaderIndex(forwardBy: 1) @@ -302,7 +302,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { } return .traceID(string) } - + case 26: if buffer.readHeaderName("lambda-runtime-deadline-ms") { buffer.moveReaderIndex(forwardBy: 1) // move forward for colon @@ -312,7 +312,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { } return .deadlineMS(deadline) } - + case 29: if buffer.readHeaderName("lambda-runtime-aws-request-id") { buffer.moveReaderIndex(forwardBy: 1) // move forward for colon @@ -325,12 +325,12 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { if buffer.readHeaderName("lambda-runtime-client-context") { return .weDontCare } - + case 31: if buffer.readHeaderName("lambda-runtime-cognito-identity") { return .weDontCare } - + case 35: if buffer.readHeaderName("lambda-runtime-invoked-function-arn") { buffer.moveReaderIndex(forwardBy: 1) @@ -339,59 +339,57 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { } return .functionARN(string) } - + default: return .weDontCare } - + return .weDontCare } - + @discardableResult mutating func decodeOptionalWhiteSpaceBeforeFieldValue(from buffer: inout ByteBuffer) throws -> Int { let startIndex = buffer.readerIndex - guard let index = buffer.readableBytesView.firstIndex(where: { ($0 != UInt8(ascii: " ") && $0 != UInt8(ascii: "\t")) }) else { + guard let index = buffer.readableBytesView.firstIndex(where: { $0 != UInt8(ascii: " ") && $0 != UInt8(ascii: "\t") }) else { throw LambdaRuntimeError.responseHeadHeaderMissingFieldValue } buffer.moveReaderIndex(to: index) return index - startIndex } - + private func decodeHeaderValue(from buffer: inout ByteBuffer) throws -> String? { func isNotOptionalWhiteSpace(_ val: UInt8) -> Bool { val != UInt8(ascii: " ") && val != UInt8(ascii: "\t") } - + guard let firstCharacterIndex = buffer.readableBytesView.firstIndex(where: isNotOptionalWhiteSpace), let lastCharacterIndex = buffer.readableBytesView.lastIndex(where: isNotOptionalWhiteSpace) else { throw LambdaRuntimeError.responseHeadHeaderMissingFieldValue } - + let string = buffer.getString(at: firstCharacterIndex, length: lastCharacterIndex + 1 - firstCharacterIndex) buffer.moveReaderIndex(to: buffer.writerIndex) return string } - } extension ControlPlaneResponseDecoder { - fileprivate struct PartialHead { var statusCode: Int var contentLength: Int? - + var requestID: LambdaRequestID? var deadlineInMillisSinceEpoch: Int? var invokedFunctionARN: String? var traceID: String? var clientContext: String? var cognitoIdentity: String? - + init(statusCode: Int) { self.statusCode = statusCode self.contentLength = nil - + self.requestID = nil self.deadlineInMillisSinceEpoch = nil self.invokedFunctionARN = nil @@ -407,25 +405,25 @@ extension ByteBuffer { let result = self.withUnsafeReadableBytes { inputBuffer in name.utf8.withContiguousStorageIfAvailable { nameBuffer -> Bool in assert(inputBuffer.count >= nameBuffer.count) - + for idx in 0 ..< nameBuffer.count { // let's hope this gets vectorised ;) - if inputBuffer[idx] & 0xdf != nameBuffer[idx] & 0xdf { + if inputBuffer[idx] & 0xDF != nameBuffer[idx] & 0xDF { return false } } return true } }! - + if result { self.moveReaderIndex(forwardBy: name.utf8.count) return true } - + return false } - + mutating func readIntegerFromHeader() -> Int? { guard let ascii = self.readInteger(as: UInt8.self), UInt8(ascii: "0") <= ascii && ascii <= UInt8(ascii: "9") else { return nil @@ -433,63 +431,60 @@ extension ByteBuffer { var value = Int(ascii - UInt8(ascii: "0")) loop: while let ascii = self.readInteger(as: UInt8.self) { switch ascii { - case UInt8(ascii: "0")...UInt8(ascii: "9"): + case UInt8(ascii: "0") ... UInt8(ascii: "9"): value = value * 10 value += Int(ascii - UInt8(ascii: "0")) - + case UInt8(ascii: " "), UInt8(ascii: "\t"): // verify that all following characters are also whitespace guard self.readableBytesView.allSatisfy({ $0 == UInt8(ascii: " ") || $0 == UInt8(ascii: "\t") }) else { return nil } return value - + default: return nil } } - + return value } - + // mutating func validateHeaderValue(_ value: String) -> Bool { // func isNotOptionalWhiteSpace(_ val: UInt8) -> Bool { // val != UInt8(ascii: " ") && val != UInt8(ascii: "\t") // } -// +// // guard let firstCharacterIndex = self.readableBytesView.firstIndex(where: isNotOptionalWhiteSpace), // let lastCharacterIndex = self.readableBytesView.lastIndex(where: isNotOptionalWhiteSpace) // else { // return false // } -// +// // self.com // } - - mutating func readOptionalWhiteSpace() { - - } + + mutating func readOptionalWhiteSpace() {} } extension Invocation { - fileprivate init(head: ControlPlaneResponseDecoder.PartialHead) throws { guard let requestID = head.requestID else { throw LambdaRuntimeError.invocationHeadMissingRequestID } - + guard let deadlineInMillisSinceEpoch = head.deadlineInMillisSinceEpoch else { throw LambdaRuntimeError.invocationHeadMissingDeadlineInMillisSinceEpoch } - + guard let invokedFunctionARN = head.invokedFunctionARN else { throw LambdaRuntimeError.invocationHeadMissingFunctionARN } - + guard let traceID = head.traceID else { throw LambdaRuntimeError.invocationHeadMissingTraceID } - + self = Invocation( requestID: requestID.lowercased, deadlineInMillisSinceEpoch: Int64(deadlineInMillisSinceEpoch), diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift b/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift index 10ee1931..6cc2866a 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift @@ -16,7 +16,7 @@ struct LambdaRuntimeError: Error, Hashable { enum Base: Hashable { case unsolicitedResponse case unexpectedStatusCode - + case responseHeadInvalidStatusLine case responseHeadMissingContentLengthOrTransferEncodingChunked case responseHeadMoreThan256BytesBeforeCRLF @@ -27,21 +27,21 @@ struct LambdaRuntimeError: Error, Hashable { case responseHeadInvalidRequestIDValue case responseHeadInvalidTraceIDValue case responseHeadInvalidDeadlineValue - + case invocationHeadMissingRequestID case invocationHeadMissingDeadlineInMillisSinceEpoch case invocationHeadMissingFunctionARN case invocationHeadMissingTraceID - + case controlPlaneErrorResponse(ErrorResponse) } - + private let base: Base - + private init(_ base: Base) { self.base = base } - + static var unsolicitedResponse = LambdaRuntimeError(.unsolicitedResponse) static var unexpectedStatusCode = LambdaRuntimeError(.unexpectedStatusCode) static var responseHeadInvalidStatusLine = LambdaRuntimeError(.responseHeadInvalidStatusLine) @@ -59,7 +59,7 @@ struct LambdaRuntimeError: Error, Hashable { static var invocationHeadMissingDeadlineInMillisSinceEpoch = LambdaRuntimeError(.invocationHeadMissingDeadlineInMillisSinceEpoch) static var invocationHeadMissingFunctionARN = LambdaRuntimeError(.invocationHeadMissingFunctionARN) static var invocationHeadMissingTraceID = LambdaRuntimeError(.invocationHeadMissingTraceID) - + static func controlPlaneErrorResponse(_ response: ErrorResponse) -> Self { LambdaRuntimeError(.controlPlaneErrorResponse(response)) } diff --git a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift index b3d9be0e..5a7d7c9f 100644 --- a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift +++ b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift @@ -14,11 +14,10 @@ @testable import AWSLambdaRuntimeCore import NIOCore -import XCTest import NIOTestUtils +import XCTest final class ControlPlaneResponseDecoderTests: XCTestCase { - func testNextAndAcceptedResponse() { let nextResponse = ByteBuffer(string: """ HTTP/1.1 200 OK\r\n\ @@ -35,14 +34,14 @@ final class ControlPlaneResponseDecoderTests: XCTestCase { ) let invocation = Invocation( requestID: "9028dc49-a01b-4b44-8ffe-4912e9dabbbd", - deadlineInMillisSinceEpoch: 1638392696671, + deadlineInMillisSinceEpoch: 1_638_392_696_671, invokedFunctionARN: "arn:aws:lambda:eu-central-1:079477498937:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x", traceID: "Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0", clientContext: nil, cognitoIdentity: nil ) let next: ControlPlaneResponse = .next(invocation, ByteBuffer(string: #"{"name":"Fabian","key2":"value2","key3":"value3"}"#)) - + let acceptedResponse = ByteBuffer(string: """ HTTP/1.1 202 Accepted\r\n\ Content-Type: application/json\r\n\ @@ -52,13 +51,13 @@ final class ControlPlaneResponseDecoderTests: XCTestCase { {"status":"OK"}\n """ ) - + let pairs: [(ByteBuffer, [ControlPlaneResponse])] = [ (nextResponse, [next]), (acceptedResponse, [.accepted]), - (nextResponse + acceptedResponse, [next, .accepted]) + (nextResponse + acceptedResponse, [next, .accepted]), ] - + XCTAssertNoThrow(try ByteToMessageDecoderVerifier.verifyDecoder( inputOutputPairs: pairs, decoderFactory: { ControlPlaneResponseDecoder() } From fdb36d3089d13b69d27472fb3e363e66859f9a1c Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Mon, 3 Jan 2022 16:49:10 +0100 Subject: [PATCH 03/13] Fix failing tests --- .../ControlPlaneResponseDecoder.swift | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift index 65d27d09..1bd2385d 100644 --- a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift +++ b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift @@ -13,11 +13,6 @@ //===----------------------------------------------------------------------===// import NIOCore -#if canImport(Darwin) -import Darwin -#else -import Glibc -#endif struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { typealias InboundOut = ControlPlaneResponse @@ -191,12 +186,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { throw LambdaRuntimeError.responseHeadInvalidStatusLine } - let cmp = buffer.readableBytesView.withUnsafeBytes { ptr in - memcmp("HTTP/1.1 ", ptr.baseAddress, 8) == 0 ? true : false - } - buffer.moveReaderIndex(forwardBy: 9) - - guard cmp else { + guard buffer.readString("HTTP/1.1 ") else { throw LambdaRuntimeError.responseHeadInvalidStatusLine } @@ -401,6 +391,28 @@ extension ControlPlaneResponseDecoder { } extension ByteBuffer { + fileprivate mutating func readString(_ string: String) -> Bool { + let result = self.withUnsafeReadableBytes { inputBuffer in + string.utf8.withContiguousStorageIfAvailable { validateBuffer -> Bool in + assert(inputBuffer.count >= validateBuffer.count) + + for idx in 0 ..< validateBuffer.count { + if inputBuffer[idx] != validateBuffer[idx] { + return false + } + } + return true + } + }! + + if result { + self.moveReaderIndex(forwardBy: string.utf8.count) + return true + } + + return false + } + fileprivate mutating func readHeaderName(_ name: String) -> Bool { let result = self.withUnsafeReadableBytes { inputBuffer in name.utf8.withContiguousStorageIfAvailable { nameBuffer -> Bool in From 001494ad03d60497a4e379183fe30afa853346a1 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Mon, 3 Jan 2022 17:08:11 +0100 Subject: [PATCH 04/13] Better naming --- .../ControlPlaneResponseDecoder.swift | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift index 1bd2385d..dc32750e 100644 --- a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift +++ b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift @@ -133,7 +133,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { case .deadlineMS(let deadline): head.deadlineInMillisSinceEpoch = deadline - case .weDontCare: + case .ignore: break // switch } } @@ -229,7 +229,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { case functionARN(String) case requestID(LambdaRequestID) - case weDontCare + case ignore case headerEnd } @@ -256,12 +256,12 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { switch colonIndex { case 4: if buffer.readHeaderName("date") { - return .weDontCare + return .ignore } case 12: if buffer.readHeaderName("content-type") { - return .weDontCare + return .ignore } case 14: @@ -313,12 +313,12 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { return .requestID(requestID) } if buffer.readHeaderName("lambda-runtime-client-context") { - return .weDontCare + return .ignore } case 31: if buffer.readHeaderName("lambda-runtime-cognito-identity") { - return .weDontCare + return .ignore } case 35: @@ -331,10 +331,10 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { } default: - return .weDontCare + return .ignore } - return .weDontCare + return .ignore } @discardableResult From 2828a227c56578c8a9f337fd6d977188efc64d85 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Mon, 3 Jan 2022 18:04:56 +0100 Subject: [PATCH 05/13] Validate header fields. --- .../ControlPlaneResponseDecoder.swift | 34 ++++++++++++++++++- .../LambdaRuntimeError.swift | 2 ++ .../ControlPlaneResponseDecoderTests.swift | 27 +++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) diff --git a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift index dc32750e..4fb48ffb 100644 --- a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift +++ b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift @@ -331,7 +331,39 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { } default: - return .ignore + // Ensure we received a valid http header: + break // fallthrough + } + + // We received a header we didn't expect, let's ensure it is valid. + let satisfy = buffer.readableBytesView[0.. Bool in + switch char { + case UInt8(ascii: "a")...UInt8(ascii: "z"), + UInt8(ascii: "A")...UInt8(ascii: "Z"), + UInt8(ascii: "0")...UInt8(ascii: "9"), + UInt8(ascii: "!"), + UInt8(ascii: "#"), + UInt8(ascii: "$"), + UInt8(ascii: "%"), + UInt8(ascii: "&"), + UInt8(ascii: "'"), + UInt8(ascii: "*"), + UInt8(ascii: "+"), + UInt8(ascii: "-"), + UInt8(ascii: "."), + UInt8(ascii: "^"), + UInt8(ascii: "_"), + UInt8(ascii: "`"), + UInt8(ascii: "|"), + UInt8(ascii: "~"): + return true + default: + return false + } + } + + guard satisfy else { + throw LambdaRuntimeError.responseHeadHeaderInvalidCharacter } return .ignore diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift b/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift index 6cc2866a..acf79c34 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift @@ -20,6 +20,7 @@ struct LambdaRuntimeError: Error, Hashable { case responseHeadInvalidStatusLine case responseHeadMissingContentLengthOrTransferEncodingChunked case responseHeadMoreThan256BytesBeforeCRLF + case responseHeadHeaderInvalidCharacter case responseHeadHeaderMissingColon case responseHeadHeaderMissingFieldValue case responseHeadInvalidHeader @@ -48,6 +49,7 @@ struct LambdaRuntimeError: Error, Hashable { static var responseHeadMissingContentLengthOrTransferEncodingChunked = LambdaRuntimeError(.responseHeadMissingContentLengthOrTransferEncodingChunked) static var responseHeadMoreThan256BytesBeforeCRLF = LambdaRuntimeError(.responseHeadMoreThan256BytesBeforeCRLF) + static var responseHeadHeaderInvalidCharacter = LambdaRuntimeError(.responseHeadHeaderInvalidCharacter) static var responseHeadHeaderMissingColon = LambdaRuntimeError(.responseHeadHeaderMissingColon) static var responseHeadHeaderMissingFieldValue = LambdaRuntimeError(.responseHeadHeaderMissingFieldValue) static var responseHeadInvalidHeader = LambdaRuntimeError(.responseHeadInvalidHeader) diff --git a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift index 5a7d7c9f..42d21588 100644 --- a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift +++ b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift @@ -63,6 +63,33 @@ final class ControlPlaneResponseDecoderTests: XCTestCase { decoderFactory: { ControlPlaneResponseDecoder() } )) } + + func testWhitespaceInHeaderIsRejected() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 200 OK\r\n\ + Content-Type: application/json\r\n\ + Lambda-Runtime Aws-Request-Id: 9028dc49-a01b-4b44-8ffe-4912e9dabbbd\r\n\ + Lambda-Runtime-Deadline-Ms: 1638392696671\r\n\ + Lambda-Runtime-Invoked-Function-Arn: arn:aws:lambda:eu-central-1:079477498937:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\r\n\ + Lambda-Runtime-Trace-Id: Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0\r\n\ + Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ + Content-Length: 49\r\n\ + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + + let pairs: [(ByteBuffer, [ControlPlaneResponse])] = [ + (nextResponse, []) + ] + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: pairs, + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadHeaderInvalidCharacter) + } + } } extension ByteBuffer { From 99cfae85e35a2ca6d8c66ac8c6a51d31cbea94b6 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Mon, 3 Jan 2022 23:08:21 +0100 Subject: [PATCH 06/13] More tests --- .../ControlPlaneResponseDecoder.swift | 30 +- .../LambdaRuntimeError.swift | 1 + .../ControlPlaneResponseDecoderTests.swift | 259 +++++++++++++++++- 3 files changed, 258 insertions(+), 32 deletions(-) diff --git a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift index 4fb48ffb..4f3a44af 100644 --- a/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift +++ b/Sources/AWSLambdaRuntimeCore/ControlPlaneResponseDecoder.swift @@ -269,7 +269,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { buffer.moveReaderIndex(forwardBy: 1) // move forward for colon try self.decodeOptionalWhiteSpaceBeforeFieldValue(from: &buffer) guard let length = buffer.readIntegerFromHeader() else { - throw LambdaRuntimeError.responseHeadInvalidDeadlineValue + throw LambdaRuntimeError.responseHeadInvalidContentLengthValue } return .contentLength(length) } @@ -334,13 +334,13 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { // Ensure we received a valid http header: break // fallthrough } - + // We received a header we didn't expect, let's ensure it is valid. - let satisfy = buffer.readableBytesView[0.. Bool in + let satisfy = buffer.readableBytesView[0 ..< colonIndex].allSatisfy { char -> Bool in switch char { - case UInt8(ascii: "a")...UInt8(ascii: "z"), - UInt8(ascii: "A")...UInt8(ascii: "Z"), - UInt8(ascii: "0")...UInt8(ascii: "9"), + case UInt8(ascii: "a") ... UInt8(ascii: "z"), + UInt8(ascii: "A") ... UInt8(ascii: "Z"), + UInt8(ascii: "0") ... UInt8(ascii: "9"), UInt8(ascii: "!"), UInt8(ascii: "#"), UInt8(ascii: "$"), @@ -361,7 +361,7 @@ struct ControlPlaneResponseDecoder: NIOSingleStepByteToMessageDecoder { return false } } - + guard satisfy else { throw LambdaRuntimeError.responseHeadHeaderInvalidCharacter } @@ -493,22 +493,6 @@ extension ByteBuffer { return value } - -// mutating func validateHeaderValue(_ value: String) -> Bool { -// func isNotOptionalWhiteSpace(_ val: UInt8) -> Bool { -// val != UInt8(ascii: " ") && val != UInt8(ascii: "\t") -// } -// -// guard let firstCharacterIndex = self.readableBytesView.firstIndex(where: isNotOptionalWhiteSpace), -// let lastCharacterIndex = self.readableBytesView.lastIndex(where: isNotOptionalWhiteSpace) -// else { -// return false -// } -// -// self.com -// } - - mutating func readOptionalWhiteSpace() {} } extension Invocation { diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift b/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift index acf79c34..4c91c90a 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaRuntimeError.swift @@ -57,6 +57,7 @@ struct LambdaRuntimeError: Error, Hashable { static var responseHeadInvalidRequestIDValue = LambdaRuntimeError(.responseHeadInvalidRequestIDValue) static var responseHeadInvalidTraceIDValue = LambdaRuntimeError(.responseHeadInvalidTraceIDValue) static var responseHeadInvalidDeadlineValue = LambdaRuntimeError(.responseHeadInvalidDeadlineValue) + static var invocationHeadMissingRequestID = LambdaRuntimeError(.invocationHeadMissingRequestID) static var invocationHeadMissingDeadlineInMillisSinceEpoch = LambdaRuntimeError(.invocationHeadMissingDeadlineInMillisSinceEpoch) static var invocationHeadMissingFunctionARN = LambdaRuntimeError(.invocationHeadMissingFunctionARN) diff --git a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift index 42d21588..45c5f61a 100644 --- a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift +++ b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift @@ -24,7 +24,7 @@ final class ControlPlaneResponseDecoderTests: XCTestCase { Content-Type: application/json\r\n\ Lambda-Runtime-Aws-Request-Id: 9028dc49-a01b-4b44-8ffe-4912e9dabbbd\r\n\ Lambda-Runtime-Deadline-Ms: 1638392696671\r\n\ - Lambda-Runtime-Invoked-Function-Arn: arn:aws:lambda:eu-central-1:079477498937:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\r\n\ + Lambda-Runtime-Invoked-Function-Arn: arn:aws:lambda:eu-central-1:000000000000:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\r\n\ Lambda-Runtime-Trace-Id: Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0\r\n\ Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ Content-Length: 49\r\n\ @@ -35,7 +35,7 @@ final class ControlPlaneResponseDecoderTests: XCTestCase { let invocation = Invocation( requestID: "9028dc49-a01b-4b44-8ffe-4912e9dabbbd", deadlineInMillisSinceEpoch: 1_638_392_696_671, - invokedFunctionARN: "arn:aws:lambda:eu-central-1:079477498937:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x", + invokedFunctionARN: "arn:aws:lambda:eu-central-1:000000000000:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x", traceID: "Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0", clientContext: nil, cognitoIdentity: nil @@ -63,14 +63,14 @@ final class ControlPlaneResponseDecoderTests: XCTestCase { decoderFactory: { ControlPlaneResponseDecoder() } )) } - + func testWhitespaceInHeaderIsRejected() { let nextResponse = ByteBuffer(string: """ HTTP/1.1 200 OK\r\n\ Content-Type: application/json\r\n\ Lambda-Runtime Aws-Request-Id: 9028dc49-a01b-4b44-8ffe-4912e9dabbbd\r\n\ Lambda-Runtime-Deadline-Ms: 1638392696671\r\n\ - Lambda-Runtime-Invoked-Function-Arn: arn:aws:lambda:eu-central-1:079477498937:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\r\n\ + Lambda-Runtime-Invoked-Function-Arn: arn:aws:lambda:eu-central-1:000000000000:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\r\n\ Lambda-Runtime-Trace-Id: Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0\r\n\ Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ Content-Length: 49\r\n\ @@ -79,17 +79,258 @@ final class ControlPlaneResponseDecoderTests: XCTestCase { """ ) - let pairs: [(ByteBuffer, [ControlPlaneResponse])] = [ - (nextResponse, []) - ] - XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( - inputOutputPairs: pairs, + inputOutputPairs: [(nextResponse, [])], decoderFactory: { ControlPlaneResponseDecoder() } )) { XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadHeaderInvalidCharacter) } } + + func testVeryLongHTTPStatusLine() { + let nextResponse = ByteBuffer(repeating: UInt8(ascii: "H"), count: 1024) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadMoreThan256BytesBeforeCRLF) + } + } + + func testVeryLongHTTPHeader() { + let acceptedResponse = ByteBuffer(string: """ + HTTP/1.1 202 Accepted\r\n\ + Content-Type: application/json\r\n\ + Date: Sun, 05 Dec 2021 11:53:40 GMT\r\n\ + Content-Length: 16\r\n + """ + ) + ByteBuffer(repeating: UInt8(ascii: "H"), count: 1024) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(acceptedResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadMoreThan256BytesBeforeCRLF) + } + } + + func testNextResponseWithoutTraceID() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 200 OK\r\n\ + Content-Type: application/json\r\n\ + Lambda-Runtime-Aws-Request-Id: 9028dc49-a01b-4b44-8ffe-4912e9dabbbd\r\n\ + Lambda-Runtime-Deadline-Ms: 1638392696671\r\n\ + Lambda-Runtime-Invoked-Function-Arn: arn:aws:lambda:eu-central-1:000000000000:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\r\n\ + Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ + Content-Length: 49\r\n\ + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .invocationHeadMissingTraceID) + } + } + + func testNextResponseWithoutRequestID() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 200 OK\r\n\ + Content-Type: application/json\r\n\ + Lambda-Runtime-Deadline-Ms: 1638392696671\r\n\ + Lambda-Runtime-Invoked-Function-Arn: arn:aws:lambda:eu-central-1:000000000000:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\r\n\ + Lambda-Runtime-Trace-Id: Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0\r\n\ + Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ + Content-Length: 49\r\n\ + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .invocationHeadMissingRequestID) + } + } + + func testNextResponseWithInvalidStatusCode() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 20 OK\r\n\ + Content-Type: application/json\r\n\ + Lambda-Runtime-Deadline-Ms: 1638392696671\r\n\ + Lambda-Runtime-Invoked-Function-Arn: arn:aws:lambda:eu-central-1:000000000000:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\r\n\ + Lambda-Runtime-Trace-Id: Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0\r\n\ + Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ + Content-Length: 49\r\n\ + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadInvalidStatusLine) + } + } + + func testNextResponseWithVersionHTTP2() { + let nextResponse = ByteBuffer(string: """ + HTTP/2.0 200 OK\r\n\ + Content-Type: application/json\r\n\ + Lambda-Runtime-Deadline-Ms: 1638392696671\r\n\ + Lambda-Runtime-Invoked-Function-Arn: arn:aws:lambda:eu-central-1:000000000000:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\r\n\ + Lambda-Runtime-Trace-Id: Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0\r\n\ + Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ + Content-Length: 49\r\n\ + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadInvalidStatusLine) + } + } + + func testNextResponseLeadingAndTrailingWhitespaceHeaders() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 200 OK\r\n\ + Content-Type: \t \t application/json\t \t \r\n\ + Lambda-Runtime-Aws-Request-Id: \t \t 9028dc49-a01b-4b44-8ffe-4912e9dabbbd\t \t \r\n\ + Lambda-Runtime-Deadline-Ms: \t \t 1638392696671\t \t \r\n\ + Lambda-Runtime-Invoked-Function-Arn: \t \t arn:aws:lambda:eu-central-1:000000000000:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x\t \t \r\n\ + Lambda-Runtime-Trace-Id: \t \t Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0\t \t \r\n\ + Date: \t \t Wed, 01 Dec 2021 21:04:53 GMT\t \t \r\n\ + Content-Length: \t \t 49\t \t \r\n\ + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + let invocation = Invocation( + requestID: "9028dc49-a01b-4b44-8ffe-4912e9dabbbd", + deadlineInMillisSinceEpoch: 1_638_392_696_671, + invokedFunctionARN: "arn:aws:lambda:eu-central-1:000000000000:function:lambda-log-http-HelloWorldLambda-NiDlzMFXtF3x", + traceID: "Root=1-61a7e375-40b3edf95b388fe75d1fa416;Parent=348bb48e251c1254;Sampled=0", + clientContext: nil, + cognitoIdentity: nil + ) + let next: ControlPlaneResponse = .next(invocation, ByteBuffer(string: #"{"name":"Fabian","key2":"value2","key3":"value3"}"#)) + + XCTAssertNoThrow(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [next])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) + } + + func testContentLengthHasTrailingCharacterSurroundedByWhitespace() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 200 OK\r\n\ + Content-Type: \t \t application/json\t \t \r\n\ + Content-Length: 49 r \r\n\ + Date: \t \t Wed, 01 Dec 2021 21:04:53 GMT\t \t \r\n\ + + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadInvalidContentLengthValue) + } + } + + func testInvalidContentLength() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 200 OK\r\n\ + Content-Type: \t \t application/json\t \t \r\n\ + Content-Length: 4u9 \r\n\ + Date: \t \t Wed, 01 Dec 2021 21:04:53 GMT\t \t \r\n\ + + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadInvalidContentLengthValue) + } + } + + func testResponseHeaderWithoutColon() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 200 OK\r\n\ + Content-Type application/json\r\n\ + Content-Length: 49\r\n\ + Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ + + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadHeaderMissingColon) + } + } + + func testResponseHeaderWithDoubleCR() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 200 OK\r\n\ + Content-Type: application/json\r\r\n\ + Content-Length: 49\r\n\ + Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ + + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadInvalidHeader) + } + } + + func testResponseHeaderWithoutValue() { + let nextResponse = ByteBuffer(string: """ + HTTP/1.1 200 OK\r\n\ + Content-Type: \r\n\ + Content-Length: 49\r\n\ + Date: Wed, 01 Dec 2021 21:04:53 GMT\r\n\ + \r\n\ + {"name":"Fabian","key2":"value2","key3":"value3"} + """ + ) + + XCTAssertThrowsError(try ByteToMessageDecoderVerifier.verifyDecoder( + inputOutputPairs: [(nextResponse, [])], + decoderFactory: { ControlPlaneResponseDecoder() } + )) { + XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadInvalidHeader) + } + } } extension ByteBuffer { From 83f284fc37fc4edb492e6626a948437f80ffb2c2 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Mon, 3 Jan 2022 23:22:13 +0100 Subject: [PATCH 07/13] New runtime --- .../ControlPlaneRequest.swift | 4 +- .../ControlPlaneRequestEncoder.swift | 8 +- .../LambdaRequestID.swift | 3 +- .../NewLambdaChannelHandler.swift | 80 ++++++++ .../NewLambdaRuntime.swift | 188 ++++++++++++++++++ .../ControlPlaneRequestEncoderTests.swift | 8 +- 6 files changed, 280 insertions(+), 11 deletions(-) create mode 100644 Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift create mode 100644 Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift diff --git a/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift b/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift index a0123467..3ef5a6fe 100644 --- a/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift +++ b/Sources/AWSLambdaRuntimeCore/ControlPlaneRequest.swift @@ -17,8 +17,8 @@ import NIOHTTP1 enum ControlPlaneRequest: Hashable { case next - case invocationResponse(String, ByteBuffer?) - case invocationError(String, ErrorResponse) + case invocationResponse(LambdaRequestID, ByteBuffer?) + case invocationError(LambdaRequestID, ErrorResponse) case initializationError(ErrorResponse) } diff --git a/Sources/AWSLambdaRuntimeCore/ControlPlaneRequestEncoder.swift b/Sources/AWSLambdaRuntimeCore/ControlPlaneRequestEncoder.swift index a91e1e44..a8ad3b64 100644 --- a/Sources/AWSLambdaRuntimeCore/ControlPlaneRequestEncoder.swift +++ b/Sources/AWSLambdaRuntimeCore/ControlPlaneRequestEncoder.swift @@ -100,15 +100,15 @@ extension String { } extension ByteBuffer { - fileprivate mutating func writeInvocationResultRequestLine(_ requestID: String) { + fileprivate mutating func writeInvocationResultRequestLine(_ requestID: LambdaRequestID) { self.writeString("POST /2018-06-01/runtime/invocation/") - self.writeString(requestID) + self.writeRequestID(requestID) self.writeString("/response HTTP/1.1\r\n") } - fileprivate mutating func writeInvocationErrorRequestLine(_ requestID: String) { + fileprivate mutating func writeInvocationErrorRequestLine(_ requestID: LambdaRequestID) { self.writeString("POST /2018-06-01/runtime/invocation/") - self.writeString(requestID) + self.writeRequestID(requestID) self.writeString("/error HTTP/1.1\r\n") } diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRequestID.swift b/Sources/AWSLambdaRuntimeCore/LambdaRequestID.swift index 86178ff4..866f0273 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaRequestID.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaRequestID.swift @@ -340,12 +340,13 @@ extension ByteBuffer { @discardableResult mutating func setRequestID(_ requestID: LambdaRequestID, at index: Int) -> Int { - var localBytes = requestID.toAsciiBytesOnStack(characters: LambdaRequestID.lowercaseLookup) + var localBytes = requestID.toAsciiBytesOnStack(characters: LambdaRequestID.uppercaseLookup) return withUnsafeBytes(of: &localBytes) { self.setBytes($0, at: index) } } + @discardableResult mutating func writeRequestID(_ requestID: LambdaRequestID) -> Int { let length = self.setRequestID(requestID, at: self.writerIndex) self.moveWriterIndex(forwardBy: length) diff --git a/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift b/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift new file mode 100644 index 00000000..0ab8ddae --- /dev/null +++ b/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift @@ -0,0 +1,80 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftAWSLambdaRuntime open source project +// +// Copyright (c) 2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore + +protocol LambdaChannelHandlerDelegate { + + func responseReceived(_: ControlPlaneResponse) + + func errorCaught(_: Error) + + func channelInactive() + +} + +final class NewLambdaChannelHandler: ChannelInboundHandler { + typealias InboundIn = ByteBuffer + typealias OutboundOut = ByteBuffer + + private let delegate: Delegate + private let requestsInFlight: CircularBuffer + + private var context: ChannelHandlerContext! + + private var encoder: ControlPlaneRequestEncoder + private var decoder: NIOSingleStepByteToMessageProcessor + + init(delegate: Delegate, host: String) { + self.delegate = delegate + self.requestsInFlight = CircularBuffer(initialCapacity: 4) + + self.encoder = ControlPlaneRequestEncoder(host: host) + self.decoder = NIOSingleStepByteToMessageProcessor(ControlPlaneResponseDecoder(), maximumBufferSize: 7 * 1024 * 1024) + } + + func sendRequest(_ request: ControlPlaneRequest) { + self.encoder.writeRequest(request, context: self.context, promise: nil) + } + + func handlerAdded(context: ChannelHandlerContext) { + self.context = context + self.encoder.writerAdded(context: context) + } + + func handlerRemoved(context: ChannelHandlerContext) { + self.context = context + self.encoder.writerRemoved(context: context) + } + + func channelRead(context: ChannelHandlerContext, data: NIOAny) { + do { + try self.decoder.process(buffer: self.unwrapInboundIn(data)) { response in + // TODO: The response matches the request + + self.delegate.responseReceived(response) + } + } catch { + + } + } + + func channelInactive(context: ChannelHandlerContext) { + self.delegate.channelInactive() + } + + func errorCaught(context: ChannelHandlerContext, error: Error) { + self.delegate.errorCaught(error) + } +} diff --git a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift new file mode 100644 index 00000000..81a516d0 --- /dev/null +++ b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift @@ -0,0 +1,188 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftAWSLambdaRuntime open source project +// +// Copyright (c) 2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Logging +import NIOConcurrencyHelpers +import NIOCore +import NIOPosix + +/// `LambdaRuntime` manages the Lambda process lifecycle. +/// +/// - note: It is intended to be used within a single `EventLoop`. For this reason this class is not thread safe. +public final class NewLambdaRuntime { + private let eventLoop: EventLoop + private let shutdownPromise: EventLoopPromise + private let logger: Logger + private let configuration: Lambda.Configuration + private let factory: (Lambda.InitializationContext) -> EventLoopFuture + + private var state: StateMachine + + init(eventLoop: EventLoop, + logger: Logger, + configuration: Lambda.Configuration, + factory: @escaping (Lambda.InitializationContext) -> EventLoopFuture + ) { + self.state = StateMachine() + self.eventLoop = eventLoop + self.shutdownPromise = eventLoop.makePromise(of: Void.self) + self.logger = logger + self.configuration = configuration + self.factory = factory + } + + deinit { + // TODO: Verify is shutdown + } + + /// The `Lifecycle` shutdown future. + /// + /// - Returns: An `EventLoopFuture` that is fulfilled after the Lambda lifecycle has fully shutdown. + public var shutdownFuture: EventLoopFuture { + self.shutdownPromise.futureResult + } + + /// Start the `LambdaRuntime`. + /// + /// - Returns: An `EventLoopFuture` that is fulfilled after the Lambda hander has been created and initiliazed, and a first run has been scheduled. + public func start(promise: EventLoopPromise?) { + if self.eventLoop.inEventLoop { + self.start0(promise: promise) + } else { + self.eventLoop.execute { + self.start0(promise: promise) + } + } + } + + /// Begin the `LambdaRuntime` shutdown. Only needed for debugging purposes, hence behind a `DEBUG` flag. + public func shutdown(promise: EventLoopPromise?) { + if self.eventLoop.inEventLoop { + self.shutdown0(promise: promise) + } else { + self.eventLoop.execute { + self.shutdown0(promise: promise) + } + } + } + + // MARK: - Private + + private func start0(promise: EventLoopPromise?) { + self.eventLoop.assertInEventLoop() + + // when starting we want to do thing in parallel: + // 1. start the connection to the control plane + // 2. create the lambda handler + + self.logger.debug("initializing lambda") + // 1. create the handler from the factory + // 2. report initialization error if one occured + let context = Lambda.InitializationContext( + logger: self.logger, + eventLoop: self.eventLoop, + allocator: ByteBufferAllocator() + ) + + self.factory(context).hop(to: self.eventLoop).whenComplete { result in + let action: StateMachine.Action + switch result { + case .success(let handler): + action = self.state.handlerCreated(handler) + case .failure(let error): + action = self.state.handlerCreationFailed(error) + } + self.run(action) + } + + let connectFuture = ClientBootstrap(group: self.eventLoop).connect( + host: self.configuration.runtimeEngine.ip, + port: self.configuration.runtimeEngine.port + ) + + connectFuture.whenComplete { result in + let action: StateMachine.Action + switch result { + case .success(let channel): + action = self.state.httpChannelConnected(channel) + case .failure(let error): + action = self.state.httpChannelConnectFailed(error) + } + self.run(action) + } + } + + private func shutdown0(promise: EventLoopPromise?) { + + } + + private func run(_ action: StateMachine.Action) { + + } +} + +extension LambdaRuntime: LambdaChannelHandlerDelegate { + func responseReceived(_ response: ControlPlaneResponse) { + + } + + func errorCaught(_: Error) { + + } + + func channelInactive() { + + } +} + +extension NewLambdaRuntime { + + struct StateMachine { + enum Action { + case none + } + + private enum State { + case initialized + case starting + case channelConnected(Channel, NewLambdaChannelHandler) + case handlerCreated(Handler) + case running(Channel, NewLambdaChannelHandler, Handler) + } + + private var markShutdown: Bool + private var state: State + + init() { + self.markShutdown = false + self.state = .initialized + } + + func handlerCreated(_ handler: Handler) -> Action { + return .none + } + + func handlerCreationFailed(_ error: Error) -> Action { + return .none + } + + func httpChannelConnected(_ channel: Channel) -> Action { + return .none + } + + func httpChannelConnectFailed(_ error: Error) -> Action { + return .none + } + } +} diff --git a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneRequestEncoderTests.swift b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneRequestEncoderTests.swift index ac6c0838..2d10094b 100644 --- a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneRequestEncoderTests.swift +++ b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneRequestEncoderTests.swift @@ -54,7 +54,7 @@ final class ControlPlaneRequestEncoderTests: XCTestCase { } func testPostInvocationSuccessWithoutBody() { - let requestID = UUID().uuidString + let requestID = LambdaRequestID() var request: NIOHTTPServerRequestFull? XCTAssertNoThrow(request = try self.sendRequest(.invocationResponse(requestID, nil))) @@ -70,7 +70,7 @@ final class ControlPlaneRequestEncoderTests: XCTestCase { } func testPostInvocationSuccessWithBody() { - let requestID = UUID().uuidString + let requestID = LambdaRequestID() let payload = ByteBuffer(string: "hello swift lambda!") var request: NIOHTTPServerRequestFull? @@ -89,7 +89,7 @@ final class ControlPlaneRequestEncoderTests: XCTestCase { } func testPostInvocationErrorWithBody() { - let requestID = UUID().uuidString + let requestID = LambdaRequestID() let error = ErrorResponse(errorType: "SomeError", errorMessage: "An error happened") var request: NIOHTTPServerRequestFull? XCTAssertNoThrow(request = try self.sendRequest(.invocationError(requestID, error))) @@ -137,7 +137,7 @@ final class ControlPlaneRequestEncoderTests: XCTestCase { XCTAssertEqual(nextRequest?.head.method, .GET) XCTAssertEqual(nextRequest?.head.uri, "/2018-06-01/runtime/invocation/next") - let requestID = UUID().uuidString + let requestID = LambdaRequestID() let payload = ByteBuffer(string: "hello swift lambda!") var successRequest: NIOHTTPServerRequestFull? XCTAssertNoThrow(successRequest = try self.sendRequest(.invocationResponse(requestID, payload))) From 490c546a53240382121b97b0eed4bdbb420303e5 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Wed, 9 Mar 2022 16:21:02 +0100 Subject: [PATCH 08/13] Final design --- Sources/AWSLambdaRuntimeCore/Lambda.swift | 19 +- .../LambdaRuntime+StateMachine.swift | 300 ++++++++++++++++++ .../NewLambdaChannelHandler.swift | 9 +- .../NewLambdaRuntime.swift | 223 +++++++++---- .../LambdaRequestIDTests.swift | 2 +- .../NewLambdaChannelHandlerTests.swift | 215 +++++++++++++ 6 files changed, 690 insertions(+), 78 deletions(-) create mode 100644 Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift create mode 100644 Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift diff --git a/Sources/AWSLambdaRuntimeCore/Lambda.swift b/Sources/AWSLambdaRuntimeCore/Lambda.swift index 1bf4f0dc..7ff66436 100644 --- a/Sources/AWSLambdaRuntimeCore/Lambda.swift +++ b/Sources/AWSLambdaRuntimeCore/Lambda.swift @@ -32,20 +32,12 @@ public enum Lambda { return String(cString: value) } - /// Run a Lambda defined by implementing the ``ByteBufferLambdaHandler`` protocol. - /// The Runtime will manage the Lambdas application lifecycle automatically. It will invoke the - /// ``ByteBufferLambdaHandler/makeHandler(context:)`` to create a new Handler. - /// - /// - parameters: - /// - configuration: A Lambda runtime configuration object - /// - handlerType: The Handler to create and invoke. - /// - /// - note: This is a blocking operation that will run forever, as its lifecycle is managed by the AWS Lambda Runtime Engine. + // for testing and internal use internal static func run( configuration: Configuration = .init(), handlerType: Handler.Type ) -> Result { - let _run = { (configuration: Configuration) -> Result in + let _run = { (configuration: Configuration, handlerType: Handler.Type) -> Result in Backtrace.install() var logger = Logger(label: "Lambda") logger.logLevel = configuration.general.logLevel @@ -84,16 +76,17 @@ public enum Lambda { if Lambda.env("LOCAL_LAMBDA_SERVER_ENABLED").flatMap(Bool.init) ?? false { do { return try Lambda.withLocalServer { - _run(configuration) + _run(configuration, handlerType) } } catch { return .failure(error) } } else { - return _run(configuration) + return _run(configuration, handlerType) } #else - return _run(configuration) + return _run(configuration, factory) #endif } + } diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift b/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift new file mode 100644 index 00000000..40c6cf7a --- /dev/null +++ b/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift @@ -0,0 +1,300 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftAWSLambdaRuntime open source project +// +// Copyright (c) 2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore + +extension NewLambdaRuntime { + struct Connection { + var channel: Channel + var handler: NewLambdaChannelHandler + } + + struct StateMachine { + enum Action { + case none + case createHandler(andConnection: Bool) + + case requestNextInvocation(NewLambdaChannelHandler, succeedStartPromise: EventLoopPromise?) + + case reportInvocationResult(LambdaRequestID, Result, pipelineNextInvocationRequest: Bool, NewLambdaChannelHandler) + case reportStartupError(Error, NewLambdaChannelHandler) + + case invokeHandler(Handler, Invocation, ByteBuffer) + + case failRuntime(Error) + } + + private enum State { + case initialized + case starting(EventLoopPromise?) + case connected(Connection, EventLoopPromise?) + case handlerCreated(Handler, EventLoopPromise?) + case handlerCreationFailed(Error, EventLoopPromise?) + case reportingStartupError(Connection, Error, EventLoopPromise?) + + case waitingForInvocation(Connection, Handler) + case executingInvocation(Connection, Handler, LambdaRequestID) + case reportingInvocationResult(Connection, Handler, nextInvocationRequestPipelined: Bool) + + case failed(Error) + } + + private var markShutdown: Bool + private var state: State + + init() { + self.markShutdown = false + self.state = .initialized + } + + mutating func start(connection: Connection?, promise: EventLoopPromise?) -> Action { + switch self.state { + case .initialized: + if let connection = connection { + self.state = .connected(connection, promise) + return .createHandler(andConnection: false) + } + + self.state = .starting(promise) + return .createHandler(andConnection: true) + + case .starting, + .connected, + .handlerCreated, + .handlerCreationFailed, + .reportingStartupError, + .waitingForInvocation, + .executingInvocation, + .reportingInvocationResult, + .failed: + preconditionFailure("Invalid state: \(self.state)") + } + } + + mutating func handlerCreated(_ handler: Handler) -> Action { + switch self.state { + case .initialized, + .handlerCreated, + .handlerCreationFailed, + .waitingForInvocation, + .executingInvocation, + .reportingInvocationResult, + .reportingStartupError: + preconditionFailure("Invalid state: \(self.state)") + + case .starting(let promise): + self.state = .handlerCreated(handler, promise) + return .none + + case .connected(let connection, let promise): + self.state = .waitingForInvocation(connection, handler) + return .requestNextInvocation(connection.handler, succeedStartPromise: promise) + + case .failed: + return .none + } + } + + mutating func handlerCreationFailed(_ error: Error) -> Action { + switch self.state { + case .initialized, + .handlerCreated, + .handlerCreationFailed, + .waitingForInvocation, + .executingInvocation, + .reportingInvocationResult, + .reportingStartupError: + preconditionFailure("Invalid state: \(self.state)") + + case .starting(let promise): + self.state = .handlerCreationFailed(error, promise) + return .none + + case .connected(let connection, let promise): + self.state = .reportingStartupError(connection, error, promise) + return .reportStartupError(error, connection.handler) + + case .failed: + return .none + } + } + + mutating func httpConnectionCreated( + _ connection: Connection + ) -> Action { + switch self.state { + case .initialized, + .connected, + .waitingForInvocation, + .executingInvocation, + .reportingInvocationResult, + .reportingStartupError: + preconditionFailure("Invalid state: \(self.state)") + + case .starting(let promise): + self.state = .connected(connection, promise) + return .none + + case .handlerCreated(let handler, let promise): + self.state = .waitingForInvocation(connection, handler) + return .requestNextInvocation(connection.handler, succeedStartPromise: promise) + + case .handlerCreationFailed(let error, let promise): + self.state = .reportingStartupError(connection, error, promise) + return .reportStartupError(error, connection.handler) + + case .failed: + return .none + } + } + + mutating func httpChannelConnectFailed(_ error: Error) -> Action { + switch self.state { + case .initialized, + .connected, + .waitingForInvocation, + .executingInvocation, + .reportingInvocationResult, + .reportingStartupError: + preconditionFailure("Invalid state: \(self.state)") + + case .starting: + self.state = .failed(error) + return .failRuntime(error) + + case .handlerCreated(let handler, let promise): + self.state = .failed(error) + return .failRuntime(error) + + case .handlerCreationFailed(let error, let promise): + self.state = .failed(error) + return .failRuntime(error) + + case .failed: + return .none + } + } + + mutating func newInvocationReceived(_ invocation: Invocation, _ body: ByteBuffer) -> Action { + switch self.state { + case .initialized, + .starting, + .connected, + .handlerCreated, + .handlerCreationFailed, + .executingInvocation, + .reportingInvocationResult, + .reportingStartupError: + preconditionFailure("Invalid state: \(self.state)") + + case .waitingForInvocation(let connection, let handler): + self.state = .executingInvocation(connection, handler, .init(uuidString: invocation.requestID)!) + return .invokeHandler(handler, invocation, body) + + case .failed: + return .none + } + } + + mutating func acceptedReceived() -> Action { + switch self.state { + case .initialized, + .starting, + .connected, + .handlerCreated, + .handlerCreationFailed, + .executingInvocation: + preconditionFailure("Invalid state: \(self.state)") + + case .waitingForInvocation: + preconditionFailure("TODO: fixme") + + case .reportingStartupError(_, let error, let promise): + self.state = .failed(error) + return .failRuntime(error) + + case .reportingInvocationResult(let connection, let handler, true): + self.state = .waitingForInvocation(connection, handler) + return .none + + case .reportingInvocationResult(let connection, let handler, false): + self.state = .waitingForInvocation(connection, handler) + return .requestNextInvocation(connection.handler, succeedStartPromise: nil) + + case .failed: + return .none + } + } + + mutating func errorResponseReceived(_ errorResponse: ErrorResponse) -> Action { + switch self.state { + case .initialized, + .starting, + .connected, + .handlerCreated, + .handlerCreationFailed, + .executingInvocation: + preconditionFailure("Invalid state: \(self.state)") + + case .waitingForInvocation: + let error = LambdaRuntimeError.controlPlaneErrorResponse(errorResponse) + self.state = .failed(error) + return .failRuntime(error) + + case .reportingStartupError(_, let error, let promise): + let error = LambdaRuntimeError.controlPlaneErrorResponse(errorResponse) + self.state = .failed(error) + return .failRuntime(error) + + case .reportingInvocationResult(let connection, let handler, _): + let error = LambdaRuntimeError.controlPlaneErrorResponse(errorResponse) + self.state = .failed(error) + return .failRuntime(error) + + case .failed: + return .none + } + } + + mutating func handlerError(_ error: Error) { + + } + + mutating func channelInactive() { + + } + + mutating func invocationFinished(_ result: Result) -> Action { + switch self.state { + case .initialized, + .starting, + .handlerCreated, + .handlerCreationFailed, + .connected, + .waitingForInvocation, + .reportingStartupError, + .reportingInvocationResult: + preconditionFailure("Invalid state: \(self.state)") + + case .failed: + return .none + + case .executingInvocation(let connection, let handler, let requestID): + let pipelining = true + self.state = .reportingInvocationResult(connection, handler, nextInvocationRequestPipelined: pipelining) + return .reportInvocationResult(requestID, result, pipelineNextInvocationRequest: pipelining, connection.handler) + } + } + } +} diff --git a/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift b/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift index 0ab8ddae..4868cba7 100644 --- a/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift +++ b/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift @@ -29,7 +29,7 @@ final class NewLambdaChannelHandler: Cha typealias OutboundOut = ByteBuffer private let delegate: Delegate - private let requestsInFlight: CircularBuffer + private var requestsInFlight: CircularBuffer private var context: ChannelHandlerContext! @@ -45,6 +45,7 @@ final class NewLambdaChannelHandler: Cha } func sendRequest(_ request: ControlPlaneRequest) { + self.requestsInFlight.append(request) self.encoder.writeRequest(request, context: self.context, promise: nil) } @@ -61,12 +62,14 @@ final class NewLambdaChannelHandler: Cha func channelRead(context: ChannelHandlerContext, data: NIOAny) { do { try self.decoder.process(buffer: self.unwrapInboundIn(data)) { response in - // TODO: The response matches the request + guard self.requestsInFlight.popFirst() != nil else { + throw LambdaRuntimeError.unsolicitedResponse + } self.delegate.responseReceived(response) } } catch { - + self.delegate.errorCaught(error) } } diff --git a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift index 81a516d0..4845aa53 100644 --- a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift +++ b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift @@ -16,30 +16,29 @@ import Logging import NIOConcurrencyHelpers import NIOCore import NIOPosix +import Backtrace /// `LambdaRuntime` manages the Lambda process lifecycle. /// -/// - note: It is intended to be used within a single `EventLoop`. For this reason this class is not thread safe. +/// - note: All state changes are dispatched onto the supplied EventLoop. public final class NewLambdaRuntime { private let eventLoop: EventLoop private let shutdownPromise: EventLoopPromise private let logger: Logger private let configuration: Lambda.Configuration - private let factory: (Lambda.InitializationContext) -> EventLoopFuture private var state: StateMachine init(eventLoop: EventLoop, logger: Logger, configuration: Lambda.Configuration, - factory: @escaping (Lambda.InitializationContext) -> EventLoopFuture + handlerType: Handler.Type ) { self.state = StateMachine() self.eventLoop = eventLoop self.shutdownPromise = eventLoop.makePromise(of: Void.self) self.logger = logger self.configuration = configuration - self.factory = factory } deinit { @@ -52,6 +51,12 @@ public final class NewLambdaRuntime { public var shutdownFuture: EventLoopFuture { self.shutdownPromise.futureResult } + + public func start() -> EventLoopFuture { + let promise = self.eventLoop.makePromise(of: Void.self) + self.start(promise: promise) + return promise.futureResult + } /// Start the `LambdaRuntime`. /// @@ -66,6 +71,17 @@ public final class NewLambdaRuntime { } } + public func __testOnly_start(channel: Channel, promise: EventLoopPromise?) { + precondition(channel.eventLoop === self.eventLoop, "Channel must be created on the supplied EventLoop.") + if self.eventLoop.inEventLoop { + self.__testOnly_start0(channel: channel, promise: promise) + } else { + self.eventLoop.execute { + self.__testOnly_start0(channel: channel, promise: promise) + } + } + } + /// Begin the `LambdaRuntime` shutdown. Only needed for debugging purposes, hence behind a `DEBUG` flag. public func shutdown(promise: EventLoopPromise?) { if self.eventLoop.inEventLoop { @@ -77,7 +93,7 @@ public final class NewLambdaRuntime { } } - // MARK: - Private + // MARK: - Private - private func start0(promise: EventLoopPromise?) { self.eventLoop.assertInEventLoop() @@ -87,25 +103,81 @@ public final class NewLambdaRuntime { // 2. create the lambda handler self.logger.debug("initializing lambda") - // 1. create the handler from the factory - // 2. report initialization error if one occured - let context = Lambda.InitializationContext( - logger: self.logger, - eventLoop: self.eventLoop, - allocator: ByteBufferAllocator() - ) - self.factory(context).hop(to: self.eventLoop).whenComplete { result in - let action: StateMachine.Action + let action = self.state.start(connection: nil, promise: promise) + self.run(action) + } + + private func shutdown0(promise: EventLoopPromise?) { + + } + + private func __testOnly_start0(channel: Channel, promise: EventLoopPromise?) { + channel.eventLoop.preconditionInEventLoop() + assert(channel.isActive) + + do { + let connection = try self.setupConnection(channel: channel) + let action = self.state.start(connection: connection, promise: promise) + self.run(action) + } catch { + promise?.fail(error) + } + } + + private func run(_ action: StateMachine.Action) { + switch action { + case .createHandler(andConnection: let andConnection): + self.createHandler() + if andConnection { + self.createConnection() + } + + case .invokeHandler(let handler, let invocation, let event): + let context = LambdaContext( + logger: self.logger, + eventLoop: self.eventLoop, + allocator: .init(), + invocation: invocation + ) + handler.handle(event, context: context).whenComplete { result in + let action = self.state.invocationFinished(result) + self.run(action) + } + + case .failRuntime(let error): + self.shutdownPromise.fail(error) + + case .requestNextInvocation(let handler, let startPromise): + handler.sendRequest(.next) + startPromise?.succeed(()) + + case .reportInvocationResult(let requestID, let result, let pipelineNextInvocationRequest, let handler): switch result { - case .success(let handler): - action = self.state.handlerCreated(handler) + case .success(let body): + handler.sendRequest(.invocationResponse(requestID, body)) + case .failure(let error): - action = self.state.handlerCreationFailed(error) + let errorString = String(describing: error) + let errorResponse = ErrorResponse(errorType: errorString, errorMessage: errorString) + handler.sendRequest(.invocationError(requestID, errorResponse)) } - self.run(action) - } + + if pipelineNextInvocationRequest { + handler.sendRequest(.next) + } + + case .reportStartupError(let error, let handler): + let errorString = String(describing: error) + handler.sendRequest(.initializationError(.init(errorType: errorString, errorMessage: errorString))) + + case .none: + break + } + } + + private func createConnection() { let connectFuture = ClientBootstrap(group: self.eventLoop).connect( host: self.configuration.runtimeEngine.ip, port: self.configuration.runtimeEngine.port @@ -115,7 +187,12 @@ public final class NewLambdaRuntime { let action: StateMachine.Action switch result { case .success(let channel): - action = self.state.httpChannelConnected(channel) + do { + let connection = try self.setupConnection(channel: channel) + action = self.state.httpConnectionCreated(connection) + } catch { + action = self.state.httpChannelConnectFailed(error) + } case .failure(let error): action = self.state.httpChannelConnectFailed(error) } @@ -123,66 +200,90 @@ public final class NewLambdaRuntime { } } - private func shutdown0(promise: EventLoopPromise?) { - + private func setupConnection(channel: Channel) throws -> Connection { + let handler = NewLambdaChannelHandler(delegate: self, host: self.configuration.runtimeEngine.ip) + try channel.pipeline.syncOperations.addHandler(handler) + return Connection(channel: channel, handler: handler) } - private func run(_ action: StateMachine.Action) { + private func createHandler() { + let context = Lambda.InitializationContext( + logger: self.logger, + eventLoop: self.eventLoop, + allocator: ByteBufferAllocator() + ) + Handler.factory(context: context).hop(to: self.eventLoop).whenComplete { result in + let action: StateMachine.Action + switch result { + case .success(let handler): + action = self.state.handlerCreated(handler) + case .failure(let error): + action = self.state.handlerCreationFailed(error) + } + self.run(action) + } } } -extension LambdaRuntime: LambdaChannelHandlerDelegate { +extension NewLambdaRuntime: LambdaChannelHandlerDelegate { func responseReceived(_ response: ControlPlaneResponse) { + let action: StateMachine.Action + switch response { + case .next(let invocation, let byteBuffer): + action = self.state.newInvocationReceived(invocation, byteBuffer) + + case .accepted: + action = self.state.acceptedReceived() + + case .error(let errorResponse): + action = self.state.errorResponseReceived(errorResponse) + } + self.run(action) } - func errorCaught(_: Error) { - + func errorCaught(_ error: Error) { + self.state.handlerError(error) } func channelInactive() { - + self.state.channelInactive() } } extension NewLambdaRuntime { - struct StateMachine { - enum Action { - case none - } - - private enum State { - case initialized - case starting - case channelConnected(Channel, NewLambdaChannelHandler) - case handlerCreated(Handler) - case running(Channel, NewLambdaChannelHandler, Handler) - } - - private var markShutdown: Bool - private var state: State - - init() { - self.markShutdown = false - self.state = .initialized - } + static func run(handlerType: Handler.Type) { + Backtrace.install() - func handlerCreated(_ handler: Handler) -> Action { - return .none - } - - func handlerCreationFailed(_ error: Error) -> Action { - return .none - } - - func httpChannelConnected(_ channel: Channel) -> Action { - return .none - } - - func httpChannelConnectFailed(_ error: Error) -> Action { - return .none + let configuration = Lambda.Configuration() + var logger = Logger(label: "Lambda") + logger.logLevel = configuration.general.logLevel + + MultiThreadedEventLoopGroup.withCurrentThreadAsEventLoop { eventLoop in + let runtime = NewLambdaRuntime(eventLoop: eventLoop, logger: logger, configuration: configuration, handlerType: Handler.self) + + #if DEBUG + let signalSource = trap(signal: configuration.lifecycle.stopSignal) { signal in + logger.info("intercepted signal: \(signal)") + runtime.shutdown(promise: nil) + } + #endif + + runtime.start().flatMap { + runtime.shutdownFuture + }.whenComplete { lifecycleResult in + #if DEBUG + signalSource.cancel() + #endif + eventLoop.shutdownGracefully { error in + if let error = error { + preconditionFailure("Failed to shutdown eventloop: \(error)") + } + logger.info("shutdown completed") + } + } } } } diff --git a/Tests/AWSLambdaRuntimeCoreTests/LambdaRequestIDTests.swift b/Tests/AWSLambdaRuntimeCoreTests/LambdaRequestIDTests.swift index 7849fe09..656fbfd6 100644 --- a/Tests/AWSLambdaRuntimeCoreTests/LambdaRequestIDTests.swift +++ b/Tests/AWSLambdaRuntimeCoreTests/LambdaRequestIDTests.swift @@ -30,7 +30,7 @@ final class LambdaRequestIDTest: XCTestCase { func testInitFromLowercaseStringSuccess() { let string = "E621E1F8-C36C-495A-93FC-0C247A3E6E5F".lowercased() - var originalBuffer = ByteBuffer(string: string) + var originalBuffer = ByteBuffer(string: string.uppercased()) let requestID = originalBuffer.readRequestID() XCTAssertEqual(originalBuffer.readerIndex, 36) diff --git a/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift b/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift new file mode 100644 index 00000000..594a336a --- /dev/null +++ b/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift @@ -0,0 +1,215 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftAWSLambdaRuntime open source project +// +// Copyright (c) 2022 Apple Inc. and the SwiftAWSLambdaRuntime project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftAWSLambdaRuntime project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import XCTest +import NIOCore +import NIOEmbedded +import NIOHTTP1 +@testable import AWSLambdaRuntimeCore +import SwiftUI + +final class NewLambdaChannelHandlerTests: XCTestCase { + let host = "192.168.0.1" + + var delegate: EmbeddedLambdaChannelHandlerDelegate! + var handler: NewLambdaChannelHandler! + var client: EmbeddedChannel! + var server: EmbeddedChannel! + + override func setUp() { + self.delegate = EmbeddedLambdaChannelHandlerDelegate() + self.handler = NewLambdaChannelHandler(delegate: self.delegate, host: "127.0.0.1") + + self.client = EmbeddedChannel(handler: self.handler) + self.server = EmbeddedChannel(handlers: [ + NIOHTTPServerRequestAggregator(maxContentLength: 1024 * 1024), + ]) + + XCTAssertNoThrow(try self.server.pipeline.syncOperations.configureHTTPServerPipeline(position: .first)) + + XCTAssertNoThrow(try self.server.bind(to: .init(ipAddress: "127.0.0.1", port: 0), promise: nil)) + XCTAssertNoThrow(try self.client.connect(to: .init(ipAddress: "127.0.0.1", port: 0), promise: nil)) + } + + func testPipelineRequests() { + self.handler.sendRequest(.next) + + self.assertInteract() + + var nextRequest: NIOHTTPServerRequestFull? + XCTAssertNoThrow(nextRequest = try self.server.readInbound(as: NIOHTTPServerRequestFull.self)) + XCTAssertEqual(nextRequest?.head.uri, "/2018-06-01/runtime/invocation/next") + XCTAssertEqual(nextRequest?.head.method, .GET) + + XCTAssertNil(try self.server.readInbound(as: NIOHTTPServerRequestFull.self)) + + let requestID = LambdaRequestID() + let traceID = "foo" + let functionARN = "arn" + let deadline = UInt(Date().timeIntervalSince1970 * 1000) + 3000 + let requestBody = ByteBuffer(string: "foo bar") + + XCTAssertNoThrow(try self.server.writeOutboundInvocation( + requestID: requestID, + traceID: traceID, + functionARN: functionARN, + deadline: deadline, + body: requestBody + )) + + self.assertInteract() + + var response: (Invocation, ByteBuffer)? + XCTAssertNoThrow(response = try self.delegate.readNextResponse()) + + XCTAssertEqual(response?.0.requestID, requestID.lowercased) + XCTAssertEqual(response?.0.traceID, traceID) + XCTAssertEqual(response?.0.invokedFunctionARN, functionARN) + XCTAssertEqual(response?.0.deadlineInMillisSinceEpoch, Int64(deadline)) + XCTAssertEqual(response?.1, requestBody) + + let responseBody = ByteBuffer(string: "hello world") + + self.handler.sendRequest(.invocationResponse(requestID, responseBody)) + + self.assertInteract() + + var responseRequest: NIOHTTPServerRequestFull? + XCTAssertNoThrow(responseRequest = try self.server.readInbound(as: NIOHTTPServerRequestFull.self)) + XCTAssertEqual(responseRequest?.head.uri, "/2018-06-01/runtime/invocation/\(requestID.uppercased)/response") + XCTAssertEqual(responseRequest?.head.method, .POST) + XCTAssertEqual(responseRequest?.body, responseBody) + } + + func assertInteract(file: StaticString = #file, line: UInt = #line) { + XCTAssertNoThrow(try { + while let clientBuffer = try self.client.readOutbound(as: ByteBuffer.self) { + try self.server.writeInbound(clientBuffer) + } + + while let serverBuffer = try self.server.readOutbound(as: ByteBuffer.self) { + try self.client.writeInbound(serverBuffer) + } + }(), file: file, line: line) + } +} + +final class EmbeddedLambdaChannelHandlerDelegate: LambdaChannelHandlerDelegate { + + enum Error: Swift.Error { + case missingEvent + case wrongEventType + case wrongResponseType + } + + private enum Event { + case channelInactive + case error(Swift.Error) + case response(ControlPlaneResponse) + } + + private var events: CircularBuffer + + init() { + self.events = CircularBuffer(initialCapacity: 8) + } + + func channelInactive() { + self.events.append(.channelInactive) + } + + func errorCaught(_ error: Swift.Error) { + self.events.append(.error(error)) + } + + func responseReceived(_ response: ControlPlaneResponse) { + self.events.append(.response(response)) + } + + func readResponse() throws -> ControlPlaneResponse { + guard case .response(let response) = try self.popNextEvent() else { + throw Error.wrongEventType + } + return response + } + + func readNextResponse() throws -> (Invocation, ByteBuffer) { + guard case .next(let invocation, let body) = try self.readResponse() else { + throw Error.wrongResponseType + } + return (invocation, body) + } + + func assertAcceptResponse() throws { + guard case .accepted = try self.readResponse() else { + throw Error.wrongResponseType + } + } + + func readErrorResponse() throws -> ErrorResponse { + guard case .error(let errorResponse) = try self.readResponse() else { + throw Error.wrongResponseType + } + return errorResponse + } + + func readError() throws -> Swift.Error { + guard case .error(let error) = try self.popNextEvent() else { + throw Error.wrongEventType + } + return error + } + + func assertChannelInactive() throws { + guard case .channelInactive = try self.popNextEvent() else { + throw Error.wrongEventType + } + } + + private func popNextEvent() throws -> Event { + guard let event = self.events.popFirst() else { + throw Error.missingEvent + } + return event + } +} + +extension EmbeddedChannel { + + func writeOutboundInvocation( + requestID: LambdaRequestID = LambdaRequestID(), + traceID: String = "Root=\(DispatchTime.now().uptimeNanoseconds);Parent=\(DispatchTime.now().uptimeNanoseconds);Sampled=1", + functionARN: String = "", + deadline: UInt = UInt(Date().timeIntervalSince1970 * 1000) + 3000, + body: ByteBuffer? + ) throws { + let head = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: [ + "content-length": "\(body?.readableBytes ?? 0)", + "lambda-runtime-deadline-ms": "\(deadline)", + "lambda-runtime-trace-id": "\(traceID)", + "lambda-runtime-aws-request-id": "\(requestID)", + "lambda-runtime-invoked-function-arn": "\(functionARN)" + ] + ) + + try self.writeOutbound(HTTPServerResponsePart.head(head)) + if let body = body { + try self.writeOutbound(HTTPServerResponsePart.body(.byteBuffer(body))) + } + try self.writeOutbound(HTTPServerResponsePart.end(nil)) + } +} From cc6f6367eddba9c3d5c3883eea2c9d7fc1870425 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Wed, 9 Mar 2022 16:22:04 +0100 Subject: [PATCH 09/13] Lambda factory as a protocol requirement. # Conflicts: # Sources/AWSLambdaRuntimeCore/Lambda.swift # Sources/AWSLambdaRuntimeCore/LambdaHandler.swift --- Sources/AWSLambdaRuntimeCore/Lambda.swift | 21 +++++++++++-------- .../AWSLambdaRuntimeCore/LambdaRunner.swift | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Sources/AWSLambdaRuntimeCore/Lambda.swift b/Sources/AWSLambdaRuntimeCore/Lambda.swift index 7ff66436..c6163fd3 100644 --- a/Sources/AWSLambdaRuntimeCore/Lambda.swift +++ b/Sources/AWSLambdaRuntimeCore/Lambda.swift @@ -32,12 +32,16 @@ public enum Lambda { return String(cString: value) } - // for testing and internal use - internal static func run( - configuration: Configuration = .init(), - handlerType: Handler.Type - ) -> Result { - let _run = { (configuration: Configuration, handlerType: Handler.Type) -> Result in + /// Run a Lambda defined by implementing the ``ByteBufferLambdaHandler`` protocol. + /// The Runtime will manage the Lambdas application lifecycle automatically. It will invoke the + /// ``ByteBufferLambdaHandler/factory(context:)`` to create a new Handler. + /// + /// - parameters: + /// - factory: A `ByteBufferLambdaHandler` factory. + /// + /// - note: This is a blocking operation that will run forever, as its lifecycle is managed by the AWS Lambda Runtime Engine. + internal static func run(configuration: Configuration = .init(), handlerType: Handler.Type) -> Result { + let _run = { (configuration: Configuration) -> Result in Backtrace.install() var logger = Logger(label: "Lambda") logger.logLevel = configuration.general.logLevel @@ -76,17 +80,16 @@ public enum Lambda { if Lambda.env("LOCAL_LAMBDA_SERVER_ENABLED").flatMap(Bool.init) ?? false { do { return try Lambda.withLocalServer { - _run(configuration, handlerType) + _run(configuration) } } catch { return .failure(error) } } else { - return _run(configuration, handlerType) + return _run(configuration) } #else return _run(configuration, factory) #endif } - } diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRunner.swift b/Sources/AWSLambdaRuntimeCore/LambdaRunner.swift index 38499a05..2fa9d0f8 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaRunner.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaRunner.swift @@ -41,7 +41,7 @@ extension Lambda { let context = InitializationContext(logger: logger, eventLoop: self.eventLoop, allocator: self.allocator) - return Handler.makeHandler(context: context) + return Handler.factory(context: context) // Hopping back to "our" EventLoop is important in case the factory returns a future // that originated from a foreign EventLoop/EventLoopGroup. // This can happen if the factory uses a library (let's say a database client) that manages its own threads/loops From 7c66455f7693acd118cce3086794d76d918f8a57 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Wed, 9 Mar 2022 16:14:35 +0100 Subject: [PATCH 10/13] Fix test and code format --- Sources/AWSLambdaRuntimeCore/LambdaRunner.swift | 2 +- Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRunner.swift b/Sources/AWSLambdaRuntimeCore/LambdaRunner.swift index 2fa9d0f8..38499a05 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaRunner.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaRunner.swift @@ -41,7 +41,7 @@ extension Lambda { let context = InitializationContext(logger: logger, eventLoop: self.eventLoop, allocator: self.allocator) - return Handler.factory(context: context) + return Handler.makeHandler(context: context) // Hopping back to "our" EventLoop is important in case the factory returns a future // that originated from a foreign EventLoop/EventLoopGroup. // This can happen if the factory uses a library (let's say a database client) that manages its own threads/loops diff --git a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift index 4845aa53..196b2dff 100644 --- a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift +++ b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift @@ -213,7 +213,7 @@ public final class NewLambdaRuntime { allocator: ByteBufferAllocator() ) - Handler.factory(context: context).hop(to: self.eventLoop).whenComplete { result in + Handler.makeHandler(context: context).hop(to: self.eventLoop).whenComplete { result in let action: StateMachine.Action switch result { case .success(let handler): From 4582587b8eaf5398a2229edc4d238a5d4e80d8c6 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Fri, 7 Jan 2022 09:15:39 +0100 Subject: [PATCH 11/13] New runtime works on happy path --- .../Lambda+LocalServer.swift | 2 +- Sources/AWSLambdaRuntimeCore/Lambda.swift | 2 +- .../AWSLambdaRuntimeCore/LambdaHandler.swift | 20 ++++++++++++++++++ .../LambdaRequestID.swift | 13 ++++-------- .../NewLambdaChannelHandler.swift | 3 ++- .../NewLambdaRuntime.swift | 21 ++++++++++++++++++- 6 files changed, 48 insertions(+), 13 deletions(-) diff --git a/Sources/AWSLambdaRuntimeCore/Lambda+LocalServer.swift b/Sources/AWSLambdaRuntimeCore/Lambda+LocalServer.swift index 1e09d867..7f12546a 100644 --- a/Sources/AWSLambdaRuntimeCore/Lambda+LocalServer.swift +++ b/Sources/AWSLambdaRuntimeCore/Lambda+LocalServer.swift @@ -131,7 +131,7 @@ private enum LocalLambda { guard let work = request.body else { return self.writeResponse(context: context, response: .init(status: .badRequest)) } - let requestID = "\(DispatchTime.now().uptimeNanoseconds)" // FIXME: + let requestID = LambdaRequestID().lowercased let promise = context.eventLoop.makePromise(of: Response.self) promise.futureResult.whenComplete { result in switch result { diff --git a/Sources/AWSLambdaRuntimeCore/Lambda.swift b/Sources/AWSLambdaRuntimeCore/Lambda.swift index c6163fd3..43e299e7 100644 --- a/Sources/AWSLambdaRuntimeCore/Lambda.swift +++ b/Sources/AWSLambdaRuntimeCore/Lambda.swift @@ -89,7 +89,7 @@ public enum Lambda { return _run(configuration) } #else - return _run(configuration, factory) + return _run(configuration) #endif } } diff --git a/Sources/AWSLambdaRuntimeCore/LambdaHandler.swift b/Sources/AWSLambdaRuntimeCore/LambdaHandler.swift index 3c2697ff..31ec9078 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaHandler.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaHandler.swift @@ -201,7 +201,27 @@ extension ByteBufferLambdaHandler { /// The lambda runtime provides a default implementation of the method that manages the launch /// process. public static func main() { + #if false _ = Lambda.run(configuration: .init(), handlerType: Self.self) + #else + + #if DEBUG + if Lambda.env("LOCAL_LAMBDA_SERVER_ENABLED").flatMap(Bool.init) ?? false { + do { + return try Lambda.withLocalServer { + NewLambdaRuntime.run(handlerType: Self.self) + } + } catch { + print(error) + exit(1) + } + } else { + NewLambdaRuntime.run(handlerType: Self.self) + } + #else + NewLambdaRuntime.run(handlerType: Self.self) + #endif + #endif } } diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRequestID.swift b/Sources/AWSLambdaRuntimeCore/LambdaRequestID.swift index 866f0273..031d8c3f 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaRequestID.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaRequestID.swift @@ -64,11 +64,6 @@ struct LambdaRequestID { private let _uuid: uuid_t - /// Returns a string representation for the `LambdaRequestID`, such as "E621E1F8-C36C-495A-93FC-0C247A3E6E5F" - var uuidString: String { - self.uppercased - } - /// Returns a lowercase string representation for the `LambdaRequestID`, such as "e621e1f8-c36c-495a-93fc-0c247a3e6e5f" var lowercased: String { var bytes = self.toAsciiBytesOnStack(characters: Self.lowercaseLookup) @@ -144,13 +139,13 @@ extension LambdaRequestID: Hashable { extension LambdaRequestID: CustomStringConvertible { var description: String { - self.uuidString + self.lowercased } } extension LambdaRequestID: CustomDebugStringConvertible { var debugDescription: String { - self.uuidString + self.lowercased } } @@ -170,7 +165,7 @@ extension LambdaRequestID: Decodable { extension LambdaRequestID: Encodable { func encode(to encoder: Encoder) throws { var container = encoder.singleValueContainer() - try container.encode(self.uuidString) + try container.encode(self.lowercased) } } @@ -340,7 +335,7 @@ extension ByteBuffer { @discardableResult mutating func setRequestID(_ requestID: LambdaRequestID, at index: Int) -> Int { - var localBytes = requestID.toAsciiBytesOnStack(characters: LambdaRequestID.uppercaseLookup) + var localBytes = requestID.toAsciiBytesOnStack(characters: LambdaRequestID.lowercaseLookup) return withUnsafeBytes(of: &localBytes) { self.setBytes($0, at: index) } diff --git a/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift b/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift index 4868cba7..6f940c3b 100644 --- a/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift +++ b/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift @@ -61,7 +61,8 @@ final class NewLambdaChannelHandler: Cha func channelRead(context: ChannelHandlerContext, data: NIOAny) { do { - try self.decoder.process(buffer: self.unwrapInboundIn(data)) { response in + let buffer = self.unwrapInboundIn(data) + try self.decoder.process(buffer: buffer) { response in guard self.requestsInFlight.popFirst() != nil else { throw LambdaRuntimeError.unsolicitedResponse } diff --git a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift index 196b2dff..31445a8d 100644 --- a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift +++ b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift @@ -18,6 +18,10 @@ import NIOCore import NIOPosix import Backtrace +#if canImport(Glibc) +import Glibc +#endif + /// `LambdaRuntime` manages the Lambda process lifecycle. /// /// - note: All state changes are dispatched onto the supplied EventLoop. @@ -134,6 +138,7 @@ public final class NewLambdaRuntime { } case .invokeHandler(let handler, let invocation, let event): + self.logger.trace("invoking handler") let context = LambdaContext( logger: self.logger, eventLoop: self.eventLoop, @@ -149,15 +154,22 @@ public final class NewLambdaRuntime { self.shutdownPromise.fail(error) case .requestNextInvocation(let handler, let startPromise): + self.logger.trace("requesting next invocation") handler.sendRequest(.next) startPromise?.succeed(()) case .reportInvocationResult(let requestID, let result, let pipelineNextInvocationRequest, let handler): switch result { case .success(let body): + self.logger.trace("reporting invocation success", metadata: [ + "lambda-request-id": "\(requestID)" + ]) handler.sendRequest(.invocationResponse(requestID, body)) case .failure(let error): + self.logger.trace("reporting invocation failure", metadata: [ + "lambda-request-id": "\(requestID)" + ]) let errorString = String(describing: error) let errorResponse = ErrorResponse(errorType: errorString, errorMessage: errorString) handler.sendRequest(.invocationError(requestID, errorResponse)) @@ -262,7 +274,14 @@ extension NewLambdaRuntime { logger.logLevel = configuration.general.logLevel MultiThreadedEventLoopGroup.withCurrentThreadAsEventLoop { eventLoop in - let runtime = NewLambdaRuntime(eventLoop: eventLoop, logger: logger, configuration: configuration, handlerType: Handler.self) + let runtime = NewLambdaRuntime( + eventLoop: eventLoop, + logger: logger, + configuration: configuration, + handlerType: Handler.self + ) + + logger.info("lambda runtime starting with \(configuration)") #if DEBUG let signalSource = trap(signal: configuration.lifecycle.stopSignal) { signal in From d375c38e1ff306dd667f09d9a1da0e8a6bf951db Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Wed, 9 Mar 2022 16:04:53 +0100 Subject: [PATCH 12/13] Make tests succeed. --- .../LambdaRuntime+StateMachine.swift | 23 +++++++++---------- .../NewLambdaRuntime.swift | 13 +++++++---- .../ControlPlaneResponseDecoderTests.swift | 3 ++- .../LambdaRequestIDTests.swift | 20 ++++++++-------- .../NewLambdaChannelHandlerTests.swift | 2 +- 5 files changed, 32 insertions(+), 29 deletions(-) diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift b/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift index 40c6cf7a..43e8400a 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift @@ -32,7 +32,7 @@ extension NewLambdaRuntime { case invokeHandler(Handler, Invocation, ByteBuffer) - case failRuntime(Error) + case failRuntime(Error, startPomise: EventLoopPromise?) } private enum State { @@ -169,17 +169,17 @@ extension NewLambdaRuntime { .reportingStartupError: preconditionFailure("Invalid state: \(self.state)") - case .starting: + case .starting(let promise): self.state = .failed(error) - return .failRuntime(error) + return .failRuntime(error, startPomise: promise) - case .handlerCreated(let handler, let promise): + case .handlerCreated(_, let promise): self.state = .failed(error) - return .failRuntime(error) + return .failRuntime(error, startPomise: promise) case .handlerCreationFailed(let error, let promise): self.state = .failed(error) - return .failRuntime(error) + return .failRuntime(error, startPomise: promise) case .failed: return .none @@ -222,7 +222,7 @@ extension NewLambdaRuntime { case .reportingStartupError(_, let error, let promise): self.state = .failed(error) - return .failRuntime(error) + return .failRuntime(error, startPomise: promise) case .reportingInvocationResult(let connection, let handler, true): self.state = .waitingForInvocation(connection, handler) @@ -250,17 +250,16 @@ extension NewLambdaRuntime { case .waitingForInvocation: let error = LambdaRuntimeError.controlPlaneErrorResponse(errorResponse) self.state = .failed(error) - return .failRuntime(error) + return .failRuntime(error, startPomise: nil) case .reportingStartupError(_, let error, let promise): - let error = LambdaRuntimeError.controlPlaneErrorResponse(errorResponse) self.state = .failed(error) - return .failRuntime(error) + return .failRuntime(error, startPomise: promise) - case .reportingInvocationResult(let connection, let handler, _): + case .reportingInvocationResult: let error = LambdaRuntimeError.controlPlaneErrorResponse(errorResponse) self.state = .failed(error) - return .failRuntime(error) + return .failRuntime(error, startPomise: nil) case .failed: return .none diff --git a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift index 31445a8d..c6cac746 100644 --- a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift +++ b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift @@ -55,16 +55,18 @@ public final class NewLambdaRuntime { public var shutdownFuture: EventLoopFuture { self.shutdownPromise.futureResult } - + + /// Start the `LambdaRuntime`. + /// + /// - Returns: An `EventLoopFuture` that is fulfilled after the Lambda hander has been created + /// and initiliazed, and a first run has been scheduled. public func start() -> EventLoopFuture { let promise = self.eventLoop.makePromise(of: Void.self) self.start(promise: promise) return promise.futureResult } - /// Start the `LambdaRuntime`. - /// - /// - Returns: An `EventLoopFuture` that is fulfilled after the Lambda hander has been created and initiliazed, and a first run has been scheduled. + public func start(promise: EventLoopPromise?) { if self.eventLoop.inEventLoop { self.start0(promise: promise) @@ -150,7 +152,8 @@ public final class NewLambdaRuntime { self.run(action) } - case .failRuntime(let error): + case .failRuntime(let error, let startPromise): + startPromise?.fail(error) self.shutdownPromise.fail(error) case .requestNextInvocation(let handler, let startPromise): diff --git a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift index 45c5f61a..27cea84c 100644 --- a/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift +++ b/Tests/AWSLambdaRuntimeCoreTests/ControlPlaneResponseDecoderTests.swift @@ -328,7 +328,8 @@ final class ControlPlaneResponseDecoderTests: XCTestCase { inputOutputPairs: [(nextResponse, [])], decoderFactory: { ControlPlaneResponseDecoder() } )) { - XCTAssertEqual($0 as? LambdaRuntimeError, .responseHeadInvalidHeader) + // TODO: This should return an invalid header function error (.responseHeadInvalidHeader) + XCTAssertEqual($0 as? LambdaRuntimeError, .invocationHeadMissingRequestID) } } } diff --git a/Tests/AWSLambdaRuntimeCoreTests/LambdaRequestIDTests.swift b/Tests/AWSLambdaRuntimeCoreTests/LambdaRequestIDTests.swift index 656fbfd6..e3d89dae 100644 --- a/Tests/AWSLambdaRuntimeCoreTests/LambdaRequestIDTests.swift +++ b/Tests/AWSLambdaRuntimeCoreTests/LambdaRequestIDTests.swift @@ -24,18 +24,18 @@ final class LambdaRequestIDTest: XCTestCase { let requestID = buffer.readRequestID() XCTAssertEqual(buffer.readerIndex, 36) XCTAssertEqual(buffer.readableBytes, 0) - XCTAssertEqual(requestID?.uuidString, UUID(uuidString: string)?.uuidString) + XCTAssertEqual(requestID?.uppercased, UUID(uuidString: string)?.uuidString) XCTAssertEqual(requestID?.uppercased, string) } func testInitFromLowercaseStringSuccess() { let string = "E621E1F8-C36C-495A-93FC-0C247A3E6E5F".lowercased() - var originalBuffer = ByteBuffer(string: string.uppercased()) + var originalBuffer = ByteBuffer(string: string) let requestID = originalBuffer.readRequestID() XCTAssertEqual(originalBuffer.readerIndex, 36) XCTAssertEqual(originalBuffer.readableBytes, 0) - XCTAssertEqual(requestID?.uuidString, UUID(uuidString: string)?.uuidString) + XCTAssertEqual(requestID?.uppercased, UUID(uuidString: string)?.uuidString) XCTAssertEqual(requestID?.lowercased, string) var newBuffer = ByteBuffer() @@ -109,7 +109,7 @@ final class LambdaRequestIDTest: XCTestCase { // achieve this though at the moment // XCTAssertFalse((nsString as String).isContiguousUTF8) let requestID = LambdaRequestID(uuidString: nsString as String) - XCTAssertEqual(requestID?.uuidString, LambdaRequestID(uuidString: nsString as String)?.uuidString) + XCTAssertEqual(requestID?.lowercased, LambdaRequestID(uuidString: nsString as String)?.lowercased) XCTAssertEqual(requestID?.uppercased, nsString as String) } @@ -121,10 +121,10 @@ final class LambdaRequestIDTest: XCTestCase { func testDescription() { let requestID = LambdaRequestID() - let fduuid = UUID(uuid: requestID.uuid) + let uuid = UUID(uuid: requestID.uuid) - XCTAssertEqual(fduuid.description, requestID.description) - XCTAssertEqual(fduuid.debugDescription, requestID.debugDescription) + XCTAssertEqual(uuid.description.lowercased(), requestID.description) + XCTAssertEqual(uuid.debugDescription.lowercased(), requestID.debugDescription) } func testFoundationInteropFromFoundation() { @@ -190,7 +190,7 @@ final class LambdaRequestIDTest: XCTestCase { var data: Data? XCTAssertNoThrow(data = try JSONEncoder().encode(test)) - XCTAssertEqual(try String(decoding: XCTUnwrap(data), as: Unicode.UTF8.self), #"{"requestID":"\#(requestID.uuidString)"}"#) + XCTAssertEqual(try String(decoding: XCTUnwrap(data), as: Unicode.UTF8.self), #"{"requestID":"\#(requestID.lowercased)"}"#) } func testDecodingSuccess() { @@ -198,7 +198,7 @@ final class LambdaRequestIDTest: XCTestCase { let requestID: LambdaRequestID } let requestID = LambdaRequestID() - let data = #"{"requestID":"\#(requestID.uuidString)"}"#.data(using: .utf8) + let data = #"{"requestID":"\#(requestID.lowercased)"}"#.data(using: .utf8) var result: Test? XCTAssertNoThrow(result = try JSONDecoder().decode(Test.self, from: XCTUnwrap(data))) @@ -210,7 +210,7 @@ final class LambdaRequestIDTest: XCTestCase { let requestID: LambdaRequestID } let requestID = LambdaRequestID() - var requestIDString = requestID.uuidString + var requestIDString = requestID.lowercased _ = requestIDString.removeLast() let data = #"{"requestID":"\#(requestIDString)"}"#.data(using: .utf8) diff --git a/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift b/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift index 594a336a..660ec53d 100644 --- a/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift +++ b/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift @@ -87,7 +87,7 @@ final class NewLambdaChannelHandlerTests: XCTestCase { var responseRequest: NIOHTTPServerRequestFull? XCTAssertNoThrow(responseRequest = try self.server.readInbound(as: NIOHTTPServerRequestFull.self)) - XCTAssertEqual(responseRequest?.head.uri, "/2018-06-01/runtime/invocation/\(requestID.uppercased)/response") + XCTAssertEqual(responseRequest?.head.uri, "/2018-06-01/runtime/invocation/\(requestID.lowercased)/response") XCTAssertEqual(responseRequest?.head.method, .POST) XCTAssertEqual(responseRequest?.body, responseBody) } From 2385a74ec7ce775f4af1e1281ad8eb1475a65e86 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Wed, 9 Mar 2022 17:14:37 +0100 Subject: [PATCH 13/13] Fix compile errors --- .../AWSLambdaRuntimeCore/LambdaHandler.swift | 5 + .../LambdaRuntime+StateMachine.swift | 124 +++++++++--------- .../NewLambdaChannelHandler.swift | 30 ++--- .../NewLambdaRuntime.swift | 74 +++++------ .../NewLambdaChannelHandlerTests.swift | 73 +++++------ 5 files changed, 148 insertions(+), 158 deletions(-) diff --git a/Sources/AWSLambdaRuntimeCore/LambdaHandler.swift b/Sources/AWSLambdaRuntimeCore/LambdaHandler.swift index 31ec9078..621056e5 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaHandler.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaHandler.swift @@ -14,6 +14,11 @@ import Dispatch import NIOCore +#if canImport(Darwin) +import Darwin +#else +import Glibc +#endif // MARK: - LambdaHandler diff --git a/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift b/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift index 43e8400a..ab5c7dd6 100644 --- a/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift +++ b/Sources/AWSLambdaRuntimeCore/LambdaRuntime+StateMachine.swift @@ -19,22 +19,22 @@ extension NewLambdaRuntime { var channel: Channel var handler: NewLambdaChannelHandler } - + struct StateMachine { enum Action { case none case createHandler(andConnection: Bool) - + case requestNextInvocation(NewLambdaChannelHandler, succeedStartPromise: EventLoopPromise?) - + case reportInvocationResult(LambdaRequestID, Result, pipelineNextInvocationRequest: Bool, NewLambdaChannelHandler) case reportStartupError(Error, NewLambdaChannelHandler) - + case invokeHandler(Handler, Invocation, ByteBuffer) - + case failRuntime(Error, startPomise: EventLoopPromise?) } - + private enum State { case initialized case starting(EventLoopPromise?) @@ -42,22 +42,22 @@ extension NewLambdaRuntime { case handlerCreated(Handler, EventLoopPromise?) case handlerCreationFailed(Error, EventLoopPromise?) case reportingStartupError(Connection, Error, EventLoopPromise?) - + case waitingForInvocation(Connection, Handler) case executingInvocation(Connection, Handler, LambdaRequestID) case reportingInvocationResult(Connection, Handler, nextInvocationRequestPipelined: Bool) - + case failed(Error) } - + private var markShutdown: Bool private var state: State - + init() { self.markShutdown = false self.state = .initialized } - + mutating func start(connection: Connection?, promise: EventLoopPromise?) -> Action { switch self.state { case .initialized: @@ -65,10 +65,10 @@ extension NewLambdaRuntime { self.state = .connected(connection, promise) return .createHandler(andConnection: false) } - + self.state = .starting(promise) return .createHandler(andConnection: true) - + case .starting, .connected, .handlerCreated, @@ -81,7 +81,7 @@ extension NewLambdaRuntime { preconditionFailure("Invalid state: \(self.state)") } } - + mutating func handlerCreated(_ handler: Handler) -> Action { switch self.state { case .initialized, @@ -92,20 +92,20 @@ extension NewLambdaRuntime { .reportingInvocationResult, .reportingStartupError: preconditionFailure("Invalid state: \(self.state)") - + case .starting(let promise): self.state = .handlerCreated(handler, promise) return .none - + case .connected(let connection, let promise): self.state = .waitingForInvocation(connection, handler) return .requestNextInvocation(connection.handler, succeedStartPromise: promise) - + case .failed: return .none } } - + mutating func handlerCreationFailed(_ error: Error) -> Action { switch self.state { case .initialized, @@ -116,20 +116,20 @@ extension NewLambdaRuntime { .reportingInvocationResult, .reportingStartupError: preconditionFailure("Invalid state: \(self.state)") - + case .starting(let promise): self.state = .handlerCreationFailed(error, promise) return .none - + case .connected(let connection, let promise): self.state = .reportingStartupError(connection, error, promise) return .reportStartupError(error, connection.handler) - + case .failed: return .none } } - + mutating func httpConnectionCreated( _ connection: Connection ) -> Action { @@ -141,24 +141,24 @@ extension NewLambdaRuntime { .reportingInvocationResult, .reportingStartupError: preconditionFailure("Invalid state: \(self.state)") - + case .starting(let promise): self.state = .connected(connection, promise) return .none - + case .handlerCreated(let handler, let promise): self.state = .waitingForInvocation(connection, handler) return .requestNextInvocation(connection.handler, succeedStartPromise: promise) - + case .handlerCreationFailed(let error, let promise): self.state = .reportingStartupError(connection, error, promise) return .reportStartupError(error, connection.handler) - + case .failed: return .none } } - + mutating func httpChannelConnectFailed(_ error: Error) -> Action { switch self.state { case .initialized, @@ -168,24 +168,24 @@ extension NewLambdaRuntime { .reportingInvocationResult, .reportingStartupError: preconditionFailure("Invalid state: \(self.state)") - + case .starting(let promise): self.state = .failed(error) return .failRuntime(error, startPomise: promise) - + case .handlerCreated(_, let promise): self.state = .failed(error) return .failRuntime(error, startPomise: promise) - + case .handlerCreationFailed(let error, let promise): self.state = .failed(error) return .failRuntime(error, startPomise: promise) - + case .failed: return .none } } - + mutating func newInvocationReceived(_ invocation: Invocation, _ body: ByteBuffer) -> Action { switch self.state { case .initialized, @@ -197,16 +197,16 @@ extension NewLambdaRuntime { .reportingInvocationResult, .reportingStartupError: preconditionFailure("Invalid state: \(self.state)") - + case .waitingForInvocation(let connection, let handler): - self.state = .executingInvocation(connection, handler, .init(uuidString: invocation.requestID)!) + self.state = .executingInvocation(connection, handler, LambdaRequestID(uuidString: invocation.requestID)!) return .invokeHandler(handler, invocation, body) - + case .failed: return .none } } - + mutating func acceptedReceived() -> Action { switch self.state { case .initialized, @@ -216,27 +216,27 @@ extension NewLambdaRuntime { .handlerCreationFailed, .executingInvocation: preconditionFailure("Invalid state: \(self.state)") - + case .waitingForInvocation: preconditionFailure("TODO: fixme") - + case .reportingStartupError(_, let error, let promise): self.state = .failed(error) return .failRuntime(error, startPomise: promise) - + case .reportingInvocationResult(let connection, let handler, true): self.state = .waitingForInvocation(connection, handler) return .none - + case .reportingInvocationResult(let connection, let handler, false): self.state = .waitingForInvocation(connection, handler) return .requestNextInvocation(connection.handler, succeedStartPromise: nil) - + case .failed: return .none } } - + mutating func errorResponseReceived(_ errorResponse: ErrorResponse) -> Action { switch self.state { case .initialized, @@ -246,49 +246,45 @@ extension NewLambdaRuntime { .handlerCreationFailed, .executingInvocation: preconditionFailure("Invalid state: \(self.state)") - + case .waitingForInvocation: let error = LambdaRuntimeError.controlPlaneErrorResponse(errorResponse) self.state = .failed(error) return .failRuntime(error, startPomise: nil) - + case .reportingStartupError(_, let error, let promise): self.state = .failed(error) return .failRuntime(error, startPomise: promise) - + case .reportingInvocationResult: let error = LambdaRuntimeError.controlPlaneErrorResponse(errorResponse) self.state = .failed(error) return .failRuntime(error, startPomise: nil) - + case .failed: return .none } } - - mutating func handlerError(_ error: Error) { - - } - - mutating func channelInactive() { - - } - + + mutating func handlerError(_: Error) {} + + mutating func channelInactive() {} + mutating func invocationFinished(_ result: Result) -> Action { switch self.state { case .initialized, - .starting, - .handlerCreated, - .handlerCreationFailed, - .connected, - .waitingForInvocation, - .reportingStartupError, - .reportingInvocationResult: + .starting, + .handlerCreated, + .handlerCreationFailed, + .connected, + .waitingForInvocation, + .reportingStartupError, + .reportingInvocationResult: preconditionFailure("Invalid state: \(self.state)") - + case .failed: return .none - + case .executingInvocation(let connection, let handler, let requestID): let pipelining = true self.state = .reportingInvocationResult(connection, handler, nextInvocationRequestPipelined: pipelining) diff --git a/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift b/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift index 6f940c3b..128ec5cc 100644 --- a/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift +++ b/Sources/AWSLambdaRuntimeCore/NewLambdaChannelHandler.swift @@ -15,50 +15,48 @@ import NIOCore protocol LambdaChannelHandlerDelegate { - func responseReceived(_: ControlPlaneResponse) - + func errorCaught(_: Error) - + func channelInactive() - } final class NewLambdaChannelHandler: ChannelInboundHandler { typealias InboundIn = ByteBuffer typealias OutboundOut = ByteBuffer - + private let delegate: Delegate private var requestsInFlight: CircularBuffer - + private var context: ChannelHandlerContext! - + private var encoder: ControlPlaneRequestEncoder private var decoder: NIOSingleStepByteToMessageProcessor - + init(delegate: Delegate, host: String) { self.delegate = delegate self.requestsInFlight = CircularBuffer(initialCapacity: 4) - + self.encoder = ControlPlaneRequestEncoder(host: host) self.decoder = NIOSingleStepByteToMessageProcessor(ControlPlaneResponseDecoder(), maximumBufferSize: 7 * 1024 * 1024) } - + func sendRequest(_ request: ControlPlaneRequest) { self.requestsInFlight.append(request) self.encoder.writeRequest(request, context: self.context, promise: nil) } - + func handlerAdded(context: ChannelHandlerContext) { self.context = context self.encoder.writerAdded(context: context) } - + func handlerRemoved(context: ChannelHandlerContext) { self.context = context self.encoder.writerRemoved(context: context) } - + func channelRead(context: ChannelHandlerContext, data: NIOAny) { do { let buffer = self.unwrapInboundIn(data) @@ -66,18 +64,18 @@ final class NewLambdaChannelHandler: Cha guard self.requestsInFlight.popFirst() != nil else { throw LambdaRuntimeError.unsolicitedResponse } - + self.delegate.responseReceived(response) } } catch { self.delegate.errorCaught(error) } } - + func channelInactive(context: ChannelHandlerContext) { self.delegate.channelInactive() } - + func errorCaught(context: ChannelHandlerContext, error: Error) { self.delegate.errorCaught(error) } diff --git a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift index c6cac746..4f84808f 100644 --- a/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift +++ b/Sources/AWSLambdaRuntimeCore/NewLambdaRuntime.swift @@ -12,11 +12,11 @@ // //===----------------------------------------------------------------------===// +import Backtrace import Logging import NIOConcurrencyHelpers import NIOCore import NIOPosix -import Backtrace #if canImport(Glibc) import Glibc @@ -36,8 +36,7 @@ public final class NewLambdaRuntime { init(eventLoop: EventLoop, logger: Logger, configuration: Lambda.Configuration, - handlerType: Handler.Type - ) { + handlerType: Handler.Type) { self.state = StateMachine() self.eventLoop = eventLoop self.shutdownPromise = eventLoop.makePromise(of: Void.self) @@ -66,7 +65,6 @@ public final class NewLambdaRuntime { return promise.futureResult } - public func start(promise: EventLoopPromise?) { if self.eventLoop.inEventLoop { self.start0(promise: promise) @@ -76,7 +74,7 @@ public final class NewLambdaRuntime { } } } - + public func __testOnly_start(channel: Channel, promise: EventLoopPromise?) { precondition(channel.eventLoop === self.eventLoop, "Channel must be created on the supplied EventLoop.") if self.eventLoop.inEventLoop { @@ -87,7 +85,7 @@ public final class NewLambdaRuntime { } } } - + /// Begin the `LambdaRuntime` shutdown. Only needed for debugging purposes, hence behind a `DEBUG` flag. public func shutdown(promise: EventLoopPromise?) { if self.eventLoop.inEventLoop { @@ -98,30 +96,28 @@ public final class NewLambdaRuntime { } } } - + // MARK: - Private - - + private func start0(promise: EventLoopPromise?) { self.eventLoop.assertInEventLoop() // when starting we want to do thing in parallel: // 1. start the connection to the control plane // 2. create the lambda handler - + self.logger.debug("initializing lambda") - + let action = self.state.start(connection: nil, promise: promise) self.run(action) } - - private func shutdown0(promise: EventLoopPromise?) { - - } - + + private func shutdown0(promise: EventLoopPromise?) {} + private func __testOnly_start0(channel: Channel, promise: EventLoopPromise?) { channel.eventLoop.preconditionInEventLoop() assert(channel.isActive) - + do { let connection = try self.setupConnection(channel: channel) let action = self.state.start(connection: connection, promise: promise) @@ -130,7 +126,7 @@ public final class NewLambdaRuntime { promise?.fail(error) } } - + private func run(_ action: StateMachine.Action) { switch action { case .createHandler(andConnection: let andConnection): @@ -138,7 +134,7 @@ public final class NewLambdaRuntime { if andConnection { self.createConnection() } - + case .invokeHandler(let handler, let invocation, let event): self.logger.trace("invoking handler") let context = LambdaContext( @@ -151,47 +147,46 @@ public final class NewLambdaRuntime { let action = self.state.invocationFinished(result) self.run(action) } - + case .failRuntime(let error, let startPromise): startPromise?.fail(error) self.shutdownPromise.fail(error) - + case .requestNextInvocation(let handler, let startPromise): self.logger.trace("requesting next invocation") handler.sendRequest(.next) startPromise?.succeed(()) - + case .reportInvocationResult(let requestID, let result, let pipelineNextInvocationRequest, let handler): switch result { case .success(let body): self.logger.trace("reporting invocation success", metadata: [ - "lambda-request-id": "\(requestID)" + "lambda-request-id": "\(requestID)", ]) handler.sendRequest(.invocationResponse(requestID, body)) - + case .failure(let error): self.logger.trace("reporting invocation failure", metadata: [ - "lambda-request-id": "\(requestID)" + "lambda-request-id": "\(requestID)", ]) let errorString = String(describing: error) let errorResponse = ErrorResponse(errorType: errorString, errorMessage: errorString) handler.sendRequest(.invocationError(requestID, errorResponse)) } - + if pipelineNextInvocationRequest { handler.sendRequest(.next) } - + case .reportStartupError(let error, let handler): let errorString = String(describing: error) handler.sendRequest(.initializationError(.init(errorType: errorString, errorMessage: errorString))) - + case .none: break - } } - + private func createConnection() { let connectFuture = ClientBootstrap(group: self.eventLoop).connect( host: self.configuration.runtimeEngine.ip, @@ -214,20 +209,20 @@ public final class NewLambdaRuntime { self.run(action) } } - + private func setupConnection(channel: Channel) throws -> Connection { let handler = NewLambdaChannelHandler(delegate: self, host: self.configuration.runtimeEngine.ip) try channel.pipeline.syncOperations.addHandler(handler) return Connection(channel: channel, handler: handler) } - + private func createHandler() { let context = Lambda.InitializationContext( logger: self.logger, eventLoop: self.eventLoop, allocator: ByteBufferAllocator() ) - + Handler.makeHandler(context: context).hop(to: self.eventLoop).whenComplete { result in let action: StateMachine.Action switch result { @@ -254,24 +249,23 @@ extension NewLambdaRuntime: LambdaChannelHandlerDelegate { case .error(let errorResponse): action = self.state.errorResponseReceived(errorResponse) } - + self.run(action) } - + func errorCaught(_ error: Error) { self.state.handlerError(error) } - + func channelInactive() { self.state.channelInactive() } } extension NewLambdaRuntime { - static func run(handlerType: Handler.Type) { Backtrace.install() - + let configuration = Lambda.Configuration() var logger = Logger(label: "Lambda") logger.logLevel = configuration.general.logLevel @@ -285,17 +279,17 @@ extension NewLambdaRuntime { ) logger.info("lambda runtime starting with \(configuration)") - + #if DEBUG let signalSource = trap(signal: configuration.lifecycle.stopSignal) { signal in logger.info("intercepted signal: \(signal)") runtime.shutdown(promise: nil) } #endif - + runtime.start().flatMap { runtime.shutdownFuture - }.whenComplete { lifecycleResult in + }.whenComplete { _ in #if DEBUG signalSource.cancel() #endif diff --git a/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift b/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift index 660ec53d..f2c9fc33 100644 --- a/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift +++ b/Tests/AWSLambdaRuntimeCoreTests/NewLambdaChannelHandlerTests.swift @@ -12,12 +12,11 @@ // //===----------------------------------------------------------------------===// -import XCTest +@testable import AWSLambdaRuntimeCore import NIOCore import NIOEmbedded import NIOHTTP1 -@testable import AWSLambdaRuntimeCore -import SwiftUI +import XCTest final class NewLambdaChannelHandlerTests: XCTestCase { let host = "192.168.0.1" @@ -30,36 +29,36 @@ final class NewLambdaChannelHandlerTests: XCTestCase { override func setUp() { self.delegate = EmbeddedLambdaChannelHandlerDelegate() self.handler = NewLambdaChannelHandler(delegate: self.delegate, host: "127.0.0.1") - + self.client = EmbeddedChannel(handler: self.handler) self.server = EmbeddedChannel(handlers: [ NIOHTTPServerRequestAggregator(maxContentLength: 1024 * 1024), ]) - + XCTAssertNoThrow(try self.server.pipeline.syncOperations.configureHTTPServerPipeline(position: .first)) - + XCTAssertNoThrow(try self.server.bind(to: .init(ipAddress: "127.0.0.1", port: 0), promise: nil)) XCTAssertNoThrow(try self.client.connect(to: .init(ipAddress: "127.0.0.1", port: 0), promise: nil)) } - + func testPipelineRequests() { self.handler.sendRequest(.next) - + self.assertInteract() - + var nextRequest: NIOHTTPServerRequestFull? XCTAssertNoThrow(nextRequest = try self.server.readInbound(as: NIOHTTPServerRequestFull.self)) XCTAssertEqual(nextRequest?.head.uri, "/2018-06-01/runtime/invocation/next") XCTAssertEqual(nextRequest?.head.method, .GET) - + XCTAssertNil(try self.server.readInbound(as: NIOHTTPServerRequestFull.self)) - + let requestID = LambdaRequestID() let traceID = "foo" let functionARN = "arn" let deadline = UInt(Date().timeIntervalSince1970 * 1000) + 3000 let requestBody = ByteBuffer(string: "foo bar") - + XCTAssertNoThrow(try self.server.writeOutboundInvocation( requestID: requestID, traceID: traceID, @@ -67,37 +66,37 @@ final class NewLambdaChannelHandlerTests: XCTestCase { deadline: deadline, body: requestBody )) - + self.assertInteract() - + var response: (Invocation, ByteBuffer)? XCTAssertNoThrow(response = try self.delegate.readNextResponse()) - + XCTAssertEqual(response?.0.requestID, requestID.lowercased) XCTAssertEqual(response?.0.traceID, traceID) XCTAssertEqual(response?.0.invokedFunctionARN, functionARN) XCTAssertEqual(response?.0.deadlineInMillisSinceEpoch, Int64(deadline)) XCTAssertEqual(response?.1, requestBody) - + let responseBody = ByteBuffer(string: "hello world") - + self.handler.sendRequest(.invocationResponse(requestID, responseBody)) - + self.assertInteract() - + var responseRequest: NIOHTTPServerRequestFull? XCTAssertNoThrow(responseRequest = try self.server.readInbound(as: NIOHTTPServerRequestFull.self)) XCTAssertEqual(responseRequest?.head.uri, "/2018-06-01/runtime/invocation/\(requestID.lowercased)/response") XCTAssertEqual(responseRequest?.head.method, .POST) XCTAssertEqual(responseRequest?.body, responseBody) } - + func assertInteract(file: StaticString = #file, line: UInt = #line) { XCTAssertNoThrow(try { while let clientBuffer = try self.client.readOutbound(as: ByteBuffer.self) { try self.server.writeInbound(clientBuffer) } - + while let serverBuffer = try self.server.readOutbound(as: ByteBuffer.self) { try self.client.writeInbound(serverBuffer) } @@ -106,77 +105,76 @@ final class NewLambdaChannelHandlerTests: XCTestCase { } final class EmbeddedLambdaChannelHandlerDelegate: LambdaChannelHandlerDelegate { - enum Error: Swift.Error { case missingEvent case wrongEventType case wrongResponseType } - + private enum Event { case channelInactive case error(Swift.Error) case response(ControlPlaneResponse) } - + private var events: CircularBuffer - + init() { self.events = CircularBuffer(initialCapacity: 8) } - + func channelInactive() { self.events.append(.channelInactive) } - + func errorCaught(_ error: Swift.Error) { self.events.append(.error(error)) } - + func responseReceived(_ response: ControlPlaneResponse) { self.events.append(.response(response)) } - + func readResponse() throws -> ControlPlaneResponse { guard case .response(let response) = try self.popNextEvent() else { throw Error.wrongEventType } return response } - + func readNextResponse() throws -> (Invocation, ByteBuffer) { guard case .next(let invocation, let body) = try self.readResponse() else { throw Error.wrongResponseType } return (invocation, body) } - + func assertAcceptResponse() throws { guard case .accepted = try self.readResponse() else { throw Error.wrongResponseType } } - + func readErrorResponse() throws -> ErrorResponse { guard case .error(let errorResponse) = try self.readResponse() else { throw Error.wrongResponseType } return errorResponse } - + func readError() throws -> Swift.Error { guard case .error(let error) = try self.popNextEvent() else { throw Error.wrongEventType } return error } - + func assertChannelInactive() throws { guard case .channelInactive = try self.popNextEvent() else { throw Error.wrongEventType } } - + private func popNextEvent() throws -> Event { guard let event = self.events.popFirst() else { throw Error.missingEvent @@ -186,7 +184,6 @@ final class EmbeddedLambdaChannelHandlerDelegate: LambdaChannelHandlerDelegate { } extension EmbeddedChannel { - func writeOutboundInvocation( requestID: LambdaRequestID = LambdaRequestID(), traceID: String = "Root=\(DispatchTime.now().uptimeNanoseconds);Parent=\(DispatchTime.now().uptimeNanoseconds);Sampled=1", @@ -202,10 +199,10 @@ extension EmbeddedChannel { "lambda-runtime-deadline-ms": "\(deadline)", "lambda-runtime-trace-id": "\(traceID)", "lambda-runtime-aws-request-id": "\(requestID)", - "lambda-runtime-invoked-function-arn": "\(functionARN)" + "lambda-runtime-invoked-function-arn": "\(functionARN)", ] ) - + try self.writeOutbound(HTTPServerResponsePart.head(head)) if let body = body { try self.writeOutbound(HTTPServerResponsePart.body(.byteBuffer(body)))