From f26f11f1ad5cb8620478c7a9468d44f749da97ef Mon Sep 17 00:00:00 2001 From: David Hart Date: Tue, 18 Feb 2020 13:24:50 +0100 Subject: [PATCH] Add JSONMessageStreamingParser to help parse swiftc & xcbuild --- Sources/TSCUtility/CMakeLists.txt | 8 +- .../JSONMessageStreamingParser.swift | 167 +++++++++++ .../JSONMessageStreamingParserTests.swift | 279 ++++++++++++++++++ 3 files changed, 451 insertions(+), 3 deletions(-) create mode 100644 Sources/TSCUtility/JSONMessageStreamingParser.swift create mode 100644 Tests/TSCUtilityTests/JSONMessageStreamingParserTests.swift diff --git a/Sources/TSCUtility/CMakeLists.txt b/Sources/TSCUtility/CMakeLists.txt index 56920140..a1dbc613 100644 --- a/Sources/TSCUtility/CMakeLists.txt +++ b/Sources/TSCUtility/CMakeLists.txt @@ -13,12 +13,15 @@ add_library(TSCUtility BuildFlags.swift CollectionExtensions.swift Diagnostics.swift + dlopen.swift Downloader.swift - FSWatch.swift FloatingPointExtensions.swift + FSWatch.swift Git.swift IndexStore.swift InterruptHandler.swift + JSONMessageStreamingParser.swift + misc.swift OSLog.swift PkgConfig.swift Platform.swift @@ -30,8 +33,7 @@ add_library(TSCUtility Verbosity.swift Version.swift Versioning.swift - dlopen.swift - misc.swift) +) target_link_libraries(TSCUtility PUBLIC TSCBasic) # NOTE(compnerd) workaround for CMake not setting up include flags yet diff --git a/Sources/TSCUtility/JSONMessageStreamingParser.swift b/Sources/TSCUtility/JSONMessageStreamingParser.swift new file mode 100644 index 00000000..2e0b1f7c --- /dev/null +++ b/Sources/TSCUtility/JSONMessageStreamingParser.swift @@ -0,0 +1,167 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2020 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See http://swift.org/LICENSE.txt for license information + See http://swift.org/CONTRIBUTORS.txt for Swift project authors + */ + +import Foundation + +/// Protocol for the parser delegate to get notified of parsing events. +public protocol JSONMessageStreamingParserDelegate: class { + + /// A decodable type representing the JSON messages being parsed. + associatedtype Message: Decodable + + /// Called for each message parsed. + func jsonMessageStreamingParser(_ parser: JSONMessageStreamingParser, didParse message: Message) + + /// Called when parsing raw text instead of message size. + func jsonMessageStreamingParser(_ parser: JSONMessageStreamingParser, didParseRawText text: String) + + /// Called on an un-expected parsing error. No more events will be received after that. + func jsonMessageStreamingParser(_ parser: JSONMessageStreamingParser, didFailWith error: Error) +} + +/// Streaming parser for JSON messages seperated by integers to represent size of message. Used by the Swift compiler +/// and XCBuild to share progess information: https://github.com/apple/swift/blob/master/docs/DriverParseableOutput.rst. +public final class JSONMessageStreamingParser { + + /// The object representing the JSON message being parsed. + public typealias Message = Delegate.Message + + /// State of the parser state machine. + private enum State { + case parsingMessageSize + case parsingMessage(size: Int) + case parsingNewlineAfterMessage + case failed + } + + /// Delegate to notify of parsing events. + public weak var delegate: Delegate? + + /// Buffer containing the bytes until a full message can be parsed. + private var buffer: [UInt8] = [] + + /// The parser's state machine current state. + private var state: State = .parsingMessageSize + + /// The JSON decoder to parse messages. + private let decoder: JSONDecoder + + /// Initializes the parser. + /// - Parameters: + /// - delegate: The `JSONMessageStreamingParserDelegate` that will receive parsing event callbacks. + /// - decoder: The `JSONDecoder` to use for decoding JSON messages. + public init(delegate: Delegate, decoder: JSONDecoder = JSONDecoder()) + { + self.delegate = delegate + self.decoder = decoder + } + + /// Parse the next bytes of the stream. + /// - Note: If a parsing error is encountered, the delegate will be notified and the parser won't accept any further + /// input. + public func parse(bytes: C) where C: Collection, C.Element == UInt8 { + if case .failed = state { return } + + do { + try parseImpl(bytes: bytes) + } catch { + state = .failed + delegate?.jsonMessageStreamingParser(self, didFailWith: error) + } + } +} + +private extension JSONMessageStreamingParser { + + /// Error corresponding to invalid Swift compiler output. + struct ParsingError: LocalizedError { + + /// Text describing the specific reason for the parsing failure. + let reason: String + + /// The underlying error, if there is one. + let underlyingError: Error? + + var errorDescription: String? { + if let error = underlyingError { + return "\(reason): \(error)" + } else { + return reason + } + } + } + + /// Throwing implementation of the parse function. + func parseImpl(bytes: C) throws where C: Collection, C.Element == UInt8 { + switch state { + case .parsingMessageSize: + if let newlineIndex = bytes.firstIndex(of: newline) { + buffer.append(contentsOf: bytes[.. Message { + let data = Data(buffer) + buffer.removeAll() + state = .parsingNewlineAfterMessage + + do { + return try decoder.decode(Message.self, from: data) + } catch { + throw ParsingError(reason: "unexpected JSON message", underlyingError: error) + } + } +} + +private let newline = UInt8(ascii: "\n") diff --git a/Tests/TSCUtilityTests/JSONMessageStreamingParserTests.swift b/Tests/TSCUtilityTests/JSONMessageStreamingParserTests.swift new file mode 100644 index 00000000..7018780f --- /dev/null +++ b/Tests/TSCUtilityTests/JSONMessageStreamingParserTests.swift @@ -0,0 +1,279 @@ +/* + This source file is part of the Swift.org open source project + + Copyright (c) 2020 Apple Inc. and the Swift project authors + Licensed under Apache License v2.0 with Runtime Library Exception + + See http://swift.org/LICENSE.txt for license information + See http://swift.org/CONTRIBUTORS.txt for Swift project authors + */ + +import XCTest +import TSCTestSupport +import TSCUtility + +class JSONMessageStreamingParserTests: XCTestCase { + func testParse() throws { + let delegate = MockParserDelegate() + let parser = JSONMessageStreamingParser(delegate: delegate) + + parser.parse(bytes: "7".utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: nil) + + parser.parse(bytes: "".utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: nil) + + parser.parse(bytes: """ + 3 + { + "id": 123456, + "type": "error", + + """.utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: nil) + + parser.parse(bytes: "".utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: nil) + + parser.parse(bytes: """ + "message": "This is outrageous!" + } + 78 + + """.utf8) + delegate.assert( + messages: [MockParserDelegate.Message(id: 123456, type: "error", message: "This is outrageous!")], + rawTexts: [], + errorDescription: nil + ) + + parser.parse(bytes: """ + { + "id": 456798, + "type": "warning", + "message": "You should be careful." + } + """.utf8) + delegate.assert( + messages: [MockParserDelegate.Message(id: 456798, type: "warning", message: "You should be careful.")], + rawTexts: [], + errorDescription: nil + ) + + parser.parse(bytes: """ + + 76 + { + "id": 789123, + "type": "note", + "message": "Note to self: buy milk." + } + 64 + { + "id": 456123, + "type": "note", + "message": "...and eggs" + } + 63 + """.utf8) + delegate.assert( + messages: [ + MockParserDelegate.Message(id: 789123, type: "note", message: "Note to self: buy milk."), + MockParserDelegate.Message(id: 456123, type: "note", message: "...and eggs"), + ], + rawTexts: [], + errorDescription: nil + ) + + parser.parse(bytes: """ + + { + "id": 753869, + "type": "error", + "message": "Pancakes!" + } + + """.utf8) + delegate.assert( + messages: [ + MockParserDelegate.Message(id: 753869, type: "error", message: "Pancakes!"), + ], + rawTexts: [], + errorDescription: nil + ) + } + + func testInvalidMessageSizeBytes() { + let delegate = MockParserDelegate() + let parser = JSONMessageStreamingParser(delegate: delegate) + + parser.parse(bytes: [65, 66, 200, 67, UInt8(ascii: "\n")]) + delegate.assert(messages: [], rawTexts: [], errorDescription: "invalid UTF8 bytes") + + parser.parse(bytes: """ + 76 + { + "id": 789123, + "type": "note", + "message": "Note to self: buy milk." + } + """.utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: nil) + } + + func testInvalidMessageSizeValue() { + let delegate = MockParserDelegate() + let parser = JSONMessageStreamingParser(delegate: delegate) + + parser.parse(bytes: """ + 2A + + """.utf8) + delegate.assert(messages: [], rawTexts: ["2A"], errorDescription: nil) + + parser.parse(bytes: """ + 76 + { + "id": 789123, + "type": "note", + "message": "Note to self: buy milk." + } + """.utf8) + delegate.assert( + messages: [MockParserDelegate.Message(id: 789123, type: "note", message: "Note to self: buy milk.")], + rawTexts: [], + errorDescription: nil + ) + } + + func testInvalidMessageBytes() { + let delegate = MockParserDelegate() + let parser = JSONMessageStreamingParser(delegate: delegate) + + parser.parse(bytes: """ + 4 + + """.utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: nil) + parser.parse(bytes: [65, 66, 200, 67, UInt8(ascii: "\n")]) + delegate.assert(messages: [], rawTexts: [], errorDescription: .contains("unexpected JSON message")) + + parser.parse(bytes: """ + 76 + { + "id": 789123, + "type": "note", + "message": "Note to self: buy milk." + } + """.utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: nil) + } + + func testInvalidMessageMissingField() { + let delegate = MockParserDelegate() + let parser = JSONMessageStreamingParser(delegate: delegate) + + parser.parse(bytes: """ + 23 + { + "invalid": "json" + } + """.utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: .contains("unexpected JSON message")) + + parser.parse(bytes: """ + 76 + { + "id": 789123, + "type": "note", + "message": "Note to self: buy milk." + } + """.utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: nil) + } + + func testInvalidMessageInvalidValue() { + let delegate = MockParserDelegate() + let parser = JSONMessageStreamingParser(delegate: delegate) + + parser.parse(bytes: """ + 5 + { + "id": 789123, + "type": "note", + "message": "Note to self: buy milk." + } + """.utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: .contains("unexpected JSON message")) + + parser.parse(bytes: """ + 76 + { + "id": 789123, + "type": "note", + "message": "Note to self: buy milk." + } + """.utf8) + delegate.assert(messages: [], rawTexts: [], errorDescription: nil) + } +} + +private final class MockParserDelegate: JSONMessageStreamingParserDelegate { + struct Message: Equatable, Decodable { + let id: Int + let type: String + let message: String + } + + private var messages: [Message] = [] + private var rawTexts: [String] = [] + private var error: Error? = nil + + func jsonMessageStreamingParser( + _ parser: JSONMessageStreamingParser, + didParse message: Message + ) { + messages.append(message) + } + + func jsonMessageStreamingParser( + _ parser: JSONMessageStreamingParser, + didParseRawText text: String + ) { + rawTexts.append(text) + } + + func jsonMessageStreamingParser( + _ parser: JSONMessageStreamingParser, + didFailWith error: Error + ) { + self.error = error + } + + func assert( + messages: [Message], + rawTexts: [String], + errorDescription: StringPattern?, + file: StaticString = #file, + line: UInt = #line + ) { + XCTAssertEqual(messages, self.messages, file: file, line: line) + XCTAssertEqual(rawTexts, self.rawTexts, file: file, line: line) + + let errorReason = (self.error as? LocalizedError)?.errorDescription ?? error?.localizedDescription + switch (errorReason, errorDescription) { + case (let errorReason?, let errorDescription?): + XCTAssertMatch(errorReason, errorDescription, file: file, line: line) + case (nil, nil): + break + case (let errorReason?, nil): + XCTFail("unexpected error: \(errorReason)") + case (nil, .some): + XCTFail("unexpected success") + } + + self.messages.removeAll() + self.rawTexts.removeAll() + self.error = nil + } +}