diff --git a/Package.swift b/Package.swift index f640bd7e..469ed01e 100644 --- a/Package.swift +++ b/Package.swift @@ -31,7 +31,8 @@ let package = Package( dependencies: ["_CAsyncSequenceValidationSupport"], swiftSettings: [ .unsafeFlags([ - "-Xfrontend", "-disable-availability-checking" + "-Xfrontend", "-disable-availability-checking", + "-Xfrontend", "-enable-experimental-pairwise-build-block" ]) ]), .systemLibrary(name: "_CAsyncSequenceValidationSupport"), @@ -40,7 +41,8 @@ let package = Package( dependencies: ["AsyncAlgorithms", "AsyncSequenceValidation"], swiftSettings: [ .unsafeFlags([ - "-Xfrontend", "-disable-availability-checking" + "-Xfrontend", "-disable-availability-checking", + "-Xfrontend", "-enable-experimental-pairwise-build-block" ]) ]), ] diff --git a/Sources/AsyncSequenceValidation/AsyncSequenceValidationDiagram.swift b/Sources/AsyncSequenceValidation/AsyncSequenceValidationDiagram.swift index ecdf6d30..b88f43f3 100644 --- a/Sources/AsyncSequenceValidation/AsyncSequenceValidationDiagram.swift +++ b/Sources/AsyncSequenceValidation/AsyncSequenceValidationDiagram.swift @@ -13,49 +13,46 @@ import _CAsyncSequenceValidationSupport @resultBuilder public struct AsyncSequenceValidationDiagram : Sendable { - public static func buildBlock( - _ sequence: Operation, - _ output: String - ) -> some AsyncSequenceValidationTest where Operation.Element == String { - return Test(inputs: [], sequence: sequence, output: output) + public struct Component { + var component: T + var location: SourceLocation } - public static func buildBlock( - _ input: String, - _ sequence: Operation, - _ output: String - ) -> some AsyncSequenceValidationTest where Operation.Element == String { - return Test(inputs: [input], sequence: sequence, output: output) + public struct AccumulatedInputs { + var inputs: [Specification] = [] } - public static func buildBlock( - _ input1: String, - _ input2: String, - _ sequence: Operation, - _ output: String - ) -> some AsyncSequenceValidationTest where Operation.Element == String { - Test(inputs: [input1, input2], sequence: sequence, output: output) + public struct AccumulatedInputsWithOperation where Operation.Element == String { + var inputs: [Specification] + var operation: Operation } - public static func buildBlock( - _ input1: String, - _ input2: String, - _ input3: String, - _ sequence: Operation, - _ output: String - ) -> some AsyncSequenceValidationTest where Operation.Element == String { - Test(inputs: [input1, input2, input3], sequence: sequence, output: output) + public static func buildExpression(_ expr: String, file: StaticString = #file, line: UInt = #line) -> Component { + Component(component: expr, location: SourceLocation(file: file, line: line)) } - - public static func buildBlock( - _ input1: String, - _ input2: String, - _ input3: String, - _ input4: String, - _ sequence: Operation, - _ output: String - ) -> some AsyncSequenceValidationTest where Operation.Element == String { - Test(inputs: [input1, input2, input3, input4], sequence: sequence, output: output) + + public static func buildExpression(_ expr: S, file: StaticString = #file, line: UInt = #line) -> Component { + Component(component: expr, location: SourceLocation(file: file, line: line)) + } + + public static func buildPartialBlock(first input: Component) -> AccumulatedInputs { + return AccumulatedInputs(inputs: [Specification(specification: input.component, location: input.location)]) + } + + public static func buildPartialBlock(first operation: Component) -> AccumulatedInputsWithOperation where Operation.Element == String { + return AccumulatedInputsWithOperation(inputs: [], operation: operation.component) + } + + public static func buildPartialBlock(accumulated: AccumulatedInputs, next input: Component) -> AccumulatedInputs { + return AccumulatedInputs(inputs: accumulated.inputs + [Specification(specification: input.component, location: input.location)]) + } + + public static func buildPartialBlock(accumulated: AccumulatedInputs, next operation: Component) -> AccumulatedInputsWithOperation { + return AccumulatedInputsWithOperation(inputs: accumulated.inputs, operation: operation.component) + } + + public static func buildPartialBlock(accumulated: AccumulatedInputsWithOperation, next output: Component) -> some AsyncSequenceValidationTest { + return Test(inputs: accumulated.inputs, sequence: accumulated.operation, output: Specification(specification: output.component, location: output.location)) } let queue: WorkQueue diff --git a/Sources/AsyncSequenceValidation/Event.swift b/Sources/AsyncSequenceValidation/Event.swift index 45e5d9d1..70101475 100644 --- a/Sources/AsyncSequenceValidation/Event.swift +++ b/Sources/AsyncSequenceValidation/Event.swift @@ -12,10 +12,18 @@ extension AsyncSequenceValidationDiagram { struct Failure: Error, Equatable { } - enum ParseFailure: Error, CustomStringConvertible { - case stepInGroup(String, String.Index) - case nestedGroup(String, String.Index) - case unbalancedNesting(String, String.Index) + enum ParseFailure: Error, CustomStringConvertible, SourceFailure { + case stepInGroup(String, String.Index, SourceLocation) + case nestedGroup(String, String.Index, SourceLocation) + case unbalancedNesting(String, String.Index, SourceLocation) + + var location: SourceLocation { + switch self { + case .stepInGroup(_, _, let location): return location + case .nestedGroup(_, _, let location): return location + case .unbalancedNesting(_, _, let location): return location + } + } var description: String { switch self { @@ -56,7 +64,7 @@ extension AsyncSequenceValidationDiagram { } } - static func parse(_ dsl: String, theme: Theme) throws -> [(Clock.Instant, Event)] { + static func parse(_ dsl: String, theme: Theme, location: SourceLocation) throws -> [(Clock.Instant, Event)] { var emissions = [(Clock.Instant, Event)]() var when = Clock.Instant(when: .steps(0)) var string: String? @@ -70,7 +78,7 @@ extension AsyncSequenceValidationDiagram { if grouping == 0 { when = when.advanced(by: .steps(1)) } else { - throw ParseFailure.stepInGroup(dsl, index) + throw ParseFailure.stepInGroup(dsl, index, location) } } else { string?.append(ch) @@ -125,13 +133,13 @@ extension AsyncSequenceValidationDiagram { if grouping == 0 { when = when.advanced(by: .steps(1)) } else { - throw ParseFailure.nestedGroup(dsl, index) + throw ParseFailure.nestedGroup(dsl, index, location) } grouping += 1 case .endGroup: grouping -= 1 if grouping < 0 { - throw ParseFailure.unbalancedNesting(dsl, index) + throw ParseFailure.unbalancedNesting(dsl, index, location) } case .skip: string?.append(ch) @@ -148,7 +156,7 @@ extension AsyncSequenceValidationDiagram { } } if grouping != 0 { - throw ParseFailure.unbalancedNesting(dsl, dsl.endIndex) + throw ParseFailure.unbalancedNesting(dsl, dsl.endIndex, location) } return emissions } diff --git a/Sources/AsyncSequenceValidation/Expectation.swift b/Sources/AsyncSequenceValidation/Expectation.swift index 6b7dbda2..0f45ac5d 100644 --- a/Sources/AsyncSequenceValidation/Expectation.swift +++ b/Sources/AsyncSequenceValidation/Expectation.swift @@ -11,7 +11,12 @@ extension AsyncSequenceValidationDiagram { public struct ExpectationResult: Sendable { - public var expected: [(Clock.Instant, Result)] + public struct Event: Sendable { + public var when: Clock.Instant + public var result: Result + public var offset: String.Index + } + public var expected: [Event] public var actual: [(Clock.Instant, Result)] func reconstitute(_ result: Result, theme: Theme) -> String { @@ -61,7 +66,9 @@ extension AsyncSequenceValidationDiagram { var events = [Clock.Instant : [Result]]() var end: Clock.Instant = Clock.Instant(when: .zero) - for (when, result) in expected { + for expectation in expected { + let when = expectation.when + let result = expectation.result events[when, default: []].append(result) if when > end { end = when @@ -108,6 +115,16 @@ extension AsyncSequenceValidationDiagram { public var when: Clock.Instant public var kind: Kind + public var specification: Specification? + public var index: String.Index? + + init(when: Clock.Instant, kind: Kind, specification: Specification? = nil, index: String.Index? = nil) { + self.when = when + self.kind = kind + self.specification = specification + self.index = index + } + var reason: String { switch kind { case .expectedFinishButGotValue(let actual): diff --git a/Sources/AsyncSequenceValidation/Input.swift b/Sources/AsyncSequenceValidation/Input.swift index 5013307a..35ede268 100644 --- a/Sources/AsyncSequenceValidation/Input.swift +++ b/Sources/AsyncSequenceValidation/Input.swift @@ -10,6 +10,16 @@ //===----------------------------------------------------------------------===// extension AsyncSequenceValidationDiagram { + public struct Specification: Sendable { + public let specification: String + public let location: SourceLocation + + init(specification: String, location: SourceLocation) { + self.specification = specification + self.location = location + } + } + public struct Input: AsyncSequence, Sendable { public typealias Element = String @@ -74,8 +84,8 @@ extension AsyncSequenceValidationDiagram { Iterator(state: state, queue: queue, index: index) } - func parse(_ dsl: String, theme: Theme) throws { - let emissions = try Event.parse(dsl, theme: theme) + func parse(_ dsl: String, theme: Theme, location: SourceLocation) throws { + let emissions = try Event.parse(dsl, theme: theme, location: location) state.withCriticalRegion { state in state.emissions = emissions } diff --git a/Sources/AsyncSequenceValidation/SourceLocation.swift b/Sources/AsyncSequenceValidation/SourceLocation.swift new file mode 100644 index 00000000..f2bfa17c --- /dev/null +++ b/Sources/AsyncSequenceValidation/SourceLocation.swift @@ -0,0 +1,23 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift Async Algorithms open source project +// +// Copyright (c) 2022 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// +//===----------------------------------------------------------------------===// + +public struct SourceLocation: Sendable, CustomStringConvertible { + public var file: StaticString + public var line: UInt + + public var description: String { + return "\(file):\(line)" + } +} + +public protocol SourceFailure: Error { + var location: SourceLocation { get } +} diff --git a/Sources/AsyncSequenceValidation/Test.swift b/Sources/AsyncSequenceValidation/Test.swift index ce232b93..95bd43e9 100644 --- a/Sources/AsyncSequenceValidation/Test.swift +++ b/Sources/AsyncSequenceValidation/Test.swift @@ -19,19 +19,19 @@ internal func _swiftJobRun( ) -> () public protocol AsyncSequenceValidationTest: Sendable { - var inputs: [String] { get } - var output: String { get } + var inputs: [AsyncSequenceValidationDiagram.Specification] { get } + var output: AsyncSequenceValidationDiagram.Specification { get } - func test(with clock: C, activeTicks: [C.Instant], _ event: (String) -> Void) async throws + func test(with clock: C, activeTicks: [C.Instant], output: AsyncSequenceValidationDiagram.Specification, _ event: (String) -> Void) async throws } extension AsyncSequenceValidationDiagram { struct Test: AsyncSequenceValidationTest, @unchecked Sendable where Operation.Element == String { - let inputs: [String] + let inputs: [Specification] let sequence: Operation - let output: String + let output: Specification - func test(with clock: C, activeTicks: [C.Instant], _ event: (String) -> Void) async throws { + func test(with clock: C, activeTicks: [C.Instant], output: Specification, _ event: (String) -> Void) async throws { var iterator = sequence.makeAsyncIterator() do { for tick in activeTicks { @@ -46,10 +46,18 @@ extension AsyncSequenceValidationDiagram { } do { if let pastEnd = try await iterator.next(){ - Context.specificationFailures.append(ExpectationFailure(when: Context.clock!.now, kind: .specificationViolationGotValueAfterIteration(pastEnd))) + let failure = ExpectationFailure( + when: Context.clock!.now, + kind: .specificationViolationGotValueAfterIteration(pastEnd), + specification: output) + Context.specificationFailures.append(failure) } } catch { - Context.specificationFailures.append(ExpectationFailure(when: Context.clock!.now, kind: .specificationViolationGotFailureAfterIteration(error))) + let failure = ExpectationFailure( + when: Context.clock!.now, + kind: .specificationViolationGotFailureAfterIteration(error), + specification: output) + Context.specificationFailures.append(failure) } } catch { throw error @@ -99,9 +107,10 @@ extension AsyncSequenceValidationDiagram { } static func validate( - output: String, + inputs: [Specification], + output: Specification, theme: Theme, - expected: [(Clock.Instant, Result)], + expected: [ExpectationResult.Event], actual: [(Clock.Instant, Result)] ) -> (ExpectationResult, [ExpectationFailure]) { let result = ExpectationResult(expected: expected, actual: actual) @@ -109,13 +118,13 @@ extension AsyncSequenceValidationDiagram { Context.specificationFailures.removeAll() let actualTimes = actual.map { when, _ in when } - let expectedTimes = expected.map { when, _ in when } + let expectedTimes = expected.map { $0.when } - var expectedMap = [Clock.Instant: [Result]]() + var expectedMap = [Clock.Instant: [ExpectationResult.Event]]() var actualMap = [Clock.Instant: [Result]]() - for (when, result) in expected { - expectedMap[when, default: []].append(result) + for event in expected { + expectedMap[event.when, default: []].append(event) } for (when, result) in actual { @@ -128,27 +137,32 @@ extension AsyncSequenceValidationDiagram { let actualResults = actualMap[when] ?? [] var expectedIterator = expectedResults.makeIterator() var actualIterator = actualResults.makeIterator() - while let expectedResult = expectedIterator.next() { + while let expectedEvent = expectedIterator.next() { let actualResult = ActualResult(actualIterator.next()) - switch (expectedResult, actualResult) { + switch (expectedEvent.result, actualResult) { case (.success(let expected), .success(let actual)): switch (expected, actual) { case (.some(let expected), .some(let actual)): if expected != actual { let failure = ExpectationFailure( when: when, - kind: .expectedMismatch(expected, actual)) + kind: .expectedMismatch(expected, actual), + specification: output, + index: expectedEvent.offset) failures.append(failure) } case (.none, .some(let actual)): let failure = ExpectationFailure( when: when, - kind: .expectedFinishButGotValue(actual)) + kind: .expectedFinishButGotValue(actual), + specification: output) failures.append(failure) case (.some(let expected), .none): let failure = ExpectationFailure( when: when, - kind: .expectedValueButGotFinished(expected)) + kind: .expectedValueButGotFinished(expected), + specification: output, + index: expectedEvent.offset) failures.append(failure) case (.none, .none): break @@ -157,12 +171,16 @@ extension AsyncSequenceValidationDiagram { if let expected = expected { let failure = ExpectationFailure( when: when, - kind: .expectedValueButGotFailure(expected, actual)) + kind: .expectedValueButGotFailure(expected, actual), + specification: output, + index: expectedEvent.offset) failures.append(failure) } else { let failure = ExpectationFailure( when: when, - kind: .expectedFinishButGotFailure(actual)) + kind: .expectedFinishButGotFailure(actual), + specification: output, + index: expectedEvent.offset) failures.append(failure) } case (.success(let expected), .none): @@ -170,24 +188,32 @@ extension AsyncSequenceValidationDiagram { case .some(let expected): let failure = ExpectationFailure( when: when, - kind: .expectedValue(expected)) + kind: .expectedValue(expected), + specification: output, + index: expectedEvent.offset) failures.append(failure) case .none: let failure = ExpectationFailure( when: when, - kind: .expectedFinish) + kind: .expectedFinish, + specification: output, + index: expectedEvent.offset) failures.append(failure) } case (.failure(let expected), .success(let actual)): if let actual = actual { let failure = ExpectationFailure( when: when, - kind: .expectedFailureButGotValue(expected, actual)) + kind: .expectedFailureButGotValue(expected, actual), + specification: output, + index: expectedEvent.offset) failures.append(failure) } else { let failure = ExpectationFailure( when: when, - kind: .expectedFailureButGotFinish(expected)) + kind: .expectedFailureButGotFinish(expected), + specification: output, + index: expectedEvent.offset) failures.append(failure) } case (.failure, .failure): @@ -195,7 +221,9 @@ extension AsyncSequenceValidationDiagram { case (.failure(let expected), .none): let failure = ExpectationFailure( when: when, - kind: .expectedFailure(expected)) + kind: .expectedFailure(expected), + specification: output, + index: expectedEvent.offset) failures.append(failure) } } @@ -206,18 +234,21 @@ extension AsyncSequenceValidationDiagram { case .some(let actual): let failure = ExpectationFailure( when: when, - kind: .unexpectedValue(actual)) + kind: .unexpectedValue(actual), + specification: output) failures.append(failure) case .none: let failure = ExpectationFailure( when: when, - kind: .unexpectedFinish) + kind: .unexpectedFinish, + specification: output) failures.append(failure) } case .failure(let actual): let failure = ExpectationFailure( when: when, - kind: .unexpectedFailure(actual)) + kind: .unexpectedFailure(actual), + specification: output) failures.append(failure) } } @@ -239,10 +270,11 @@ extension AsyncSequenceValidationDiagram { } for (index, input) in diagram.inputs.enumerated() { - try input.parse(test.inputs[index], theme: theme) + let inputSpecification = test.inputs[index] + try input.parse(inputSpecification.specification, theme: theme, location: inputSpecification.location) } - let parsedOutput = try Event.parse(test.output, theme: theme) + let parsedOutput = try Event.parse(test.output.specification, theme: theme, location: test.output.location) let cancelEvents = Set(parsedOutput.filter { when, event in switch event { case .cancel: return true @@ -260,10 +292,10 @@ extension AsyncSequenceValidationDiagram { } } - var expected = [(Clock.Instant, Result)]() + var expected = [ExpectationResult.Event]() for (when, event) in parsedOutput { for result in event.results { - expected.append((when, result)) + expected.append(ExpectationResult.Event(when: when, result: result, offset: event.index)) } } let times = parsedOutput.map { when, _ in when } @@ -283,7 +315,7 @@ extension AsyncSequenceValidationDiagram { let runner = Task { do { - try await test.test(with: clock, activeTicks: activeTicks) { event in + try await test.test(with: clock, activeTicks: activeTicks, output: test.output) { event in actual.withCriticalRegion { values in values.append((clock.now, .success(event))) } @@ -323,6 +355,7 @@ extension AsyncSequenceValidationDiagram { Context.driver = nil return validate( + inputs: test.inputs, output: test.output, theme: theme, expected: expected, diff --git a/Tests/AsyncAlgorithmsTests/Support/ValidationTest.swift b/Tests/AsyncAlgorithmsTests/Support/ValidationTest.swift index 7a212f28..09f0b95f 100644 --- a/Tests/AsyncAlgorithmsTests/Support/ValidationTest.swift +++ b/Tests/AsyncAlgorithmsTests/Support/ValidationTest.swift @@ -15,23 +15,46 @@ import AsyncSequenceValidation extension XCTestCase { public func validate(theme: Theme, @AsyncSequenceValidationDiagram _ build: (AsyncSequenceValidationDiagram) -> Test, file: StaticString = #file, line: UInt = #line) { - let location = XCTSourceCodeLocation(filePath: file.description, lineNumber: Int(line)) - let context = XCTSourceCodeContext(location: location) + let baseLocation = XCTSourceCodeLocation(filePath: file.description, lineNumber: Int(line)) + let baseContext = XCTSourceCodeContext(location: baseLocation) do { let (result, failures) = try AsyncSequenceValidationDiagram.test(theme: theme, build) + var detail: String? if failures.count > 0 { + detail = """ + Expected + \(result.reconstituteExpected(theme: theme)) + Actual + \(result.reconstituteActual(theme: theme)) + """ print("Expected") print(result.reconstituteExpected(theme: theme)) print("Actual") print(result.reconstituteActual(theme: theme)) } for failure in failures { - let issue = XCTIssue(type: .assertionFailure, compactDescription: failure.description, detailedDescription: nil, sourceCodeContext: context, associatedError: nil, attachments: []) - record(issue) + if let specification = failure.specification { + print(specification.location) + let location = XCTSourceCodeLocation(filePath: specification.location.file.description, lineNumber: Int(specification.location.line)) + let context = XCTSourceCodeContext(location: location) + let issue = XCTIssue(type: .assertionFailure, compactDescription: failure.description, detailedDescription: detail, sourceCodeContext: context, associatedError: nil, attachments: []) + print(location) + record(issue) + } else { + let issue = XCTIssue(type: .assertionFailure, compactDescription: failure.description, detailedDescription: detail, sourceCodeContext: baseContext, associatedError: nil, attachments: []) + record(issue) + } } } catch { - let issue = XCTIssue(type: .system, compactDescription: "\(error)", detailedDescription: nil, sourceCodeContext: context, associatedError: nil, attachments: []) - record(issue) + if let sourceFailure = error as? SourceFailure { + let location = XCTSourceCodeLocation(filePath: sourceFailure.location.file.description, lineNumber: Int(sourceFailure.location.line)) + let context = XCTSourceCodeContext(location: location) + let issue = XCTIssue(type: .system, compactDescription: "\(error)", detailedDescription: nil, sourceCodeContext: context, associatedError: nil, attachments: []) + record(issue) + } else { + let issue = XCTIssue(type: .system, compactDescription: "\(error)", detailedDescription: nil, sourceCodeContext: baseContext, associatedError: nil, attachments: []) + record(issue) + } } } diff --git a/Tests/AsyncAlgorithmsTests/TestValidationTests.swift b/Tests/AsyncAlgorithmsTests/TestValidationTests.swift index 1126a8de..2ae3a2fa 100644 --- a/Tests/AsyncAlgorithmsTests/TestValidationTests.swift +++ b/Tests/AsyncAlgorithmsTests/TestValidationTests.swift @@ -232,6 +232,15 @@ final class TestValidationDiagram: XCTestCase { } } + func test_diagram_parse_failure_unbalanced_group_input() { + expectFailures(["validation diagram unbalanced grouping"]) + validate { + "[ab|" + $0.inputs[0] + " ab|" + } + } + func test_diagram_parse_failure_nested_group() { expectFailures(["validation diagram nested grouping"]) validate { @@ -241,6 +250,15 @@ final class TestValidationDiagram: XCTestCase { } } + func test_diagram_parse_failure_nested_group_input() { + expectFailures(["validation diagram nested grouping"]) + validate { + "[[ab|" + $0.inputs[0] + " ab|" + } + } + func test_diagram_parse_failure_step_in_group() { expectFailures(["validation diagram step symbol in group"]) validate { @@ -250,6 +268,15 @@ final class TestValidationDiagram: XCTestCase { } } + func test_diagram_parse_failure_step_in_group_input() { + expectFailures(["validation diagram step symbol in group"]) + validate { + "[a-]b|" + $0.inputs[0] + " ab|" + } + } + func test_diagram_specification_produce_past_end() { expectFailures(["specification violation got \"d\" after iteration terminated at tick 9"]) validate {