diff --git a/pom.xml b/pom.xml index 4f5321a..4bdbe4f 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ com.google.pdsl pdsl - 1.8.1-SNAPSHOT + 1.8.2 pdsl http://www.github.com/google/polymorphicDSL diff --git a/src/main/java/com/pdsl/executors/ColorizedLoggerObserver.java b/src/main/java/com/pdsl/executors/ColorizedLoggerObserver.java new file mode 100644 index 0000000..380d70b --- /dev/null +++ b/src/main/java/com/pdsl/executors/ColorizedLoggerObserver.java @@ -0,0 +1,163 @@ +package com.pdsl.executors; + +import com.pdsl.logging.AnsiTerminalColorHelper; +import com.pdsl.logging.PdslThreadSafeOutputStream; +import com.pdsl.reports.MetadataTestRunResults; +import com.pdsl.specifications.PolymorphicDslTransformationException; +import com.pdsl.testcases.TestCase; +import com.pdsl.testcases.TestSection; +import org.antlr.v4.runtime.tree.ParseTreeListener; +import org.antlr.v4.runtime.tree.ParseTreeVisitor; +import org.antlr.v4.runtime.tree.ParseTreeWalker; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.util.Collection; + +/** + * A logger for the progress of test execution where the output has color. + * + * The underlying implementation uses ANSII escape characters which may not be supported by all terminals. + */ +public class ColorizedLoggerObserver implements ExecutorObserver { + + private static final PdslThreadSafeOutputStream stream = new PdslThreadSafeOutputStream(); + private static final String exceptionMessage = "Could not log!"; + private final Charset charset; + private final byte[] RESET; + + public ColorizedLoggerObserver() { + this.charset = Charset.defaultCharset(); + this.RESET = AnsiTerminalColorHelper.RESET.getBytes(charset); + } + + public ColorizedLoggerObserver(Charset charset) { + this.charset = charset; + this.RESET = AnsiTerminalColorHelper.RESET.getBytes(charset); + } + + private void notifyStreams(InputStream inputStream) { + try { + stream.write(inputStream.readAllBytes()); + } catch (IOException e) { + throw new PolymorphicDslTransformationException(exceptionMessage, e); + } + } + + private void notifyStreams(byte[] bytes) { + try { + stream.write(bytes); + } catch (IOException e) { + throw new PolymorphicDslTransformationException(exceptionMessage, e); + } + } + + private void notifyStreams(String str) { + try { + stream.write(str.getBytes(Charset.defaultCharset())); + } catch (IOException e) { + throw new PolymorphicDslTransformationException(exceptionMessage, e); + } + } + + @Override + public void onBeforeTestCase(TestCase testCase) { + notifyStreams(AnsiTerminalColorHelper.YELLOW + + String.format("%s%n%s%n", testCase.getOriginalSource(), testCase.getTestTitle() + + AnsiTerminalColorHelper.RESET)); + } + + @Override + public void onTestCaseSuccess(TestCase testCase) { + notifyStreams( + (AnsiTerminalColorHelper.GREEN + "All Sentences are parsed." + "\n" + + AnsiTerminalColorHelper.RESET).getBytes(charset)); + } + + private void beforePhrase(TestSection testSection) { + if (testSection.getMetaData().isPresent()) { + notifyStreams(AnsiTerminalColorHelper.CYAN.getBytes(charset)); + notifyStreams(testSection.getMetaData().get()); + notifyStreams(RESET); + } + } + + @Override + public void onBeforePhrase(ParseTreeVisitor visitor, TestSection testSection) { + beforePhrase(testSection); + } + + @Override + public void onAfterPhrase(ParseTreeListener listener, ParseTreeWalker walker, TestSection testSection) { + afterPhrase(testSection); + } + + @Override + public void onAfterPhrase(ParseTreeVisitor visitor, TestSection testSection) { + afterPhrase(testSection); + } + + private void afterPhrase(TestSection testSection) { + notifyStreams( + (AnsiTerminalColorHelper.GREEN + testSection.getPhrase().getParseTree().getText() + "\n" + + AnsiTerminalColorHelper.RESET).getBytes(charset)); + } + + @Override + public void onPhraseFailure(ParseTreeListener listener, TestSection testSection, TestCase testCase, Throwable exception) { + onFailure(testSection); + } + + private void onFailure(TestSection testSection) { + notifyStreams( + (AnsiTerminalColorHelper.BRIGHT_RED + testSection.getPhrase().getParseTree().getText() + "\n" + + AnsiTerminalColorHelper.RESET).getBytes(charset)); + } + + @Override + public void onPhraseFailure(ParseTreeVisitor visitor, TestSection testSection, TestCase testCase, Throwable exception) { + onFailure(testSection); + } + + @Override + public void onBeforeTestSuite(Collection testCases, ParseTreeVisitor visitor, String context) { + logBeforeSuite(); + } + + @Override + public void onBeforeTestSuite(Collection testCases, ParseTreeListener listener, String context) { + logBeforeSuite(); + } + + @Override + public void onAfterTestSuite(Collection testCases, ParseTreeVisitor visitor, MetadataTestRunResults results, String context) { + logAfterSuite(results); + } + + @Override + public void onAfterTestSuite(Collection testCases, ParseTreeListener listener, MetadataTestRunResults results, String context) { + logAfterSuite(results); + } + + @Override + public void onDuplicateSkipped(TestCase testCase) { + notifyStreams(String.format( + "A test was skipped because after filtering it duplicated an earlier run test!%n\t%s", + testCase.getTestTitle())); + } + + private void logAfterSuite(MetadataTestRunResults results) { + if (results.failingTestTotal() == 0) { + notifyStreams(AnsiTerminalColorHelper.BRIGHT_GREEN + "All phrases successfully executed!" + + AnsiTerminalColorHelper.RESET); + } else { + notifyStreams(AnsiTerminalColorHelper.BRIGHT_RED + "There were test failures!" + + AnsiTerminalColorHelper.RESET); + } + } + + private void logBeforeSuite() { + notifyStreams(AnsiTerminalColorHelper.BRIGHT_YELLOW + "Running tests..." + AnsiTerminalColorHelper.RESET); + } +} diff --git a/src/main/java/com/pdsl/executors/DefaultPolymorphicDslTestExecutor.java b/src/main/java/com/pdsl/executors/DefaultPolymorphicDslTestExecutor.java index 2dc9d06..f5aab1b 100644 --- a/src/main/java/com/pdsl/executors/DefaultPolymorphicDslTestExecutor.java +++ b/src/main/java/com/pdsl/executors/DefaultPolymorphicDslTestExecutor.java @@ -14,7 +14,9 @@ import com.pdsl.testcases.SharedTestSuite.SharedTestCaseWithInterpreter; import com.pdsl.testcases.TestCase; import com.pdsl.testcases.TestSection; + import java.util.stream.Collectors; + import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.ParseTreeVisitor; @@ -29,379 +31,340 @@ /** * An executor that runs PDSL tests create from a TestCaseFactory. - * + *

* The default executor has colorized terminal output and prints the phrases as they execute and the * stack trace on failure. - * + *

* All metadata associated with the tests will also be printed. */ -public class DefaultPolymorphicDslTestExecutor implements TraceableTestRunExecutor,ActivePhraseObservable { - - private static final Logger logger = LoggerFactory.getLogger( - DefaultPolymorphicDslTestExecutor.class); - private final ParseTreeWalker walker = new ParseTreeWalker(); - private final Optional outputStreams = Optional.of( - new MultiOutputStream(new PdslThreadSafeOutputStream())); - private static final Charset DEFAULT_CHARSET = Charset.defaultCharset(); - private static final byte[] RESET = AnsiTerminalColorHelper.RESET.getBytes(DEFAULT_CHARSET); - - @Override - public PolymorphicDslTestRunResults runTests(Collection testCases, - ParseTreeListener listener) { - // Walk the phrase registry to make sure all phrases are defined - logger.info( - AnsiTerminalColorHelper.BRIGHT_YELLOW + "Running tests..." + AnsiTerminalColorHelper.RESET); - notifyBeforeTestSuite(testCases, listener, ""); - MetadataTestRunResults results = walk(testCases, new PhraseRegistry(listener), "NONE"); - //send results to notifyAfterTestSuite - notifyAfterTestSuite(testCases, listener,results, ""); - if (results.failingTestTotal() == 0) { - logger.info(AnsiTerminalColorHelper.BRIGHT_GREEN + "All phrases successfully executed!" - + AnsiTerminalColorHelper.RESET); - } else { - logger.error(AnsiTerminalColorHelper.BRIGHT_RED + "There were test failures!" - + AnsiTerminalColorHelper.RESET); +public class DefaultPolymorphicDslTestExecutor implements TraceableTestRunExecutor, ActivePhraseObservable { + + private final ParseTreeWalker walker = new ParseTreeWalker(); + private static final Charset DEFAULT_CHARSET = Charset.defaultCharset(); + private final List activePhraseObservers = new ArrayList<>(); + + /** + * Constructs a DefaultPolymorphicDslTestExecutor with a default ColorizedObserver. + */ + public DefaultPolymorphicDslTestExecutor() { + this.registerObserver(new ColorizedLoggerObserver()); } - return (PolymorphicDslTestRunResults) results; - } - - @Override - public TestRunResults runTests(Collection testCases, - ParseTreeVisitor subgrammarVisitor) { - logger.info( - AnsiTerminalColorHelper.BRIGHT_YELLOW + "Running tests..." + AnsiTerminalColorHelper.RESET); - notifyBeforeTestSuite(testCases, subgrammarVisitor, ""); - MetadataTestRunResults results = walk(testCases, new PhraseRegistry(subgrammarVisitor), "NONE"); - notifyAfterTestSuite(testCases, subgrammarVisitor,results, ""); - if (results.failingTestTotal() == 0) { - logger.info(AnsiTerminalColorHelper.BRIGHT_YELLOW + "All phrases successfully executed!" - + AnsiTerminalColorHelper.RESET); - } else { - logger.error(AnsiTerminalColorHelper.BRIGHT_RED + "There were test failures!" - + AnsiTerminalColorHelper.RESET); + + /** + * Creates a test executor that has all provided observers registered to it. + * + * The observers will be visited in the order they are provided in the list + * @param observers + * @return DefaultPolymorphicDslTestExecutor + */ + public static DefaultPolymorphicDslTestExecutor of(List observers) { + DefaultPolymorphicDslTestExecutor executor = new DefaultPolymorphicDslTestExecutor(); + executor.activePhraseObservers.clear(); // Remove default logger observer + observers.forEach(executor::registerObserver); + return executor; } - return (PolymorphicDslTestRunResults) results; - } - - List activePhraseObservers = new ArrayList<>(); - - @Override - public void registerObserver(ExecutorObserver observer) { - activePhraseObservers.add(observer); - } - - @Override - public void removeObserver(ExecutorObserver observer) { - activePhraseObservers.remove(observer); - } - - private void notifyBeforeListener(ParseTreeListener listener, ParseTreeWalker walker, - Phrase activePhrase) { - activePhraseObservers.forEach(o -> o.onBeforePhrase(listener, walker, activePhrase)); - } - - private void notifyBeforeVisitor(ParseTreeVisitor visitor, - Phrase activePhrase) { - activePhraseObservers.forEach(o -> o.onBeforePhrase(visitor, activePhrase)); - } - - private void notifyAfterListener(ParseTreeListener listener, ParseTreeWalker walker, - Phrase activePhrase) { - activePhraseObservers.forEach(o -> o.onAfterPhrase(listener, walker, activePhrase)); - } - - private void notifyAfterVisitor(ParseTreeVisitor visitor, - Phrase activePhrase) { - activePhraseObservers.forEach(o -> o.onAfterPhrase(visitor, activePhrase)); - } - - private void notifyOnListenerException(ParseTreeListener listener, - Phrase activePhrase, TestCase testCase, Throwable exception) { - activePhraseObservers.forEach(o -> o.onPhraseFailure(listener, activePhrase,testCase, exception)); - } - - private void notifyOnVisitorException(ParseTreeVisitor visitor, - Phrase activePhrase, TestCase testCase, Throwable exception) { - activePhraseObservers.forEach(a -> a.onPhraseFailure(visitor, activePhrase,testCase, exception)); - } - - private void notifyBeforeTestSuite(Collection testCases, ParseTreeVisitor visitor, - String context) { - activePhraseObservers.forEach(a -> a.onBeforeTestSuite(testCases,visitor, context)); - } - private void notifyBeforeTestSuite(Collection testCases, - String context) { - activePhraseObservers.forEach(a -> a.onBeforeTestSuite(testCases, context)); - } - - private void notifyAfterTestSuite(Collection testCases, ParseTreeVisitor visitor, MetadataTestRunResults results, - String context) { - activePhraseObservers.forEach(a -> a.onAfterTestSuite(testCases, visitor, results, context)); - } - - private void notifyBeforeTestSuite(Collection testCases, ParseTreeListener listener, - String context) { - activePhraseObservers.forEach(a -> a.onBeforeTestSuite(testCases, listener, context)); - } - - private void notifyAfterTestSuite(Collection testCases, ParseTreeListener listener, MetadataTestRunResults results, - String context) { - activePhraseObservers.forEach(a -> a.onAfterTestSuite(testCases, listener, results, context)); - } - private void notifyAfterTestSuite(Collection testCases, MetadataTestRunResults results, - String context) { - activePhraseObservers.forEach(a -> a.onAfterTestSuite(testCases, results, context)); - } - /** - * A container for a listener XOR a visitor. - */ - private static final class PhraseRegistry { - - private final Optional listener; - private final Optional> visitor; - - PhraseRegistry(ParseTreeListener listener) { - this.listener = Optional.of(listener); - this.visitor = Optional.empty(); + + @Override + public PolymorphicDslTestRunResults runTests(Collection testCases, + ParseTreeListener listener) { + notifyBeforeTestSuite(testCases, listener, "NONE"); + MetadataTestRunResults results = walk(testCases, new InterpreterObj(listener), "NONE"); + notifyAfterTestSuite(testCases, listener, results, "NONE"); + return (PolymorphicDslTestRunResults) results; } - PhraseRegistry(ParseTreeVisitor visitor) { - this.listener = Optional.empty(); - this.visitor = Optional.of(visitor); + @Override + public TestRunResults runTests(Collection testCases, + ParseTreeVisitor subgrammarVisitor) { + notifyBeforeTestSuite(testCases, subgrammarVisitor,"NONE"); + MetadataTestRunResults results = walk(testCases, new InterpreterObj(subgrammarVisitor), "NONE"); + notifyAfterTestSuite(testCases, subgrammarVisitor, results, "NONE"); + return results; } - } - - private MetadataTestRunResults walk(Collection testCases, PhraseRegistry phraseRegistry, - String context) { - PolymorphicDslTestRunResults results = new PolymorphicDslTestRunResults( - new PdslThreadSafeOutputStream(), context); - Set> previouslyExecutedTests = new HashSet<>(); - - String filterDuplicatesProperty = System.getProperty("pdsl.filterDuplicates"); - boolean filter = filterDuplicatesProperty != null && filterDuplicatesProperty.equalsIgnoreCase("true"); - for (TestCase testCase : testCases) { - - notifyStreams(AnsiTerminalColorHelper.YELLOW.getBytes(DEFAULT_CHARSET)); - notifyStreams(String.format("%s%n%s", testCase.getOriginalSource(), testCase.getTestTitle()) - .getBytes(DEFAULT_CHARSET)); - notifyStreams(String.format("%n").getBytes(DEFAULT_CHARSET)); - notifyStreams(RESET); - Phrase activePhrase = null; - Iterator testBody = testCase.getContextFilteredTestSectionIterator(); - int phraseIndex = 0; - try { - - if (filter && previouslyExecutedTests.contains(testCase.getContextFilteredPhraseBody())) { - logger.warn(String.format( - "A test was skipped because after filtering it duplicated an earlier run test!%n\t%s", - testCase.getTestTitle())); - StringBuilder duplicateBody = new StringBuilder(); - testCase.getContextFilteredTestSectionIterator().forEachRemaining(duplicateBody::append); - results.addTestResult(DefaultTestResult.duplicateTest(testCase)); - } else { - if (filter) { - previouslyExecutedTests.add(testCase.getContextFilteredPhraseBody()); - } - while (testBody.hasNext()) { - TestSection section = testBody.next(); - if (section.getMetaData().isPresent()) { - notifyStreams(AnsiTerminalColorHelper.CYAN.getBytes(DEFAULT_CHARSET)); - notifyStreams(section.getMetaData().get()); - notifyStreams(RESET); - } - activePhrase = section.getPhrase(); - if (phraseRegistry.listener.isPresent()) { - notifyBeforeListener(phraseRegistry.listener.get(), walker, activePhrase); - walker.walk(phraseRegistry.listener.get(), activePhrase.getParseTree()); - notifyAfterListener(phraseRegistry.listener.get(), walker, activePhrase); - } else { - notifyBeforeVisitor(phraseRegistry.visitor.get(), activePhrase); - phraseRegistry.visitor.get().visit(activePhrase.getParseTree()); - notifyAfterVisitor(phraseRegistry.visitor.get(), activePhrase); + + private MetadataTestRunResults walk(Collection testCases, InterpreterObj interpreterObj, + String context) { + PolymorphicDslTestRunResults results = new PolymorphicDslTestRunResults( + new PdslThreadSafeOutputStream(), context); + Set> previouslyExecutedTests = new HashSet<>(); + String filterDuplicatesProperty = System.getProperty("pdsl.filterDuplicates"); + boolean filter = filterDuplicatesProperty != null && filterDuplicatesProperty.equalsIgnoreCase("true"); + for (TestCase testCase : testCases) { + Optional listener = interpreterObj.getListenerSupplier().isPresent() + ? Optional.of(interpreterObj.getListenerSupplier().get().get()) + : Optional.empty(); + Optional> visitor = interpreterObj.getVisitorSupplier().isPresent() + ? Optional.of(interpreterObj.getVisitorSupplier().get().get()) + : Optional.empty(); + notifyBeforeTestCase(testCase); + TestSection testSection = testCase.getContextFilteredTestSectionIterator().next(); + Iterator testBody = testCase.getContextFilteredTestSectionIterator(); + int phraseIndex = 0; + try { + + if (filter && previouslyExecutedTests.contains(testCase.getContextFilteredPhraseBody())) { + notifyDuplicateSkipped(testCase); + results.addTestResult(DefaultTestResult.duplicateTest(testCase)); + continue; + } + if (filter) { + previouslyExecutedTests.add(testCase.getContextFilteredPhraseBody()); + } + while (testBody.hasNext()) { + testSection = testBody.next(); + Phrase activePhrase = testSection.getPhrase(); + if (listener.isPresent()) { + notifyBeforeListener(listener.get(), walker, testSection); + walker.walk(listener.get(), activePhrase.getParseTree()); + notifyAfterListener(listener.get(), walker, testSection); + } else { + notifyBeforeVisitor(visitor.orElseThrow(), testSection); + visitor.get().visit(activePhrase.getParseTree()); + notifyAfterVisitor(visitor.get(), testSection); + } + phraseIndex++; + } + results.addTestResult(DefaultTestResult.passingTest(testCase)); + notifyTestCaseSuccess(testCase); + } catch (Throwable e) { + if (listener.isPresent()) { + notifyAfterListener(listener.get(), walker, testSection); + notifyOnListenerException(listener.get(), testSection, testCase, e); + } else { + notifyAfterVisitor(visitor.get(), testSection); + notifyOnVisitorException(visitor.orElseThrow(), testSection, testCase, e); + } + + int phrasesSkippedDueToFailure = 0; + while (testBody.hasNext()) { + testBody.next(); + phrasesSkippedDueToFailure++; + } + results.addTestResult(DefaultTestResult.failedTest(testCase, testSection.getPhrase(), e, phraseIndex, + phrasesSkippedDueToFailure)); } - phraseIndex++; - notifyStreams( - (AnsiTerminalColorHelper.GREEN + activePhrase.getParseTree().getText() + "\n" - + AnsiTerminalColorHelper.RESET).getBytes(DEFAULT_CHARSET)); - } - results.addTestResult(DefaultTestResult.passingTest(testCase)); - } - } catch (Throwable e) { - if (phraseRegistry.listener.isPresent()) { - notifyOnListenerException(phraseRegistry.listener.get(), activePhrase, testCase, e); - } else { - notifyOnVisitorException(phraseRegistry.visitor.get(), activePhrase,testCase, e); + notifyAfterTestCase(testCase); } - notifyStreams( - (AnsiTerminalColorHelper.BRIGHT_RED + activePhrase.getParseTree().getText() + "\n" - + AnsiTerminalColorHelper.RESET).getBytes(DEFAULT_CHARSET)); - - int phrasesSkippedDueToFailure = 0; - while (testBody.hasNext()) { - testBody.next(); - phrasesSkippedDueToFailure++; + return results; + } + + @Override + public MetadataTestRunResults runTestsWithMetadata(Collection testCases, + ParseTreeListener subgrammarListener, String context) { + notifyBeforeTestSuite(testCases, subgrammarListener, context); + MetadataTestRunResults results = walk(testCases, new InterpreterObj(subgrammarListener), + context); + notifyAfterTestSuite(testCases, subgrammarListener, results, context); + return results; + } + + @Override + public MetadataTestRunResults runTestsWithMetadata(Collection testCases, + ParseTreeVisitor visitor, String context) { + notifyBeforeTestSuite(testCases, visitor, context); + MetadataTestRunResults results = walk(testCases, new InterpreterObj(visitor), context); + notifyAfterTestSuite(testCases, visitor, results, context); + return results; + } + + /** + * {@inheritDoc} + * + * If a supplier is used with any elements in the shared test cases, that supplier will be called to produce + * a visitor/listener once before each test case. + */ + @Override + public MetadataTestRunResults runTestsWithMetadata(Collection sharedTestCases, + String context) { + notifyBeforeTestSuite(sharedTestCases, context); + PolymorphicDslTestRunResults results = new PolymorphicDslTestRunResults( + new PdslThreadSafeOutputStream(), context); + + + for (SharedTestCase sharedTestCase : sharedTestCases) { + List listOfTestCases = sharedTestCase.getSharedTestCaseWithInterpreters().stream() + .map(SharedTestCaseWithInterpreter::getTestCase).toList(); + int size = listOfTestCases.getFirst().getUnfilteredPhraseBody().size(); + TestCase testCase = listOfTestCases.stream().findFirst().orElseThrow(); + notifyBeforeTestCase(sharedTestCase); + // Create each visitor/listener one time per test case + Map suppliedListeneres = new HashMap<>(); + Map> suppliedVisitors = new HashMap<>(); + Optional phrase = Optional.empty(); + int phraseIndex = 0; + Optional interpreterObj = Optional.empty(); + // TODO: The default implementation associates metadata with the first phrase. + // We should design for more general use cases, but we can't do this without + // severely refactoring the API. Save this for v2 + TestSection testSection = sharedTestCase.getSharedTestCaseWithInterpreters().stream() + .map(SharedTestCaseWithInterpreter::getTestCase) + .map(TestCase::getContextFilteredTestSectionIterator) + .filter(Iterator::hasNext) + .map(Iterator::next) + .findFirst() + .orElseThrow(() -> new IllegalStateException("No executable phrases were found!")); + + try { + for (int j = 0; + j < sharedTestCase.getSharedTestCaseWithInterpreters().getFirst().getTestCase() + .getUnfilteredPhraseBody().size(); j++) { + for (SharedTestCaseWithInterpreter interpreter : sharedTestCase.getSharedTestCaseWithInterpreters()) { + FilteredPhrase filteredPhrase = interpreter.getTestCase().getFilteredPhrases() + .get(phraseIndex); + + //TODO - Add implementation for the duplication checking + Optional parseTree = filteredPhrase.getParseTree(); + if (parseTree.isPresent()) { + phrase = Optional.of(new DefaultPhrase(parseTree.get(), phraseIndex)); + + interpreterObj = Optional.of(interpreter.getInterpreterObj()); + + if (interpreterObj.get().getListenerSupplier().isPresent()) { + ParseTreeListener listener = suppliedListeneres.computeIfAbsent(interpreterObj.get(), + (i) -> i.getListenerSupplier().orElseThrow().get()); + notifyBeforeListener(listener, walker, + testSection); + walker.walk(listener, parseTree.get()); + notifyAfterListener(listener, walker, + testSection); + } else { + ParseTreeVisitor visitor = suppliedVisitors.computeIfAbsent(interpreterObj.get(), + (i) -> i.getVisitorSupplier().orElseThrow().get()); + notifyBeforeVisitor(visitor, + testSection); + visitor.visit(parseTree.get()); + notifyAfterVisitor(visitor, testSection); + } + + } + } + phraseIndex++; + } + notifyTestCaseSuccess(testCase); + } catch (Throwable e) { + if (interpreterObj.isPresent() && phrase.isPresent()) { + if (interpreterObj.get().getParseTreeListener().isPresent()) { + notifyOnListenerException(interpreterObj.get().getParseTreeListener().get(), + testSection, testCase, e); + } else if (interpreterObj.get().getParseTreeVisitor().isPresent()) { + notifyOnVisitorException(interpreterObj.get().getParseTreeVisitor().get(), + testSection, testCase, e); + } + } + results.addTestResult(DefaultTestResult.failedTest(testCase, null, e, phraseIndex, + size - phraseIndex)); + } + results.addTestResult(DefaultTestResult.passingTest(testCase)); + notifyAfterTestCase(testCase); } - results.addTestResult(DefaultTestResult.failedTest(testCase, activePhrase, e, phraseIndex, - phrasesSkippedDueToFailure)); - logger.error("Phrase failure", e); - } + notifyAfterTestSuite(sharedTestCases, results, context); + return results; } - return results; - } - - private void notifyStreams(InputStream inputStream) { - if (outputStreams.isPresent()) { - try { - outputStreams.get().write(inputStream.readAllBytes()); - } catch (IOException e) { - throw new PolymorphicDslTransformationException("Could not notify streams!", e); - } + + + /** + * Adds an observer that will be notified of events as test cases are executed. + * + * beforeTestSuite - Called once before the execution of test cases + * afterTestSuite - Called once after the execution of all test cases + * beforeTestCaseTriggered - Called once before each test case + * afterTestCaseTriggered - Called once after each test case + * phraseFailure - Called once for any test that failed + * duplicateSkipped - Called once only if filtering is enabled + * fore each identical test was run earlier + * beforePhrase - Called before each phrase in each test case. + * In the event of a phrase failure, no subsequent + * phrases for that specific test case will be called + * afterPhrase - Called after each successful phrase in each test case + * In the event of a phrase failure this method will not + * be called + * testCaseSuccess - Called once after each test case only if it had no + * phrase failures + * @param observer + */ + @Override + public void registerObserver(ExecutorObserver observer) { + activePhraseObservers.add(observer); } - } - - private void notifyStreams(byte[] bytes) { - if (outputStreams.isPresent()) { - try { - outputStreams.get().write(bytes); - } catch (IOException e) { - throw new PolymorphicDslTransformationException("Could not notify streams!", e); - } + + @Override + public void removeObserver(ExecutorObserver observer) { + activePhraseObservers.remove(observer); } - } - - @Override - public MetadataTestRunResults runTestsWithMetadata(Collection testCases, - ParseTreeListener subgrammarListener, String context) { - logger.info("Running tests..."); - notifyBeforeTestSuite(testCases, subgrammarListener, ""); - MetadataTestRunResults results = walk(testCases, new PhraseRegistry(subgrammarListener), - context); - notifyAfterTestSuite(testCases, subgrammarListener,results, ""); - if (results.failingTestTotal() == 0) { - logger.info(AnsiTerminalColorHelper.BRIGHT_GREEN + "All phrases successfully executed!" - + AnsiTerminalColorHelper.RESET); - } else { - logger.error(AnsiTerminalColorHelper.BRIGHT_RED + "There were test failures!" - + AnsiTerminalColorHelper.RESET); + + private void notifyDuplicateSkipped(TestCase testCase) { + activePhraseObservers.forEach(o -> o.onDuplicateSkipped(testCase)); } - return (PolymorphicDslTestRunResults) results; - } - - @Override - public MetadataTestRunResults runTestsWithMetadata(Collection testCases, - ParseTreeVisitor visitor, String context) { - logger.info("Running tests..."); - notifyBeforeTestSuite(testCases, visitor, context); - MetadataTestRunResults results = walk(testCases, new PhraseRegistry(visitor), context); - notifyAfterTestSuite(testCases, visitor, results,context); - if (results.failingTestTotal() == 0) { - logger.info(AnsiTerminalColorHelper.BRIGHT_GREEN + "All phrases successfully executed!" - + AnsiTerminalColorHelper.RESET); - } else { - logger.error(AnsiTerminalColorHelper.BRIGHT_RED + "There were test failures!" - + AnsiTerminalColorHelper.RESET); + + private void notifyTestCaseSuccess(TestCase testCase) { + activePhraseObservers.forEach(o -> o.onTestCaseSuccess(testCase)); } - return (PolymorphicDslTestRunResults) results; - } - - @Override - public MetadataTestRunResults runTestsWithMetadata(Collection sharedTestCases, - String context) { - notifyBeforeTestSuite(sharedTestCases, context); - PolymorphicDslTestRunResults results = new PolymorphicDslTestRunResults( - new PdslThreadSafeOutputStream(), context); - //Set> previouslyExecutedTests = new HashSet<>(); - - for (SharedTestCase sharedTestCase : sharedTestCases) { - List listOfTestCases = sharedTestCase.getSharedTestCaseWithInterpreters().stream() - .map(tc -> tc.getTestCase()).collect( - Collectors.toUnmodifiableList()); - int size = listOfTestCases.get(0).getUnfilteredPhraseBody().size(); - TestCase testCase = listOfTestCases.stream().findFirst().orElseThrow(); - - notifyStreams(AnsiTerminalColorHelper.YELLOW.getBytes(DEFAULT_CHARSET)); - notifyStreams(String.format("%s%n%s", testCase.getOriginalSource(), testCase.getTestTitle()) - .getBytes(DEFAULT_CHARSET)); - notifyStreams(String.format("%n").getBytes(DEFAULT_CHARSET)); - notifyStreams(RESET); - - String filteredPhraseText = null; - Optional phrase = Optional.empty(); - int phraseIndex = 0; - Optional interpreterObj = Optional.empty(); - try { - //for (SharedTestCaseWithInterpreter interpreter : sharedTestCase.getSharedTestCaseWithInterpreters()) { - for (int j = 0; - j < sharedTestCase.getSharedTestCaseWithInterpreters().get(0).getTestCase() - .getUnfilteredPhraseBody().size(); j++) { - for (SharedTestCaseWithInterpreter interpreter : sharedTestCase.getSharedTestCaseWithInterpreters()) { - - FilteredPhrase filteredPhrase = interpreter.getTestCase().getFilteredPhrases() - .get(phraseIndex); - filteredPhraseText = filteredPhrase.getPhrase(); - - - //TODO - Add implementation for the duplication checking - Optional parseTree = filteredPhrase.getParseTree(); - if (parseTree.isPresent()) { - phrase = Optional.of(new DefaultPhrase(parseTree.get(), phraseIndex)); - - interpreterObj = Optional.of(interpreter.getInterpreterObj()); - - if (interpreterObj.get().getParseTreeListener().isPresent()) { - notifyBeforeListener(interpreterObj.get().getParseTreeListener().get(), walker, - phrase.get()); - walker.walk(interpreterObj.get().getParseTreeListener().get(), parseTree.get()); - notifyAfterListener(interpreterObj.get().getParseTreeListener().get(), walker, - phrase.get()); - } else { - notifyBeforeVisitor(interpreterObj.get().getParseTreeVisitor().get(), - phrase.get()); - interpreterObj.get().getParseTreeVisitor().get().visit(parseTree.get()); - notifyAfterVisitor(interpreterObj.get().getParseTreeVisitor().get(), - phrase.get()); - } - - notifyStreams( - (AnsiTerminalColorHelper.GREY + parseTree.get().getText() + "\n" - + AnsiTerminalColorHelper.RESET).getBytes(DEFAULT_CHARSET)); - } - } - phraseIndex++; - } - notifyStreams( - (AnsiTerminalColorHelper.GREEN + "All Sentences are parsed." + "\n" - + AnsiTerminalColorHelper.RESET).getBytes(DEFAULT_CHARSET)); - - } catch (Throwable e) { - if (interpreterObj.isPresent() && phrase.isPresent()) { - if (interpreterObj.get().getParseTreeListener().isPresent()) { - notifyOnListenerException(interpreterObj.get().getParseTreeListener().get(), - phrase.get(),testCase, - e); - } else if (interpreterObj.get().getParseTreeVisitor().isPresent()) { - notifyOnVisitorException(interpreterObj.get().getParseTreeVisitor().get(), - phrase.get(),testCase, - e); - } - } - notifyStreams( - (AnsiTerminalColorHelper.BRIGHT_RED + filteredPhraseText + "\n" - + AnsiTerminalColorHelper.RESET).getBytes(DEFAULT_CHARSET)); - results.addTestResult(DefaultTestResult.failedTest(testCase, null, e, phraseIndex, - size - phraseIndex)); - logger.error("Phrase failure", e); - } + private void notifyBeforeListener(ParseTreeListener listener, ParseTreeWalker walker, + TestSection testSection) { + activePhraseObservers.forEach(o -> o.onBeforePhrase(listener, walker, testSection)); + } - results.addTestResult(DefaultTestResult.passingTest(testCase)); + private void notifyBeforeVisitor(ParseTreeVisitor visitor, + TestSection testSection) { + activePhraseObservers.forEach(o -> o.onBeforePhrase(visitor, testSection)); } - notifyAfterTestSuite(sharedTestCases,results,context); - return results; - } + private void notifyAfterListener(ParseTreeListener listener, ParseTreeWalker walker, + TestSection testSection) { + activePhraseObservers.forEach(o -> o.onAfterPhrase(listener, walker, testSection)); + } + private void notifyBeforeTestCase(TestCase testCase) { + activePhraseObservers.forEach(o -> o.onBeforeTestCase(testCase)); + } -} + private void notifyAfterTestCase(TestCase testCase) { + activePhraseObservers.forEach(o -> o.onAfterTestCase(testCase)); + } + private void notifyAfterVisitor(ParseTreeVisitor visitor, + TestSection testSection) { + activePhraseObservers.forEach(o -> o.onAfterPhrase(visitor, testSection)); + } + + private void notifyOnListenerException(ParseTreeListener listener, + TestSection testSection, TestCase testCase, Throwable exception) { + activePhraseObservers.forEach(o -> o.onPhraseFailure(listener, testSection, testCase, exception)); + } + private void notifyOnVisitorException(ParseTreeVisitor visitor, + TestSection testSection, TestCase testCase, Throwable exception) { + activePhraseObservers.forEach(a -> a.onPhraseFailure(visitor, testSection, testCase, exception)); + } + + private void notifyBeforeTestSuite(Collection testCases, ParseTreeVisitor visitor, + String context) { + activePhraseObservers.forEach(a -> a.onBeforeTestSuite(testCases, visitor, context)); + } + + private void notifyBeforeTestSuite(Collection testCases, + String context) { + activePhraseObservers.forEach(a -> a.onBeforeTestSuite(testCases, context)); + } + + private void notifyAfterTestSuite(Collection testCases, ParseTreeVisitor visitor, MetadataTestRunResults results, + String context) { + activePhraseObservers.forEach(a -> a.onAfterTestSuite(testCases, visitor, results, context)); + } + + private void notifyBeforeTestSuite(Collection testCases, ParseTreeListener listener, + String context) { + activePhraseObservers.forEach(a -> a.onBeforeTestSuite(testCases, listener, context)); + } + + private void notifyAfterTestSuite(Collection testCases, ParseTreeListener listener, MetadataTestRunResults results, + String context) { + activePhraseObservers.forEach(a -> a.onAfterTestSuite(testCases, listener, results, context)); + } + + private void notifyAfterTestSuite(Collection testCases, MetadataTestRunResults results, + String context) { + activePhraseObservers.forEach(a -> a.onAfterTestSuite(testCases, results, context)); + } +} diff --git a/src/main/java/com/pdsl/executors/ExecutorObserver.java b/src/main/java/com/pdsl/executors/ExecutorObserver.java index 3680bc9..67c2f6a 100644 --- a/src/main/java/com/pdsl/executors/ExecutorObserver.java +++ b/src/main/java/com/pdsl/executors/ExecutorObserver.java @@ -2,64 +2,92 @@ import com.pdsl.reports.MetadataTestRunResults; import com.pdsl.specifications.Phrase; +import com.pdsl.testcases.SharedTestCase; import com.pdsl.testcases.TestCase; -import java.util.Collection; +import com.pdsl.testcases.TestSection; import org.antlr.v4.runtime.tree.ParseTreeListener; import org.antlr.v4.runtime.tree.ParseTreeVisitor; import org.antlr.v4.runtime.tree.ParseTreeWalker; +import java.util.Collection; + /** An observer that is notified when specific events occur while a Test Executor executes tests. */ public interface ExecutorObserver { + /** + * Notifies just before a test case is going to execute. + * @param testCase + */ + default void onBeforeTestCase(TestCase testCase) {} + + /** + * Notifies after a test case has executed regardless of whether it has passed or failed. + * @param testCase + */ + default void onAfterTestCase(TestCase testCase) {} + + /** + * Notifies when a test case has completed execution with no phrase failures. + * @param testCase + */ + default void onTestCaseSuccess(TestCase testCase) {} + + /** + * Notifies the implementer that a test case isn't being run because a separate test case + * already ran that did the same thing. + * @param testCase that was skipped + */ + default void onDuplicateSkipped(TestCase testCase) {} + /** Notifies the implementer of a phrase that is about to be executed with a listener. */ - void onBeforePhrase(ParseTreeListener listener, - ParseTreeWalker walker, Phrase activePhrase); + default void onBeforePhrase(ParseTreeListener listener, + ParseTreeWalker walker, TestSection phrase) {} /** Notifies the implementer of a phrase that is about to be executed with a visitor. */ - void onBeforePhrase(ParseTreeVisitor visitor, - Phrase activePhrase); + default void onBeforePhrase(ParseTreeVisitor visitor, + TestSection testSection) {} /** Notifies the implementer of a phrase that has completed execution with a listener WITHOUT an exception. */ - void onAfterPhrase(ParseTreeListener listener, - ParseTreeWalker walker, Phrase activePhrase); + default void onAfterPhrase(ParseTreeListener listener, + ParseTreeWalker walker, TestSection testSection) {} /** Notifies the implementer of a phrase that has completed execution with a visitor WITHOUT an exception. */ void onAfterPhrase(ParseTreeVisitor visitor, - Phrase activePhrase); + TestSection testSection); /** Notifies the implementer of a phrase that has failed during excecution with a listener. */ - void onPhraseFailure(ParseTreeListener listener, - Phrase activePhrase, TestCase testCase, Throwable exception); + default void onPhraseFailure(ParseTreeListener listener, + TestSection testSection, TestCase testCase, Throwable exception) {} /** Notifies the implementer of a phrase that has failed during excecution with a visitor. */ - void onPhraseFailure(ParseTreeVisitor visitor, - Phrase activePhrase, TestCase testCase, Throwable exception); + default void onPhraseFailure(ParseTreeVisitor visitor, + TestSection testSection, TestCase testCase, Throwable exception) {} - /** Notifies the implementer of a of a test suite that is about to execute with a visitor. */ - void onBeforeTestSuite(Collection testCases, ParseTreeVisitor visitor, - String context); + /** Notifies the implementer of a test suite that is about to execute with a visitor. */ + default void onBeforeTestSuite(Collection testCases, ParseTreeVisitor visitor, + String context) {} - /** Notifies the implementer of a of a test suite that is about to execute with a listener. */ - void onBeforeTestSuite(Collection testCases, ParseTreeListener listener, - String context); + /** Notifies the implementer of a test suite that is about to execute with a listener. */ + default void onBeforeTestSuite(Collection testCases, ParseTreeListener listener, + String context) {} - /** Notifies the implementer of a of a test suite that is about to execute. - * The implementation of the TestCase collection will likely be SharedTestCases in the event this method is triggered. + /** Notifies the implementer of a test suite that is about to execute. + * */ - void onBeforeTestSuite(Collection testCases, - String context); + default void onBeforeTestSuite(Collection testCases, + String context) {} - /** Notifies the implementer of a of a test suite that has completed execution with a visitor. */ - void onAfterTestSuite(Collection testCases, ParseTreeVisitor visitor, MetadataTestRunResults results, - String context); + /** Notifies the implementer of a test suite that has completed execution with a visitor. */ + default void onAfterTestSuite(Collection testCases, ParseTreeVisitor visitor, MetadataTestRunResults results, + String context) {} - /** Notifies the implementer of a of a test suite that has completed execution with a listener. */ - void onAfterTestSuite(Collection testCases, ParseTreeListener listener, MetadataTestRunResults results, - String context); + /** Notifies the implementer a test suite that has completed execution with a listener. */ + default void onAfterTestSuite(Collection testCases, ParseTreeListener listener, MetadataTestRunResults results, + String context) {} - /** Notifies the implementer of a of a test suite that is has completed execution. - * The implementation of the TestCase collection will likely be SharedTestCases in the event this method is triggered. + /** + * Notifies the implementer of a test suite that is has completed execution. */ - void onAfterTestSuite(Collection testCases, MetadataTestRunResults results, - String context); + default void onAfterTestSuite(Collection testCases, MetadataTestRunResults results, + String context) {} } diff --git a/src/main/java/com/pdsl/executors/InterpreterObj.java b/src/main/java/com/pdsl/executors/InterpreterObj.java index 9ccc931..e58206a 100644 --- a/src/main/java/com/pdsl/executors/InterpreterObj.java +++ b/src/main/java/com/pdsl/executors/InterpreterObj.java @@ -27,7 +27,7 @@ public InterpreterObj(ParseTreeVisitor parseTreeVisitor) { * Creates a InterpreterObj from the supplied parameters. * * This will use the default start rule. If a specific start rule is needed use the related - * method ofVisitor(supplier, String) + * method ofVisitor(supplier, String)} * * @return InterpreterObj */ @@ -132,5 +132,12 @@ public Optional getParseTreeListener() { public String getStartRule(){ return this.startRule; } -} + public Optional> getListenerSupplier() { + return parseTreeListener; + } + + public Optional>> getVisitorSupplier() { + return parseTreeVisitor; + } +} diff --git a/src/main/java/com/pdsl/executors/PolymorphicDslTestExecutor.java b/src/main/java/com/pdsl/executors/PolymorphicDslTestExecutor.java index e104d85..536ec96 100644 --- a/src/main/java/com/pdsl/executors/PolymorphicDslTestExecutor.java +++ b/src/main/java/com/pdsl/executors/PolymorphicDslTestExecutor.java @@ -32,7 +32,7 @@ public interface PolymorphicDslTestExecutor { * @param subgrammarVisitor the visitor to trigger work on appropriate phrases * @return the results of the test run */ - TestRunResults runTests(Collection testCases, ParseTreeVisitor subgrammarVisitor); + TestRunResults runTests(Collection testCases, ParseTreeVisitor subgrammarVisitor); diff --git a/src/main/java/com/pdsl/gherkin/executors/GherkinTestExecutor.java b/src/main/java/com/pdsl/gherkin/executors/GherkinTestExecutor.java index 89a4a4e..c466ef1 100644 --- a/src/main/java/com/pdsl/gherkin/executors/GherkinTestExecutor.java +++ b/src/main/java/com/pdsl/gherkin/executors/GherkinTestExecutor.java @@ -93,7 +93,7 @@ public TestRunResults runTests(Collection testCases, ParseTreeListener } @Override - public TestRunResults runTests(Collection testCases, ParseTreeVisitor subgrammarVisitor) { + public TestRunResults runTests(Collection testCases, ParseTreeVisitor subgrammarVisitor) { return executor.runTests(testCases, subgrammarVisitor); } diff --git a/src/main/java/com/pdsl/logging/PdslThreadSafeOutputStream.java b/src/main/java/com/pdsl/logging/PdslThreadSafeOutputStream.java index 0ad78b3..d53d6a7 100644 --- a/src/main/java/com/pdsl/logging/PdslThreadSafeOutputStream.java +++ b/src/main/java/com/pdsl/logging/PdslThreadSafeOutputStream.java @@ -47,7 +47,7 @@ public void write(byte[] bytes, int start, int stop) { // Multibyte characters may require us to have an earlier stop point //TODO: This is a bug. Find a way to handle character encoding instead of assuming // the caller wants to write the full array logged - message = new String(bytes).substring(start, stop <= message.length() ? stop : message.length()); + message = new String(bytes).substring(start, Math.min(stop, message.length())); logger.info(message); } diff --git a/src/main/java/com/pdsl/runners/SharedTestSuiteVisitor.java b/src/main/java/com/pdsl/runners/SharedTestSuiteVisitor.java index b8fe896..05b51a0 100644 --- a/src/main/java/com/pdsl/runners/SharedTestSuiteVisitor.java +++ b/src/main/java/com/pdsl/runners/SharedTestSuiteVisitor.java @@ -56,7 +56,7 @@ public SharedTestSuite recognizerParamsOperation(RecognizerParams recognizerPara // Read the files and structure them as specifications which will allow us to figure out how many // test cases to create from each file, which parts to ignore, etc. Collection specifications = getSpecifications(parser, recognizerParams, params, testResources); - // Convert the specificaitons into test cases + // Convert the specifications into test cases List testCasesForSingleInterpreter = new ArrayList<>(getTestCases(recognizerParams.providers().testCaseFactoryProvider().get(), specifications)); testCasesPerInterpreters.add(testCasesForSingleInterpreter); interpreterObjs.add(parser.interpreterProvider().get()); diff --git a/src/main/java/com/pdsl/testcases/DefaultPdslTestCase.java b/src/main/java/com/pdsl/testcases/DefaultPdslTestCase.java index 796b394..aea559b 100644 --- a/src/main/java/com/pdsl/testcases/DefaultPdslTestCase.java +++ b/src/main/java/com/pdsl/testcases/DefaultPdslTestCase.java @@ -2,6 +2,8 @@ import com.google.common.base.Preconditions; import com.pdsl.specifications.FilteredPhrase; + +import java.io.InputStream; import java.net.URI; import org.antlr.v4.runtime.tree.ParseTree; @@ -44,16 +46,16 @@ public DefaultPdslTestCase(String testCaseTitle, List testBody this.phrasesToTestSections = testBodyFragments.stream() .map(TestBodyFragment::getTestPhrases) .flatMap(Collection::stream) - .collect(Collectors.toUnmodifiableList()); + .toList(); this.unfilteredPhraseBody = phrasesToTestSections.stream() .map(FilteredPhrase::getPhrase) - .collect(Collectors.toUnmodifiableList()); + .toList(); this.contextFilteredPhraseBody = phrasesToTestSections.stream() .map(FilteredPhrase::getParseTree) .filter(Optional::isPresent) .map(Optional::get) .map(ParseTree::getText) - .collect(Collectors.toUnmodifiableList()); + .toList(); } @Override @@ -76,7 +78,7 @@ public String getTestTitle() { @Override public Iterator getContextFilteredTestSectionIterator() { return getTestSectionStream() - .collect(Collectors.toUnmodifiableList()) + .toList() .iterator(); } @@ -86,6 +88,12 @@ private Stream getTestSectionStream() { .flatMap(Collection::stream); } + public List> getUnfilteredTestSectionsMetaData() { + return testBodyFragments.stream() + .map(TestBodyFragment::getMetaData) + .toList(); + } + @Override public List getFilteredPhrases() { return phrasesToTestSections; diff --git a/src/main/java/com/pdsl/testcases/SharedTestCase.java b/src/main/java/com/pdsl/testcases/SharedTestCase.java index 272caa2..93ee270 100644 --- a/src/main/java/com/pdsl/testcases/SharedTestCase.java +++ b/src/main/java/com/pdsl/testcases/SharedTestCase.java @@ -14,31 +14,12 @@ * and associated (Lexer/Parser; Listener/Visitor) with them. * {@link com.pdsl.executors.InterpreterObj} */ -public final class SharedTestCase implements TestCase{ +public final class SharedTestCase implements TestCase { List sharedTestCaseWithInterpreters; - private final List testCases; - private final List interpreterObjs; - - private final List> iterators; - - public SharedTestCase(List testCases, List interpreterObjs) { - /** - * Take the first one (same) phrase in different Listener/Visitor (InterpreterObj). - */ - int countFirst = testCases.get(0).getUnfilteredPhraseBody().size(); - Preconditions.checkArgument(testCases.stream().allMatch(tc -> tc.getUnfilteredPhraseBody().size() == countFirst), "The size should be the same for all " + countFirst); - this.testCases = testCases; - this.interpreterObjs = interpreterObjs; - this.iterators = this.testCases.stream().map(TestCase::getFilteredPhrases).map(v-> v.iterator()).collect(Collectors.toUnmodifiableList()); - } - public SharedTestCase(List sharedTestCaseWithInterpreters) { this.sharedTestCaseWithInterpreters = sharedTestCaseWithInterpreters; - this.testCases=null; - this.interpreterObjs=null; - this.iterators = null; } public List getSharedTestCaseWithInterpreters() {return sharedTestCaseWithInterpreters;} diff --git a/src/main/java/com/pdsl/testcases/TestSection.java b/src/main/java/com/pdsl/testcases/TestSection.java index 6031ddc..33de140 100644 --- a/src/main/java/com/pdsl/testcases/TestSection.java +++ b/src/main/java/com/pdsl/testcases/TestSection.java @@ -37,7 +37,7 @@ static List convertBodyFragment(TestBodyFragment testBodyFragment) List testSections = new ArrayList<>(testBodyFragment.getTestPhrases().size()); List filteredPhrases = testBodyFragment.getTestPhrases().stream() .filter(filteredPhrase -> filteredPhrase.getParseTree().isPresent()) - .collect(Collectors.toUnmodifiableList()); + .toList(); Preconditions.checkArgument(!filteredPhrases.isEmpty(), "Test Body Fragment must have at least one filtered phrase with a parse tree present"); for (int i = 0; i < testBodyFragment.getTestPhrases().size(); i++) { @@ -46,11 +46,12 @@ static List convertBodyFragment(TestBodyFragment testBodyFragment) Phrase phrase = new DefaultPhrase(filteredPhrase.getParseTree().get(), i); testSections.add( // If its the first phrase with a parse tree add metadata, else just the phrase - filteredPhrase == filteredPhrases.get(0) && testBodyFragment.getMetaData().isPresent() + filteredPhrase == filteredPhrases.getFirst() && testBodyFragment.getMetaData().isPresent() ? new DefaultTestSection(testBodyFragment.getMetaData().get(), phrase) : new DefaultTestSection(phrase)); } } return testSections; } + } diff --git a/src/main/java/org/junit/jupiter/engine/descriptor/PdslSharedInvocationContextProvider.java b/src/main/java/org/junit/jupiter/engine/descriptor/PdslSharedInvocationContextProvider.java index 8a858b4..ec82c0f 100644 --- a/src/main/java/org/junit/jupiter/engine/descriptor/PdslSharedInvocationContextProvider.java +++ b/src/main/java/org/junit/jupiter/engine/descriptor/PdslSharedInvocationContextProvider.java @@ -64,7 +64,6 @@ public abstract class PdslSharedInvocationContextProvider private static final TraceableTestRunExecutor DEFAULT_EXECUTOR = new DefaultPolymorphicDslTestExecutor(); private static final SharedTestSuiteVisitor SHARED_TEST_SUITE_VISITOR = new SharedTestSuiteVisitor(); - private static final JUnit5DefaultPackageAccessor ACCESSOR = new JUnit5DefaultPackageAccessor(new JupiterDescriptorKey()); @Override public boolean supportsTestTemplate(ExtensionContext context) { diff --git a/src/test/java/com/pdsl/component/FrameworkTestExecutor.java b/src/test/java/com/pdsl/component/FrameworkTestExecutor.java index 9245df9..2bedaa1 100644 --- a/src/test/java/com/pdsl/component/FrameworkTestExecutor.java +++ b/src/test/java/com/pdsl/component/FrameworkTestExecutor.java @@ -1,24 +1,239 @@ package com.pdsl.component; +import com.pdsl.executors.DefaultPolymorphicDslTestExecutor; +import com.pdsl.executors.ExecutorObserver; +import com.pdsl.executors.InterpreterObj; import com.pdsl.gherkin.executors.GherkinTestExecutor; import com.pdsl.grammars.TestExecutorLexer; import com.pdsl.grammars.TestExecutorMetaParser; import com.pdsl.grammars.TestExecutorMetaParserListenerImpl; +import com.pdsl.reports.MetadataTestRunResults; import com.pdsl.reports.TestRunResults; +import com.pdsl.runners.SharedTestSuiteVisitor; +import com.pdsl.specifications.FilteredPhrase; +import com.pdsl.testcases.*; import com.pdsl.transformers.DefaultPolymorphicDslPhraseFilter; import com.pdsl.transformers.PolymorphicDslPhraseFilter; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.tree.*; +import org.jruby.ir.Interp; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.mockito.internal.stubbing.answers.AnswersValidator; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; -import java.util.HashSet; -import java.util.Set; +import java.util.*; +import java.util.logging.Filter; import static com.google.common.truth.Truth.assertThat; +import static com.ibm.icu.text.PluralRules.Operand.n; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; public class FrameworkTestExecutor { + private static final ParseTree parseTree = getParseTree(); + private static final ParseTree failingParseTree = getFailingParseTree(); + private static final ParseTreeListener listener = getListener(); + private static final ParseTreeVisitor visitor = getVisitor(); + + private static ParseTreeListener getListener() { + var listener = mock(ParseTreeListener.class); + ArgumentCaptor terminalNodeCaptor = ArgumentCaptor.forClass(TerminalNode.class); + doAnswer(invocation -> { + TerminalNode n = invocation.getArgumentAt(0, TerminalNode.class); + if (n.getText().contains("Failing")) { + throw new AssertionError("intended error for PDSL component tests"); + } + return invocation; + }).when(listener).visitTerminal(terminalNodeCaptor.capture()); + return listener; + } + + private static ParseTreeVisitor getVisitor() { + var visitor = mock(ParseTreeVisitor.class); + ArgumentCaptor terminalNodeCaptor = ArgumentCaptor.forClass(TerminalNode.class); + doAnswer(invocation -> { + TerminalNode n = invocation.getArgumentAt(0, TerminalNode.class); + if (n.getText().contains("Failing")) { + throw new AssertionError("intended error for PDSL component tests"); + } + return invocation; + }).when(visitor).visit(terminalNodeCaptor.capture()); + return visitor; + } + + + @Test + public void testExecutorObserver_listener_lifecyclesWorkProperly() { + // ARRANGE + final ExecutorLifecycleTestObserver observer = new ExecutorLifecycleTestObserver(); + DefaultPolymorphicDslTestExecutor executor = + DefaultPolymorphicDslTestExecutor.of(Collections.singletonList(observer)); + + // ACT + executor.runTests(List.of( + getTestCase(7, StubbedTestCaseStrategy.SOME_PHRASES_FILTERED), + getTestCase(5, StubbedTestCaseStrategy.ALL_PHRASES_PASSING), + getTestCase(3, StubbedTestCaseStrategy.FIRST_PHRASE_FAILS)), listener); + // ASSERT + assertThat(observer.afterTestSuite).isEqualTo(1); + assertThat(observer.beforeTestSuite).isEqualTo(1); + assertThat(observer.beforeTestCaseTriggered).isEqualTo(3); + assertThat(observer.afterTestCaseTriggered).isEqualTo(3); + assertThat(observer.phraseFailure).isEqualTo(1); + assertThat(observer.duplicateSkipped).isEqualTo(0); + assertThat(observer.beforePhrase).isEqualTo(10); + assertThat(observer.afterPhrase).isEqualTo(10); + assertThat(observer.testCaseSuccess).isEqualTo(2); + } + + @Test + public void testExecutorObserverWithListenerMetadata_lifecyclesWorkProperly() { + // ARRANGE + final ExecutorLifecycleTestObserver observer = new ExecutorLifecycleTestObserver(); + DefaultPolymorphicDslTestExecutor executor = + DefaultPolymorphicDslTestExecutor.of(Collections.singletonList(observer)); + + // ACT + executor.runTestsWithMetadata(List.of( + getTestCase(7, StubbedTestCaseStrategy.SOME_PHRASES_FILTERED), + getTestCase(5, StubbedTestCaseStrategy.ALL_PHRASES_PASSING), + getTestCase(3, StubbedTestCaseStrategy.FIRST_PHRASE_FAILS)), listener, "Component"); + // ASSERT + assertThat(observer.afterTestSuite).isEqualTo(1); + assertThat(observer.beforeTestSuite).isEqualTo(1); + assertThat(observer.beforeTestCaseTriggered).isEqualTo(3); + assertThat(observer.afterTestCaseTriggered).isEqualTo(3); + assertThat(observer.phraseFailure).isEqualTo(1); + assertThat(observer.duplicateSkipped).isEqualTo(0); + assertThat(observer.beforePhrase).isEqualTo(10); + assertThat(observer.afterPhrase).isEqualTo(10); + assertThat(observer.testCaseSuccess).isEqualTo(2); + } + + @Test + public void testExecutorObserver_visitor_lifecyclesWorkProperly() { + // ARRANGE + final ExecutorLifecycleTestObserver observer = new ExecutorLifecycleTestObserver(); + DefaultPolymorphicDslTestExecutor executor = + DefaultPolymorphicDslTestExecutor.of(Collections.singletonList(observer)); + + // ACT + executor.runTests(List.of( + getTestCase(7, StubbedTestCaseStrategy.SOME_PHRASES_FILTERED), + getTestCase(5, StubbedTestCaseStrategy.ALL_PHRASES_PASSING), + getTestCase(3, StubbedTestCaseStrategy.FIRST_PHRASE_FAILS)), visitor); + // ASSERT + assertThat(observer.afterTestSuite).isEqualTo(1); + assertThat(observer.beforeTestSuite).isEqualTo(1); + assertThat(observer.beforeTestCaseTriggered).isEqualTo(3); + assertThat(observer.afterTestCaseTriggered).isEqualTo(3); + assertThat(observer.phraseFailure).isEqualTo(1); + assertThat(observer.duplicateSkipped).isEqualTo(0); + assertThat(observer.beforePhrase).isEqualTo(10); + assertThat(observer.afterPhrase).isEqualTo(10); + assertThat(observer.testCaseSuccess).isEqualTo(2); + } + + @Test + public void testExecutorObserverVisitorMetadata_lifecyclesWorkProperly() { + // ARRANGE + final ExecutorLifecycleTestObserver observer = new ExecutorLifecycleTestObserver(); + DefaultPolymorphicDslTestExecutor executor = + DefaultPolymorphicDslTestExecutor.of(Collections.singletonList(observer)); + + // ACT + executor.runTestsWithMetadata(List.of( + getTestCase(7, StubbedTestCaseStrategy.SOME_PHRASES_FILTERED), + getTestCase(5, StubbedTestCaseStrategy.ALL_PHRASES_PASSING), + getTestCase(3, StubbedTestCaseStrategy.FIRST_PHRASE_FAILS)), visitor, "Component"); + // ASSERT + assertThat(observer.afterTestSuite).isEqualTo(1); + assertThat(observer.beforeTestSuite).isEqualTo(1); + assertThat(observer.beforeTestCaseTriggered).isEqualTo(3); + assertThat(observer.afterTestCaseTriggered).isEqualTo(3); + assertThat(observer.phraseFailure).isEqualTo(1); + assertThat(observer.duplicateSkipped).isEqualTo(0); + assertThat(observer.beforePhrase).isEqualTo(10); + assertThat(observer.afterPhrase).isEqualTo(10); + assertThat(observer.testCaseSuccess).isEqualTo(2); + } + + @Test + public void executorSharedTestCases_lifecyclesWorkProperly() { + + // ARRANGE + // 3 phrases visited + var sharedTestCaseWithInterpreter1 = mock(SharedTestSuite.SharedTestCaseWithInterpreter.class); + doReturn(getTestCase(5, StubbedTestCaseStrategy.SOME_PHRASES_FILTERED)) + .when(sharedTestCaseWithInterpreter1).getTestCase(); + doReturn(new InterpreterObj(listener)) + .when(sharedTestCaseWithInterpreter1).getInterpreterObj(); + + // 5 phrases + var sharedTestCaseWithInterpreter2 = mock(SharedTestSuite.SharedTestCaseWithInterpreter.class); + doReturn(getTestCase(5, StubbedTestCaseStrategy.ALL_PHRASES_PASSING)) + .when(sharedTestCaseWithInterpreter2).getTestCase(); + doReturn(InterpreterObj.ofListener(() -> listener)) + .when(sharedTestCaseWithInterpreter2).getInterpreterObj(); + + // 5 phrases + var sharedTestCaseWithInterpreter3 = mock(SharedTestSuite.SharedTestCaseWithInterpreter.class); + doReturn( getTestCase(5, StubbedTestCaseStrategy.ALL_PHRASES_PASSING)) + .when(sharedTestCaseWithInterpreter3).getTestCase(); + doReturn(new InterpreterObj(visitor)).when(sharedTestCaseWithInterpreter3).getInterpreterObj(); + + // 3 phrases + var sharedTestCaseWithInterpreter4 = mock(SharedTestSuite.SharedTestCaseWithInterpreter.class); + doReturn( getTestCase(5, StubbedTestCaseStrategy.SOME_PHRASES_FILTERED)) + .when(sharedTestCaseWithInterpreter4).getTestCase(); + doReturn(InterpreterObj.ofVisitor(() -> visitor)) + .when(sharedTestCaseWithInterpreter4).getInterpreterObj(); + + var sharedTestCase1 = new SharedTestCase(List.of( + sharedTestCaseWithInterpreter1, + sharedTestCaseWithInterpreter2, + sharedTestCaseWithInterpreter3, + sharedTestCaseWithInterpreter4)); + + // 5 phrases visited) + var stc1 = mock(SharedTestSuite.SharedTestCaseWithInterpreter.class); + doReturn(getTestCase(5, StubbedTestCaseStrategy.LAST_PHRASE_FAILS)) + .when(stc1).getTestCase(); + doReturn(InterpreterObj.ofListener(() -> listener)).when(stc1).getInterpreterObj(); + + // 3 phrases visited + var stc2 = mock(SharedTestSuite.SharedTestCaseWithInterpreter.class); + doReturn(getTestCase(5, StubbedTestCaseStrategy.SOME_PHRASES_FILTERED)) + .when(stc2).getTestCase(); + doReturn(InterpreterObj.ofVisitor(() -> visitor)).when(stc2).getInterpreterObj(); + + var sharedTestCase2 = new SharedTestCase(List.of(stc1, stc2)); + + final ExecutorLifecycleTestObserver observer = new ExecutorLifecycleTestObserver(); + DefaultPolymorphicDslTestExecutor executor = + DefaultPolymorphicDslTestExecutor.of(Collections.singletonList(observer)); + + // ACT + executor.runTestsWithMetadata(List.of( + sharedTestCase1, + sharedTestCase2), "Component"); + // ASSERT + assertThat(observer.afterTestSuite).isEqualTo(1); + assertThat(observer.beforeTestSuite).isEqualTo(1); + assertThat(observer.beforeTestCaseTriggered).isEqualTo(2); + assertThat(observer.afterTestCaseTriggered).isEqualTo(2); + assertThat(observer.phraseFailure).isEqualTo(1); + assertThat(observer.duplicateSkipped).isEqualTo(0); + assertThat(observer.beforePhrase).isEqualTo(22); + assertThat(observer.afterPhrase).isEqualTo(21); + assertThat(observer.testCaseSuccess).isEqualTo(1); + } + @Test public void testExecutor_meetsSpecifications() throws URISyntaxException { final URL testResources = getClass().getClassLoader() @@ -40,4 +255,189 @@ public void testExecutor_meetsSpecifications() throws URISyntaxException { assertThat(results.totalFilteredDuplicateTests()).isEqualTo(0); assertThat(results.totalPhrases()).isGreaterThan(0); } + + + private static ParseTree getParseTree() { + var tree = mock(TerminalNode.class); + when(tree.getText()).thenReturn("Mocked sentence"); + return tree; + } + + private static ParseTree getFailingParseTree() { + var tree = mock(TerminalNode.class); + when(tree.getText()).thenReturn("Failing sentence"); + return tree; + } + + private enum StubbedTestCaseStrategy { + ALL_PHRASES_PASSING, + FIRST_PHRASE_FAILS, + LAST_PHRASE_FAILS, + SOME_PHRASES_FILTERED + } + private static final FilteredPhrase passingPhrase = new FilteredPhrase() { + @Override + public String getPhrase() { + return "passing phrase"; + } + + @Override + public Optional getParseTree() { + return Optional.of(parseTree); + } + }; + + private static FilteredPhrase failingPhrase = new FilteredPhrase() { + @Override + public String getPhrase() { + return "failing phrase"; + } + + @Override + public Optional getParseTree() { + return Optional.of(failingParseTree); + } + }; + + private static FilteredPhrase emptyPhrase = new FilteredPhrase() { + @Override + public String getPhrase() { + return "empty phrase"; + } + + @Override + public Optional getParseTree() { + return Optional.empty(); + } + }; + + private static TestCase getTestCase(int totalPhrases, StubbedTestCaseStrategy strategy) { + + List phrases = new ArrayList<>(totalPhrases); + for (int i=1; i <= totalPhrases; i++) { + switch (strategy) { + case ALL_PHRASES_PASSING -> { + phrases.add(passingPhrase); + } + case FIRST_PHRASE_FAILS -> { + if (i == 1) { + phrases.add(failingPhrase); + } else { + phrases.add(passingPhrase); + } + } + case LAST_PHRASE_FAILS -> { + if (i == totalPhrases -1) { + phrases.add(failingPhrase); + } else { + phrases.add(passingPhrase); + } + } + case SOME_PHRASES_FILTERED -> { + if ((i & 1) == 1) { + phrases.add(passingPhrase); + } else { + phrases.add(emptyPhrase); + } + } + } + + } + try { + return new DefaultPdslTestCase("stubbed test case", List.of(new TestBodyFragment(phrases)), new URI("./stubbed.feature")); + } catch (URISyntaxException e) { + throw new IllegalStateException(e); + } + } + + private static class ExecutorLifecycleTestObserver implements ExecutorObserver { + + private int beforeTestCaseTriggered = 0; + private int afterTestCaseTriggered = 0; + private int testCaseSuccess = 0; + private int duplicateSkipped = 0; + private int beforePhrase = 0; + private int afterPhrase = 0; + private int phraseFailure = 0; + private int beforeTestSuite = 0; + private int afterTestSuite = 0; + + public void reset() { + beforeTestCaseTriggered = 0; + afterTestCaseTriggered = 0; + testCaseSuccess = 0; + duplicateSkipped = 0; + beforePhrase = 0; + afterPhrase = 0; + phraseFailure = 0; + beforeTestSuite = 0; + afterTestSuite = 0; + } + + + @Override + public void onBeforePhrase(ParseTreeVisitor visitor, TestSection testSection) { beforePhrase++; } + + @Override + public void onAfterPhrase(ParseTreeListener listener, ParseTreeWalker walker, TestSection testSection) { afterPhrase++; } + + @Override + public void onAfterPhrase(ParseTreeVisitor visitor, TestSection testSection) { afterPhrase++; } + + @Override + public void onPhraseFailure(ParseTreeListener listener, TestSection testSection, TestCase testCase, Throwable exception) { + phraseFailure++; + } + + @Override + public void onPhraseFailure(ParseTreeVisitor visitor, TestSection testSection, TestCase testCase, Throwable exception) { + phraseFailure++; + } + + @Override + public void onBeforeTestSuite(Collection testCases, ParseTreeVisitor visitor, String context) { + beforeTestSuite++; + } + + @Override + public void onBeforeTestSuite(Collection testCases, ParseTreeListener listener, String context) { + beforeTestSuite++; + } + + @Override + public void onBeforeTestSuite(Collection testCases, String context) { + beforeTestSuite++; + } + + @Override + public void onAfterTestSuite(Collection testCases, ParseTreeVisitor visitor, MetadataTestRunResults results, String context) { + afterTestSuite++; + } + + @Override + public void onAfterTestSuite(Collection testCases, ParseTreeListener listener, MetadataTestRunResults results, String context) { + afterTestSuite++; + } + + @Override + public void onAfterTestSuite(Collection testCases, MetadataTestRunResults results, String context) { + afterTestSuite++; + } + + @Override + public void onBeforeTestCase(TestCase testCase) {beforeTestCaseTriggered++;} + + @Override + public void onAfterTestCase(TestCase testCase) { afterTestCaseTriggered++; } + + @Override + public void onTestCaseSuccess(TestCase testCase) { testCaseSuccess++; } + + @Override + public void onDuplicateSkipped(TestCase testCase) { duplicateSkipped++; } + + @Override + public void onBeforePhrase(ParseTreeListener listener, ParseTreeWalker walker, TestSection phrase) { beforePhrase++; } + + } } diff --git a/src/test/java/com/pdsl/uat/java/sut/ConfigurableExecutorRunner.java b/src/test/java/com/pdsl/uat/java/sut/ConfigurableExecutorRunner.java index 69dc0ee..e5dbed6 100644 --- a/src/test/java/com/pdsl/uat/java/sut/ConfigurableExecutorRunner.java +++ b/src/test/java/com/pdsl/uat/java/sut/ConfigurableExecutorRunner.java @@ -86,7 +86,7 @@ public int totalFilteredDuplicateTests() { } @Override - public TestRunResults runTests(Collection testCases, ParseTreeVisitor subgrammarVisitor) { + public TestRunResults runTests(Collection testCases, ParseTreeVisitor subgrammarVisitor) { throw new UnsupportedOperationException("Not implemented for this test case!"); }