Wrap each Core test step with a decorator

java/server/src/org/openqa/selenium/server/htmlrunner/CoreStep.java

@@ -20,5 +20,5 @@
 import com.thoughtworks.selenium.Selenium;
 
 public interface CoreStep {
-  Object execute(Selenium selenium);
+  NextStepDecorator execute(Selenium selenium);
 }

java/server/src/org/openqa/selenium/server/htmlrunner/CoreStepFactory.java

@@ -21,5 +21,5 @@
 import java.util.List;
 
 interface CoreStepFactory {
-  CoreStep create(Iterator<List<String>> remainingSteps, String locator, String value);
+  CoreStep create(String locator, String value);
 }

java/server/src/org/openqa/selenium/server/htmlrunner/CoreTestCase.java

@@ -28,11 +28,15 @@
 import org.openqa.selenium.JavascriptExecutor;
 import org.openqa.selenium.WebDriver;
 
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.logging.Logger;
 
 public class CoreTestCase {
 
+  private static final Logger LOG = Logger.getLogger(CoreTestCase.class.getName());
+
   private static final ImmutableMap<String, CoreStepFactory> STEP_FACTORY =
     ImmutableMap.<String, CoreStepFactory>builder()
     .putAll(new ReflectivelyDiscoveredSteps().get())
@@ -50,18 +54,20 @@ public void run(Results results, WebDriver driver, Selenium selenium) {
       driver.get(url);
     }
 
-    List<CoreStep> steps = findCommands(driver);
-    for (CoreStep step : steps) {
-      try {
-        step.execute(selenium);
-      } catch (SeleniumException e) {
-        results.addTestFailure();
-        return;
+    List<LoggableStep> steps = findCommands(driver);
+    List<StepResult> testResults = new ArrayList<>(steps.size());
+    NextStepDecorator decorator = NextStepDecorator.IDENTITY;
+    for (LoggableStep step : steps) {
+      LOG.info(step.toString());
+      decorator = Preconditions.checkNotNull(decorator.evaluate(step, selenium), step);
+      testResults.add(new StepResult(step, null));
+      if (!decorator.isOkayToContinueTest()) {
+        break;
       }
     }
   }
 
-  private List<CoreStep> findCommands(WebDriver driver) {
+  private List<LoggableStep> findCommands(WebDriver driver) {
     // Let's just run and hide in the horror that is JS for the sake of speed.
     List<List<String>> rawSteps = (List<List<String>>) ((JavascriptExecutor) driver).executeScript(
       "var toReturn = [];\n" +
@@ -77,15 +83,65 @@ private List<CoreStep> findCommands(WebDriver driver) {
       "}\n" +
       "return toReturn;");
 
-    ImmutableList.Builder<CoreStep> steps = ImmutableList.builder();
+    ImmutableList.Builder<LoggableStep> steps = ImmutableList.builder();
     Iterator<List<String>> stepIterator = rawSteps.iterator();
     while (stepIterator.hasNext()) {
       List<String> step =  stepIterator.next();
       if (!STEP_FACTORY.containsKey(step.get(0))) {
         throw new SeleniumException("Unknown command: " + step.get(0));
       }
-      steps.add(STEP_FACTORY.get(step.get(0)).create(stepIterator, step.get(1), step.get(2)));
+      steps.add(new LoggableStep(
+        STEP_FACTORY.get(step.get(0)).create(step.get(1), step.get(2)),
+        step.get(0),
+        step.get(1),
+        step.get(2)));
     }
     return steps.build();
   }
+
+  private static class LoggableStep implements CoreStep {
+    private final CoreStep actualStep;
+    private final String command;
+    private final String locator;
+    private final String value;
+
+    public LoggableStep(CoreStep toWrap, String command, String locator, String value) {
+      this.actualStep = toWrap;
+      this.command = command;
+      this.locator = locator;
+      this.value = value;
+    }
+
+    @Override
+    public NextStepDecorator execute(Selenium selenium) {
+      return actualStep.execute(selenium);
+    }
+
+    @Override
+    public String toString() {
+      return String.format("|%s | %s | %s |", command, locator, value);
+    }
+  }
+
+  private static class StepResult {
+    private final LoggableStep step;
+    private final Throwable cause;
+
+    public StepResult(LoggableStep step, Throwable cause) {
+      this.step = Preconditions.checkNotNull(step);
+      this.cause = cause;
+    }
+
+    public boolean isSuccessful() {
+      return cause == null;
+    }
+
+    public boolean isError() {
+      return cause instanceof SeleniumException;
+    }
+
+    public boolean isFailure() {
+      return !isSuccessful() && !isError();
+    }
+  }
 }

java/server/src/org/openqa/selenium/server/htmlrunner/NextStepDecorator.java

@@ -0,0 +1,63 @@
+// Licensed to the Software Freedom Conservancy (SFC) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The SFC licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.openqa.selenium.server.htmlrunner;
+
+
+import com.thoughtworks.selenium.Selenium;
+
+abstract class NextStepDecorator {
+
+  static NextStepDecorator IDENTITY = new NextStepDecorator() {
+
+    @Override
+    public boolean isOkayToContinueTest() {
+      return true;
+    }
+  };
+
+  static NextStepDecorator ASSERTION_FAILED = new NextStepDecorator() {
+
+    @Override
+    public boolean isOkayToContinueTest() {
+      return false;
+    }
+  };
+
+  static NextStepDecorator VERIFICATION_FAILED = new NextStepDecorator() {
+
+    @Override
+    public boolean isOkayToContinueTest() {
+      return true;
+    }
+  };
+
+  public abstract  boolean isOkayToContinueTest();
+
+  public NextStepDecorator evaluate(CoreStep nextStep, Selenium selenium) {
+    return nextStep.execute(selenium);
+  }
+
+  public static NextStepDecorator ERROR(Throwable cause) {
+    return new NextStepDecorator() {
+      @Override
+      public boolean isOkayToContinueTest() {
+        return false;
+      }
+    };
+  }
+}

java/server/src/org/openqa/selenium/server/htmlrunner/NonReflectiveSteps.java

@@ -21,18 +21,14 @@
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.primitives.Booleans;
 
-import com.thoughtworks.selenium.SeleneseTestBase;
+import com.thoughtworks.selenium.Selenium;
 import com.thoughtworks.selenium.SeleniumException;
 
-import java.util.List;
 import java.util.logging.Logger;
 
 class NonReflectiveSteps {
   private static final Logger LOG = Logger.getLogger("Selenium Core Step");
-  private static final ImmutableMap<String, CoreStepFactory> wrappableSteps =
-    new ReflectivelyDiscoveredSteps().get();
 
   private static Supplier<ImmutableMap<String, CoreStepFactory>> STEPS =
     Suppliers.memoize(() -> build());
@@ -44,49 +40,47 @@ public ImmutableMap<String, CoreStepFactory> get() {
   private static ImmutableMap<String, CoreStepFactory> build() {
     ImmutableMap.Builder<String, CoreStepFactory> steps = ImmutableMap.builder();
 
-    CoreStepFactory nextCommandFails = (remainingSteps, locator, value) -> {
-      if (!remainingSteps.hasNext()) {
-        throw new SeleniumException("Next command not present. Unable to assert failure");
-      }
-      List<String> toWrap = remainingSteps.next();
-      if (!wrappableSteps.containsKey(toWrap.get(0))) {
-        throw new SeleniumException("Unable to wrap: " + toWrap.get(0));
-      }
-
-      return (selenium -> {
-        Object result;
-
-        try {
-          result = wrappableSteps.get(toWrap.get(0)).create(null, locator, value).execute(selenium);
-        } catch (SeleniumException e) {
-          result = e.getMessage();
-        }
-
-        SeleneseTestBase.assertEquals(value, String.valueOf(result));
-        return null;
-      });
-    };
-    // Not ideal, but it'll help us move things forward
+    CoreStepFactory nextCommandFails = (locator, value) -> (selenium) -> new NextCommandFails();
     steps.put("assertErrorOnNext", nextCommandFails);
     steps.put("assertFailureOnNext", nextCommandFails);
 
-    steps.put("echo", ((remainingSteps, locator, value) -> (selenium) -> {
+    steps.put("echo", ((locator, value) -> (selenium) -> {
       LOG.info(locator);
-      return null;
+      return NextStepDecorator.IDENTITY;
     }));
 
-    steps.put("pause", ((remainingSteps, locator, value) -> (selenium) -> {
+    steps.put("pause", ((locator, value) -> (selenium) -> {
       try {
         long timeout = Long.parseLong(locator);
         Thread.sleep(timeout);
-        return null;
+        return NextStepDecorator.IDENTITY;
       } catch (NumberFormatException e) {
-        throw new SeleniumException("Unable to parse timeout: " + locator);
+        return NextStepDecorator.ERROR(
+          new SeleniumException("Unable to parse timeout: " + locator));
       } catch (InterruptedException e) {
         System.exit(255);
         throw new CoreRunnerError("We never get this far");
       }
     }));
     return steps.build();
   }
+
+  private static class NextCommandFails extends NextStepDecorator {
+
+    @Override
+    public NextStepDecorator evaluate(CoreStep nextStep, Selenium selenium) {
+      NextStepDecorator actualResult = nextStep.execute(selenium);
+
+      // This is kind of fragile. Oh well.
+      if (actualResult.equals(NextStepDecorator.IDENTITY)) {
+        return NextStepDecorator.ASSERTION_FAILED;
+      }
+      return NextStepDecorator.IDENTITY;
+    }
+
+    @Override
+    public boolean isOkayToContinueTest() {
+      return true;
+    }
+  }
 }

java/server/src/org/openqa/selenium/server/htmlrunner/ReflectivelyDiscoveredSteps.java

@@ -68,7 +68,7 @@ private static ImmutableMap<String, CoreStepFactory> discover() {
         continue;
       }
 
-      CoreStepFactory factory = ((remainingSteps, locator, value) -> (selenium) ->
+      CoreStepFactory factory = ((locator, value) -> (selenium) ->
         invokeMethod(method, selenium, buildArgs(method, locator, value)));
 
       factories.put(method.getName(), factory);
@@ -88,48 +88,62 @@ private static ImmutableMap<String, CoreStepFactory> discover() {
       if (shortName != null && method.getParameterCount() < 2) {
         String negatedName = negateName(shortName);
 
-        factories.put("assert" + shortName, ((remainingSteps, locator, value) -> (selenium) -> {
+        factories.put("assert" + shortName, ((locator, value) -> (selenium) -> {
           Object seen = invokeMethod(method, selenium, buildArgs(method, locator, value));
           String expected = getExpectedValue(method, locator, value);
 
-          SeleneseTestBase.assertEquals(expected, seen);
-          return null;
+          try {
+            SeleneseTestBase.assertEquals(expected, seen);
+            return NextStepDecorator.IDENTITY;
+          } catch (AssertionError e) {
+            return NextStepDecorator.ASSERTION_FAILED;
+          }
         }));
 
-        factories.put("assert" + negatedName, ((remainingSteps, locator, value) -> (selenium) -> {
+        factories.put("assert" + negatedName, ((locator, value) -> (selenium) -> {
           Object seen = invokeMethod(method, selenium, buildArgs(method, locator, value));
           String expected = getExpectedValue(method, locator, value);
 
-          SeleneseTestBase.assertNotEquals(expected, seen);
-          return null;
+          try {
+            SeleneseTestBase.assertNotEquals(expected, seen);
+            return NextStepDecorator.IDENTITY;
+          } catch (AssertionError e) {
+            return NextStepDecorator.ASSERTION_FAILED;
+          }
         }));
 
-        factories.put("verify" + shortName, ((remainingSteps, locator, value) -> (selenium) -> {
+        factories.put("verify" + shortName, ((locator, value) -> (selenium) -> {
           Object seen = invokeMethod(method, selenium, buildArgs(method, locator, value));
           String expected = getExpectedValue(method, locator, value);
 
-          // TODO: Not this. Actual verification.
-          SeleneseTestBase.assertEquals(expected, seen);
-          return null;
+          try {
+            SeleneseTestBase.assertEquals(expected, seen);
+            return NextStepDecorator.IDENTITY;
+          } catch (AssertionError e) {
+            return NextStepDecorator.VERIFICATION_FAILED;
+          }
         }));
 
-        factories.put("verify" + negatedName, ((remainingSteps, locator, value) -> (selenium) -> {
+        factories.put("verify" + negatedName, ((locator, value) -> (selenium) -> {
           Object seen = invokeMethod(method, selenium, buildArgs(method, locator, value));
           String expected = getExpectedValue(method, locator, value);
 
-          // TODO: Not this. Actual verification.
-          SeleneseTestBase.assertNotEquals(expected, seen);
-          return null;
+          try {
+            SeleneseTestBase.assertNotEquals(expected, seen);
+            return NextStepDecorator.IDENTITY;
+          } catch (AssertionError e) {
+            return NextStepDecorator.VERIFICATION_FAILED;
+          }
         }));
       }
 
       factories.put(
         method.getName() + "AndWait",
-        ((remainingSteps, locator, value) -> (selenium -> {
+        ((locator, value) -> (selenium -> {
           Object result = invokeMethod(method, selenium, buildArgs(method, locator, value));
           // TODO: Hard coding this is obviously bogus
           selenium.waitForPageToLoad("30000");
-          return result;
+          return NextStepDecorator.IDENTITY;
         })));
     }
 
@@ -176,13 +190,14 @@ private static String getExpectedValue(Method method, String locator, String val
     }
   }
 
-  private static Object invokeMethod(Method method, Selenium selenium, String[] args) {
+  private static NextStepDecorator invokeMethod(Method method, Selenium selenium, String[] args) {
     try {
-      return method.invoke(selenium, args);
+      method.invoke(selenium, args);
+      return NextStepDecorator.IDENTITY;
     } catch (ReflectiveOperationException e) {
       for (Throwable cause = e; cause != null; cause = cause.getCause()) {
         if (cause instanceof SeleniumException) {
-          throw (SeleniumException) cause;
+          return NextStepDecorator.ERROR(cause);
         }
       }
       throw new CoreRunnerError(