Rename expected-to-fail? to expected-to-fail-test

components.dylan

@@ -77,8 +77,10 @@ define generic test-requires-assertions? (r :: <runnable>) => (required? :: <boo
 define abstract class <runnable> (<component>)
   constant slot test-function :: <function>,
     required-init-keyword: function:;
-  constant slot %expected-to-fail? :: type-union(<boolean>, <function>) = #f,
-    init-keyword: expected-to-fail?:;
+  // A test is expected to fail if this function returns true or if
+  // expected-to-fail-reason is true.
+  constant slot expected-to-fail-test :: false-or(<function>) = #f,
+    init-keyword: expected-to-fail-test:;
   constant slot expected-to-fail-reason :: false-or(<string>) = #f,
     init-keyword: expected-to-fail-reason:;
   // Benchmarks don't require assertions.  Needs to be an instance variable,
@@ -92,35 +94,24 @@ define abstract class <runnable> (<component>)
 end class <runnable>;
 
 define method make
-    (class :: subclass(<runnable>),
-     #rest args,
-     #key name, tags, expected-to-fail?, expected-to-fail-reason)
+    (class :: subclass(<runnable>), #rest args, #key name, tags)
  => (runnable :: <runnable>)
   let tags = map(make-tag, tags | #[]);
   let negative = choose(tag-negated?, tags);
   if (~empty?(negative))
     error("tags associated with tests or benchmarks may not be negated."
             " test = %s, tags = %s", name, negative);
   end;
-  if (expected-to-fail-reason & ~expected-to-fail?)
-    expected-to-fail? := #t;
-  end;
-  if (expected-to-fail? & ~expected-to-fail-reason)
-    error("the expected-to-fail-reason init argument must be supplied for"
-            " tests that are expected to fail. test = %s", name);
-  end;
-  apply(next-method, class,
-        tags: tags,
-        expected-to-fail?: expected-to-fail?,
-        expected-to-fail-reason: expected-to-fail-reason,
-        args)
+  apply(next-method, class, tags: tags, args)
 end method;
 
-define method expected-to-fail? (r :: <runnable>)
-  select (r.%expected-to-fail? by instance?)
-    <boolean> => r.%expected-to-fail?;
-    <function> => r.%expected-to-fail?();
-  end select
+define method expected-to-fail? (runnable :: <runnable>)
+  let test = runnable.expected-to-fail-test;
+  if (test)
+    test()
+  else
+    runnable.expected-to-fail-reason
+  end
 end method;
 
 define class <test> (<runnable>)

documentation/users-guide/source/reference.rst

@@ -20,32 +20,39 @@ Suites, Tests, and Benchmarks
 
    Define a new test.
 
-   :signature: define test *test-name* (#key *expected-to-fail?, expected-to-fail-reason, tags*) *body* end
+   :signature: define test *test-name*
+                   (#key *expected-to-fail-reason, expected-to-fail-test, tags*)
+                 *body*
+               end
    :parameter test-name: Name of the test; a Dylan variable name.
-   :parameter #key expected-to-fail?: An instance of either :drm:`<boolean>` or
-      :drm:`<function>`. This indicates whether or not the test is expected to
-      fail.
-   :parameter #key expected-to-fail-reason: A :drm:`<string>` or ``#f``. Must
-      be supplied if ``expected-to-fail?`` is true. A good reason usually
-      references a bug.
+   :parameter #key expected-to-fail-reason: A :drm:`<string>` or ``#f``.
+      The reason this test is expected to fail.
+   :parameter #key expected-to-fail-test: An instance of :drm:`<function>`.
+      A function to decide whether the test is expected to fail.
    :parameter #key tags: A list of strings to tag this test.
 
-   Tests may contain arbitrary code, plus any number of assertions.
-   If any assertion fails the test will fail, but any remaining
-   assertions in the test will still be executed.  If code outside of
-   an assertion signals an error, the test is marked as "crashed" and
-   remaining assertions are skipped.
-
-   If *expected-to-fail?* is set to ``#t`` or a function that when executed
-   returns a true value, then the test will be expected to fail. Such a failure
-   is treated as a successful test run. If the test passes rather than failing,
-   it is considered a test failure. This option has no effect on tests which
-   are *not implemented* or which have *crashed*.
-
-   *expected-to-fail-reason* is required if the test is expected to
-   fail. Normally it should reference a bug (a URL or at least a bug number).
-   If *expected-to-fail-reason* is supplied, *expected-to-fail?* may be
-   omitted because it is implied to be ``#t``.
+   Tests may contain arbitrary code, and must have at least one assertion. That
+   is they must call one of the ``assert-*`` or ``check-*`` macros. If they
+   have no assertions they are given a ``NOT IMPLEMENTED`` result. If any
+   assertion fails the test fails, but any remaining assertions in the test are
+   still executed.  If code outside of an assertion signals an error, the test
+   is marked as ``CRASHED`` and the rest of the test is skipped.
+
+   *expected-to-fail-test*, if supplied, is a function of no arguments that
+   returns a true value if the test is expected to fail. Such a failure is then
+   classified as an ``EXPECTED FAILURE`` result (which is a passing result). If
+   the test passes rather than failing, it has an ``UNEXPECTED SUCCESS`` result
+   (which is a type of failure). A common usage is to make expected failure
+   conditional on the OS: ``method () $os-name == #"win32" end``.  This option
+   has no effect on tests which are ``NOT IMPLEMENTED`` or ``CRASHED``.
+
+   *expected-to-fail-reason* should be supplied if the test is expected to
+   fail. It is good practice to reference a bug (a URL or at least a bug
+   number).
+
+   .. note:: If *expected-to-fail-reason* is supplied without
+             *expected-to-fail-test* the test is implicitly always expected to
+             fail.
 
    *tags* provide a way to select or filter out specific tests during a test
    run.  The Testworks command-line (provided by :func:`run-test-application`)
@@ -56,17 +63,21 @@ Suites, Tests, and Benchmarks
 
    Define a new benchmark.
 
-   :signature: define benchmark *name* (#key *expected-to-fail?, tags*) *body* end
-   :parameter name: Name of the benchmark; a Dylan variable name.
-   :parameter #key expected-to-fail?: An instance of either :drm:`<boolean>` or
-      :drm:`<function>`. This indicates whether or not the test is expected to
-      fail.
+   :signature: define benchmark *benchmark-name*
+                   (#key *expected-to-fail-reason, expected-to-fail-test, tags*)
+                 *body*
+               end
+   :parameter benchmark-name: Name of the benchmark; a Dylan variable name.
+   :parameter #key expected-to-fail-reason: A :drm:`<string>` or ``#f``.
+      The reason this benchmark is expected to fail.
+   :parameter #key expected-to-fail-test: An instance of :drm:`<function>`.
+      A function to decide whether the benchmark is expected to fail.
    :parameter #key tags: A list of strings to tag this benchmark.
 
-   Benchmarks may contain arbitrary code and do not require any
-   assertions.  If the benchmark signals an error it is marked as
-   "crashed". Other than this, and some differences in how the results
-   are displayed, benchmarks are the same as tests.
+   Benchmarks may contain arbitrary code and do not require any assertions.  If
+   the benchmark signals an error it is marked as ``CRASHED``. Other than this,
+   and minor differences in how the results are displayed, benchmarks are the
+   same as tests.
 
 .. macro:: benchmark-repeat
 

documentation/users-guide/source/usage.rst

@@ -235,6 +235,15 @@ marking a test as expected to fail, ``expected-to-fail-reason:`` is
 **required** and ``expected-to-fail?:`` is optional, and normally
 unnecessary. An example of a good reason is a bug URL or other bug reference.
 
+.. note:: When providing a value for ``expected-to-fail?:`` always provide a
+          method of no arguments. For example, instead of ``expected-to-fail?:
+          $os-name == #"win32"`` use ``expected-to-fail?: method () $os-name ==
+          #"win32" end``. The former is equivalent to ``expected-to-fail?: #f``
+          on non-Windows platforms and results in an ``UNEXPECTED SUCCESS``
+          result. This is because the (required) reason string is used as
+          shorthand to indicate that failure is expected even when
+          ``expected-to-fail?:`` is ``#f``.
+
 Test setup and teardown is accomplished with normal Dylan code using
 ``block () ... cleanup ... end;``...
 

run.dylan

@@ -309,7 +309,8 @@ define function decide-status
       => if (test.expected-to-fail?)
            let reason = format-to-string("%s passed but was expected to fail due to %=",
                                          test.component-type-name,
-                                         test.expected-to-fail-reason);
+                                         test.expected-to-fail-reason
+                                           | "<no reason supplied>");
            values($unexpected-success, reason)
          else
            $passed

tests/test-command-line.dylan

@@ -10,7 +10,12 @@ define test command-line-options-test ()
                   list("--debug=failures", debug-runner?, #t, #f),
                   list("--debug=foo", debug-runner?, #f, #t),
                   list("--options key1 = val1 --options key2 = val2", runner-options,
-                       tabling(<string-table>, "key1" => "val1", "key2" => "val2"),
+                       begin
+                         let t = make(<string-table>);
+                         t["key1"] := "val1";
+                         t["key2"] := "val2";
+                         t
+                       end,
                        #f),
                   list("key", runner-options, #f, #t)  // error, not key=val form
                   );

tests/testworks-test-suite.dylan

@@ -392,22 +392,22 @@ define constant test-expected-to-fail-always
   = make(<test>,
          name: "test-expected-to-fail-always",
          function: method () assert-true(#f) end,
-         // Intentionally not passing `expected-to-fail?: #t` here. It should
-         // be set to `#t` because a reason is provided below.
+         // Intentionally not passing `expected-to-fail-test:` here. The test
+         // should be expected to fail because a reason is provided.
          expected-to-fail-reason: "because of assert-true(#f)");
 
 define constant test-expected-to-fail-maybe
   = make(<test>,
          name: "test-expected-to-fail-maybe",
          function: method () assert-true(#f) end,
-         expected-to-fail?: method () #t end,
+         expected-to-fail-test: method () #t end,
          expected-to-fail-reason: "because of assert-true(#f)");
 
 define constant test-expected-to-crash-always
   = make(<test>,
          name: "test-expected-to-crash-always",
          function: curry(error, "test-expected-to-crash-always"),
-         expected-to-fail?: method () #t end,
+         expected-to-fail-test: method () #t end,
          expected-to-fail-reason: "because of error(...)");
 
 define constant expected-to-fail-suite
@@ -421,7 +421,7 @@ define constant test-unexpected-success
   = make(<test>,
          name: "test-unexpected-success",
          function: method () assert-true(#t) end,
-         expected-to-fail?: #t,
+         expected-to-fail-test: always(#t),
          expected-to-fail-reason: "because of assert-true(#t)");
 
 define constant unexpected-success-suite