Add support for "--failed" in ExUnit

lib/ex_unit/lib/ex_unit.ex

@@ -214,6 +214,9 @@ defmodule ExUnit do
     * `:exclude` - specifies which tests are run by skipping tests that match the
       filter;
 
+    * `:failures_manifest_file` - specifies a path to the file used to store failures
+      between runs;
+
     * `:formatters` - the formatters that will print results,
       defaults to `[ExUnit.CLIFormatter]`;
 
@@ -230,6 +233,9 @@ defmodule ExUnit do
     * `:module_load_timeout` - the timeout to be used when loading a test module,
       defaults to `60_000` milliseconds;
 
+    * `:only_test_ids` - a list of `{module_name, test_name}` tuples that limits
+      what tests get run;
+
     * `:refute_receive_timeout` - the timeout to be used on `refute_receive`
       calls, defaults to `100` milliseconds;
 

lib/ex_unit/lib/ex_unit/failures_manifest.ex

@@ -0,0 +1,103 @@
+defmodule ExUnit.FailuresManifest do
+  @moduledoc false
+
+  @type test_id :: {module, name :: atom}
+  @opaque t :: %{test_id => test_file :: Path.t()}
+
+  @manifest_vsn 1
+
+  @spec new() :: t
+  def new, do: %{}
+
+  @spec files_with_failures(t) :: MapSet.t(Path.t())
+  def files_with_failures(%{} = manifest) do
+    manifest
+    |> Map.values()
+    |> MapSet.new()
+  end
+
+  @spec failed_test_ids(t) :: MapSet.t(test_id)
+  def failed_test_ids(%{} = manifest) do
+    manifest
+    |> Map.keys()
+    |> MapSet.new()
+  end
+
+  @spec put_test(t, ExUnit.Test.t()) :: t
+  # TODO: figure out a way to prevent ExUnit from allowing the `file` tag
+  #       to be set incorrectly instead of using this function clause
+  #       (which only ignores values of the wrong type, but not wrong binary values)
+  def put_test(%{} = manifest, %ExUnit.Test{tags: %{file: file}})
+      when not is_binary(file),
+      do: manifest
+
+  def put_test(%{} = manifest, %ExUnit.Test{state: {ignored_state, _}})
+      when ignored_state in [:skipped, :excluded],
+      do: manifest
+
+  def put_test(%{} = manifest, %ExUnit.Test{state: nil} = test) do
+    Map.delete(manifest, {test.module, test.name})
+  end
+
+  def put_test(%{} = manifest, %ExUnit.Test{state: {failed_state, _}} = test)
+      when failed_state in [:failed, :invalid] do
+    Map.put(manifest, {test.module, test.name}, test.tags.file)
+  end
+
+  @spec write!(t, Path.t()) :: :ok
+  def write!(manifest, file) when is_binary(file) do
+    manifest = prune_deleted_tests(manifest)
+    binary = :erlang.term_to_binary({@manifest_vsn, manifest})
+    Path.dirname(file) |> File.mkdir_p!()
+    File.write!(file, binary)
+  end
+
+  @spec read(Path.t()) :: t
+  def read(file) when is_binary(file) do
+    with {:ok, binary} <- File.read(file),
+         {:ok, {@manifest_vsn, manifest}} when is_map(manifest) <- safe_binary_to_term(binary) do
+      manifest
+    else
+      _ -> new()
+    end
+  end
+
+  defp safe_binary_to_term(binary) do
+    {:ok, :erlang.binary_to_term(binary)}
+  rescue
+    ArgumentError ->
+      :error
+  end
+
+  defp prune_deleted_tests(manifest) do
+    Map.drop(manifest, find_deleted_tests(Enum.to_list(manifest), %{}, []))
+  end
+
+  defp find_deleted_tests([], _file_existence, deleted_tests), do: deleted_tests
+
+  defp find_deleted_tests([{{mod, name} = id, file} | rest] = all, file_existence, acc) do
+    file_exists = Map.fetch(file_existence, file)
+
+    cond do
+      file_exists == :error ->
+        # This is the first time we've looked up the existence of the file.
+        # Cache the result and try again.
+        file_existence = Map.put(file_existence, file, File.regular?(file))
+        find_deleted_tests(all, file_existence, acc)
+
+      file_exists == {:ok, false} ->
+        # The file does not exist, so the test has been deleted.
+        find_deleted_tests(rest, file_existence, [id | acc])
+
+      :code.is_loaded(mod) != false and not function_exported?(mod, name, 1) ->
+        # The test module has been loaded, but the test no longer exists.
+        find_deleted_tests(rest, file_existence, [id | acc])
+
+      true ->
+        # The file exists and the test module was not loaded (which means the test
+        # *might* still exist) or the function is exported (which means the test
+        # *definitely* still exists). Either way, we do not want to prune it.
+        find_deleted_tests(rest, file_existence, acc)
+    end
+  end
+end

lib/ex_unit/lib/ex_unit/filters.ex

@@ -1,4 +1,6 @@
 defmodule ExUnit.Filters do
+  alias ExUnit.FailuresManifest
+
   @moduledoc """
   Conveniences for parsing and evaluating filters.
   """
@@ -88,6 +90,27 @@ defmodule ExUnit.Filters do
     end)
   end
 
+  @doc """
+  Returns a tuple containing useful information about test failures from the
+  manifest. The tuple contains:
+
+    - A set of files that contain tests that failed the last time they ran.
+      The paths are absolute paths.
+    - A set of test ids that failed the last time they ran
+  """
+  @spec failure_info(Path.t()) :: {
+          MapSet.t(Path.t()),
+          MapSet.t(FailuresManifest.test_id())
+        }
+  def failure_info(manifest_file) do
+    manifest = FailuresManifest.read(manifest_file)
+
+    {
+      FailuresManifest.files_with_failures(manifest),
+      FailuresManifest.failed_test_ids(manifest)
+    }
+  end
+
   @doc """
   Evaluates the `include` and `exclude` filters against the given `tags` to
   determine if tests should be skipped or excluded.

lib/ex_unit/lib/ex_unit/runner.ex

@@ -34,6 +34,7 @@ defmodule ExUnit.Runner do
       include: opts[:include],
       manager: manager,
       max_cases: opts[:max_cases],
+      only_test_ids: opts[:only_test_ids],
       seed: opts[:seed],
       modules: :async,
       timeout: opts[:timeout],
@@ -132,8 +133,9 @@ defmodule ExUnit.Runner do
     tests = shuffle(config, tests)
     include = config.include
     exclude = config.exclude
+    test_ids = config.only_test_ids
 
-    for test <- tests do
+    for test <- tests, include_test?(test_ids, test) do
       tags = Map.merge(test.tags, %{test: test.name, module: test.module})
 
       case ExUnit.Filters.eval(include, exclude, tags, tests) do
@@ -143,6 +145,12 @@ defmodule ExUnit.Runner do
     end
   end
 
+  defp include_test?(nil, _test), do: true
+
+  defp include_test?(test_ids, test) do
+    MapSet.member?(test_ids, {test.module, test.name})
+  end
+
   defp spawn_module(config, test_module, tests) do
     parent = self()
 

lib/ex_unit/lib/ex_unit/runner_stats.ex

@@ -2,31 +2,22 @@ defmodule ExUnit.RunnerStats do
   @moduledoc false
 
   use GenServer
-  alias ExUnit.{Manifest, Test, TestModule}
+  alias ExUnit.{FailuresManifest, Test, TestModule}
 
   def stats(pid) do
     GenServer.call(pid, :stats, :infinity)
   end
 
   # Callbacks
 
-  @manifest ".ex_unit_results.elixir"
-
   def init(opts) do
-    manifest_file =
-      case Keyword.fetch(opts, :manifest_path) do
-        :error -> nil
-        {:ok, manifest_path} -> Path.join(manifest_path, @manifest)
-      end
-
     state = %{
       total: 0,
       failures: 0,
       skipped: 0,
       excluded: 0,
-      manifest_file: manifest_file,
-      old_manifest: nil,
-      new_manifest: Manifest.new()
+      failures_manifest_file: opts[:failures_manifest_file],
+      failures_manifest: FailuresManifest.new()
     }
 
     {:ok, state}
@@ -40,7 +31,7 @@ defmodule ExUnit.RunnerStats do
   def handle_cast({:test_finished, %Test{} = test}, state) do
     state =
       state
-      |> Map.update!(:new_manifest, &Manifest.add_test(&1, test))
+      |> Map.update!(:failures_manifest, &FailuresManifest.put_test(&1, test))
       |> Map.update!(:total, &(&1 + 1))
       |> increment_status_counter(test.state)
 
@@ -53,17 +44,15 @@ defmodule ExUnit.RunnerStats do
     {:noreply, %{state | failures: failures + test_count, total: total + test_count}}
   end
 
-  def handle_cast({:suite_started, _opts}, %{old_manifest: nil, manifest_file: file} = state)
+  def handle_cast({:suite_started, _opts}, %{failures_manifest_file: file} = state)
       when is_binary(file) do
-    state = %{state | old_manifest: Manifest.read(file)}
+    state = %{state | failures_manifest: FailuresManifest.read(file)}
     {:noreply, state}
   end
 
-  def handle_cast({:suite_finished, _, _}, %{manifest_file: file} = state) when is_binary(file) do
-    state.old_manifest
-    |> Manifest.merge(state.new_manifest)
-    |> Manifest.write!(file)
-
+  def handle_cast({:suite_finished, _, _}, %{failures_manifest_file: file} = state)
+      when is_binary(file) do
+    FailuresManifest.write!(state.failures_manifest, file)
     {:noreply, state}
   end