make tests repeatable (--repeat-until-failure) (#5843)

This allows to use the new `mix test --repeat-until-failure` feature
to run the unit tests.

I also found some more issues where mix compile was called in tests
leading to missing modules if you had bad luck. This is now fixed by
always passing `--no-compile`.

Furthermore, all directories in the `tmp` folder are now properly
cleaned up. #5623
already tried to do this, but this still left directories if the
generated test name or random string contained a `/` character.

installer/test/mix_helper.exs

@@ -11,24 +11,26 @@ defmodule MixHelper do
   end
 
   defp random_string(len) do
-    len |> :crypto.strong_rand_bytes() |> Base.encode64() |> binary_part(0, len)
+    len |> :crypto.strong_rand_bytes() |> Base.url_encode64() |> binary_part(0, len)
   end
 
   def in_tmp(which, function) do
-    path = Path.join([tmp_path(), random_string(10) <> to_string(which)])
+    base = Path.join([tmp_path(), random_string(10)])
+    path = Path.join([base, to_string(which)])
 
     try do
       File.rm_rf!(path)
       File.mkdir_p!(path)
       File.cd!(path, function)
     after
-      File.rm_rf!(path)
+      File.rm_rf!(base)
     end
   end
 
   def in_tmp_project(which, function) do
     conf_before = Application.get_env(:phoenix, :generators) || []
-    path = Path.join([tmp_path(), random_string(10) <> to_string(which)])
+    base = Path.join([tmp_path(), random_string(10)])
+    path = Path.join([base, to_string(which)])
 
     try do
       File.rm_rf!(path)
@@ -47,14 +49,15 @@ defmodule MixHelper do
         function.()
       end)
     after
-      File.rm_rf!(path)
+      File.rm_rf!(base)
       Application.put_env(:phoenix, :generators, conf_before)
     end
   end
 
   def in_tmp_umbrella_project(which, function) do
     conf_before = Application.get_env(:phoenix, :generators) || []
-    path = Path.join([tmp_path(), random_string(10) <> to_string(which)])
+    base = Path.join([tmp_path(), random_string(10)])
+    path = Path.join([base, to_string(which)])
 
     try do
       apps_path = Path.join(path, "apps")
@@ -72,7 +75,7 @@ defmodule MixHelper do
       File.cd!(apps_path, function)
     after
       Application.put_env(:phoenix, :generators, conf_before)
-      File.rm_rf!(path)
+      File.rm_rf!(base)
     end
   end
 
@@ -81,7 +84,10 @@ defmodule MixHelper do
 
     try do
       capture_io(:stderr, fn ->
-        Mix.Project.in_project(app, path, [prune_code_paths: false], fun)
+        Mix.Project.in_project(app, path, [prune_code_paths: false], fn mod ->
+          fun.(mod)
+          Mix.Project.clear_deps_cache()
+        end)
       end)
     after
       Mix.Project.push(name, file)

test/mix/tasks/phx.digest.clean_test.exs

@@ -2,19 +2,24 @@ defmodule Mix.Tasks.Phx.Digest.CleanTest do
   use ExUnit.Case
 
   test "fails when the given paths are invalid" do
-    Mix.Tasks.Phx.Digest.Clean.run(["--output", "invalid_path"])
+    Mix.Tasks.Phx.Digest.Clean.run(["--output", "invalid_path", "--no-compile"])
 
     assert_received {:mix_shell, :error, ["The output path \"invalid_path\" does not exist"]}
   end
 
   test "removes old versions", config do
     output_path = Path.join("tmp", to_string(config.test))
     input_path = "priv/static"
-    :ok = File.mkdir_p!(output_path)
 
-    Mix.Tasks.Phx.Digest.Clean.run([input_path, "-o", output_path, "--no-compile"])
+    try do
+      :ok = File.mkdir_p!(output_path)
 
-    msg = "Clean complete for \"#{output_path}\""
-    assert_received {:mix_shell, :info, [^msg]}
+      Mix.Tasks.Phx.Digest.Clean.run([input_path, "-o", output_path, "--no-compile"])
+
+      msg = "Clean complete for \"#{output_path}\""
+      assert_received {:mix_shell, :info, [^msg]}
+    after
+      File.rm_rf!(output_path)
+    end
   end
 end

test/mix/tasks/phx.digest_test.exs

@@ -5,7 +5,7 @@ defmodule Mix.Tasks.Phx.DigestTest do
   import MixHelper
 
   test "logs when the path is invalid" do
-    Mix.Tasks.Phx.Digest.run(["invalid_path", "--no-deps-check"])
+    Mix.Tasks.Phx.Digest.run(["invalid_path", "--no-deps-check", "--no-compile"])
     assert_received {:mix_shell, :error, ["The input path \"invalid_path\" does not exist"]}
   end
 

test/mix/tasks/phx.routes_test.exs

@@ -24,37 +24,37 @@ defmodule Mix.Tasks.Phx.RoutesTest do
   use ExUnit.Case, async: true
 
   test "format routes for specific router" do
-    Mix.Tasks.Phx.Routes.run(["PhoenixTestWeb.Router"])
+    Mix.Tasks.Phx.Routes.run(["PhoenixTestWeb.Router", "--no-compile"])
     assert_received {:mix_shell, :info, [routes]}
     assert routes =~ "page_path  GET  /  PageController :index"
   end
 
   test "prints error when explicit router cannot be found" do
     assert_raise Mix.Error, "the provided router, Foo.UnknownBar.CantFindBaz, does not exist", fn ->
-      Mix.Tasks.Phx.Routes.run(["Foo.UnknownBar.CantFindBaz"])
+      Mix.Tasks.Phx.Routes.run(["Foo.UnknownBar.CantFindBaz", "--no-compile"])
     end
   end
 
   test "prints error when implicit router cannot be found" do
     assert_raise Mix.Error, ~r/no router found at FooWeb.Router or Foo.Router/, fn ->
-      Mix.Tasks.Phx.Routes.run([], Foo)
+      Mix.Tasks.Phx.Routes.run(["--no-compile"], Foo)
     end
   end
 
   test "implicit router detection for web namespace" do
-    Mix.Tasks.Phx.Routes.run([], PhoenixTest)
+    Mix.Tasks.Phx.Routes.run(["--no-compile"], PhoenixTest)
     assert_received {:mix_shell, :info, [routes]}
     assert routes =~ "page_path  GET  /  PageController :index"
   end
 
   test "implicit router detection fallback for old namespace" do
-    Mix.Tasks.Phx.Routes.run([], PhoenixTestOld)
+    Mix.Tasks.Phx.Routes.run(["--no-compile"], PhoenixTestOld)
     assert_received {:mix_shell, :info, [routes]}
     assert routes =~ "page_path  GET  /old  PageController :index"
   end
 
   test "overrides module name for route with :log_module metadata" do
-    Mix.Tasks.Phx.Routes.run(["PhoenixTestLiveWeb.Router"])
+    Mix.Tasks.Phx.Routes.run(["PhoenixTestLiveWeb.Router", "--no-compile"])
     assert_received {:mix_shell, :info, [routes]}
     assert routes =~ "page_path  GET  /  PageLive.Index :index"
   end

test/phoenix/controller/render_test.exs

@@ -176,6 +176,7 @@ defmodule Phoenix.Controller.RenderTest do
 
     setup context do
       :telemetry.attach_many(context.test, @render_events, &__MODULE__.message_pid/4, self())
+      on_exit(fn -> :telemetry.detach(context.test) end)
     end
 
     def message_pid(event, measures, metadata, test_pid) do

test/phoenix/digester_test.exs

@@ -13,6 +13,7 @@ defmodule Phoenix.DigesterTest do
 
   setup do
     File.rm_rf!(@output_path)
+    on_exit(fn -> File.rm_rf!(@output_path) end)
     :ok
   end
 
@@ -90,6 +91,8 @@ defmodule Phoenix.DigesterTest do
 
       assert json["digests"][key]["sha512"] ==
                "93pY5dBa8nHHi0Zfj75O/vXCBXb+UvEVCyU7Yd3pzOJ7o1wkYBWbvs3pVXhBChEmo8MDANT11vsggo2+bnYqoQ=="
+    after
+      File.rm_rf!("tmp/digest")
     end
 
     test "excludes compiled files" do
@@ -133,6 +136,8 @@ defmodule Phoenix.DigesterTest do
       assert_in_delta json["digests"]["foo-1198fd3c7ecf0e8f4a33a6e4fc5ae168.css"]["mtime"],
                       now(),
                       2
+    after
+      File.rm_rf!("tmp/digest")
     end
 
     test "excludes files that no longer exist from cache manifest" do
@@ -150,6 +155,8 @@ defmodule Phoenix.DigesterTest do
 
       json = Path.join(input_path, "cache_manifest.json") |> json_read!()
       assert json["digests"] == %{}
+    after
+      File.rm_rf!("tmp/digest")
     end
 
     test "digests and compress nested files" do
@@ -172,38 +179,46 @@ defmodule Phoenix.DigesterTest do
       input_path = Path.join("tmp", "phoenix_digest_twice")
       input_file = Path.join(input_path, "file.js")
 
-      File.rm_rf!(input_path)
-      File.mkdir_p!(input_path)
-      File.mkdir_p!(@output_path)
+      try do
+        File.rm_rf!(input_path)
+        File.mkdir_p!(input_path)
+        File.mkdir_p!(@output_path)
 
-      File.write!(input_file, "console.log('test');")
-      assert :ok = Phoenix.Digester.compile(input_path, @output_path, true)
+        File.write!(input_file, "console.log('test');")
+        assert :ok = Phoenix.Digester.compile(input_path, @output_path, true)
 
-      json1 = Path.join(@output_path, "cache_manifest.json") |> json_read!()
-      assert Enum.count(json1["digests"]) == 1
+        json1 = Path.join(@output_path, "cache_manifest.json") |> json_read!()
+        assert Enum.count(json1["digests"]) == 1
 
-      File.write!(input_file, "console.log('test2');")
-      assert :ok = Phoenix.Digester.compile(input_path, @output_path, true)
+        File.write!(input_file, "console.log('test2');")
+        assert :ok = Phoenix.Digester.compile(input_path, @output_path, true)
 
-      json2 = Path.join(@output_path, "cache_manifest.json") |> json_read!()
-      assert Enum.count(json2["digests"]) == 2
+        json2 = Path.join(@output_path, "cache_manifest.json") |> json_read!()
+        assert Enum.count(json2["digests"]) == 2
+      after
+        File.rm_rf!(input_path)
+      end
     end
 
     test "doesn't duplicate files when digesting and compressing twice" do
       input_path = Path.join("tmp", "phoenix_digest_twice")
       input_file = Path.join(input_path, "file.js")
 
-      File.rm_rf!(input_path)
-      File.mkdir_p!(input_path)
-      File.write!(input_file, "console.log('test');")
-
-      assert :ok = Phoenix.Digester.compile(input_path, input_path, true)
-      assert :ok = Phoenix.Digester.compile(input_path, input_path, true)
-
-      output_files = assets_files(input_path)
-      refute "file.js.gz.gz" in output_files
-      refute "cache_manifest.json.gz" in output_files
-      refute Enum.any?(output_files, &(&1 =~ ~r/file-#{@hash_regex}.[\w|\d]*.[-#{@hash_regex}/))
+      try do
+        File.rm_rf!(input_path)
+        File.mkdir_p!(input_path)
+        File.write!(input_file, "console.log('test');")
+
+        assert :ok = Phoenix.Digester.compile(input_path, input_path, true)
+        assert :ok = Phoenix.Digester.compile(input_path, input_path, true)
+
+        output_files = assets_files(input_path)
+        refute "file.js.gz.gz" in output_files
+        refute "cache_manifest.json.gz" in output_files
+        refute Enum.any?(output_files, &(&1 =~ ~r/file-#{@hash_regex}.[\w|\d]*.[-#{@hash_regex}/))
+      after
+        File.rm_rf!(input_path)
+      end
     end
 
     test "digests only absolute and relative asset paths found within stylesheets with vsn" do

test/phoenix/endpoint/endpoint_test.exs

@@ -214,6 +214,9 @@ defmodule Phoenix.Endpoint.EndpointTest do
     UrlEndpoint.start_link()
     assert UrlEndpoint.path("/phoenix.png") =~ "/api/phoenix.png"
     assert UrlEndpoint.static_path("/phoenix.png") =~ "/api/phoenix.png"
+  after
+    :code.purge(__MODULE__.UrlEndpoint)
+    :code.delete(__MODULE__.UrlEndpoint)
   end
 
   @tag :capture_log
@@ -227,6 +230,9 @@ defmodule Phoenix.Endpoint.EndpointTest do
     StaticEndpoint.start_link()
     assert StaticEndpoint.path("/phoenix.png") =~ "/phoenix.png"
     assert StaticEndpoint.static_path("/phoenix.png") =~ "/static/phoenix.png"
+  after
+    :code.purge(__MODULE__.StaticEndpoint)
+    :code.delete(__MODULE__.StaticEndpoint)
   end
 
   @tag :capture_log
@@ -243,6 +249,9 @@ defmodule Phoenix.Endpoint.EndpointTest do
     AddressEndpoint.start_link()
     assert {:ok, {{127, 0, 0, 1}, port}} = AddressEndpoint.server_info(:http)
     assert is_integer(port)
+  after
+    :code.purge(__MODULE__.AddressEndpoint)
+    :code.delete(__MODULE__.AddressEndpoint)
   end
 
   test "injects pubsub broadcast with configured server" do

test/phoenix/param_test.exs

@@ -32,6 +32,9 @@ defmodule Phoenix.ParamTest do
     end
     assert to_param(struct(Foo, id: 1)) == "1"
     assert to_param(struct(Foo, id: "foo")) == "foo"
+  after
+    :code.purge(__MODULE__.Foo)
+    :code.delete(__MODULE__.Foo)
   end
 
   test "to_param for derivable structs without id" do
@@ -55,5 +58,10 @@ defmodule Phoenix.ParamTest do
     assert_raise ArgumentError, msg, fn ->
       to_param(struct(Bar, uuid: nil))
     end
+  after
+    :code.purge(Module.concat(Phoenix.Param, __MODULE__.Bar))
+    :code.delete(Module.concat(Phoenix.Param, __MODULE__.Bar))
+    :code.purge(__MODULE__.Bar)
+    :code.delete(__MODULE__.Bar)
   end
 end

test/phoenix/router/routing_test.exs

@@ -323,6 +323,7 @@ defmodule Phoenix.Router.RoutingTest do
         end,
         nil
       )
+      on_exit(fn -> :telemetry.detach(test_name) end)
     end
 
     test "phoenix.router_dispatch.start and .stop are emitted on success" do