diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b4bce06377..5007392979 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,7 +9,7 @@ concurrency:
jobs:
mix_test:
- name: mix test (OTP ${{matrix.otp}} | Elixir ${{matrix.elixir}})
+ name: mix test (Elixir ${{matrix.elixir}} | OTP ${{matrix.otp}})
env:
MIX_ENV: test
@@ -72,7 +72,7 @@ jobs:
if: ${{ matrix.lint }}
installer_test:
- name: installer test (OTP ${{matrix.otp}} | Elixir ${{matrix.elixir}})
+ name: installer test (Elixir ${{matrix.elixir}} | OTP ${{matrix.otp}})
env:
MIX_ENV: test
@@ -223,14 +223,16 @@ jobs:
run: mix docs --warnings-as-errors
integration-test-elixir:
- name: integration test (OTP ${{matrix.otp}} | Elixir ${{matrix.elixir}})
+ name: integration test [${{ matrix.service == 'none' && 'sqlite3 + no-db' || matrix.service }}] (Elixir ${{ matrix.versions.elixir }} | OTP ${{ matrix.versions.otp }})
runs-on: ubuntu-24.04
- timeout-minutes: 30
+ timeout-minutes: 15
strategy:
+ fail-fast: false
matrix:
- include:
+ service: [postgresql, mysql, mssql, none]
+ versions:
- elixir: "1.18.4"
otp: "27.3.4.3"
@@ -247,42 +249,6 @@ jobs:
MIX_OS_DEPS_COMPILE_PARTITION_COUNT: 2
MAKEFLAGS: "-j2"
- services:
- postgres:
- image: postgres:18
- ports:
- - 5432:5432
- env:
- POSTGRES_PASSWORD: postgres
- options: >-
- --health-cmd "pg_isready -U postgres"
- --health-interval 2s
- --health-timeout 3s
- --health-retries 10
- mysql:
- image: mysql:26
- ports:
- - 3306:3306
- env:
- MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
- options: >-
- --health-cmd "mysqladmin ping -h localhost"
- --health-interval 2s
- --health-timeout 3s
- --health-retries 10
- mssql:
- image: mcr.microsoft.com/mssql/server:2019-latest
- env:
- ACCEPT_EULA: Y
- SA_PASSWORD: some!Password
- ports:
- - 1433:1433
- options: >-
- --health-cmd "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -C -Q 'SELECT 1' 2>/dev/null || /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -Q 'SELECT 1' 2>/dev/null"
- --health-interval 2s
- --health-timeout 3s
- --health-retries 25
-
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -294,11 +260,16 @@ jobs:
mkdir -p installer/tmp
sudo mount -t tmpfs -o size=4G,uid=$(id -u),gid=$(id -g) tmpfs installer/tmp
+ - name: Start database container (${{ matrix.service }})
+ if: matrix.service != 'none'
+ run: |
+ docker compose -f integration_test/docker-compose.yml up -d --wait ${{ matrix.service }}
+
- name: Set up Elixir
uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
with:
- elixir-version: ${{ matrix.elixir }}
- otp-version: ${{ matrix.otp }}
+ elixir-version: ${{ matrix.versions.elixir }}
+ otp-version: ${{ matrix.versions.otp }}
- name: Restore deps and _build cache
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
@@ -306,9 +277,9 @@ jobs:
path: |
integration_test/deps
integration_test/_build
- key: integration-deps-${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('integration_test/mix.lock') }}
+ key: integration-deps-${{ runner.os }}-${{ matrix.versions.otp }}-${{ matrix.versions.elixir }}-${{ hashFiles('integration_test/mix.lock') }}
restore-keys: |
- integration-deps-${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}
+ integration-deps-${{ runner.os }}-${{ matrix.versions.otp }}-${{ matrix.versions.elixir }}
- name: Fetch dependencies
working-directory: integration_test
@@ -323,6 +294,59 @@ jobs:
- name: Run integration tests
working-directory: integration_test
run: |
- mix test --include database \
+ if [ "${{ matrix.service }}" = "none" ]; then
+ FLAGS="--include database:sqlite3"
+ else
+ FLAGS="--include database:${{ matrix.service }} --only database:${{ matrix.service }}"
+ fi
+
+ mix test $FLAGS \
--formatter ExUnit.CLIFormatter \
--formatter Phoenix.Integration.SummaryFormatter
+ env:
+ PHX_TEST_SERVICE: ${{ matrix.service }}
+ PHX_ELIXIR_VERSION: ${{ matrix.versions.elixir }}
+ PHX_OTP_VERSION: ${{ matrix.versions.otp }}
+ PHX_INTEGRATION_SUMMARY_JSON: ${{ runner.temp }}/summary-${{ matrix.service }}-${{ matrix.versions.otp }}-${{ matrix.versions.elixir }}.json
+
+ - name: Upload summary artifact
+ if: always()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: summary-${{ matrix.service }}-${{ matrix.versions.otp }}-${{ matrix.versions.elixir }}
+ path: ${{ runner.temp }}/summary-${{ matrix.service }}-${{ matrix.versions.otp }}-${{ matrix.versions.elixir }}.json
+ retention-days: 1
+ compression-level: 0
+ overwrite: true
+ if-no-files-found: warn
+
+ integration-test-summary:
+ name: integration test results
+ runs-on: ubuntu-24.04
+ timeout-minutes: 2
+ needs: integration-test-elixir
+ if: always()
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Set up Elixir
+ uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
+ with:
+ elixir-version: "1.20.4"
+ otp-version: "29.0.5"
+
+ - name: Download summary artifacts
+ uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+ with:
+ pattern: summary-*
+ merge-multiple: true
+ path: tmp/summaries
+
+ - name: Aggregate test summaries
+ run: elixir integration_test/aggregate_summary.exs tmp/summaries
+ env:
+ MATRIX_RESULT: ${{ needs.integration-test-elixir.result }}
diff --git a/integration_test/aggregate_summary.exs b/integration_test/aggregate_summary.exs
new file mode 100644
index 0000000000..c870cc354f
--- /dev/null
+++ b/integration_test/aggregate_summary.exs
@@ -0,0 +1,677 @@
+# Script to aggregate integration test shard JSON summaries into a single GitHub Actions summary.
+#
+# Usage:
+# elixir integration_test/aggregate_summary.exs [directory_with_json_files]
+#
+
+defmodule Phoenix.Integration.AggregateSummary do
+ @service_order ["postgresql", "mysql", "mssql", "none"]
+
+ def run(argv) do
+ summaries_dir = List.first(argv) || "tmp/summaries"
+
+ json_files = Path.wildcard(Path.join(summaries_dir, "**/*.json"))
+
+ if json_files == [] do
+ IO.puts(:stderr, "No summary JSON files found in #{summaries_dir}")
+
+ if summary_file = System.get_env("GITHUB_STEP_SUMMARY") do
+ File.write!(
+ summary_file,
+ "## Phoenix Integration Tests: No summary artifacts found\n\n",
+ [:append]
+ )
+ end
+
+ System.halt(1)
+ end
+
+ summaries =
+ Enum.map(json_files, fn file ->
+ file
+ |> File.read!()
+ |> decode_json!()
+ end)
+
+ markdown = format_report(summaries)
+
+ if summary_file = System.get_env("GITHUB_STEP_SUMMARY") do
+ case File.write(summary_file, markdown, [:append]) do
+ :ok ->
+ :ok
+
+ {:error, reason} ->
+ IO.warn(
+ "Failed to write integration test summary to #{summary_file}: #{inspect(reason)}"
+ )
+ end
+ end
+
+ IO.puts(markdown)
+
+ any_failed? =
+ Enum.any?(summaries, fn s ->
+ s["status"] != "passed" or (s["total_failures"] || 0) > 0
+ end)
+
+ matrix_failed? = System.get_env("MATRIX_RESULT") in ["failure", "cancelled"]
+
+ if any_failed? or matrix_failed? do
+ System.halt(1)
+ else
+ System.halt(0)
+ end
+ end
+
+ defp decode_json!(content) do
+ cond do
+ Code.ensure_loaded?(JSON) ->
+ apply(JSON, :decode!, [content])
+
+ Code.ensure_loaded?(Jason) ->
+ apply(Jason, :decode!, [content])
+
+ true ->
+ raise "Neither JSON nor Jason available for decoding summary"
+ end
+ end
+
+ def format_report(summaries) do
+ versions =
+ summaries
+ |> Enum.map(fn s -> {s["elixir"], s["otp"]} end)
+ |> Enum.uniq()
+ |> Enum.sort_by(fn {elixir, _otp} -> elixir end)
+
+ version_headers = format_version_headers(versions)
+
+ peak_wall_time =
+ summaries
+ |> Enum.map(&(&1["wall_time_ms"] || 0))
+ |> Enum.max(fn -> 0 end)
+
+ total_shards = length(summaries)
+ total_suites = length(versions)
+
+ any_failed? =
+ Enum.any?(summaries, fn s ->
+ s["status"] != "passed" or (s["total_failures"] || 0) > 0
+ end)
+
+ overall_status = if any_failed?, do: "Failed", else: "Passed"
+
+ summary_header = """
+ ## Phoenix Integration Tests Summary
+
+ | Total Suites | Total Shards | Overall Status | Peak Wall Time |
+ | :---: | :---: | :---: | :---: |
+ | **#{total_suites}** | **#{total_shards}** | **#{overall_status}** | **`#{format_duration(peak_wall_time)}`** |
+ """
+
+ overview_table = format_combined_overview_table(summaries, versions)
+ slowest_table = format_combined_slowest_table(summaries, versions, version_headers)
+ shard_details = format_combined_shard_details(summaries, versions, version_headers)
+
+ [summary_header, overview_table, slowest_table, shard_details]
+ |> Enum.reject(&(&1 in ["", nil]))
+ |> Enum.join("\n\n")
+ |> String.trim()
+ |> Kernel.<>("\n\n")
+ end
+
+ defp format_combined_overview_table(summaries, versions) do
+ grouped =
+ summaries
+ |> Enum.group_by(fn s -> {s["elixir"], s["otp"]} end)
+
+ table_rows =
+ Enum.map_join(versions, "\n", fn {elixir, otp} = ver ->
+ shard_summaries = Map.get(grouped, ver, [])
+ sorted_shards = sort_shards(shard_summaries)
+
+ total_executed = Enum.sum(Enum.map(sorted_shards, &(&1["executed_tests"] || 0)))
+ total_passed = Enum.sum(Enum.map(sorted_shards, &(&1["passed_tests"] || 0)))
+ total_failed = Enum.sum(Enum.map(sorted_shards, &(&1["total_failures"] || 0)))
+ max_wall_time = Enum.max(Enum.map(sorted_shards, &(&1["wall_time_ms"] || 0)), fn -> 0 end)
+
+ status_str = if total_failed == 0, do: "Passed", else: "Failed (#{total_failed})"
+
+ overall_slowest =
+ sorted_shards
+ |> Enum.flat_map(&(&1["slowest_tests"] || []))
+ |> Enum.max_by(&(&1["time_us"] || 0), fn -> nil end)
+
+ overall_slowest_desc = format_test_desc(overall_slowest)
+
+ total_row =
+ "| **Elixir #{elixir} / OTP #{otp}** | **All** | **#{status_str}** | **#{total_passed}/#{total_executed}** | **`#{format_duration(max_wall_time)}`** | #{overall_slowest_desc} |"
+
+ shard_rows =
+ Enum.map_join(sorted_shards, "\n", fn shard ->
+ service_label = service_name(shard["service"])
+ status = shard_status_cell(shard)
+ executed = shard["executed_tests"] || 0
+ wall_time = format_duration(shard["wall_time_ms"] || 0)
+ slowest_desc = format_shard_slowest(shard["slowest_tests"])
+
+ "| | #{service_label} | #{status} | #{executed} | `#{wall_time}` | #{slowest_desc} |"
+ end)
+
+ "#{total_row}\n#{shard_rows}"
+ end)
+
+ """
+ | Elixir / OTP | Database | Status | Tests | Wall Time | Slowest Test |
+ | :--- | :--- | :---: | :---: | :---: | :--- |
+ #{table_rows}
+ """
+ |> String.trim()
+ end
+
+ defp format_combined_slowest_table(summaries, versions, version_headers) do
+ all_tests =
+ for s <- summaries,
+ t <- s["slowest_tests"] || [] do
+ Map.merge(t, %{
+ "service" => s["service"],
+ "elixir" => s["elixir"],
+ "otp" => s["otp"]
+ })
+ end
+
+ grouped_tests =
+ all_tests
+ |> Enum.group_by(fn t -> {t["module"], t["name"]} end)
+ |> Enum.map(fn {{mod, name}, instances} ->
+ first = hd(instances)
+
+ durations_by_ver =
+ Enum.into(instances, %{}, fn inst ->
+ ms = inst["duration_ms"] || div(inst["time_us"] || 0, 1000)
+ {{inst["elixir"], inst["otp"]}, ms}
+ end)
+
+ max_duration_ms =
+ durations_by_ver
+ |> Map.values()
+ |> Enum.max(fn -> 0 end)
+
+ statuses = Enum.map(instances, & &1["status"])
+ all_passed? = Enum.all?(statuses, &(&1 == "passed"))
+ status = if all_passed?, do: "passed", else: "failed"
+
+ %{
+ module: mod,
+ name: name,
+ service: first["service"],
+ file: first["file"],
+ line: first["line"],
+ durations_by_ver: durations_by_ver,
+ max_duration_ms: max_duration_ms,
+ status: status
+ }
+ end)
+ |> Enum.sort_by(& &1.max_duration_ms, :desc)
+ |> Enum.take(10)
+
+ if grouped_tests != [] do
+ ver_header_cols = Enum.map_join(version_headers, " | ", fn {_, h} -> h end)
+ ver_align_cols = Enum.map_join(version_headers, " | ", fn _ -> ":---" end)
+
+ rows =
+ Enum.map_join(grouped_tests, "\n", fn t ->
+ ver_duration_cols =
+ Enum.map_join(versions, " | ", fn ver ->
+ case Map.get(t.durations_by_ver, ver) do
+ nil -> "-"
+ ms -> "`#{format_duration(ms)}`"
+ end
+ end)
+
+ "| #{t.status} | `#{format_duration(t.max_duration_ms)}` | #{ver_duration_cols} | #{escape_markdown(t.name)} | `#{t.module}` | #{service_name(t.service)} | `#{t.file}:#{t.line}` |"
+ end)
+
+ """
+
+ Top 10 Slowest Tests
+
+ | Status | Max Duration | #{ver_header_cols} | Test | Module | Database | Location |
+ | :---: | :--- | #{ver_align_cols} | :--- | :--- | :---: | :--- |
+ #{rows}
+
+
+ """
+ |> String.trim()
+ else
+ ""
+ end
+ end
+
+ defp format_combined_shard_details(summaries, versions, version_headers) do
+ shards_by_service = Enum.group_by(summaries, & &1["service"])
+
+ services =
+ @service_order
+ |> Enum.filter(&Map.has_key?(shards_by_service, &1))
+ |> Kernel.++(Enum.sort(Map.keys(shards_by_service) -- @service_order))
+
+ Enum.map_join(services, "\n\n", fn service ->
+ service_summaries =
+ shards_by_service
+ |> Map.get(service, [])
+ |> Enum.sort_by(fn s -> {s["elixir"], s["otp"]} end)
+
+ format_service_accordion(service, service_summaries, versions, version_headers)
+ end)
+ end
+
+ defp format_service_accordion(service, service_summaries, versions, version_headers) do
+ service_label = service_name(service)
+
+ total_failures =
+ Enum.sum(Enum.map(service_summaries, &(&1["total_failures"] || 0)))
+
+ max_tests =
+ Enum.max(Enum.map(service_summaries, &(&1["executed_tests"] || 0)), fn -> 0 end)
+
+ details_tag = if total_failures > 0, do: "", else: ""
+ status_label = if total_failures == 0, do: "passed", else: "failed (#{total_failures})"
+
+ wall_times_summary =
+ Enum.map_join(service_summaries, " | ", fn s ->
+ "#{format_duration(s["wall_time_ms"] || 0)} in #{short_version(s["elixir"])}"
+ end)
+
+ modules_section = format_service_modules_table(service_summaries, versions, version_headers)
+ timelines_section = format_service_timelines(service_summaries)
+ tests_section = format_service_tests_table(service_summaries, versions, version_headers)
+
+ """
+ #{details_tag}
+ #{service_label} Details: #{status_label} — #{max_tests} tests (#{wall_times_summary})
+
+ #{timelines_section}
+
+ #{modules_section}
+
+ #{tests_section}
+
+ """
+ |> String.trim()
+ end
+
+ defp format_service_modules_table(service_summaries, versions, version_headers) do
+ all_modules =
+ for s <- service_summaries,
+ m <- s["modules"] || [] do
+ Map.merge(m, %{"elixir" => s["elixir"], "otp" => s["otp"]})
+ end
+
+ grouped_modules =
+ all_modules
+ |> Enum.group_by(& &1["module"])
+ |> Enum.map(fn {mod, instances} ->
+ test_count = Enum.max(Enum.map(instances, &(&1["test_count"] || 0)), fn -> 0 end)
+
+ durations_by_ver =
+ Enum.into(instances, %{}, fn inst ->
+ ms = div(inst["total_us"] || 0, 1000)
+ {{inst["elixir"], inst["otp"]}, ms}
+ end)
+
+ max_duration_ms =
+ durations_by_ver
+ |> Map.values()
+ |> Enum.max(fn -> 0 end)
+
+ avg_ms = if test_count > 0, do: div(max_duration_ms, test_count), else: 0
+ max_test_ms = Enum.max(Enum.map(instances, &div(&1["max_us"] || 0, 1000)), fn -> 0 end)
+
+ statuses = Enum.map(instances, & &1["status"])
+ all_passed? = Enum.all?(statuses, &(&1 == "passed"))
+ status = if all_passed?, do: "passed", else: "failed"
+
+ %{
+ module: mod,
+ status: status,
+ test_count: test_count,
+ durations_by_ver: durations_by_ver,
+ max_duration_ms: max_duration_ms,
+ avg_ms: avg_ms,
+ max_test_ms: max_test_ms
+ }
+ end)
+ |> Enum.sort_by(& &1.max_duration_ms, :desc)
+
+ if grouped_modules != [] do
+ ver_header_cols = Enum.map_join(version_headers, " | ", fn {_, h} -> h end)
+ ver_align_cols = Enum.map_join(version_headers, " | ", fn _ -> ":---" end)
+
+ rows =
+ Enum.map_join(grouped_modules, "\n", fn m ->
+ ver_duration_cols =
+ Enum.map_join(versions, " | ", fn ver ->
+ case Map.get(m.durations_by_ver, ver) do
+ nil -> "-"
+ ms -> "`#{format_duration(ms)}`"
+ end
+ end)
+
+ "| `#{m.module}` | #{m.status} | #{m.test_count} | #{ver_duration_cols} | `#{format_duration(m.avg_ms)}` | `#{format_duration(m.max_test_ms)}` |"
+ end)
+
+ """
+
+ Module Breakdown
+
+ | Module | Status | Tests | #{ver_header_cols} | Avg / Test | Max / Test |
+ | :--- | :---: | :---: | #{ver_align_cols} | :--- | :--- |
+ #{rows}
+
+
+ """
+ |> String.trim()
+ else
+ ""
+ end
+ end
+
+ defp format_service_timelines(service_summaries) do
+ timelines =
+ Enum.map(service_summaries, fn s ->
+ modules = s["modules"] || []
+ gantt = format_shard_mermaid_gantt(modules)
+ {s["elixir"], s["otp"], gantt}
+ end)
+ |> Enum.reject(fn {_, _, gantt} -> gantt == "" end)
+
+ if timelines != [] do
+ content =
+ Enum.map_join(timelines, "\n\n", fn {elixir, otp, gantt} ->
+ """
+ **Elixir #{elixir} / OTP #{otp}**
+
+ #{gantt}
+ """
+ end)
+
+ """
+
+ Module Execution Timelines
+
+ #{content}
+
+ """
+ |> String.trim()
+ else
+ ""
+ end
+ end
+
+ defp format_service_tests_table(service_summaries, versions, version_headers) do
+ all_tests =
+ for s <- service_summaries,
+ t <- s["slowest_tests"] || [] do
+ Map.merge(t, %{
+ "elixir" => s["elixir"],
+ "otp" => s["otp"]
+ })
+ end
+
+ grouped_tests =
+ all_tests
+ |> Enum.group_by(fn t -> {t["module"], t["name"]} end)
+ |> Enum.map(fn {{mod, name}, instances} ->
+ first = hd(instances)
+
+ durations_by_ver =
+ Enum.into(instances, %{}, fn inst ->
+ ms = inst["duration_ms"] || div(inst["time_us"] || 0, 1000)
+ {{inst["elixir"], inst["otp"]}, ms}
+ end)
+
+ max_duration_ms =
+ durations_by_ver
+ |> Map.values()
+ |> Enum.max(fn -> 0 end)
+
+ statuses = Enum.map(instances, & &1["status"])
+ all_passed? = Enum.all?(statuses, &(&1 == "passed"))
+ status = if all_passed?, do: "passed", else: "failed"
+
+ %{
+ module: mod,
+ name: name,
+ file: first["file"],
+ line: first["line"],
+ durations_by_ver: durations_by_ver,
+ max_duration_ms: max_duration_ms,
+ status: status
+ }
+ end)
+ |> Enum.sort_by(& &1.max_duration_ms, :desc)
+
+ if grouped_tests != [] do
+ ver_header_cols = Enum.map_join(version_headers, " | ", fn {_, h} -> h end)
+ ver_align_cols = Enum.map_join(version_headers, " | ", fn _ -> ":---" end)
+
+ rows =
+ Enum.map_join(grouped_tests, "\n", fn t ->
+ ver_duration_cols =
+ Enum.map_join(versions, " | ", fn ver ->
+ case Map.get(t.durations_by_ver, ver) do
+ nil -> "-"
+ ms -> "`#{format_duration(ms)}`"
+ end
+ end)
+
+ "| #{t.status} | `#{format_duration(t.max_duration_ms)}` | #{ver_duration_cols} | #{escape_markdown(t.name)} | `#{t.module}` | `#{t.file}:#{t.line}` |"
+ end)
+
+ """
+
+ Test Durations (Slowest to Fastest)
+
+ | Status | Max Duration | #{ver_header_cols} | Test | Module | Location |
+ | :---: | :--- | #{ver_align_cols} | :--- | :--- | :--- |
+ #{rows}
+
+
+ """
+ |> String.trim()
+ else
+ ""
+ end
+ end
+
+ defp format_shard_mermaid_gantt(modules) do
+ sorted_modules =
+ Enum.sort_by(modules, fn m ->
+ s = m["start_ms"] || 0
+ f = m["finish_ms"] || s
+ {s, f, m["module"]}
+ end)
+
+ lanes =
+ Enum.reduce(sorted_modules, [], fn item, acc_lanes ->
+ assign_to_lane(acc_lanes, item, [])
+ end)
+ |> Enum.map(&Enum.reverse/1)
+
+ slowest_mod =
+ Enum.max_by(
+ modules,
+ fn m -> (m["finish_ms"] || m["start_ms"] || 0) - (m["start_ms"] || 0) end,
+ fn -> nil end
+ )
+
+ slowest_mod_name = slowest_mod && slowest_mod["module"]
+
+ section_rows =
+ lanes
+ |> Enum.with_index(1)
+ |> Enum.map(fn {lane, idx} ->
+ tasks =
+ Enum.map(lane, fn m ->
+ module_name = format_gantt_module_name(m["module"])
+ s = m["start_ms"] || 0
+ f = m["finish_ms"] || s
+ duration_ms = max(1000, f - s)
+ finish_ms = s + duration_ms
+
+ start_str = format_gantt_time(s)
+ finish_str = format_gantt_time(finish_ms)
+ tag = if m["module"] == slowest_mod_name, do: ":crit, active,", else: ":active,"
+
+ " #{module_name} #{tag} #{start_str}, #{finish_str}"
+ end)
+
+ " section Lane #{idx}\n" <> Enum.join(tasks, "\n")
+ end)
+
+ """
+ ```mermaid
+ ---
+ displayMode: compact
+ ---
+ gantt
+ title Module Execution Timeline
+ dateFormat mm:ss
+ axisFormat %M:%S
+ todayMarker off
+ #{Enum.join(section_rows, "\n")}
+ ```
+ """
+ |> String.trim()
+ end
+
+ defp assign_to_lane([], item, acc) do
+ Enum.reverse([[item] | acc])
+ end
+
+ defp assign_to_lane(
+ [[last_item | _] = lane | rest],
+ item,
+ acc
+ ) do
+ last_finish = last_item["finish_ms"] || last_item["start_ms"] || 0
+ item_start = item["start_ms"] || 0
+
+ if last_finish <= item_start do
+ Enum.reverse(acc) ++ [[item | lane] | rest]
+ else
+ assign_to_lane(rest, item, [lane | acc])
+ end
+ end
+
+ defp format_gantt_module_name(nil), do: "Unknown"
+
+ defp format_gantt_module_name(mod) do
+ mod
+ |> to_string()
+ |> String.replace_prefix("UmbrellaAppWith", "Umbrella")
+ |> String.replace_prefix("AppWith", "")
+ |> String.replace("Adapter", "")
+ |> String.replace_suffix("Test", "")
+ end
+
+ defp format_gantt_time(ms) do
+ total_seconds = div(ms, 1000)
+ mins = div(total_seconds, 60)
+ secs = rem(total_seconds, 60)
+
+ "#{String.pad_leading(Integer.to_string(mins), 2, "0")}:#{String.pad_leading(Integer.to_string(secs), 2, "0")}"
+ end
+
+ defp format_version_headers(versions) do
+ short_versions =
+ Enum.map(versions, fn {elixir, _otp} ->
+ short_version(elixir)
+ end)
+
+ unique? = length(Enum.uniq(short_versions)) == length(versions)
+
+ Enum.map(versions, fn {elixir, otp} ->
+ label = if unique?, do: short_version(elixir), else: elixir
+ {{elixir, otp}, "Duration (#{label})"}
+ end)
+ end
+
+ defp short_version(version) do
+ version
+ |> to_string()
+ |> String.split(".")
+ |> Enum.take(2)
+ |> Enum.join(".")
+ end
+
+ defp sort_shards(shards) do
+ Enum.sort_by(shards, fn s ->
+ idx = Enum.find_index(@service_order, &(&1 == s["service"]))
+ {idx || 99, s["service"]}
+ end)
+ end
+
+ defp shard_status_cell(shard) do
+ failures = shard["total_failures"] || 0
+ if failures == 0, do: "passed", else: "**failed (#{failures})**"
+ end
+
+ defp format_shard_slowest(nil), do: "-"
+ defp format_shard_slowest([]), do: "-"
+
+ defp format_shard_slowest([slowest | _]) do
+ format_test_desc(slowest)
+ end
+
+ defp format_test_desc(nil), do: "-"
+
+ defp format_test_desc(test) do
+ name = escape_markdown(test["name"] || "unknown")
+ mod = test["module"] || "unknown"
+ dur = format_duration(test["duration_ms"] || 0)
+ "`#{mod}`: #{truncate_text(name, 45)} (`#{dur}`)"
+ end
+
+ defp service_name("postgresql"), do: "PostgreSQL"
+ defp service_name("mysql"), do: "MySQL"
+ defp service_name("mssql"), do: "MSSQL"
+ defp service_name("none"), do: "sqlite3 + no-db"
+ defp service_name(other), do: to_string(other)
+
+ defp format_duration(ms) when ms < 1000, do: "#{ms}ms"
+
+ defp format_duration(ms) do
+ total_seconds = round(ms / 1000)
+ mins = div(total_seconds, 60)
+ secs = rem(total_seconds, 60)
+
+ if mins == 0 do
+ "#{secs}s"
+ else
+ "#{mins}m #{String.pad_leading(Integer.to_string(secs), 2, "0")}s"
+ end
+ end
+
+ defp truncate_text(text, max_len) do
+ if String.length(text) > max_len do
+ String.slice(text, 0, max_len - 3) <> "..."
+ else
+ text
+ end
+ end
+
+ defp escape_markdown(text) do
+ text
+ |> to_string()
+ |> String.replace("&", "&")
+ |> String.replace("<", "<")
+ |> String.replace(">", ">")
+ |> String.replace("\"", """)
+ |> String.replace("|", "\\|")
+ |> String.replace("\r\n", " ")
+ |> String.replace("\n", " ")
+ end
+end
+
+Phoenix.Integration.AggregateSummary.run(System.argv())
diff --git a/integration_test/docker-compose.yml b/integration_test/docker-compose.yml
index c483f05836..c51d394857 100644
--- a/integration_test/docker-compose.yml
+++ b/integration_test/docker-compose.yml
@@ -1,21 +1,43 @@
-version: '3'
services:
- postgres:
- image: postgres
+ postgresql:
+ image: postgres:18
ports:
- "5432:5432"
environment:
POSTGRES_PASSWORD: postgres
+ tmpfs:
+ - /var/lib/postgresql
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U postgres"]
+ interval: 2s
+ timeout: 3s
+ retries: 10
+
mysql:
- image: mysql
+ image: mysql:26
ports:
- "3306:3306"
environment:
MYSQL_ALLOW_EMPTY_PASSWORD: "yes"
+ tmpfs:
+ - /var/lib/mysql
+ healthcheck:
+ test: ["CMD-SHELL", "mysqladmin ping -h localhost --silent"]
+ interval: 2s
+ timeout: 3s
+ retries: 10
+
mssql:
image: mcr.microsoft.com/mssql/server:2019-latest
environment:
- ACCEPT_EULA: Y
- SA_PASSWORD: some!Password
+ ACCEPT_EULA: "Y"
+ SA_PASSWORD: "some!Password"
ports:
- "1433:1433"
+ tmpfs:
+ - /var/opt/mssql
+ healthcheck:
+ test: ["CMD-SHELL", "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P \"$$SA_PASSWORD\" -C -Q 'SELECT 1' 2>/dev/null || /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$$SA_PASSWORD\" -Q 'SELECT 1' 2>/dev/null"]
+ interval: 2s
+ timeout: 3s
+ retries: 25
diff --git a/integration_test/test.sh b/integration_test/test.sh
index 2934e880eb..d80dc7e78e 100755
--- a/integration_test/test.sh
+++ b/integration_test/test.sh
@@ -7,7 +7,7 @@ mix local.hex --force
apk add --no-progress --update git socat make gcc libc-dev cmake g++
# Set up local proxies
-socat TCP-LISTEN:5432,fork TCP-CONNECT:postgres:5432&
+socat TCP-LISTEN:5432,fork TCP-CONNECT:postgresql:5432&
socat TCP-LISTEN:3306,fork TCP-CONNECT:mysql:3306&
socat TCP-LISTEN:1433,fork TCP-CONNECT:mssql:1433&
diff --git a/integration_test/test/support/summary_formatter.ex b/integration_test/test/support/summary_formatter.ex
index a6a5767163..59a51718b5 100644
--- a/integration_test/test/support/summary_formatter.ex
+++ b/integration_test/test/support/summary_formatter.ex
@@ -1,19 +1,14 @@
defmodule Phoenix.Integration.SummaryFormatter do
@moduledoc false
#
- # Custom ExUnit formatter that generates a markdown summary for GitHub Actions.
+ # Custom ExUnit formatter that outputs structured JSON test metrics.
#
- # When the `GITHUB_STEP_SUMMARY` environment variable is set (which GitHub Actions
- # automatically populates with a path to a step summary file), this formatter appends
- # a markdown summary containing overall test metrics, a concurrent module execution
- # timeline grouped into virtual worker lanes, a per-module duration breakdown, and a table
- # of test durations sorted from slowest to fastest.
+ # By collecting test events in this custom formatter, we retain full concurrency
+ # across test modules while gathering granular execution durations and concurrency
+ # intervals.
#
- # This formatter takes inspiration from `mix test --slowest` and `--slowest-modules`.
- # We do not use those built-in flags directly because they implicitly enable `--trace`,
- # which forces all tests to run synchronously/sequentially and significantly slows down
- # the integration test suite. By collecting test events in this custom formatter, we
- # retain full concurrency across test modules while still getting granular timing insights.
+ # On suite finish, it writes a JSON summary file to `PHX_INTEGRATION_SUMMARY_JSON`
+ # (defaulting to "tmp/integration_test_summary.json").
#
use GenServer
@@ -60,28 +55,30 @@ defmodule Phoenix.Integration.SummaryFormatter do
end
def handle_cast({:suite_finished, run_data}, %{tests: tests} = state) do
- summary_file = System.get_env("GITHUB_STEP_SUMMARY")
+ json_path =
+ System.get_env("PHX_INTEGRATION_SUMMARY_JSON") || "tmp/integration_test_summary.json"
- if is_binary(summary_file) and summary_file != "" do
- executed =
- Enum.reject(tests, fn test ->
- match?({:excluded, _}, test.state) or match?({:skipped, _}, test.state)
- end)
+ executed =
+ Enum.reject(tests, fn test ->
+ match?({:excluded, _}, test.state) or match?({:skipped, _}, test.state)
+ end)
+
+ slowest = Enum.sort_by(executed, &(&1.time || 0), :desc)
+ executed_modules = MapSet.new(slowest, & &1.module)
- slowest = Enum.sort_by(executed, &(&1.time || 0), :desc)
- write_github_summary(summary_file, run_data, state, slowest)
- end
+ write_json_summary(json_path, run_data, state, slowest, executed_modules)
{:noreply, state}
end
def handle_cast(_event, state), do: {:noreply, state}
- defp write_github_summary(
- summary_file,
+ defp write_json_summary(
+ json_path,
run_data,
%{tests: tests, modules: modules, suite_start: suite_start},
- slowest
+ slowest,
+ executed_modules
) do
total = length(tests)
failed = Enum.count(tests, &match?({:failed, _}, &1.state))
@@ -91,7 +88,7 @@ defmodule Phoenix.Integration.SummaryFormatter do
passed = Enum.count(tests, &is_nil(&1.state))
total_failures = failed + invalid
- status = if total_failures == 0, do: "Passed", else: "Failed (#{total_failures})"
+ status = if total_failures == 0, do: "passed", else: "failed"
wall_time_ms =
case run_data do
@@ -104,6 +101,7 @@ defmodule Phoenix.Integration.SummaryFormatter do
modules_stats =
tests
|> Enum.group_by(& &1.module)
+ |> Enum.filter(fn {mod, _tests} -> MapSet.member?(executed_modules, mod) end)
|> Enum.map(fn {mod, mod_tests} ->
mod_executed =
Enum.reject(mod_tests, fn test ->
@@ -117,233 +115,88 @@ defmodule Phoenix.Integration.SummaryFormatter do
mod_failed = Enum.count(mod_tests, &match?({:failed, _}, &1.state))
mod_invalid = Enum.count(mod_tests, &match?({:invalid, _}, &1.state))
mod_failures = mod_failed + mod_invalid
- mod_status = if mod_failures == 0, do: "passed", else: "failed (#{mod_failures})"
+ mod_status = if mod_failures == 0, do: "passed", else: "failed"
+
+ suite_start_ms = suite_start || 0
+ mod_timing = Map.get(modules, mod, %{})
+ start_ms = Map.get(mod_timing, :start, suite_start_ms)
+ finish_ms = Map.get(mod_timing, :finish, start_ms)
+ module_name = if mod, do: mod |> Module.split() |> List.last(), else: "Unknown"
%{
- module: mod,
+ module: module_name,
+ full_module: if(mod, do: inspect(mod), else: "Unknown"),
status: mod_status,
- test_count: length(mod_tests),
+ failures: mod_failures,
+ test_count: length(mod_executed),
total_us: total_us,
avg_us: avg_us,
- max_us: max_us
+ max_us: max_us,
+ start_ms: max(0, start_ms - suite_start_ms),
+ finish_ms: max(0, finish_ms - suite_start_ms)
}
end)
|> Enum.sort_by(& &1.total_us, :desc)
- env_info = "Elixir #{System.version()} / OTP #{System.otp_release()}"
-
- sections = [
- """
- ## Phoenix Integration Tests (#{env_info}): #{status}
-
- | Total | Passed | Failed | Excluded | Skipped | Total Wall Time |
- | :---: | :---: | :---: | :---: | :---: | :---: |
- | **#{total}** | **#{passed}** | **#{total_failures}** | **#{excluded}** | **#{skipped}** | **#{format_duration(wall_time_ms)}** |
- """,
- format_mermaid_gantt(modules, suite_start),
- """
-
- Module Breakdown
-
- | Module | Status | Tests | Total Duration | % of Total | Avg / Test | Max / Test |
- | :--- | :---: | :---: | :--- | :---: | :--- | :--- |
- #{Enum.map_join(modules_stats, "\n", &format_module_markdown_row(&1, total_executed_time_us))}
-
-
- """,
- """
-
- Test Durations (Slowest to Fastest)
-
- | Status | Duration | Test | Module | Location |
- | :--- | :--- | :--- | :--- | :--- |
- #{Enum.map_join(slowest, "\n", &format_markdown_row/1)}
-
-
- """
- ]
-
- markdown =
- sections
- |> Enum.reject(&(&1 in ["", nil]))
- |> Enum.map_join("\n\n", &String.trim/1)
- |> Kernel.<>("\n\n")
-
- case File.write(summary_file, markdown, [:append]) do
- :ok ->
- :ok
-
- {:error, reason} ->
- IO.warn("Failed to write integration test summary to #{summary_file}: #{inspect(reason)}")
- end
- end
-
- defp format_mermaid_gantt(modules, suite_start) when map_size(modules) > 0 do
- suite_start =
- suite_start ||
- modules
- |> Map.values()
- |> Enum.map(& &1.start)
- |> Enum.min(fn -> 0 end)
-
- sorted_modules =
- modules
- |> Enum.sort_by(fn {mod, %{start: s, finish: f}} -> {s, f || s, mod} end)
-
- lanes =
- Enum.reduce(sorted_modules, [], fn item, acc_lanes ->
- assign_to_lane(acc_lanes, item, [])
- end)
- |> Enum.map(&Enum.reverse/1)
-
- {slowest_mod, _} =
- Enum.max_by(
- modules,
- fn {_mod, %{start: s, finish: f}} -> (f || s) - s end,
- fn -> {nil, nil} end
- )
-
- section_rows =
- lanes
- |> Enum.with_index(1)
- |> Enum.map(fn {lane, idx} ->
- tasks =
- Enum.map(lane, fn {mod, %{start: s, finish: f}} ->
- module_name = format_gantt_module_name(mod)
- duration_ms = max(1000, (f || s) - s)
- start_ms = max(0, s - suite_start)
- finish_ms = start_ms + duration_ms
-
- start_str = format_gantt_time(start_ms)
- finish_str = format_gantt_time(finish_ms)
- tag = if mod == slowest_mod, do: ":crit, active,", else: ":active,"
-
- " #{module_name} #{tag} #{start_str}, #{finish_str}"
- end)
+ slowest_tests =
+ Enum.map(slowest, fn %ExUnit.Test{
+ name: name,
+ module: mod,
+ time: time,
+ tags: tags,
+ state: state
+ } ->
+ test_name = name |> to_string() |> String.replace_prefix("test ", "")
+ module_name = if mod, do: mod |> Module.split() |> List.last(), else: "Unknown"
+ file = if f = tags[:file], do: Path.relative_to(f, File.cwd!()), else: "unknown"
+ line = tags[:line] || 0
+ status = if is_nil(state), do: "passed", else: "failed"
- " section Lane #{idx}\n" <> Enum.join(tasks, "\n")
+ %{
+ name: test_name,
+ module: module_name,
+ duration_ms: div(time || 0, 1000),
+ time_us: time || 0,
+ file: file,
+ line: line,
+ status: status
+ }
end)
- """
-
- Module Execution Timeline
-
- ```mermaid
- ---
- displayMode: compact
- ---
- gantt
- title Module Execution Timeline
- dateFormat mm:ss
- axisFormat %M:%S
- todayMarker off
- #{Enum.join(section_rows, "\n")}
- ```
-
-
- """
- end
-
- defp format_mermaid_gantt(_modules, _suite_start), do: ""
-
- defp assign_to_lane([], item, acc) do
- Enum.reverse([[item] | acc])
- end
-
- defp assign_to_lane(
- [[{_last_mod, %{finish: last_finish, start: last_start}} | _] = lane | rest],
- {_mod, %{start: s}} = item,
- acc
- ) do
- if (last_finish || last_start) <= s do
- Enum.reverse(acc) ++ [[item | lane] | rest]
- else
- assign_to_lane(rest, item, [lane | acc])
- end
- end
-
- defp format_gantt_module_name(nil), do: "Unknown"
-
- defp format_gantt_module_name(mod) do
- mod
- |> Module.split()
- |> List.last()
- |> String.replace_prefix("UmbrellaAppWith", "Umbrella")
- |> String.replace_prefix("AppWith", "")
- |> String.replace("Adapter", "")
- |> String.replace_suffix("Test", "")
- end
-
- defp format_gantt_time(ms) do
- total_seconds = div(ms, 1000)
- mins = div(total_seconds, 60)
- secs = rem(total_seconds, 60)
-
- "#{String.pad_leading(Integer.to_string(mins), 2, "0")}:#{String.pad_leading(Integer.to_string(secs), 2, "0")}"
- end
-
- defp format_module_markdown_row(
- %{
- module: mod,
- status: status,
- test_count: count,
- total_us: total_us,
- avg_us: avg_us,
- max_us: max_us
- },
- total_suite_us
- ) do
- module_name = if mod, do: mod |> Module.split() |> List.last(), else: "Unknown"
- total_ms = div(total_us, 1000)
- avg_ms = div(avg_us, 1000)
- max_ms = div(max_us, 1000)
-
- pct =
- if total_suite_us > 0 do
- Float.round(total_us / total_suite_us * 100, 1)
- else
- 0.0
+ service =
+ System.get_env("PHX_TEST_SERVICE") || System.get_env("PHX_TEST_GROUP") || "none"
+
+ payload = %{
+ service: service,
+ elixir: System.get_env("PHX_ELIXIR_VERSION") || System.version(),
+ otp: System.get_env("PHX_OTP_VERSION") || System.otp_release(),
+ status: status,
+ total_failures: total_failures,
+ total_tests: total,
+ executed_tests: length(slowest),
+ passed_tests: passed,
+ failed_tests: total_failures,
+ skipped_tests: skipped,
+ excluded_tests: excluded,
+ wall_time_ms: wall_time_ms,
+ total_executed_time_us: total_executed_time_us,
+ modules: modules_stats,
+ slowest_tests: slowest_tests
+ }
+
+ json_str =
+ cond do
+ Code.ensure_loaded?(JSON) ->
+ apply(JSON, :encode!, [payload])
+
+ Code.ensure_loaded?(Jason) ->
+ apply(Jason, :encode!, [payload])
+
+ true ->
+ raise "Neither JSON nor Jason available for encoding summary payload"
end
- "| `#{module_name}` | #{status} | #{count} | `#{format_duration(total_ms)}` | #{pct}% | `#{format_duration(avg_ms)}` | `#{format_duration(max_ms)}` |"
- end
-
- defp format_markdown_row(%ExUnit.Test{
- name: name,
- module: mod,
- time: time,
- tags: tags,
- state: state
- }) do
- ms = div(time || 0, 1000)
- test_name = name |> to_string() |> String.replace_prefix("test ", "")
- module_name = if mod, do: mod |> Module.split() |> List.last(), else: "Unknown"
- file = if f = tags[:file], do: Path.relative_to(f, File.cwd!()), else: "unknown"
-
- status = if is_nil(state), do: "passed", else: "failed"
-
- "| #{status} | `#{format_duration(ms)}` | #{escape_markdown(test_name)} | `#{module_name}` | `#{file}:#{tags[:line]}` |"
- end
-
- defp format_duration(ms) when ms < 1000, do: "#{ms}ms"
-
- defp format_duration(ms) do
- total_seconds = round(ms / 1000)
- mins = div(total_seconds, 60)
- secs = rem(total_seconds, 60)
-
- if mins == 0 do
- "#{secs}s"
- else
- "#{mins}m #{String.pad_leading(Integer.to_string(secs), 2, "0")}s"
- end
- end
-
- defp escape_markdown(text) do
- text
- |> Plug.HTML.html_escape()
- |> String.replace("|", "\\|")
- |> String.replace("\r\n", " ")
- |> String.replace("\n", " ")
+ json_path |> Path.dirname() |> File.mkdir_p!()
+ File.write!(json_path, json_str)
end
end