diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4bce06377..5007392979 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,7 +9,7 @@ concurrency: jobs: mix_test: - name: mix test (OTP ${{matrix.otp}} | Elixir ${{matrix.elixir}}) + name: mix test (Elixir ${{matrix.elixir}} | OTP ${{matrix.otp}}) env: MIX_ENV: test @@ -72,7 +72,7 @@ jobs: if: ${{ matrix.lint }} installer_test: - name: installer test (OTP ${{matrix.otp}} | Elixir ${{matrix.elixir}}) + name: installer test (Elixir ${{matrix.elixir}} | OTP ${{matrix.otp}}) env: MIX_ENV: test @@ -223,14 +223,16 @@ jobs: run: mix docs --warnings-as-errors integration-test-elixir: - name: integration test (OTP ${{matrix.otp}} | Elixir ${{matrix.elixir}}) + name: integration test [${{ matrix.service == 'none' && 'sqlite3 + no-db' || matrix.service }}] (Elixir ${{ matrix.versions.elixir }} | OTP ${{ matrix.versions.otp }}) runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 15 strategy: + fail-fast: false matrix: - include: + service: [postgresql, mysql, mssql, none] + versions: - elixir: "1.18.4" otp: "27.3.4.3" @@ -247,42 +249,6 @@ jobs: MIX_OS_DEPS_COMPILE_PARTITION_COUNT: 2 MAKEFLAGS: "-j2" - services: - postgres: - image: postgres:18 - ports: - - 5432:5432 - env: - POSTGRES_PASSWORD: postgres - options: >- - --health-cmd "pg_isready -U postgres" - --health-interval 2s - --health-timeout 3s - --health-retries 10 - mysql: - image: mysql:26 - ports: - - 3306:3306 - env: - MYSQL_ALLOW_EMPTY_PASSWORD: "yes" - options: >- - --health-cmd "mysqladmin ping -h localhost" - --health-interval 2s - --health-timeout 3s - --health-retries 10 - mssql: - image: mcr.microsoft.com/mssql/server:2019-latest - env: - ACCEPT_EULA: Y - SA_PASSWORD: some!Password - ports: - - 1433:1433 - options: >- - --health-cmd "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -C -Q 'SELECT 1' 2>/dev/null || /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -Q 'SELECT 1' 2>/dev/null" - --health-interval 2s - --health-timeout 3s - --health-retries 25 - steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -294,11 +260,16 @@ jobs: mkdir -p installer/tmp sudo mount -t tmpfs -o size=4G,uid=$(id -u),gid=$(id -g) tmpfs installer/tmp + - name: Start database container (${{ matrix.service }}) + if: matrix.service != 'none' + run: | + docker compose -f integration_test/docker-compose.yml up -d --wait ${{ matrix.service }} + - name: Set up Elixir uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 with: - elixir-version: ${{ matrix.elixir }} - otp-version: ${{ matrix.otp }} + elixir-version: ${{ matrix.versions.elixir }} + otp-version: ${{ matrix.versions.otp }} - name: Restore deps and _build cache uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -306,9 +277,9 @@ jobs: path: | integration_test/deps integration_test/_build - key: integration-deps-${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }}-${{ hashFiles('integration_test/mix.lock') }} + key: integration-deps-${{ runner.os }}-${{ matrix.versions.otp }}-${{ matrix.versions.elixir }}-${{ hashFiles('integration_test/mix.lock') }} restore-keys: | - integration-deps-${{ runner.os }}-${{ matrix.otp }}-${{ matrix.elixir }} + integration-deps-${{ runner.os }}-${{ matrix.versions.otp }}-${{ matrix.versions.elixir }} - name: Fetch dependencies working-directory: integration_test @@ -323,6 +294,59 @@ jobs: - name: Run integration tests working-directory: integration_test run: | - mix test --include database \ + if [ "${{ matrix.service }}" = "none" ]; then + FLAGS="--include database:sqlite3" + else + FLAGS="--include database:${{ matrix.service }} --only database:${{ matrix.service }}" + fi + + mix test $FLAGS \ --formatter ExUnit.CLIFormatter \ --formatter Phoenix.Integration.SummaryFormatter + env: + PHX_TEST_SERVICE: ${{ matrix.service }} + PHX_ELIXIR_VERSION: ${{ matrix.versions.elixir }} + PHX_OTP_VERSION: ${{ matrix.versions.otp }} + PHX_INTEGRATION_SUMMARY_JSON: ${{ runner.temp }}/summary-${{ matrix.service }}-${{ matrix.versions.otp }}-${{ matrix.versions.elixir }}.json + + - name: Upload summary artifact + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: summary-${{ matrix.service }}-${{ matrix.versions.otp }}-${{ matrix.versions.elixir }} + path: ${{ runner.temp }}/summary-${{ matrix.service }}-${{ matrix.versions.otp }}-${{ matrix.versions.elixir }}.json + retention-days: 1 + compression-level: 0 + overwrite: true + if-no-files-found: warn + + integration-test-summary: + name: integration test results + runs-on: ubuntu-24.04 + timeout-minutes: 2 + needs: integration-test-elixir + if: always() + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Elixir + uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1 + with: + elixir-version: "1.20.4" + otp-version: "29.0.5" + + - name: Download summary artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: summary-* + merge-multiple: true + path: tmp/summaries + + - name: Aggregate test summaries + run: elixir integration_test/aggregate_summary.exs tmp/summaries + env: + MATRIX_RESULT: ${{ needs.integration-test-elixir.result }} diff --git a/integration_test/aggregate_summary.exs b/integration_test/aggregate_summary.exs new file mode 100644 index 0000000000..c870cc354f --- /dev/null +++ b/integration_test/aggregate_summary.exs @@ -0,0 +1,677 @@ +# Script to aggregate integration test shard JSON summaries into a single GitHub Actions summary. +# +# Usage: +# elixir integration_test/aggregate_summary.exs [directory_with_json_files] +# + +defmodule Phoenix.Integration.AggregateSummary do + @service_order ["postgresql", "mysql", "mssql", "none"] + + def run(argv) do + summaries_dir = List.first(argv) || "tmp/summaries" + + json_files = Path.wildcard(Path.join(summaries_dir, "**/*.json")) + + if json_files == [] do + IO.puts(:stderr, "No summary JSON files found in #{summaries_dir}") + + if summary_file = System.get_env("GITHUB_STEP_SUMMARY") do + File.write!( + summary_file, + "## Phoenix Integration Tests: No summary artifacts found\n\n", + [:append] + ) + end + + System.halt(1) + end + + summaries = + Enum.map(json_files, fn file -> + file + |> File.read!() + |> decode_json!() + end) + + markdown = format_report(summaries) + + if summary_file = System.get_env("GITHUB_STEP_SUMMARY") do + case File.write(summary_file, markdown, [:append]) do + :ok -> + :ok + + {:error, reason} -> + IO.warn( + "Failed to write integration test summary to #{summary_file}: #{inspect(reason)}" + ) + end + end + + IO.puts(markdown) + + any_failed? = + Enum.any?(summaries, fn s -> + s["status"] != "passed" or (s["total_failures"] || 0) > 0 + end) + + matrix_failed? = System.get_env("MATRIX_RESULT") in ["failure", "cancelled"] + + if any_failed? or matrix_failed? do + System.halt(1) + else + System.halt(0) + end + end + + defp decode_json!(content) do + cond do + Code.ensure_loaded?(JSON) -> + apply(JSON, :decode!, [content]) + + Code.ensure_loaded?(Jason) -> + apply(Jason, :decode!, [content]) + + true -> + raise "Neither JSON nor Jason available for decoding summary" + end + end + + def format_report(summaries) do + versions = + summaries + |> Enum.map(fn s -> {s["elixir"], s["otp"]} end) + |> Enum.uniq() + |> Enum.sort_by(fn {elixir, _otp} -> elixir end) + + version_headers = format_version_headers(versions) + + peak_wall_time = + summaries + |> Enum.map(&(&1["wall_time_ms"] || 0)) + |> Enum.max(fn -> 0 end) + + total_shards = length(summaries) + total_suites = length(versions) + + any_failed? = + Enum.any?(summaries, fn s -> + s["status"] != "passed" or (s["total_failures"] || 0) > 0 + end) + + overall_status = if any_failed?, do: "Failed", else: "Passed" + + summary_header = """ + ## Phoenix Integration Tests Summary + + | Total Suites | Total Shards | Overall Status | Peak Wall Time | + | :---: | :---: | :---: | :---: | + | **#{total_suites}** | **#{total_shards}** | **#{overall_status}** | **`#{format_duration(peak_wall_time)}`** | + """ + + overview_table = format_combined_overview_table(summaries, versions) + slowest_table = format_combined_slowest_table(summaries, versions, version_headers) + shard_details = format_combined_shard_details(summaries, versions, version_headers) + + [summary_header, overview_table, slowest_table, shard_details] + |> Enum.reject(&(&1 in ["", nil])) + |> Enum.join("\n\n") + |> String.trim() + |> Kernel.<>("\n\n") + end + + defp format_combined_overview_table(summaries, versions) do + grouped = + summaries + |> Enum.group_by(fn s -> {s["elixir"], s["otp"]} end) + + table_rows = + Enum.map_join(versions, "\n", fn {elixir, otp} = ver -> + shard_summaries = Map.get(grouped, ver, []) + sorted_shards = sort_shards(shard_summaries) + + total_executed = Enum.sum(Enum.map(sorted_shards, &(&1["executed_tests"] || 0))) + total_passed = Enum.sum(Enum.map(sorted_shards, &(&1["passed_tests"] || 0))) + total_failed = Enum.sum(Enum.map(sorted_shards, &(&1["total_failures"] || 0))) + max_wall_time = Enum.max(Enum.map(sorted_shards, &(&1["wall_time_ms"] || 0)), fn -> 0 end) + + status_str = if total_failed == 0, do: "Passed", else: "Failed (#{total_failed})" + + overall_slowest = + sorted_shards + |> Enum.flat_map(&(&1["slowest_tests"] || [])) + |> Enum.max_by(&(&1["time_us"] || 0), fn -> nil end) + + overall_slowest_desc = format_test_desc(overall_slowest) + + total_row = + "| **Elixir #{elixir} / OTP #{otp}** | **All** | **#{status_str}** | **#{total_passed}/#{total_executed}** | **`#{format_duration(max_wall_time)}`** | #{overall_slowest_desc} |" + + shard_rows = + Enum.map_join(sorted_shards, "\n", fn shard -> + service_label = service_name(shard["service"]) + status = shard_status_cell(shard) + executed = shard["executed_tests"] || 0 + wall_time = format_duration(shard["wall_time_ms"] || 0) + slowest_desc = format_shard_slowest(shard["slowest_tests"]) + + "| | #{service_label} | #{status} | #{executed} | `#{wall_time}` | #{slowest_desc} |" + end) + + "#{total_row}\n#{shard_rows}" + end) + + """ + | Elixir / OTP | Database | Status | Tests | Wall Time | Slowest Test | + | :--- | :--- | :---: | :---: | :---: | :--- | + #{table_rows} + """ + |> String.trim() + end + + defp format_combined_slowest_table(summaries, versions, version_headers) do + all_tests = + for s <- summaries, + t <- s["slowest_tests"] || [] do + Map.merge(t, %{ + "service" => s["service"], + "elixir" => s["elixir"], + "otp" => s["otp"] + }) + end + + grouped_tests = + all_tests + |> Enum.group_by(fn t -> {t["module"], t["name"]} end) + |> Enum.map(fn {{mod, name}, instances} -> + first = hd(instances) + + durations_by_ver = + Enum.into(instances, %{}, fn inst -> + ms = inst["duration_ms"] || div(inst["time_us"] || 0, 1000) + {{inst["elixir"], inst["otp"]}, ms} + end) + + max_duration_ms = + durations_by_ver + |> Map.values() + |> Enum.max(fn -> 0 end) + + statuses = Enum.map(instances, & &1["status"]) + all_passed? = Enum.all?(statuses, &(&1 == "passed")) + status = if all_passed?, do: "passed", else: "failed" + + %{ + module: mod, + name: name, + service: first["service"], + file: first["file"], + line: first["line"], + durations_by_ver: durations_by_ver, + max_duration_ms: max_duration_ms, + status: status + } + end) + |> Enum.sort_by(& &1.max_duration_ms, :desc) + |> Enum.take(10) + + if grouped_tests != [] do + ver_header_cols = Enum.map_join(version_headers, " | ", fn {_, h} -> h end) + ver_align_cols = Enum.map_join(version_headers, " | ", fn _ -> ":---" end) + + rows = + Enum.map_join(grouped_tests, "\n", fn t -> + ver_duration_cols = + Enum.map_join(versions, " | ", fn ver -> + case Map.get(t.durations_by_ver, ver) do + nil -> "-" + ms -> "`#{format_duration(ms)}`" + end + end) + + "| #{t.status} | `#{format_duration(t.max_duration_ms)}` | #{ver_duration_cols} | #{escape_markdown(t.name)} | `#{t.module}` | #{service_name(t.service)} | `#{t.file}:#{t.line}` |" + end) + + """ +
+ Top 10 Slowest Tests + + | Status | Max Duration | #{ver_header_cols} | Test | Module | Database | Location | + | :---: | :--- | #{ver_align_cols} | :--- | :--- | :---: | :--- | + #{rows} + +
+ """ + |> String.trim() + else + "" + end + end + + defp format_combined_shard_details(summaries, versions, version_headers) do + shards_by_service = Enum.group_by(summaries, & &1["service"]) + + services = + @service_order + |> Enum.filter(&Map.has_key?(shards_by_service, &1)) + |> Kernel.++(Enum.sort(Map.keys(shards_by_service) -- @service_order)) + + Enum.map_join(services, "\n\n", fn service -> + service_summaries = + shards_by_service + |> Map.get(service, []) + |> Enum.sort_by(fn s -> {s["elixir"], s["otp"]} end) + + format_service_accordion(service, service_summaries, versions, version_headers) + end) + end + + defp format_service_accordion(service, service_summaries, versions, version_headers) do + service_label = service_name(service) + + total_failures = + Enum.sum(Enum.map(service_summaries, &(&1["total_failures"] || 0))) + + max_tests = + Enum.max(Enum.map(service_summaries, &(&1["executed_tests"] || 0)), fn -> 0 end) + + details_tag = if total_failures > 0, do: "
", else: "
" + status_label = if total_failures == 0, do: "passed", else: "failed (#{total_failures})" + + wall_times_summary = + Enum.map_join(service_summaries, " | ", fn s -> + "#{format_duration(s["wall_time_ms"] || 0)} in #{short_version(s["elixir"])}" + end) + + modules_section = format_service_modules_table(service_summaries, versions, version_headers) + timelines_section = format_service_timelines(service_summaries) + tests_section = format_service_tests_table(service_summaries, versions, version_headers) + + """ + #{details_tag} + #{service_label} Details: #{status_label} — #{max_tests} tests (#{wall_times_summary}) + + #{timelines_section} + + #{modules_section} + + #{tests_section} +
+ """ + |> String.trim() + end + + defp format_service_modules_table(service_summaries, versions, version_headers) do + all_modules = + for s <- service_summaries, + m <- s["modules"] || [] do + Map.merge(m, %{"elixir" => s["elixir"], "otp" => s["otp"]}) + end + + grouped_modules = + all_modules + |> Enum.group_by(& &1["module"]) + |> Enum.map(fn {mod, instances} -> + test_count = Enum.max(Enum.map(instances, &(&1["test_count"] || 0)), fn -> 0 end) + + durations_by_ver = + Enum.into(instances, %{}, fn inst -> + ms = div(inst["total_us"] || 0, 1000) + {{inst["elixir"], inst["otp"]}, ms} + end) + + max_duration_ms = + durations_by_ver + |> Map.values() + |> Enum.max(fn -> 0 end) + + avg_ms = if test_count > 0, do: div(max_duration_ms, test_count), else: 0 + max_test_ms = Enum.max(Enum.map(instances, &div(&1["max_us"] || 0, 1000)), fn -> 0 end) + + statuses = Enum.map(instances, & &1["status"]) + all_passed? = Enum.all?(statuses, &(&1 == "passed")) + status = if all_passed?, do: "passed", else: "failed" + + %{ + module: mod, + status: status, + test_count: test_count, + durations_by_ver: durations_by_ver, + max_duration_ms: max_duration_ms, + avg_ms: avg_ms, + max_test_ms: max_test_ms + } + end) + |> Enum.sort_by(& &1.max_duration_ms, :desc) + + if grouped_modules != [] do + ver_header_cols = Enum.map_join(version_headers, " | ", fn {_, h} -> h end) + ver_align_cols = Enum.map_join(version_headers, " | ", fn _ -> ":---" end) + + rows = + Enum.map_join(grouped_modules, "\n", fn m -> + ver_duration_cols = + Enum.map_join(versions, " | ", fn ver -> + case Map.get(m.durations_by_ver, ver) do + nil -> "-" + ms -> "`#{format_duration(ms)}`" + end + end) + + "| `#{m.module}` | #{m.status} | #{m.test_count} | #{ver_duration_cols} | `#{format_duration(m.avg_ms)}` | `#{format_duration(m.max_test_ms)}` |" + end) + + """ +
+ Module Breakdown + + | Module | Status | Tests | #{ver_header_cols} | Avg / Test | Max / Test | + | :--- | :---: | :---: | #{ver_align_cols} | :--- | :--- | + #{rows} + +
+ """ + |> String.trim() + else + "" + end + end + + defp format_service_timelines(service_summaries) do + timelines = + Enum.map(service_summaries, fn s -> + modules = s["modules"] || [] + gantt = format_shard_mermaid_gantt(modules) + {s["elixir"], s["otp"], gantt} + end) + |> Enum.reject(fn {_, _, gantt} -> gantt == "" end) + + if timelines != [] do + content = + Enum.map_join(timelines, "\n\n", fn {elixir, otp, gantt} -> + """ + **Elixir #{elixir} / OTP #{otp}** + + #{gantt} + """ + end) + + """ +
+ Module Execution Timelines + + #{content} +
+ """ + |> String.trim() + else + "" + end + end + + defp format_service_tests_table(service_summaries, versions, version_headers) do + all_tests = + for s <- service_summaries, + t <- s["slowest_tests"] || [] do + Map.merge(t, %{ + "elixir" => s["elixir"], + "otp" => s["otp"] + }) + end + + grouped_tests = + all_tests + |> Enum.group_by(fn t -> {t["module"], t["name"]} end) + |> Enum.map(fn {{mod, name}, instances} -> + first = hd(instances) + + durations_by_ver = + Enum.into(instances, %{}, fn inst -> + ms = inst["duration_ms"] || div(inst["time_us"] || 0, 1000) + {{inst["elixir"], inst["otp"]}, ms} + end) + + max_duration_ms = + durations_by_ver + |> Map.values() + |> Enum.max(fn -> 0 end) + + statuses = Enum.map(instances, & &1["status"]) + all_passed? = Enum.all?(statuses, &(&1 == "passed")) + status = if all_passed?, do: "passed", else: "failed" + + %{ + module: mod, + name: name, + file: first["file"], + line: first["line"], + durations_by_ver: durations_by_ver, + max_duration_ms: max_duration_ms, + status: status + } + end) + |> Enum.sort_by(& &1.max_duration_ms, :desc) + + if grouped_tests != [] do + ver_header_cols = Enum.map_join(version_headers, " | ", fn {_, h} -> h end) + ver_align_cols = Enum.map_join(version_headers, " | ", fn _ -> ":---" end) + + rows = + Enum.map_join(grouped_tests, "\n", fn t -> + ver_duration_cols = + Enum.map_join(versions, " | ", fn ver -> + case Map.get(t.durations_by_ver, ver) do + nil -> "-" + ms -> "`#{format_duration(ms)}`" + end + end) + + "| #{t.status} | `#{format_duration(t.max_duration_ms)}` | #{ver_duration_cols} | #{escape_markdown(t.name)} | `#{t.module}` | `#{t.file}:#{t.line}` |" + end) + + """ +
+ Test Durations (Slowest to Fastest) + + | Status | Max Duration | #{ver_header_cols} | Test | Module | Location | + | :---: | :--- | #{ver_align_cols} | :--- | :--- | :--- | + #{rows} + +
+ """ + |> String.trim() + else + "" + end + end + + defp format_shard_mermaid_gantt(modules) do + sorted_modules = + Enum.sort_by(modules, fn m -> + s = m["start_ms"] || 0 + f = m["finish_ms"] || s + {s, f, m["module"]} + end) + + lanes = + Enum.reduce(sorted_modules, [], fn item, acc_lanes -> + assign_to_lane(acc_lanes, item, []) + end) + |> Enum.map(&Enum.reverse/1) + + slowest_mod = + Enum.max_by( + modules, + fn m -> (m["finish_ms"] || m["start_ms"] || 0) - (m["start_ms"] || 0) end, + fn -> nil end + ) + + slowest_mod_name = slowest_mod && slowest_mod["module"] + + section_rows = + lanes + |> Enum.with_index(1) + |> Enum.map(fn {lane, idx} -> + tasks = + Enum.map(lane, fn m -> + module_name = format_gantt_module_name(m["module"]) + s = m["start_ms"] || 0 + f = m["finish_ms"] || s + duration_ms = max(1000, f - s) + finish_ms = s + duration_ms + + start_str = format_gantt_time(s) + finish_str = format_gantt_time(finish_ms) + tag = if m["module"] == slowest_mod_name, do: ":crit, active,", else: ":active," + + " #{module_name} #{tag} #{start_str}, #{finish_str}" + end) + + " section Lane #{idx}\n" <> Enum.join(tasks, "\n") + end) + + """ + ```mermaid + --- + displayMode: compact + --- + gantt + title Module Execution Timeline + dateFormat mm:ss + axisFormat %M:%S + todayMarker off + #{Enum.join(section_rows, "\n")} + ``` + """ + |> String.trim() + end + + defp assign_to_lane([], item, acc) do + Enum.reverse([[item] | acc]) + end + + defp assign_to_lane( + [[last_item | _] = lane | rest], + item, + acc + ) do + last_finish = last_item["finish_ms"] || last_item["start_ms"] || 0 + item_start = item["start_ms"] || 0 + + if last_finish <= item_start do + Enum.reverse(acc) ++ [[item | lane] | rest] + else + assign_to_lane(rest, item, [lane | acc]) + end + end + + defp format_gantt_module_name(nil), do: "Unknown" + + defp format_gantt_module_name(mod) do + mod + |> to_string() + |> String.replace_prefix("UmbrellaAppWith", "Umbrella") + |> String.replace_prefix("AppWith", "") + |> String.replace("Adapter", "") + |> String.replace_suffix("Test", "") + end + + defp format_gantt_time(ms) do + total_seconds = div(ms, 1000) + mins = div(total_seconds, 60) + secs = rem(total_seconds, 60) + + "#{String.pad_leading(Integer.to_string(mins), 2, "0")}:#{String.pad_leading(Integer.to_string(secs), 2, "0")}" + end + + defp format_version_headers(versions) do + short_versions = + Enum.map(versions, fn {elixir, _otp} -> + short_version(elixir) + end) + + unique? = length(Enum.uniq(short_versions)) == length(versions) + + Enum.map(versions, fn {elixir, otp} -> + label = if unique?, do: short_version(elixir), else: elixir + {{elixir, otp}, "Duration (#{label})"} + end) + end + + defp short_version(version) do + version + |> to_string() + |> String.split(".") + |> Enum.take(2) + |> Enum.join(".") + end + + defp sort_shards(shards) do + Enum.sort_by(shards, fn s -> + idx = Enum.find_index(@service_order, &(&1 == s["service"])) + {idx || 99, s["service"]} + end) + end + + defp shard_status_cell(shard) do + failures = shard["total_failures"] || 0 + if failures == 0, do: "passed", else: "**failed (#{failures})**" + end + + defp format_shard_slowest(nil), do: "-" + defp format_shard_slowest([]), do: "-" + + defp format_shard_slowest([slowest | _]) do + format_test_desc(slowest) + end + + defp format_test_desc(nil), do: "-" + + defp format_test_desc(test) do + name = escape_markdown(test["name"] || "unknown") + mod = test["module"] || "unknown" + dur = format_duration(test["duration_ms"] || 0) + "`#{mod}`: #{truncate_text(name, 45)} (`#{dur}`)" + end + + defp service_name("postgresql"), do: "PostgreSQL" + defp service_name("mysql"), do: "MySQL" + defp service_name("mssql"), do: "MSSQL" + defp service_name("none"), do: "sqlite3 + no-db" + defp service_name(other), do: to_string(other) + + defp format_duration(ms) when ms < 1000, do: "#{ms}ms" + + defp format_duration(ms) do + total_seconds = round(ms / 1000) + mins = div(total_seconds, 60) + secs = rem(total_seconds, 60) + + if mins == 0 do + "#{secs}s" + else + "#{mins}m #{String.pad_leading(Integer.to_string(secs), 2, "0")}s" + end + end + + defp truncate_text(text, max_len) do + if String.length(text) > max_len do + String.slice(text, 0, max_len - 3) <> "..." + else + text + end + end + + defp escape_markdown(text) do + text + |> to_string() + |> String.replace("&", "&") + |> String.replace("<", "<") + |> String.replace(">", ">") + |> String.replace("\"", """) + |> String.replace("|", "\\|") + |> String.replace("\r\n", " ") + |> String.replace("\n", " ") + end +end + +Phoenix.Integration.AggregateSummary.run(System.argv()) diff --git a/integration_test/docker-compose.yml b/integration_test/docker-compose.yml index c483f05836..c51d394857 100644 --- a/integration_test/docker-compose.yml +++ b/integration_test/docker-compose.yml @@ -1,21 +1,43 @@ -version: '3' services: - postgres: - image: postgres + postgresql: + image: postgres:18 ports: - "5432:5432" environment: POSTGRES_PASSWORD: postgres + tmpfs: + - /var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 2s + timeout: 3s + retries: 10 + mysql: - image: mysql + image: mysql:26 ports: - "3306:3306" environment: MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + tmpfs: + - /var/lib/mysql + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h localhost --silent"] + interval: 2s + timeout: 3s + retries: 10 + mssql: image: mcr.microsoft.com/mssql/server:2019-latest environment: - ACCEPT_EULA: Y - SA_PASSWORD: some!Password + ACCEPT_EULA: "Y" + SA_PASSWORD: "some!Password" ports: - "1433:1433" + tmpfs: + - /var/opt/mssql + healthcheck: + test: ["CMD-SHELL", "/opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P \"$$SA_PASSWORD\" -C -Q 'SELECT 1' 2>/dev/null || /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$$SA_PASSWORD\" -Q 'SELECT 1' 2>/dev/null"] + interval: 2s + timeout: 3s + retries: 25 diff --git a/integration_test/test.sh b/integration_test/test.sh index 2934e880eb..d80dc7e78e 100755 --- a/integration_test/test.sh +++ b/integration_test/test.sh @@ -7,7 +7,7 @@ mix local.hex --force apk add --no-progress --update git socat make gcc libc-dev cmake g++ # Set up local proxies -socat TCP-LISTEN:5432,fork TCP-CONNECT:postgres:5432& +socat TCP-LISTEN:5432,fork TCP-CONNECT:postgresql:5432& socat TCP-LISTEN:3306,fork TCP-CONNECT:mysql:3306& socat TCP-LISTEN:1433,fork TCP-CONNECT:mssql:1433& diff --git a/integration_test/test/support/summary_formatter.ex b/integration_test/test/support/summary_formatter.ex index a6a5767163..59a51718b5 100644 --- a/integration_test/test/support/summary_formatter.ex +++ b/integration_test/test/support/summary_formatter.ex @@ -1,19 +1,14 @@ defmodule Phoenix.Integration.SummaryFormatter do @moduledoc false # - # Custom ExUnit formatter that generates a markdown summary for GitHub Actions. + # Custom ExUnit formatter that outputs structured JSON test metrics. # - # When the `GITHUB_STEP_SUMMARY` environment variable is set (which GitHub Actions - # automatically populates with a path to a step summary file), this formatter appends - # a markdown summary containing overall test metrics, a concurrent module execution - # timeline grouped into virtual worker lanes, a per-module duration breakdown, and a table - # of test durations sorted from slowest to fastest. + # By collecting test events in this custom formatter, we retain full concurrency + # across test modules while gathering granular execution durations and concurrency + # intervals. # - # This formatter takes inspiration from `mix test --slowest` and `--slowest-modules`. - # We do not use those built-in flags directly because they implicitly enable `--trace`, - # which forces all tests to run synchronously/sequentially and significantly slows down - # the integration test suite. By collecting test events in this custom formatter, we - # retain full concurrency across test modules while still getting granular timing insights. + # On suite finish, it writes a JSON summary file to `PHX_INTEGRATION_SUMMARY_JSON` + # (defaulting to "tmp/integration_test_summary.json"). # use GenServer @@ -60,28 +55,30 @@ defmodule Phoenix.Integration.SummaryFormatter do end def handle_cast({:suite_finished, run_data}, %{tests: tests} = state) do - summary_file = System.get_env("GITHUB_STEP_SUMMARY") + json_path = + System.get_env("PHX_INTEGRATION_SUMMARY_JSON") || "tmp/integration_test_summary.json" - if is_binary(summary_file) and summary_file != "" do - executed = - Enum.reject(tests, fn test -> - match?({:excluded, _}, test.state) or match?({:skipped, _}, test.state) - end) + executed = + Enum.reject(tests, fn test -> + match?({:excluded, _}, test.state) or match?({:skipped, _}, test.state) + end) + + slowest = Enum.sort_by(executed, &(&1.time || 0), :desc) + executed_modules = MapSet.new(slowest, & &1.module) - slowest = Enum.sort_by(executed, &(&1.time || 0), :desc) - write_github_summary(summary_file, run_data, state, slowest) - end + write_json_summary(json_path, run_data, state, slowest, executed_modules) {:noreply, state} end def handle_cast(_event, state), do: {:noreply, state} - defp write_github_summary( - summary_file, + defp write_json_summary( + json_path, run_data, %{tests: tests, modules: modules, suite_start: suite_start}, - slowest + slowest, + executed_modules ) do total = length(tests) failed = Enum.count(tests, &match?({:failed, _}, &1.state)) @@ -91,7 +88,7 @@ defmodule Phoenix.Integration.SummaryFormatter do passed = Enum.count(tests, &is_nil(&1.state)) total_failures = failed + invalid - status = if total_failures == 0, do: "Passed", else: "Failed (#{total_failures})" + status = if total_failures == 0, do: "passed", else: "failed" wall_time_ms = case run_data do @@ -104,6 +101,7 @@ defmodule Phoenix.Integration.SummaryFormatter do modules_stats = tests |> Enum.group_by(& &1.module) + |> Enum.filter(fn {mod, _tests} -> MapSet.member?(executed_modules, mod) end) |> Enum.map(fn {mod, mod_tests} -> mod_executed = Enum.reject(mod_tests, fn test -> @@ -117,233 +115,88 @@ defmodule Phoenix.Integration.SummaryFormatter do mod_failed = Enum.count(mod_tests, &match?({:failed, _}, &1.state)) mod_invalid = Enum.count(mod_tests, &match?({:invalid, _}, &1.state)) mod_failures = mod_failed + mod_invalid - mod_status = if mod_failures == 0, do: "passed", else: "failed (#{mod_failures})" + mod_status = if mod_failures == 0, do: "passed", else: "failed" + + suite_start_ms = suite_start || 0 + mod_timing = Map.get(modules, mod, %{}) + start_ms = Map.get(mod_timing, :start, suite_start_ms) + finish_ms = Map.get(mod_timing, :finish, start_ms) + module_name = if mod, do: mod |> Module.split() |> List.last(), else: "Unknown" %{ - module: mod, + module: module_name, + full_module: if(mod, do: inspect(mod), else: "Unknown"), status: mod_status, - test_count: length(mod_tests), + failures: mod_failures, + test_count: length(mod_executed), total_us: total_us, avg_us: avg_us, - max_us: max_us + max_us: max_us, + start_ms: max(0, start_ms - suite_start_ms), + finish_ms: max(0, finish_ms - suite_start_ms) } end) |> Enum.sort_by(& &1.total_us, :desc) - env_info = "Elixir #{System.version()} / OTP #{System.otp_release()}" - - sections = [ - """ - ## Phoenix Integration Tests (#{env_info}): #{status} - - | Total | Passed | Failed | Excluded | Skipped | Total Wall Time | - | :---: | :---: | :---: | :---: | :---: | :---: | - | **#{total}** | **#{passed}** | **#{total_failures}** | **#{excluded}** | **#{skipped}** | **#{format_duration(wall_time_ms)}** | - """, - format_mermaid_gantt(modules, suite_start), - """ -
- Module Breakdown - - | Module | Status | Tests | Total Duration | % of Total | Avg / Test | Max / Test | - | :--- | :---: | :---: | :--- | :---: | :--- | :--- | - #{Enum.map_join(modules_stats, "\n", &format_module_markdown_row(&1, total_executed_time_us))} - -
- """, - """ -
- Test Durations (Slowest to Fastest) - - | Status | Duration | Test | Module | Location | - | :--- | :--- | :--- | :--- | :--- | - #{Enum.map_join(slowest, "\n", &format_markdown_row/1)} - -
- """ - ] - - markdown = - sections - |> Enum.reject(&(&1 in ["", nil])) - |> Enum.map_join("\n\n", &String.trim/1) - |> Kernel.<>("\n\n") - - case File.write(summary_file, markdown, [:append]) do - :ok -> - :ok - - {:error, reason} -> - IO.warn("Failed to write integration test summary to #{summary_file}: #{inspect(reason)}") - end - end - - defp format_mermaid_gantt(modules, suite_start) when map_size(modules) > 0 do - suite_start = - suite_start || - modules - |> Map.values() - |> Enum.map(& &1.start) - |> Enum.min(fn -> 0 end) - - sorted_modules = - modules - |> Enum.sort_by(fn {mod, %{start: s, finish: f}} -> {s, f || s, mod} end) - - lanes = - Enum.reduce(sorted_modules, [], fn item, acc_lanes -> - assign_to_lane(acc_lanes, item, []) - end) - |> Enum.map(&Enum.reverse/1) - - {slowest_mod, _} = - Enum.max_by( - modules, - fn {_mod, %{start: s, finish: f}} -> (f || s) - s end, - fn -> {nil, nil} end - ) - - section_rows = - lanes - |> Enum.with_index(1) - |> Enum.map(fn {lane, idx} -> - tasks = - Enum.map(lane, fn {mod, %{start: s, finish: f}} -> - module_name = format_gantt_module_name(mod) - duration_ms = max(1000, (f || s) - s) - start_ms = max(0, s - suite_start) - finish_ms = start_ms + duration_ms - - start_str = format_gantt_time(start_ms) - finish_str = format_gantt_time(finish_ms) - tag = if mod == slowest_mod, do: ":crit, active,", else: ":active," - - " #{module_name} #{tag} #{start_str}, #{finish_str}" - end) + slowest_tests = + Enum.map(slowest, fn %ExUnit.Test{ + name: name, + module: mod, + time: time, + tags: tags, + state: state + } -> + test_name = name |> to_string() |> String.replace_prefix("test ", "") + module_name = if mod, do: mod |> Module.split() |> List.last(), else: "Unknown" + file = if f = tags[:file], do: Path.relative_to(f, File.cwd!()), else: "unknown" + line = tags[:line] || 0 + status = if is_nil(state), do: "passed", else: "failed" - " section Lane #{idx}\n" <> Enum.join(tasks, "\n") + %{ + name: test_name, + module: module_name, + duration_ms: div(time || 0, 1000), + time_us: time || 0, + file: file, + line: line, + status: status + } end) - """ -
- Module Execution Timeline - - ```mermaid - --- - displayMode: compact - --- - gantt - title Module Execution Timeline - dateFormat mm:ss - axisFormat %M:%S - todayMarker off - #{Enum.join(section_rows, "\n")} - ``` - -
- """ - end - - defp format_mermaid_gantt(_modules, _suite_start), do: "" - - defp assign_to_lane([], item, acc) do - Enum.reverse([[item] | acc]) - end - - defp assign_to_lane( - [[{_last_mod, %{finish: last_finish, start: last_start}} | _] = lane | rest], - {_mod, %{start: s}} = item, - acc - ) do - if (last_finish || last_start) <= s do - Enum.reverse(acc) ++ [[item | lane] | rest] - else - assign_to_lane(rest, item, [lane | acc]) - end - end - - defp format_gantt_module_name(nil), do: "Unknown" - - defp format_gantt_module_name(mod) do - mod - |> Module.split() - |> List.last() - |> String.replace_prefix("UmbrellaAppWith", "Umbrella") - |> String.replace_prefix("AppWith", "") - |> String.replace("Adapter", "") - |> String.replace_suffix("Test", "") - end - - defp format_gantt_time(ms) do - total_seconds = div(ms, 1000) - mins = div(total_seconds, 60) - secs = rem(total_seconds, 60) - - "#{String.pad_leading(Integer.to_string(mins), 2, "0")}:#{String.pad_leading(Integer.to_string(secs), 2, "0")}" - end - - defp format_module_markdown_row( - %{ - module: mod, - status: status, - test_count: count, - total_us: total_us, - avg_us: avg_us, - max_us: max_us - }, - total_suite_us - ) do - module_name = if mod, do: mod |> Module.split() |> List.last(), else: "Unknown" - total_ms = div(total_us, 1000) - avg_ms = div(avg_us, 1000) - max_ms = div(max_us, 1000) - - pct = - if total_suite_us > 0 do - Float.round(total_us / total_suite_us * 100, 1) - else - 0.0 + service = + System.get_env("PHX_TEST_SERVICE") || System.get_env("PHX_TEST_GROUP") || "none" + + payload = %{ + service: service, + elixir: System.get_env("PHX_ELIXIR_VERSION") || System.version(), + otp: System.get_env("PHX_OTP_VERSION") || System.otp_release(), + status: status, + total_failures: total_failures, + total_tests: total, + executed_tests: length(slowest), + passed_tests: passed, + failed_tests: total_failures, + skipped_tests: skipped, + excluded_tests: excluded, + wall_time_ms: wall_time_ms, + total_executed_time_us: total_executed_time_us, + modules: modules_stats, + slowest_tests: slowest_tests + } + + json_str = + cond do + Code.ensure_loaded?(JSON) -> + apply(JSON, :encode!, [payload]) + + Code.ensure_loaded?(Jason) -> + apply(Jason, :encode!, [payload]) + + true -> + raise "Neither JSON nor Jason available for encoding summary payload" end - "| `#{module_name}` | #{status} | #{count} | `#{format_duration(total_ms)}` | #{pct}% | `#{format_duration(avg_ms)}` | `#{format_duration(max_ms)}` |" - end - - defp format_markdown_row(%ExUnit.Test{ - name: name, - module: mod, - time: time, - tags: tags, - state: state - }) do - ms = div(time || 0, 1000) - test_name = name |> to_string() |> String.replace_prefix("test ", "") - module_name = if mod, do: mod |> Module.split() |> List.last(), else: "Unknown" - file = if f = tags[:file], do: Path.relative_to(f, File.cwd!()), else: "unknown" - - status = if is_nil(state), do: "passed", else: "failed" - - "| #{status} | `#{format_duration(ms)}` | #{escape_markdown(test_name)} | `#{module_name}` | `#{file}:#{tags[:line]}` |" - end - - defp format_duration(ms) when ms < 1000, do: "#{ms}ms" - - defp format_duration(ms) do - total_seconds = round(ms / 1000) - mins = div(total_seconds, 60) - secs = rem(total_seconds, 60) - - if mins == 0 do - "#{secs}s" - else - "#{mins}m #{String.pad_leading(Integer.to_string(secs), 2, "0")}s" - end - end - - defp escape_markdown(text) do - text - |> Plug.HTML.html_escape() - |> String.replace("|", "\\|") - |> String.replace("\r\n", " ") - |> String.replace("\n", " ") + json_path |> Path.dirname() |> File.mkdir_p!() + File.write!(json_path, json_str) end end