Split integration test suites into focused async modules

[rhcarvalho]

Previously, integration tests were bundled into 7 monolithic test files. Because ExUnit parallelizes test execution across modules while executing tests serially within each module, this structure created two major bottlenecks:

1. Low concurrency at startup: Only 7 worker processes could run at T=0, underutilizing runners configured with higher concurrency (e.g. 8 workers in GitHub Actions).
2. Idle worker tail: Faster modules (such as SQLite or basic generators) finished early, leaving most worker processes sitting completely idle while the suite's wall-clock time was bounded by the slowest monolithic modules (AppWithScopesTest and UmbrellaAppWithDefaultsTest).

This splits the integration test suites into 33 focused, granular modules all marked with async: true:

• Preserve all 53 tests.
• Individual test modules should run under 3m00s.
• Fix a missing @tag database: :sqlite3 on test "has a passing test suite (--no-live)".
• File/module names follow the pattern app_with_{postgres,mysql,mssql,sqlite3}_adapter{,_auth_{html,live},_html,_json,_live,_scopes}_test.exs (and umbrella counterparts).
• Postgres adapter test modules explicitly named for symmetry with the other database adapters (instead of "Default").
• Short, systematic app names per module to guarantee test database isolation when executing concurrently against PostgreSQL, MySQL, and MSSQL while ensuring all generated code strictly satisfies mix format line-length limits.
• Update documentation in integration_test/README.md with rationale.

Testing

To gain confidence that the split is mechanical and adheres to the changes documented above, I've used an adhoc script to compare the body of each of the 53 tests between main and the PR branch.

Details

defmodule CompareSuites do
  def run do
    main_files = git_files("upstream/main")
    head_files = git_files("HEAD")

    main_tests = Enum.flat_map(main_files, &parse_git_file("upstream/main", &1))
    head_tests = Enum.flat_map(head_files, &parse_git_file("HEAD", &1))

    IO.puts("Main total tests: #{length(main_tests)}")
    IO.puts("HEAD total tests: #{length(head_tests)}")

    matches = match_tests(main_tests, head_tests)

    IO.puts("Successfully paired #{length(matches)} tests.\n")

    Enum.each(Enum.with_index(matches, 1), fn {{m, h}, idx} ->
      IO.puts("================================================================================")
      IO.puts("[#{idx}/53] #{m.name}")
      IO.puts("  MAIN: #{m.module} (#{m.file})")
      IO.puts("        describe: #{inspect(m.describe)}, tags: #{inspect(m.tags)}")
      IO.puts("  HEAD: #{h.module} (#{h.file})")
      IO.puts("        describe: #{inspect(h.describe)}, tags: #{inspect(h.tags)}")

      # 1. Tags diff
      if m.tags != h.tags do
        IO.puts("  ⚠️ TAG DIFFERENCE: main=#{inspect(m.tags)} vs head=#{inspect(h.tags)}")
      end

      # 2. Body normalization & diff
      diff_bodies(m, h)
    end)
  end

  defp git_files(rev) do
    {out, 0} = System.cmd("git", ["ls-tree", "-r", "--name-only", rev, "integration_test/test/code_generation"])
    out |> String.split("\n", trim: true) |> Enum.filter(&String.ends_with?(&1, ".exs"))
  end

  defp parse_git_file(rev, file) do
    {content, 0} = System.cmd("git", ["show", "#{rev}:#{file}"])
    {:ok, ast} = Code.string_to_quoted(content, columns: true)
    extract_tests(ast, file)
  end

  defp extract_tests(ast, file) do
    {_, tests} = Macro.prewalk(ast, [], fn
      {:defmodule, _, [{:__aliases__, _, mod_parts}, [do: mod_body]]} = node, acc ->
        mod_name = Module.concat(mod_parts)
        mod_tests = extract_tests_from_block(mod_body, file, mod_name, nil, [])
        {node, acc ++ mod_tests}
      node, acc ->
        {node, acc}
    end)
    tests
  end

  defp extract_tests_from_block(body, file, mod_name, current_describe, current_tags) do
    stmts =
      case body do
        {:__block__, _, stmts} -> stmts
        single -> [single]
      end

    {tests, _remaining_tags} =
      Enum.reduce(stmts, {[], current_tags}, fn stmt, {tests_acc, tags_acc} ->
        case stmt do
          {:@, _, [{:tag, _, [tag_val]}]} ->
            {tests_acc, [tag_val | tags_acc]}

          {:@, _, [{:moduletag, _, [_tag_val]}]} ->
            {tests_acc, tags_acc}

          {:@, _, [{:describetag, _, [_tag_val]}]} ->
            {tests_acc, tags_acc}

          {:describe, _, [desc_name, [do: desc_body]]} ->
            desc_tests = extract_tests_from_block(desc_body, file, mod_name, desc_name, tags_acc)
            {tests_acc ++ desc_tests, []}

          {:test, _, [test_name, [do: test_body]]} ->
            name_str = if is_binary(test_name), do: test_name, else: Macro.to_string(test_name)
            test_info = %{
              file: file,
              module: mod_name,
              describe: current_describe,
              name: name_str,
              tags: Enum.reverse(tags_acc),
              body_ast: test_body,
              body_code: Macro.to_string(test_body)
            }
            {tests_acc ++ [test_info], []}

          {:test, _, [test_name, args, [do: test_body]]} ->
            name_str = if is_binary(test_name), do: test_name, else: Macro.to_string(test_name)
            test_info = %{
              file: file,
              module: mod_name,
              describe: current_describe,
              name: name_str,
              args: Macro.to_string(args),
              tags: Enum.reverse(tags_acc),
              body_ast: test_body,
              body_code: Macro.to_string(test_body)
            }
            {tests_acc ++ [test_info], []}

          _other ->
            {tests_acc, tags_acc}
        end
      end)

    tests
  end

  defp match_tests(main_tests, head_tests) do
    Enum.map(main_tests, fn m ->
      h = Enum.find(head_tests, fn h ->
        h.name == m.name and match_mod(m, h)
      end)
      if h == nil do
        raise "Could not match main test: #{inspect(m.module)} #{m.name} (#{m.file})"
      end
      {m, h}
    end)
  end

  defp match_mod(m, h) do
    m_str = String.downcase(inspect(m.module))
    h_str = String.downcase(inspect(h.module))
    h_file = String.downcase(h.file)
    m_desc = String.downcase(m.describe || "")

    cond do
      m_str =~ "nooptions" and h_str =~ "minimal" ->
        true

      m_str =~ "umbrella" and h_str =~ "umbrella" ->
        match_feature(m_desc, h_file)

      m_str =~ "appwithdefaults" and not (m_str =~ "umbrella") and h_str =~ "postgres" and not (h_str =~ "umbrella") ->
        match_feature(m_desc, h_file)

      m_str =~ "appwithscopes" and h_str =~ "appwithscopes" ->
        match_scopes_feature(m, h)

      m_str =~ "mysql" and h_str =~ "mysql" ->
        match_db_feature(m_desc, h_file)

      m_str =~ "mssql" and h_str =~ "mssql" ->
        match_db_feature(m_desc, h_file)

      m_str =~ "sqlite3" and h_str =~ "sqlite3" ->
        match_db_feature(m_desc, h_file)

      true ->
        false
    end
  end

  defp match_feature(m_desc, h_file) do
    cond do
      m_desc =~ "html" and h_file =~ "html" -> true
      m_desc =~ "json" and h_file =~ "json" -> true
      m_desc =~ "live" and h_file =~ "live" -> true
      m_desc =~ "auth" and h_file =~ "auth" -> true
      (m_desc =~ "defaults" or m_desc =~ "umbrella") and not (h_file =~ "html" or h_file =~ "json" or h_file =~ "live" or h_file =~ "auth") -> true
      true -> false
    end
  end

  defp match_db_feature(m_desc, h_file) do
    cond do
      m_desc =~ "html" and h_file =~ "html" -> true
      m_desc =~ "json" and h_file =~ "json" -> true
      m_desc =~ "live with scope" and h_file =~ "scopes" -> true
      m_desc =~ "live" and not (m_desc =~ "scope") and h_file =~ "live" -> true
      m_desc =~ "auth" and h_file =~ "auth" -> true
      true -> false
    end
  end

  defp match_scopes_feature(m, h) do
    m_desc = String.downcase(m.describe || "")
    h_file = String.downcase(h.file)
    cond do
      m.name =~ "phx.gen.live" and h_file =~ "live" -> true
      m.name =~ "phx.gen.html" and h_file =~ "html" -> true
      m.name =~ "phx.gen.json" and m_desc =~ "auth" and h_file =~ "json" and h.describe =~ "auth" -> true
      m.name =~ "phx.gen.json" and m_desc =~ "custom" and h_file =~ "json" and h.describe =~ "custom" -> true
      m.name =~ "route_prefix" and h_file =~ "custom_routes" -> true
      true -> false
    end
  end

  defp diff_bodies(m, h) do
    m_code = format_code(m.body_code)
    h_code = format_code(h.body_code)

    if m_code == h_code do
      IO.puts("  ✅ Body IDENTICAL (byte-for-byte)")
    else
      m_app = extract_app_name(m.body_ast)
      h_app = extract_app_name(h.body_ast)

      m_norm = replace_app_references(m_code, m_app)
      h_norm = replace_app_references(h_code, h_app)

      if m_norm == h_norm do
        IO.puts("  ✅ Body EQUIVALENT (differs ONLY by app name `#{m_app}` -> `#{h_app}`)")
      else
        IO.puts("  🚨 UNEXPECTED CODE DIFFERENCE DETECTED:")
        IO.puts("  Main app name: #{m_app}, Head app name: #{h_app}")
        print_diff(m_norm, h_norm)
      end
    end
  end

  defp extract_app_name(ast) do
    Macro.prewalk(ast, nil, fn
      {:generate_phoenix_app, _, [_tmp, name | _]}, nil when is_binary(name) ->
        {nil, name}
      node, acc ->
        {node, acc}
    end)
    |> elem(1)
  end

  defp replace_app_references(code, nil), do: code
  defp replace_app_references(code, app_name) do
    pascal = Macro.camelize(app_name)
    code
    |> String.replace(~r/with_installer_tmp\("[^"]+",/, "with_installer_tmp(\"TMP\",")
    |> String.replace(app_name, "__APP__")
    |> String.replace(pascal, "__APP_MODULE__")
    |> String.replace(~r/assert response\.body =~ "[^"]+"/, "assert response.body =~ \"__APP_MODULE__\"")
    |> format_code()
  end

  defp format_code(code) do
    try do
      Code.format_string!(code) |> IO.iodata_to_binary()
    rescue
      _ -> code
    end
  end

  defp print_diff(s1, s2) do
    File.write!("/tmp/m_diff.txt", s1)
    File.write!("/tmp/h_diff.txt", s2)
    {out, _} = System.cmd("diff", ["-u", "/tmp/m_diff.txt", "/tmp/h_diff.txt"])
    IO.puts(out)
  end
end

CompareSuites.run()

Intended follow ups

These are a few of the ideas to reduce wall clock time for integration tests (and the overall CI checks) down from 9-10min to roughly 3-4min:

1. [This PR] Split existing modules into predictable and consistent modules with <3min serial execution time on the slowest run Elixir/OTP matrix combination). All existing 53 tests preserved.
2. Remove redundant CPU-bound work (compilation) by combining tests that build on the same app scaffolding (same phx.new flags) following the pattern:

   assert_passes_formatter_check(app_root_path)
   assert_no_compilation_warnings(app_root_path)
   drop_test_database(app_root_path)
   assert_tests_pass(app_root_path)

3. Shard integration tests into more jobs to be able to use more runners/CPUs in parallel, either using test partitions or explicit splits (e.g. per database)

---

Assisted by Antigravity / Gemini 3.7 Flash.

[rhcarvalho]

CI failed because with a bit longer app names the mix format --check-formatted check fails. This relates to #6015.

Still, we get a preview of how tests get scheduled:

Before

gantt
    title Module Execution Timeline
    dateFormat mm:ss
    axisFormat %M:%S
    section Modules
    UmbrellaAppWithDefaultsTest :crit, active, 00:00, 08:14
    AppWithDefaultsTest :active, 00:00, 08:04
    AppWithMySqlAdapterTest :active, 00:00, 07:16
    AppWithScopesTest :active, 00:00, 06:39
    AppWithMSSQLAdapterTest :active, 00:00, 06:27
    AppWithSQLite3AdapterTest :active, 00:00, 06:17
    AppWithNoOptionsTest :active, 00:00, 01:34

Loading

After

gantt
    title Module Execution Timeline
    dateFormat mm:ss
    axisFormat %M:%S
    section Modules
    UmbrellaAppWithDefaultsAuthTest :crit, active, 02:21, 07:00
    AppWithMySqlAdapterAuthTest :active, 02:37, 07:11
    AppWithSQLite3AdapterAuthTest :active, 00:00, 04:31
    AppWithMSSQLAdapterAuthTest :active, 03:09, 07:21
    AppWithScopesJsonTest :active, 04:29, 07:01
    AppWithDefaultsAuthTest :active, 05:22, 07:53
    AppWithDefaultsHtmlTest :active, 04:31, 06:44
    AppWithDefaultsJsonTest :active, 00:00, 02:12
    UmbrellaAppWithDefaultsLiveTest :active, 00:00, 02:10
    UmbrellaAppWithDefaultsHtmlTest :active, 00:00, 02:10
    AppWithDefaultsLiveTest :active, 00:00, 02:08
    UmbrellaAppWithDefaultsJsonTest :active, 01:08, 03:09
    AppWithNoOptionsTest :active, 05:08, 07:02
    UmbrellaAppWithDefaultsTest :active, 02:10, 04:02
    AppWithScopesLiveTest :active, 00:00, 01:41
    AppWithDefaultsTest :active, 05:32, 07:11
    AppWithScopesHtmlTest :active, 03:43, 05:22
    AppWithMySqlAdapterScopesTest :active, 02:10, 03:43
    AppWithScopesCustomRoutesTest :active, 02:08, 03:29
    AppWithMySqlAdapterLiveTest :active, 00:00, 01:19
    AppWithMSSQLAdapterJsonTest :active, 04:22, 05:32
    AppWithSQLite3AdapterHtmlTest :active, 00:00, 01:08
    AppWithSQLite3AdapterJsonTest :active, 03:14, 04:22
    AppWithMSSQLAdapterLiveTest :active, 04:02, 05:08
    AppWithMSSQLAdapterHtmlTest :active, 01:19, 02:21
    AppWithMySqlAdapterHtmlTest :active, 02:12, 03:14
    AppWithSQLite3AdapterLiveTest :active, 03:29, 04:29
    AppWithMySqlAdapterJsonTest :active, 01:41, 02:37

Loading

(the "critical path" highlighting is probably broken, my bad...)

[rhcarvalho]

CI failed because with a bit longer app names the mix format --check-formatted check fails.

I overcame this by methodically naming the apps (same name per module) and also naming the modules in a methodical way, which makes it easier to see the gaps in test coverage.

[rhcarvalho]

@SteffenDE as a follow up we can then use test partitions to utilize even more cores in parallel. Each job gets 8 vCPUs, and the repo gets up-to 20 concurrent jobs (which we saturate using a matrix of elixir/otp versions and partition numbers).

• https://docs.github.com/en/actions/reference/runners/github-hosted-runners#standard-github-hosted-runners-for-public-repositories
• https://docs.github.com/en/actions/reference/limits#job-concurrency-limits-for-github-hosted-runners

[SteffenDE]

Interesting, so the overall runtime seems pretty much the same? Did you try increasing max_cases?

[rhcarvalho]

the overall runtime seems pretty much the same?

Yes, the gain is not yet meaningful, though we have better CPU utilization. Here's the execution chart of that old run (#6826 (comment)) organized into 8 lanes:

---
displayMode: compact
---
gantt
    title Module Execution Timeline (8 Concurrent Lanes)
    dateFormat mm:ss
    axisFormat %M:%S
    todayMarker off
    section Lane 1
    AppWithSQLite3AdapterAuthTest :active, 00:00, 04:31
    AppWithDefaultsHtmlTest :active, 04:31, 06:44
    section Lane 2
    AppWithDefaultsJsonTest :active, 00:00, 02:12
    AppWithMySqlAdapterHtmlTest :active, 02:12, 03:14
    AppWithSQLite3AdapterJsonTest :active, 03:14, 04:22
    AppWithMSSQLAdapterJsonTest :active, 04:22, 05:32
    AppWithDefaultsTest :active, 05:32, 07:11
    section Lane 3
    UmbrellaAppWithDefaultsLiveTest :active, 00:00, 02:10
    UmbrellaAppWithDefaultsTest :active, 02:10, 04:02
    AppWithMSSQLAdapterLiveTest :active, 04:02, 05:08
    AppWithNoOptionsTest :active, 05:08, 07:02
    section Lane 4
    UmbrellaAppWithDefaultsHtmlTest :crit, active, 00:00, 02:10
    AppWithMySqlAdapterScopesTest :crit, active, 02:10, 03:43
    AppWithScopesHtmlTest :crit, active, 03:43, 05:22
    AppWithDefaultsAuthTest :crit, active, 05:22, 07:53
    section Lane 5
    AppWithDefaultsLiveTest :active, 00:00, 02:08
    AppWithScopesCustomRoutesTest :active, 02:08, 03:29
    AppWithSQLite3AdapterLiveTest :active, 03:29, 04:29
    AppWithScopesJsonTest :active, 04:29, 07:01
    section Lane 6
    AppWithScopesLiveTest :active, 00:00, 01:41
    AppWithMySqlAdapterJsonTest :active, 01:41, 02:37
    AppWithMySqlAdapterAuthTest :active, 02:37, 07:11
    section Lane 7
    AppWithMySqlAdapterLiveTest :active, 00:00, 01:19
    AppWithMSSQLAdapterHtmlTest :active, 01:19, 02:21
    UmbrellaAppWithDefaultsAuthTest :active, 02:21, 07:00
    section Lane 8
    AppWithSQLite3AdapterHtmlTest :active, 00:00, 01:08
    UmbrellaAppWithDefaultsJsonTest :active, 01:08, 03:09
    AppWithMSSQLAdapterAuthTest :active, 03:09, 07:21

Loading

Running 4 of the new modules sequentially yielded a similar time as the previous longest running test module.
Anyway, splitting the test modules unlocks running multiple jobs/test partitions in parallel.

Did you try increasing max_cases?

Not yet.

Have focused on correctness, first gain confidence we're not accidentally changing tests in unintended ways. I think now we're in a good spot to experiment further.

[rhcarvalho]

Did you try increasing max_cases?

I think it will not be useful because each test runs CPU bound tasks (compilation), we'll probably overbook the 4 vCPUs.

But let's try it anyway, will send it in a separate commit we can decide to keep or drop.

[rhcarvalho]

--max-cases 16 (up from 8) caused individual test module run times to increase, overall time not improve, and one of the tests timed out:

https://github.com/phoenixframework/phoenix/actions/runs/33566236525/attempts/1#summary-100049951994

---
displayMode: compact
---
gantt
    title Module Execution Timeline
    dateFormat mm:ss
    axisFormat %M:%S
    todayMarker off
    section Lane 1
    PostgresHtml :active, 00:00, 03:10
    PostgresJson :active, 03:10, 05:53
    section Lane 2
    PostgresLive :active, 00:00, 03:44
    MSSQLHtml :active, 03:44, 05:22
    section Lane 3
    ScopesJson :active, 00:00, 04:11
    section Lane 4
    MSSQLAuth :active, 00:00, 06:13
    section Lane 5
    SQLite3Live :active, 00:00, 01:48
    UmbrellaPostgres :active, 01:48, 04:58
    section Lane 6
    UmbrellaPostgresAuth :crit, active, 00:00, 06:13
    section Lane 7
    MySQLHtml :active, 00:00, 01:56
    SQLite3Auth :active, 01:56, 06:21
    section Lane 8
    MySQLScopes :active, 00:00, 02:07
    UmbrellaPostgresHtml :active, 02:07, 05:29
    section Lane 9
    MSSQLJson :active, 00:00, 01:51
    MySQLLive :active, 01:51, 03:42
    MinimalApp :active, 03:42, 05:53
    section Lane 10
    UmbrellaPostgresJson :active, 00:00, 03:45
    SQLite3Html :active, 03:45, 05:23
    section Lane 11
    ScopesCustomRoutes :active, 00:00, 02:58
    ScopesLive :active, 02:58, 05:07
    section Lane 12
    MySQLAuth :active, 00:00, 06:12
    section Lane 13
    ScopesHtml :active, 00:00, 02:09
    MySQLJson :active, 02:09, 04:03
    section Lane 14
    MSSQLLive :active, 00:00, 01:56
    SQLite3Json :active, 01:56, 03:45
    section Lane 15
    PostgresAuth :active, 00:00, 05:55
    section Lane 16
    Postgres :active, 00:00, 03:24
    UmbrellaPostgresLive :active, 03:24, 05:55

Loading

I'll drop the --max-cases 16 commit.

[rhcarvalho]

Before (main@b4056c3)

https://github.com/phoenixframework/phoenix/actions/runs/33510641219/attempts/1#summary-99865178194

Phoenix Integration Tests (Elixir 1.18.4 / OTP 27): Passed

Total	Passed	Failed	Excluded	Skipped	Total Wall Time
53	53	0	0	0	7m 48s

Module Execution Timeline

gantt
    title Module Execution Timeline
    dateFormat mm:ss
    axisFormat %M:%S
    section Modules
    UmbrellaAppWithDefaultsTest :crit, active, 00:00, 07:47
    AppWithDefaultsTest :active, 00:00, 07:47
    AppWithMySqlAdapterTest :active, 00:00, 07:03
    AppWithScopesTest :active, 00:00, 06:22
    AppWithMSSQLAdapterTest :active, 00:00, 06:04
    AppWithSQLite3AdapterTest :active, 00:00, 05:54
    AppWithNoOptionsTest :active, 00:00, 01:23

Loading

Phoenix Integration Tests (Elixir 1.20.4 / OTP 29): Passed

Total	Passed	Failed	Excluded	Skipped	Total Wall Time
53	53	0	0	0	7m 32s

Module Execution Timeline

gantt
    title Module Execution Timeline
    dateFormat mm:ss
    axisFormat %M:%S
    section Modules
    UmbrellaAppWithDefaultsTest :crit, active, 00:00, 07:31
    AppWithDefaultsTest :active, 00:00, 07:12
    AppWithMySqlAdapterTest :active, 00:00, 06:38
    AppWithMSSQLAdapterTest :active, 00:00, 06:01
    AppWithScopesTest :active, 00:00, 05:59
    AppWithSQLite3AdapterTest :active, 00:00, 05:56
    AppWithNoOptionsTest :active, 00:00, 01:21

Loading

After (this PR)

https://github.com/phoenixframework/phoenix/actions/runs/33567436374/attempts/1#summary-100053715601

Phoenix Integration Tests (Elixir 1.18.4 / OTP 27): Passed

Total	Passed	Failed	Excluded	Skipped	Total Wall Time
53	53	0	0	0	7m 48s

Module Execution Timeline

---
displayMode: compact
---
gantt
    title Module Execution Timeline
    dateFormat mm:ss
    axisFormat %M:%S
    todayMarker off
    section Lane 1
    SQLite3Live :active, 00:00, 01:10
    Postgres :active, 01:10, 02:34
    ScopesCustomRoutes :active, 02:34, 04:36
    MySQLLive :active, 04:36, 05:36
    MSSQLAuth :active, 05:36, 07:48
    section Lane 2
    SQLite3Html :active, 00:00, 01:12
    MySQLAuth :active, 01:12, 06:02
    section Lane 3
    MySQLJson :active, 00:00, 01:11
    PostgresAuth :active, 01:11, 05:55
    MinimalApp :active, 05:55, 07:12
    section Lane 4
    PostgresLive :active, 00:00, 02:16
    ScopesLive :active, 02:16, 03:53
    MySQLScopes :active, 03:53, 05:20
    UmbrellaPostgresJson :active, 05:20, 06:51
    section Lane 5
    SQLite3Json :active, 00:00, 01:04
    UmbrellaPostgresAuth :crit, active, 01:04, 05:57
    section Lane 6
    UmbrellaPostgresHtml :active, 00:00, 02:09
    MSSQLJson :active, 02:09, 03:03
    MySQLHtml :active, 03:03, 04:05
    UmbrellaPostgres :active, 04:05, 05:54
    PostgresJson :active, 05:54, 07:13
    section Lane 7
    UmbrellaPostgresLive :active, 00:00, 02:16
    ScopesHtml :active, 02:16, 03:46
    PostgresHtml :active, 03:46, 05:34
    MSSQLHtml :active, 05:34, 06:40
    section Lane 8
    SQLite3Auth :active, 00:00, 04:22
    MSSQLLive :active, 04:22, 05:23
    ScopesJson :active, 05:23, 07:31

Loading

Phoenix Integration Tests (Elixir 1.20.4 / OTP 29): Passed

Total	Passed	Failed	Excluded	Skipped	Total Wall Time
53	53	0	0	0	4m 39s

Module Execution Timeline

---
displayMode: compact
---
gantt
    title Module Execution Timeline
    dateFormat mm:ss
    axisFormat %M:%S
    todayMarker off
    section Lane 1
    MySQLHtml :active, 00:00, 00:39
    ScopesCustomRoutes :active, 00:39, 01:59
    UmbrellaPostgresAuth :active, 01:59, 04:16
    section Lane 2
    MSSQLLive :active, 00:00, 00:47
    ScopesJson :active, 00:47, 02:26
    PostgresHtml :active, 02:26, 03:43
    section Lane 3
    UmbrellaPostgresHtml :active, 00:00, 01:05
    MSSQLJson :active, 01:05, 01:46
    UmbrellaPostgres :active, 01:46, 02:51
    MySQLLive :active, 02:51, 03:23
    SQLite3Html :active, 03:23, 03:59
    section Lane 4
    UmbrellaPostgresJson :active, 00:00, 01:14
    MySQLScopes :active, 01:14, 02:18
    SQLite3Live :active, 02:18, 02:58
    SQLite3Json :active, 02:58, 03:35
    MySQLJson :active, 03:35, 04:04
    section Lane 5
    ScopesLive :active, 00:00, 01:00
    SQLite3Auth :crit, active, 01:00, 03:45
    section Lane 6
    UmbrellaPostgresLive :active, 00:00, 01:14
    Postgres :active, 01:14, 02:18
    PostgresAuth :active, 02:18, 04:27
    section Lane 7
    PostgresLive :active, 00:00, 01:14
    MinimalApp :active, 01:14, 02:12
    MySQLAuth :active, 02:12, 04:25
    section Lane 8
    PostgresJson :active, 00:00, 01:14
    MSSQLHtml :active, 01:14, 01:57
    ScopesHtml :active, 01:57, 02:57
    MSSQLAuth :active, 02:57, 04:39

Loading

[SteffenDE]

wow, newer Elixir has a noticeable performance improvement indeed with the parallel compilation. So it makes sense that more cases don't help if compiling is the bottleneck. Since the generated apps share almost all deps, I'm wondering if we should add a "warmup" run and then copy the _build folder into the other tests, such that they only need to compile the project itself, which should be much faster.

[rhcarvalho]

Since the generated apps share almost all deps, I'm wondering if we should add a "warmup" run and then copy the _build folder into the other tests, such that they only need to compile the project itself, which should be much faster.

Do you mean this?

phoenix/integration_test/test/support/code_generator_case.ex

Lines 19 to 24 in b4056c3

	for path <- ~w(mix.lock deps _build) do
	File.cp_r!(
	Path.join(integration_test_root_path, path),
	Path.join(app_root_path, path)
	)
	end

A further improvement to that will be using cp -al on Linux/CI which might be faster than File.cp_r! (tracking locally among the potential CI improvements).

[rhcarvalho]

As a hopefully final update to this PR, I've further split the phx.gen.auth related tests into --live and --no-live, which should make all modules run serially in < 3 minutes.

I've also documented the testing strategy and intended follow ups in the PR description.

[SteffenDE]

Do you mean this?

Ah, I forgot that we already do this. So maybe it works fine already (and only compiles the app files themselves) :)