From 9f3b5df96f78be88420d6339f0a175b17189b5bd Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 11 Feb 2026 18:18:32 +0000 Subject: [PATCH 1/5] Register MPI.Op for addition of Float128 Avoids errors when running on ARM chips. --- moment_kinetics/src/runge_kutta.jl | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/moment_kinetics/src/runge_kutta.jl b/moment_kinetics/src/runge_kutta.jl index 0d49156a1b..9b3c6430bb 100644 --- a/moment_kinetics/src/runge_kutta.jl +++ b/moment_kinetics/src/runge_kutta.jl @@ -15,6 +15,7 @@ using ..looping using ..type_definitions: mk_float, MPISharedArray using MPI +using Quadmath using StatsBase: mean """ @@ -1295,6 +1296,11 @@ function local_error_norm(f_loworder::MPISharedArray{mk_float,6}, end end +# Need to 'register' this operation explicitly to use it in MPI reduction operations on +# non-x86 processor architectures +# (https://juliaparallel.org/MPI.jl/stable/knownissues/#Custom-reduction-operators). +MPI.@RegisterOp(+, Float128) + """ adaptive_timestep_update_t_params!(t_params, CFL_limits, error_norms, total_points, error_norm_method, success, @@ -1351,13 +1357,19 @@ function adaptive_timestep_update_t_params!(t_params, CFL_limits, error_norms, elseif error_norm_method == "L2" # Get overall maximum error on the shared-memory block error_norms_vec = [l for l ∈ values(error_norms)] - MPI.Reduce!(error_norms_vec, +, comm_block[]; root=0) + # In case we are using Float128 for error_norms, need to explicitly call + # MPI.Op() to use the method registered with MPI.@RegisterOp above. + MPI.Reduce!(error_norms_vec, MPI.Op(+, eltype(error_norms_vec)), comm_block[]; + root=0) error_norm = Ref{mk_float}(0.0) max_error_variable_index = -1 @serial_region begin # Get maximum error over all blocks - MPI.Allreduce!(error_norms_vec, +, comm_inter_block[]) + # In case we are using Float128 for error_norms, need to explicitly call + # MPI.Op() to use the method registered with MPI.@RegisterOp above. + MPI.Allreduce!(error_norms_vec, MPI.Op(+, eltype(error_norms_vec)), + comm_inter_block[]) # So far `error_norms_vec` is the sum of squares of the errors. Now that summation # is finished, need to divide by total number of points and take square-root. @@ -1420,13 +1432,19 @@ function adaptive_timestep_update_t_params!(t_params, CFL_limits, error_norms, elseif error_norm_method == "L2" # Get overall maximum error on the shared-memory block error_norms_vec = [l for l ∈ values(error_norms)] - MPI.Reduce!(error_norms_vec, +, comm_anyzv_subblock[]; root=0) + # In case we are using Float128 for error_norms, need to explicitly call + # MPI.Op() to use the method registered with MPI.@RegisterOp above. + MPI.Reduce!(error_norms_vec, MPI.Op(+, eltype(error_norms_vec)), + comm_anyzv_subblock[]; root=0) error_norm = Ref{mk_float}(0.0) max_error_variable_index = -1 @anyzv_serial_region begin # Get maximum error over all blocks - MPI.Allreduce!(error_norms_vec, +, z.comm) + # In case we are using Float128 for error_norms, need to explicitly call + # MPI.Op() to use the method registered with MPI.@RegisterOp above. + MPI.Allreduce!(error_norms_vec, MPI.Op(+, eltype(error_norms_vec)), + z.comm) # So far `error_norms_vec` is the sum of squares of the errors. Now that summation # is finished, need to divide by total number of points and take square-root. From c2cea2f2ea3379597dd3ee19b5e99b645f1e25c0 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Tue, 17 Feb 2026 13:07:36 +0000 Subject: [PATCH 2/5] Remove version pin on HDF5_jll in macOS parallel CI job No longer needed, and causes MPI.jl to be restricted to an old version, when we need a newer one for MPI.@RegisterOp. --- .github/workflows/parallel_test.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/parallel_test.yml b/.github/workflows/parallel_test.yml index 16f8b78c76..36d287125e 100644 --- a/.github/workflows/parallel_test.yml +++ b/.github/workflows/parallel_test.yml @@ -44,11 +44,7 @@ jobs: - uses: julia-actions/cache@v2 - run: | touch Project.toml - # Pin versions of HDF5_jll for now because HDF5_jll@1.14.6+0 causes - # linker errors, when also using MPI. Should remove this pin when the - # latest version is fixed, see - # https://github.com/JuliaIO/HDF5.jl/issues/1191. - julia --project -O3 -e 'import Pkg; Pkg.add(name="HDF5_jll", version="1.14.2"); Pkg.add(["MPI", "MPIPreferences"]); using MPIPreferences; MPIPreferences.use_jll_binary("OpenMPI_jll")' + julia --project -O3 -e 'import Pkg; Pkg.add(["MPI", "MPIPreferences"]); using MPIPreferences; MPIPreferences.use_jll_binary("OpenMPI_jll")' julia --project -O3 -e 'using MPI; MPI.install_mpiexecjl(; destdir=".")' julia --project -O3 -e 'import Pkg; Pkg.add(["NCDatasets", "Random", "SpecialFunctions", "StatsBase", "Test"]); Pkg.develop(path="moment_kinetics/")' julia --project -O3 -e 'import Pkg; Pkg.precompile()' From ac10e30a8413140581d8e658b65a5446310a9299 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 18 Feb 2026 10:02:26 +0000 Subject: [PATCH 3/5] Remove matplotlib version pin in CI as PyPlot.jl should now be fixed --- .github/workflows/documentation.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index eae62b2e71..db34fe7627 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -21,12 +21,6 @@ jobs: with: version: '1.11' - uses: julia-actions/cache@v2 - - name: Install dependencies - run: | - # Version 3.9.0 of matplotlib causes an error with PyPlot.jl, so pin - # matplotlib version to 3.8.3 until this is fixed. See - # https://github.com/JuliaPy/PyPlot.jl/issues/582). - pip3 install --user "matplotlib==3.8.3" - name: Build and deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Authenticate with GitHub Actions token @@ -39,11 +33,5 @@ jobs: - uses: julia-actions/setup-julia@latest with: version: '1.11' - - name: Install dependencies - run: | - # Version 3.9.0 of matplotlib causes an error with PyPlot.jl, so pin - # matplotlib version to 3.8.3 until this is fixed. See - # https://github.com/JuliaPy/PyPlot.jl/issues/582). - pip3 install --user "matplotlib==3.8.3" - name: Build and deploy run: julia --project=docs/ docs/make-pdf.jl From 214abbc584a38add9346f7c9a99d44f772030476 Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 18 Feb 2026 10:16:12 +0000 Subject: [PATCH 4/5] Update Julia to 1.12 in CI This is needed for the docs build, as Something is preventing MPI.jl from using the latest version in that workflow. Updating Julia apparently updates whatever was blocking MPI.jl from updating. --- .github/workflows/debug_checks.yml | 2 +- .github/workflows/documentation.yml | 4 ++-- .github/workflows/examples.yml | 2 +- .github/workflows/longtest.yml | 4 ++-- .github/workflows/parallel_test.yml | 4 ++-- .github/workflows/test.yml | 2 +- .github/workflows/test_scripts.yml | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/debug_checks.yml b/.github/workflows/debug_checks.yml index ec7608536d..00efcac51c 100644 --- a/.github/workflows/debug_checks.yml +++ b/.github/workflows/debug_checks.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.11' + version: '1.12' arch: x64 - uses: julia-actions/cache@v2 - name: Debug test diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index db34fe7627..9c84057ce2 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.11' + version: '1.12' - uses: julia-actions/cache@v2 - name: Build and deploy env: @@ -32,6 +32,6 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.11' + version: '1.12' - name: Build and deploy run: julia --project=docs/ docs/make-pdf.jl diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 214a52950a..29483b7ae3 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.11' + version: '1.12' - uses: julia-actions/cache@v2 - name: Test examples run: | diff --git a/.github/workflows/longtest.yml b/.github/workflows/longtest.yml index c199fdcf09..30ac9e0d8e 100644 --- a/.github/workflows/longtest.yml +++ b/.github/workflows/longtest.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.11' + version: '1.12' - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@v1 with: @@ -58,7 +58,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.11' + version: '1.12' arch: x64 - uses: julia-actions/cache@v2 - run: | diff --git a/.github/workflows/parallel_test.yml b/.github/workflows/parallel_test.yml index 36d287125e..1c2b8f96b4 100644 --- a/.github/workflows/parallel_test.yml +++ b/.github/workflows/parallel_test.yml @@ -16,7 +16,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.11' + version: '1.12' arch: x64 - uses: julia-actions/cache@v2 - run: | @@ -40,7 +40,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.11' + version: '1.12' - uses: julia-actions/cache@v2 - run: | touch Project.toml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cbc366554b..fae97d0fa8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.11' + version: '1.12' - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@v1 with: diff --git a/.github/workflows/test_scripts.yml b/.github/workflows/test_scripts.yml index 9b8f7ee012..980b79411c 100644 --- a/.github/workflows/test_scripts.yml +++ b/.github/workflows/test_scripts.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@latest with: - version: '1.11' + version: '1.12' - uses: julia-actions/cache@v2 - name: Test test_scripts run: | From c007eef341e7f5495978a800d3364e77d78ef2ae Mon Sep 17 00:00:00 2001 From: John Omotani Date: Wed, 18 Feb 2026 15:28:31 +0000 Subject: [PATCH 5/5] Relax fokker_planck_time_evolution_tests.jl tolerances slightly Update to Julia-1.12 has increased rounding errors slightly in CI. --- moment_kinetics/test/fokker_planck_time_evolution_tests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moment_kinetics/test/fokker_planck_time_evolution_tests.jl b/moment_kinetics/test/fokker_planck_time_evolution_tests.jl index 883be32b43..c8d4f68776 100644 --- a/moment_kinetics/test/fokker_planck_time_evolution_tests.jl +++ b/moment_kinetics/test/fokker_planck_time_evolution_tests.jl @@ -477,9 +477,9 @@ function runtests(; highres=false) println("Fokker Planck dFdt = C[F,F] relaxation test") test_cases = Any[(false, false, false, 2.0e-14, 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 1.0e-4 : 5.0e-12, highres ? 4.0e-3 : 1.0e-5)] - @long push!(test_cases, (true, false, false, highres ? 2.0e-8 : 2.0e-14, highres ? 2.0e-8 : 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 1.0e-4 : 5.0e-12, highres ? 4.0e-3 : 1.0e-5)) + @long push!(test_cases, (true, false, false, highres ? 2.0e-8 : 4.0e-14, highres ? 2.0e-8 : 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 1.0e-4 : 5.0e-12, highres ? 4.0e-3 : 1.0e-5)) @long push!(test_cases, (true, true, false, highres ? 2.0e-3 : 4.0e-14, highres ? 3.0e-4 : 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 3.0e-4 : 5.0e-12, highres ? 4.0e-3 : 1.0e-5)) - push!(test_cases, (true, true, true, highres ? 2.0e-3 : 2.0e-14, highres ? 2.0e-3 : 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 2.0e-3 : 5.0e-12, highres ? 4.0e-3 : 1.0e-3)) + push!(test_cases, (true, true, true, highres ? 2.0e-3 : 4.0e-14, highres ? 2.0e-3 : 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 2.0e-3 : 5.0e-12, highres ? 4.0e-3 : 1.0e-3)) @testset "evolve_density=$evolve_density, evolve_upar=$evolve_upar, evolve_p=$evolve_p" for (evolve_density, evolve_upar, evolve_p, tol1, tol2, tol3, tol4, tol5) ∈ test_cases println(" evolve_density=$evolve_density, evolve_upar=$evolve_upar, evolve_p=$evolve_p:")