Fix MPI.Reduce! operations in some test cases when running on ARM chips

.github/workflows/debug_checks.yml

@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.11'
+          version: '1.12'
           arch: x64
       - uses: julia-actions/cache@v2
       - name: Debug test

.github/workflows/documentation.yml

@@ -19,14 +19,8 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.11'
+          version: '1.12'
       - uses: julia-actions/cache@v2
-      - name: Install dependencies
-        run: |
-          # Version 3.9.0 of matplotlib causes an error with PyPlot.jl, so pin
-          # matplotlib version to 3.8.3 until this is fixed. See
-          # https://github.com/JuliaPy/PyPlot.jl/issues/582).
-          pip3 install --user "matplotlib==3.8.3"
       - name: Build and deploy
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Authenticate with GitHub Actions token
@@ -38,12 +32,6 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.11'
-      - name: Install dependencies
-        run: |
-          # Version 3.9.0 of matplotlib causes an error with PyPlot.jl, so pin
-          # matplotlib version to 3.8.3 until this is fixed. See
-          # https://github.com/JuliaPy/PyPlot.jl/issues/582).
-          pip3 install --user "matplotlib==3.8.3"
+          version: '1.12'
       - name: Build and deploy
         run: julia --project=docs/ docs/make-pdf.jl

.github/workflows/examples.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.11'
+          version: '1.12'
       - uses: julia-actions/cache@v2
       - name: Test examples
         run: |

.github/workflows/longtest.yml

@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.11'
+          version: '1.12'
       - uses: julia-actions/cache@v2
       - uses: julia-actions/julia-buildpkg@v1
         with:
@@ -58,7 +58,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.11'
+          version: '1.12'
           arch: x64
       - uses: julia-actions/cache@v2
       - run: |

.github/workflows/parallel_test.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.11'
+          version: '1.12'
           arch: x64
       - uses: julia-actions/cache@v2
       - run: |
@@ -40,15 +40,11 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.11'
+          version: '1.12'
       - uses: julia-actions/cache@v2
       - run: |
           touch Project.toml
-          # Pin versions of HDF5_jll for now because HDF5_jll@1.14.6+0 causes
-          # linker errors, when also using MPI. Should remove this pin when the
-          # latest version is fixed, see
-          # https://github.com/JuliaIO/HDF5.jl/issues/1191.
-          julia --project -O3 -e 'import Pkg; Pkg.add(name="HDF5_jll", version="1.14.2"); Pkg.add(["MPI", "MPIPreferences"]); using MPIPreferences; MPIPreferences.use_jll_binary("OpenMPI_jll")'
+          julia --project -O3 -e 'import Pkg; Pkg.add(["MPI", "MPIPreferences"]); using MPIPreferences; MPIPreferences.use_jll_binary("OpenMPI_jll")'
           julia --project -O3 -e 'using MPI; MPI.install_mpiexecjl(; destdir=".")'
           julia --project -O3 -e 'import Pkg; Pkg.add(["NCDatasets", "Random", "SpecialFunctions", "StatsBase", "Test"]); Pkg.develop(path="moment_kinetics/")'
           julia --project -O3 -e 'import Pkg; Pkg.precompile()'

.github/workflows/test.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.11'
+          version: '1.12'
       - uses: julia-actions/cache@v2
       - uses: julia-actions/julia-buildpkg@v1
         with:

.github/workflows/test_scripts.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.11'
+          version: '1.12'
       - uses: julia-actions/cache@v2
       - name: Test test_scripts
         run: |

moment_kinetics/src/runge_kutta.jl

@@ -15,6 +15,7 @@ using ..looping
 using ..type_definitions: mk_float, MPISharedArray
 
 using MPI
+using Quadmath
 using StatsBase: mean
 
 """
@@ -1295,6 +1296,11 @@ function local_error_norm(f_loworder::MPISharedArray{mk_float,6},
     end
 end
 
+# Need to 'register' this operation explicitly to use it in MPI reduction operations on
+# non-x86 processor architectures
+# (https://juliaparallel.org/MPI.jl/stable/knownissues/#Custom-reduction-operators).
+MPI.@RegisterOp(+, Float128)
+
 """
     adaptive_timestep_update_t_params!(t_params, CFL_limits, error_norms,
                                        total_points, error_norm_method, success,
@@ -1351,13 +1357,19 @@ function adaptive_timestep_update_t_params!(t_params, CFL_limits, error_norms,
         elseif error_norm_method == "L2"
             # Get overall maximum error on the shared-memory block
             error_norms_vec = [l for l ∈ values(error_norms)]
-            MPI.Reduce!(error_norms_vec, +, comm_block[]; root=0)
+            # In case we are using Float128 for error_norms, need to explicitly call
+            # MPI.Op() to use the method registered with MPI.@RegisterOp above.
+            MPI.Reduce!(error_norms_vec, MPI.Op(+, eltype(error_norms_vec)), comm_block[];
+                        root=0)
 
             error_norm = Ref{mk_float}(0.0)
             max_error_variable_index = -1
             @serial_region begin
                 # Get maximum error over all blocks
-                MPI.Allreduce!(error_norms_vec, +, comm_inter_block[])
+                # In case we are using Float128 for error_norms, need to explicitly call
+                # MPI.Op() to use the method registered with MPI.@RegisterOp above.
+                MPI.Allreduce!(error_norms_vec, MPI.Op(+, eltype(error_norms_vec)),
+                               comm_inter_block[])
 
                 # So far `error_norms_vec` is the sum of squares of the errors. Now that summation
                 # is finished, need to divide by total number of points and take square-root.
@@ -1420,13 +1432,19 @@ function adaptive_timestep_update_t_params!(t_params, CFL_limits, error_norms,
         elseif error_norm_method == "L2"
             # Get overall maximum error on the shared-memory block
             error_norms_vec = [l for l ∈ values(error_norms)]
-            MPI.Reduce!(error_norms_vec, +, comm_anyzv_subblock[]; root=0)
+            # In case we are using Float128 for error_norms, need to explicitly call
+            # MPI.Op() to use the method registered with MPI.@RegisterOp above.
+            MPI.Reduce!(error_norms_vec, MPI.Op(+, eltype(error_norms_vec)),
+                        comm_anyzv_subblock[]; root=0)
 
             error_norm = Ref{mk_float}(0.0)
             max_error_variable_index = -1
             @anyzv_serial_region begin
                 # Get maximum error over all blocks
-                MPI.Allreduce!(error_norms_vec, +, z.comm)
+                # In case we are using Float128 for error_norms, need to explicitly call
+                # MPI.Op() to use the method registered with MPI.@RegisterOp above.
+                MPI.Allreduce!(error_norms_vec, MPI.Op(+, eltype(error_norms_vec)),
+                               z.comm)
 
                 # So far `error_norms_vec` is the sum of squares of the errors. Now that summation
                 # is finished, need to divide by total number of points and take square-root.

moment_kinetics/test/fokker_planck_time_evolution_tests.jl

@@ -477,9 +477,9 @@ function runtests(; highres=false)
         println("Fokker Planck dFdt = C[F,F] relaxation test")
 
         test_cases = Any[(false, false, false, 2.0e-14, 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 1.0e-4 : 5.0e-12, highres ? 4.0e-3 : 1.0e-5)]
-        @long push!(test_cases, (true, false, false, highres ? 2.0e-8 : 2.0e-14, highres ? 2.0e-8 : 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 1.0e-4 : 5.0e-12, highres ? 4.0e-3 : 1.0e-5))
+        @long push!(test_cases, (true, false, false, highres ? 2.0e-8 : 4.0e-14, highres ? 2.0e-8 : 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 1.0e-4 : 5.0e-12, highres ? 4.0e-3 : 1.0e-5))
         @long push!(test_cases, (true, true, false, highres ? 2.0e-3 : 4.0e-14, highres ? 3.0e-4 : 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 3.0e-4 : 5.0e-12, highres ? 4.0e-3 : 1.0e-5))
-        push!(test_cases, (true, true, true, highres ? 2.0e-3 : 2.0e-14, highres ? 2.0e-3 : 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 2.0e-3 : 5.0e-12, highres ? 4.0e-3 : 1.0e-3))
+        push!(test_cases, (true, true, true, highres ? 2.0e-3 : 4.0e-14, highres ? 2.0e-3 : 2.0e-14, highres ? 4.0e-3 : 1.0e-5, highres ? 2.0e-3 : 5.0e-12, highres ? 4.0e-3 : 1.0e-3))
         @testset "evolve_density=$evolve_density, evolve_upar=$evolve_upar, evolve_p=$evolve_p" for
                 (evolve_density, evolve_upar, evolve_p, tol1, tol2, tol3, tol4, tol5) ∈ test_cases
             println("  evolve_density=$evolve_density, evolve_upar=$evolve_upar, evolve_p=$evolve_p:")