From 7da1e3223e2d76f634cdc0240c6cf4e5c5ddd9d0 Mon Sep 17 00:00:00 2001 From: Charles Kawczynski Date: Wed, 13 Nov 2024 22:28:37 -0500 Subject: [PATCH 1/7] Implement and test vertical mass borrowing limiters Update src/Limiters/vertical_mass_borrowing_limiter.jl Co-authored-by: Tapio Schneider Update src/Limiters/vertical_mass_borrowing_limiter.jl Co-authored-by: Tapio Schneider Update src/Limiters/vertical_mass_borrowing_limiter.jl Co-authored-by: Tapio Schneider Use density-dz for pressure thickness --- docs/refs.bib | 11 ++ src/Limiters/Limiters.jl | 1 + .../vertical_mass_borrowing_limiter.jl | 133 +++++++++++++++ .../vertical_mass_borrowing_limiter.jl | 151 ++++++++++++++++++ ...rtical_mass_borrowing_limiter_advection.jl | 145 +++++++++++++++++ 5 files changed, 441 insertions(+) create mode 100644 src/Limiters/vertical_mass_borrowing_limiter.jl create mode 100644 test/Limiters/vertical_mass_borrowing_limiter.jl create mode 100644 test/Limiters/vertical_mass_borrowing_limiter_advection.jl diff --git a/docs/refs.bib b/docs/refs.bib index fb5bab7d2c..3700198cd9 100644 --- a/docs/refs.bib +++ b/docs/refs.bib @@ -271,3 +271,14 @@ @article{Wiin1967 year = {1967}, publisher = {Wiley Online Library} } + +@article{zhang2018impact, + title={Impact of numerical choices on water conservation in the E3SM Atmosphere Model version 1 (EAMv1)}, + author={Zhang, Kai and Rasch, Philip J and Taylor, Mark A and Wan, Hui and Leung, Ruby and Ma, Po-Lun and Golaz, Jean-Christophe and Wolfe, Jon and Lin, Wuyin and Singh, Balwinder and others}, + journal={Geoscientific Model Development}, + volume={11}, + number={5}, + pages={1971--1988}, + year={2018}, + publisher={Copernicus Publications G{\"o}ttingen, Germany} +} diff --git a/src/Limiters/Limiters.jl b/src/Limiters/Limiters.jl index bd9116cf0d..a731f38fcc 100644 --- a/src/Limiters/Limiters.jl +++ b/src/Limiters/Limiters.jl @@ -20,5 +20,6 @@ abstract type AbstractLimiter end # implementations include("quasimonotone.jl") +include("vertical_mass_borrowing_limiter.jl") end # end module diff --git a/src/Limiters/vertical_mass_borrowing_limiter.jl b/src/Limiters/vertical_mass_borrowing_limiter.jl new file mode 100644 index 0000000000..07054c7e72 --- /dev/null +++ b/src/Limiters/vertical_mass_borrowing_limiter.jl @@ -0,0 +1,133 @@ +import .DataLayouts as DL + +""" + VerticalMassBorrowingLimiter(f::Fields.Field, q_min) + +A vertical-only mass borrowing limiter. + +The mass borrower borrows tracer mass from an adjacent, lower layer. +It conserves the total tracer mass and can avoid negative tracers. + +At level k, it will first borrow the mass from the layer k+1 (lower level). +If the mass is not sufficient in layer k+1, it will borrow mass from +layer k+2. The borrower will proceed this process until the bottom layer. +If the tracer mass in the bottom layer goes negative, it will repeat the +process from the bottom to the top. In this way, the borrower works for +any shape of mass profiles. + +This code was adapted from [E3SM](https://github.com/E3SM-Project/E3SM/blob/2c377c5ec9a5585170524b366ad85074ab1b1a5c/components/eam/src/physics/cam/massborrow.F90) + +References: + - [zhang2018impact](@cite) +""" +struct VerticalMassBorrowingLimiter{F, T} + bmass::F + ic::F + q_min::T +end +function VerticalMassBorrowingLimiter(f::Fields.Field, q_min) + bmass = similar(Spaces.level(f, 1)) + ic = similar(Spaces.level(f, 1)) + return VerticalMassBorrowingLimiter(bmass, ic, q_min) +end + + +""" + apply_limiter!(q::Fields.Field, ρ::Fields.Field, lim::VerticalMassBorrowingLimiter) + +Apply the vertical mass borrowing +limiter `lim` to field `q`, given +density `ρ`. +""" +apply_limiter!( + q::Fields.Field, + ρ::Fields.Field, + lim::VerticalMassBorrowingLimiter, +) = apply_limiter!(q, ρ, axes(q), lim, ClimaComms.device(axes(q))) + +function apply_limiter!( + q::Fields.Field, + ρ::Fields.Field, + space::Spaces.FiniteDifferenceSpace, + lim::VerticalMassBorrowingLimiter, + device::ClimaComms.AbstractCPUDevice, +) + cache = (; bmass = lim.bmass, ic = lim.ic, q_min = lim.q_min) + columnwise_massborrow_cpu(q, ρ, cache) +end + +function apply_limiter!( + q::Fields.Field, + ρ::Fields.Field, + space::Spaces.ExtrudedFiniteDifferenceSpace, + lim::VerticalMassBorrowingLimiter, + device::ClimaComms.AbstractCPUDevice, +) + Fields.bycolumn(axes(q)) do colidx + cache = (; + bmass = lim.bmass[colidx], + ic = lim.ic[colidx], + q_min = lim.q_min, + ) + columnwise_massborrow_cpu(q[colidx], ρ[colidx], cache) + end +end + +# TODO: can we improve the performance? +# `bycolumn` on the CPU may be better here since we could multithread it. +function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # column fields + (; bmass, ic, q_min) = cache + + Δz = Fields.Δz_field(q) + Δz_vals = Fields.field_values(Δz) + ρ_vals = Fields.field_values(ρ) + # loop over tracers + nlevels = Spaces.nlevels(axes(q)) + @. ic = 0 + @. bmass = 0 + q_vals = Fields.field_values(q) + # top to bottom + for f in 1:DataLayouts.ncomponents(q_vals) + for v in 1:nlevels + CI = CartesianIndex(1, 1, f, v, 1) + # new mass in the current layer + ρΔz_v = + DL.getindex_field(Δz_vals, CI) * DL.getindex_field(ρ_vals, CI) + nmass = DL.getindex_field(q_vals, CI) + bmass[] / ρΔz_v + if nmass > q_min[f] + # if new mass in the current layer is positive, don't borrow mass any more + DL.setindex_field!(q_vals, nmass, CI) + bmass[] = 0 + else + # set mass to q_min in the current layer, and save bmass + bmass[] = (nmass - q_min[f]) * ρΔz_v + DL.setindex_field!(q_vals, q_min[f], CI) + ic[] = ic[] + 1 + end + end + + # bottom to top + for v in nlevels:-1:1 + CI = CartesianIndex(1, 1, f, v, 1) + # if the surface layer still needs to borrow mass + if bmass[] < 0 + ρΔz_v = + DL.getindex_field(Δz_vals, CI) * + DL.getindex_field(ρ_vals, CI) + # new mass in the current layer + nmass = DL.getindex_field(q_vals, CI) + bmass[] / ρΔz_v + if nmass > q_min[f] + # if new mass in the current layer is positive, don't borrow mass any more + DL.setindex_field!(q_vals, nmass, CI) + bmass[] = 0 + else + # if new mass in the current layer is negative, continue to borrow mass + bmass[] = (nmass - q_min[f]) * ρΔz_v + DL.setindex_field!(q_vals, q_min[f], CI) + end + end + end + end + + return nothing +end diff --git a/test/Limiters/vertical_mass_borrowing_limiter.jl b/test/Limiters/vertical_mass_borrowing_limiter.jl new file mode 100644 index 0000000000..6087f2327c --- /dev/null +++ b/test/Limiters/vertical_mass_borrowing_limiter.jl @@ -0,0 +1,151 @@ +#= +julia --project +using Revise; include(joinpath("test", "Limiters", "vertical_mass_borrowing_limiter.jl")) +=# +using ClimaComms +ClimaComms.@import_required_backends +using ClimaCore: Fields, Spaces, Limiters +using ClimaCore.RecursiveApply +using ClimaCore.Geometry +using ClimaCore.Grids +using ClimaCore.CommonGrids +using Test +using Random + +function perturb_field!(f::Fields.Field; perturb_radius) + device = ClimaComms.device(f) + ArrayType = ClimaComms.array_type(device) + rand_data = ArrayType(rand(size(parent(f))...)) # [0-1] + rand_data = rand_data .- sum(rand_data) / length(rand_data) # make centered about 0 ([-0.5:0.5]) + rand_data = (rand_data ./ maximum(rand_data)) .* perturb_radius # rand_data now in [-perturb_radius:perturb_radius] + parent(f) .= parent(f) .+ rand_data # f in [f ± perturb_radius] + return nothing +end + +@testset "Vertical mass borrowing limiter - column" begin + Random.seed!(1234) + z_elem = 10 + z_min = 0 + z_max = 1 + device = ClimaComms.device() + grid = ColumnGrid(; z_elem, z_min, z_max, device) + cspace = Spaces.FiniteDifferenceSpace(grid, Grids.CellCenter()) + tol = 0.01 + perturb_q = 0.3 + perturb_ρ = 0.2 + + # Initialize fields + coords = Fields.coordinate_field(cspace) + ρ = map(coord -> 1.0, coords) + q = map(coord -> 0.1, coords) + (; z) = coords + perturb_field!(q; perturb_radius = perturb_q) + perturb_field!(ρ; perturb_radius = perturb_ρ) + ρq_init = ρ .⊠ q + sum_ρq_init = sum(ρq_init) + + # Test that the minimum is below 0 + @test length(parent(q)) == z_elem == 10 + @test 0.3 ≤ count(x -> x < 0, parent(q)) / 10 ≤ 0.5 # ensure reasonable percentage of points are negative + + @test -2 * perturb_q ≤ minimum(q) ≤ -tol + limiter = Limiters.VerticalMassBorrowingLimiter(q, (0.0,)) + Limiters.apply_limiter!(q, ρ, limiter) + @test 0 ≤ minimum(q) + ρq = ρ .⊠ q + @test isapprox(sum(ρq), sum_ρq_init; atol = 1e-15) + @test isapprox(sum(ρq), sum_ρq_init; rtol = 1e-10) + # @show sum(ρq) # 0.07388931313511024 + # @show sum_ρq_init # 0.07388931313511025 +end + +@testset "Vertical mass borrowing limiter - sphere" begin + z_elem = 10 + z_min = 0 + z_max = 1 + radius = 10 + h_elem = 10 + n_quad_points = 4 + grid = ExtrudedCubedSphereGrid(; + z_elem, + z_min, + z_max, + radius, + h_elem, + n_quad_points, + ) + cspace = Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) + tol = 0.01 + perturb_q = 0.2 + perturb_ρ = 0.1 + + # Initialize fields + coords = Fields.coordinate_field(cspace) + ρ = map(coord -> 1.0, coords) + q = map(coord -> 0.1, coords) + + perturb_field!(q; perturb_radius = perturb_q) + perturb_field!(ρ; perturb_radius = perturb_ρ) + ρq_init = ρ .⊠ q + sum_ρq_init = sum(ρq_init) + + # Test that the minimum is below 0 + @test length(parent(q)) == z_elem * h_elem^2 * 6 * n_quad_points^2 == 96000 + @test 0.10 ≤ count(x -> x < 0.0001, parent(q)) / 96000 ≤ 1 # ensure 10% of points are negative + + @test -2 * perturb_q ≤ minimum(q) ≤ -tol + limiter = Limiters.VerticalMassBorrowingLimiter(q, (0.0,)) + Limiters.apply_limiter!(q, ρ, limiter) + @test 0 ≤ minimum(q) + ρq = ρ .⊠ q + @test isapprox(sum(ρq), sum_ρq_init; atol = 0.029) + @test isapprox(sum(ρq), sum_ρq_init; rtol = 0.00023) + # @show sum(ρq) # 125.5483524237572 + # @show sum_ρq_init # 125.52052986152977 +end + +@testset "Vertical mass borrowing limiter - deep atmosphere" begin + z_elem = 10 + z_min = 0 + z_max = 1 + radius = 10 + h_elem = 10 + n_quad_points = 4 + grid = ExtrudedCubedSphereGrid(; + z_elem, + z_min, + z_max, + radius, + global_geometry = Geometry.DeepSphericalGlobalGeometry(radius), + h_elem, + n_quad_points, + ) + cspace = Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) + tol = 0.01 + perturb_q = 0.2 + perturb_ρ = 0.1 + + # Initialize fields + coords = Fields.coordinate_field(cspace) + ρ = map(coord -> 1.0, coords) + q = map(coord -> 0.1, coords) + + perturb_field!(q; perturb_radius = perturb_q) + perturb_field!(ρ; perturb_radius = perturb_ρ) + ρq_init = ρ .⊠ q + sum_ρq_init = sum(ρq_init) + + # Test that the minimum is below 0 + @test length(parent(q)) == z_elem * h_elem^2 * 6 * n_quad_points^2 == 96000 + @test 0.10 ≤ count(x -> x < 0.0001, parent(q)) / 96000 ≤ 1 # ensure 10% of points are negative + + @test -2 * perturb_q ≤ minimum(q) ≤ -tol + limiter = Limiters.VerticalMassBorrowingLimiter(q, (0.0,)) + Limiters.apply_limiter!(q, ρ, limiter) + @test 0 ≤ minimum(q) + ρq = ρ .⊠ q + @test isapprox(sum(ρq), sum_ρq_init; atol = 0.269) + @test isapprox(sum(ρq), sum_ρq_init; rtol = 0.00199) + # @show sum(ρq) # 138.90494931641584 + # @show sum_ρq_init # 139.1735731377394 +end diff --git a/test/Limiters/vertical_mass_borrowing_limiter_advection.jl b/test/Limiters/vertical_mass_borrowing_limiter_advection.jl new file mode 100644 index 0000000000..5068612d07 --- /dev/null +++ b/test/Limiters/vertical_mass_borrowing_limiter_advection.jl @@ -0,0 +1,145 @@ +#= +julia --project=.buildkite +using Revise; include(joinpath("test", "Limiters", "vertical_mass_borrowing_limiter_advection.jl")) +=# +using Test +using LinearAlgebra +import ClimaComms +ClimaComms.@import_required_backends +using OrdinaryDiffEqSSPRK: ODEProblem, solve, SSPRK33 +using ClimaTimeSteppers + +import ClimaCore: + Fields, + Domains, + Limiters, + Topologies, + Meshes, + DataLayouts, + Operators, + Geometry, + Spaces + + +# Advection Equation, with constant advective velocity (so advection form = flux form) +# ∂_t y + w ∂_z y = 0 +# the solution translates to the right at speed w, +# so at time t, the solution is y(z - w * t) + +# visualization artifacts +ENV["GKSwstype"] = "nul" +using ClimaCorePlots, Plots +Plots.GRBackend() +dir = "vert_mass_borrow_advection" +path = joinpath(@__DIR__, "output", dir) +mkpath(path) + +function lim!(y, parameters, t, y_ref) + (; w, Δt, limiter) = parameters + Limiters.apply_limiter!(y.q, limiter) + return nothing +end + +function tendency!(yₜ, y, parameters, t) + (; w, Δt, limiter) = parameters + FT = Spaces.undertype(axes(y.q)) + bcvel = pulse(-π, t, z₀, zₕ, z₁) + divf2c = Operators.DivergenceF2C( + bottom = Operators.SetValue(Geometry.WVector(FT(bcvel))), + top = Operators.SetValue(Geometry.WVector(FT(0))), + ) + upwind1 = Operators.UpwindBiasedProductC2F( + bottom = Operators.Extrapolate(), + top = Operators.Extrapolate(), + ) + upwind3 = Operators.Upwind3rdOrderBiasedProductC2F( + bottom = Operators.ThirdOrderOneSided(), + top = Operators.ThirdOrderOneSided(), + ) + If = Operators.InterpolateC2F() + @. yₜ.q = + # -divf2c(w * If(y.q)) + -divf2c(upwind1(w, y.q) * If(y.q)) + # -divf2c(upwind3(w, y.q) * If(y.q)) + return nothing +end + +# Define a pulse wave or square wave + +FT = Float64 +t₀ = FT(0.0) +Δt = 0.0001 * 25 +t₁ = 200Δt +z₀ = FT(0.0) +zₕ = FT(1.0) +z₁ = FT(1.0) +speed = FT(-1.0) +pulse(z, t, z₀, zₕ, z₁) = abs(z - speed * t) ≤ zₕ ? z₁ : z₀ + +n = 2 .^ 6 + +domain = Domains.IntervalDomain( + Geometry.ZPoint{FT}(-π), + Geometry.ZPoint{FT}(π); + boundary_names = (:bottom, :top), +) + +# stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(FT(7.0))) +stretch_fns = (Meshes.Uniform(),) +plot_string = ["uniform", "stretched"] + +@testset "VerticalMassBorrowingLimiter on advection" begin + for (i, stretch_fn) in enumerate(stretch_fns) + mesh = Meshes.IntervalMesh(domain, stretch_fn; nelems = n) + cent_space = Spaces.CenterFiniteDifferenceSpace(mesh) + face_space = Spaces.FaceFiniteDifferenceSpace(cent_space) + z = Fields.coordinate_field(cent_space).z + O = ones(FT, face_space) + + # Initial condition + q_init = pulse.(z, 0.0, z₀, zₕ, z₁) + q = q_init + y = Fields.FieldVector(; q) + limiter = Limiters.VerticalMassBorrowingLimiter(q, (0.0,)) + + # Unitary, constant advective velocity + w = Geometry.WVector.(speed .* O) + + # Solve the ODE + parameters = (; w, Δt, limiter) + prob = ODEProblem( + ClimaODEFunction(; lim!, T_lim! = tendency!), + y, + (t₀, t₁), + parameters, + ) + sol = solve( + prob, + ExplicitAlgorithm(SSP33ShuOsher()), + dt = Δt, + saveat = Δt, + ) + + q_init = sol.u[1].q + q_final = sol.u[end].q + q_analytic = pulse.(z, t₁, z₀, zₕ, z₁) + err = norm(q_final .- q_analytic) + rel_mass_err = norm((sum(q_final) - sum(q_init)) / sum(q_init)) + + + p = plot() + Plots.plot!(q_init, label = "init") + Plots.plot!(q_final, label = "computed") + Plots.plot!(q_analytic, label = "analytic") + Plots.plot!(; legend = :topright) + Plots.plot!(; xlabel = "q", title = "VerticalMassBorrowingLimiter") + f = joinpath( + path, + "VerticalMassBorrowingLimiter_comparison_$(plot_string[i]).png", + ) + Plots.png(p, f) + @test err ≤ 0.25 + @test rel_mass_err ≤ 10eps() + @test minimum(q_final) ≥ 0 + end +end From 9e883ac5301d3c53aee62d485d367ba1ec7a8d15 Mon Sep 17 00:00:00 2001 From: Charles Kawczynski Date: Thu, 21 Nov 2024 12:16:28 -0500 Subject: [PATCH 2/7] Fixes --- .../vertical_mass_borrowing_limiter.jl | 19 +-- .../vertical_mass_borrowing_limiter.jl | 31 ++++- ...rtical_mass_borrowing_limiter_advection.jl | 130 +++++++++++++----- 3 files changed, 131 insertions(+), 49 deletions(-) diff --git a/src/Limiters/vertical_mass_borrowing_limiter.jl b/src/Limiters/vertical_mass_borrowing_limiter.jl index 07054c7e72..c33d79d148 100644 --- a/src/Limiters/vertical_mass_borrowing_limiter.jl +++ b/src/Limiters/vertical_mass_borrowing_limiter.jl @@ -80,6 +80,9 @@ function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # c Δz = Fields.Δz_field(q) Δz_vals = Fields.field_values(Δz) + (; J) = Fields.local_geometry_field(ρ) + # ΔV_vals = Fields.field_values(J) + ΔV_vals = Δz_vals ρ_vals = Fields.field_values(ρ) # loop over tracers nlevels = Spaces.nlevels(axes(q)) @@ -91,16 +94,16 @@ function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # c for v in 1:nlevels CI = CartesianIndex(1, 1, f, v, 1) # new mass in the current layer - ρΔz_v = - DL.getindex_field(Δz_vals, CI) * DL.getindex_field(ρ_vals, CI) - nmass = DL.getindex_field(q_vals, CI) + bmass[] / ρΔz_v + ρΔV_lev = + DL.getindex_field(ΔV_vals, CI) * DL.getindex_field(ρ_vals, CI) + nmass = DL.getindex_field(q_vals, CI) + bmass[] / ρΔV_lev if nmass > q_min[f] # if new mass in the current layer is positive, don't borrow mass any more DL.setindex_field!(q_vals, nmass, CI) bmass[] = 0 else # set mass to q_min in the current layer, and save bmass - bmass[] = (nmass - q_min[f]) * ρΔz_v + bmass[] = (nmass - q_min[f]) * ρΔV_lev DL.setindex_field!(q_vals, q_min[f], CI) ic[] = ic[] + 1 end @@ -111,18 +114,18 @@ function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # c CI = CartesianIndex(1, 1, f, v, 1) # if the surface layer still needs to borrow mass if bmass[] < 0 - ρΔz_v = - DL.getindex_field(Δz_vals, CI) * + ρΔV_lev = + DL.getindex_field(ΔV_vals, CI) * DL.getindex_field(ρ_vals, CI) # new mass in the current layer - nmass = DL.getindex_field(q_vals, CI) + bmass[] / ρΔz_v + nmass = DL.getindex_field(q_vals, CI) + bmass[] / ρΔV_lev if nmass > q_min[f] # if new mass in the current layer is positive, don't borrow mass any more DL.setindex_field!(q_vals, nmass, CI) bmass[] = 0 else # if new mass in the current layer is negative, continue to borrow mass - bmass[] = (nmass - q_min[f]) * ρΔz_v + bmass[] = (nmass - q_min[f]) * ρΔV_lev DL.setindex_field!(q_vals, q_min[f], CI) end end diff --git a/test/Limiters/vertical_mass_borrowing_limiter.jl b/test/Limiters/vertical_mass_borrowing_limiter.jl index 6087f2327c..e8c9702ce4 100644 --- a/test/Limiters/vertical_mass_borrowing_limiter.jl +++ b/test/Limiters/vertical_mass_borrowing_limiter.jl @@ -1,5 +1,5 @@ #= -julia --project +julia --project=.buildkite using Revise; include(joinpath("test", "Limiters", "vertical_mass_borrowing_limiter.jl")) =# using ClimaComms @@ -12,6 +12,20 @@ using ClimaCore.CommonGrids using Test using Random +#= +import Plots +import ClimaCorePlots +function plot_results(f, f₀) + col = Fields.ColumnIndex((1, 1), 1) + fcol = f[col] + f₀col = f₀[col] + p = Plots.plot() + Plots.plot(fcol; label = "field") + Plots.plot!(f₀col; label = "initial") +end +plot_results(ρq, ρq_init) +=# + function perturb_field!(f::Fields.Field; perturb_radius) device = ClimaComms.device(f) ArrayType = ClimaComms.array_type(device) @@ -23,12 +37,13 @@ function perturb_field!(f::Fields.Field; perturb_radius) end @testset "Vertical mass borrowing limiter - column" begin + FT = Float64 Random.seed!(1234) z_elem = 10 z_min = 0 z_max = 1 device = ClimaComms.device() - grid = ColumnGrid(; z_elem, z_min, z_max, device) + grid = ColumnGrid(FT; z_elem, z_min, z_max, device) cspace = Spaces.FiniteDifferenceSpace(grid, Grids.CellCenter()) tol = 0.01 perturb_q = 0.3 @@ -60,13 +75,14 @@ end end @testset "Vertical mass borrowing limiter - sphere" begin + FT = Float64 z_elem = 10 z_min = 0 z_max = 1 radius = 10 h_elem = 10 n_quad_points = 4 - grid = ExtrudedCubedSphereGrid(; + grid = ExtrudedCubedSphereGrid(FT; z_elem, z_min, z_max, @@ -98,25 +114,26 @@ end Limiters.apply_limiter!(q, ρ, limiter) @test 0 ≤ minimum(q) ρq = ρ .⊠ q - @test isapprox(sum(ρq), sum_ρq_init; atol = 0.029) - @test isapprox(sum(ρq), sum_ρq_init; rtol = 0.00023) + @test isapprox(sum(ρq), sum_ρq_init; atol = 0.07) + @test isapprox(sum(ρq), sum_ρq_init; rtol = 0.07) # @show sum(ρq) # 125.5483524237572 # @show sum_ρq_init # 125.52052986152977 end @testset "Vertical mass borrowing limiter - deep atmosphere" begin + FT = Float64 z_elem = 10 z_min = 0 z_max = 1 radius = 10 h_elem = 10 n_quad_points = 4 - grid = ExtrudedCubedSphereGrid(; + grid = ExtrudedCubedSphereGrid(FT; z_elem, z_min, z_max, radius, - global_geometry = Geometry.DeepSphericalGlobalGeometry(radius), + global_geometry = Geometry.DeepSphericalGlobalGeometry{FT}(radius), h_elem, n_quad_points, ) diff --git a/test/Limiters/vertical_mass_borrowing_limiter_advection.jl b/test/Limiters/vertical_mass_borrowing_limiter_advection.jl index 5068612d07..884d1f2076 100644 --- a/test/Limiters/vertical_mass_borrowing_limiter_advection.jl +++ b/test/Limiters/vertical_mass_borrowing_limiter_advection.jl @@ -7,6 +7,8 @@ using LinearAlgebra import ClimaComms ClimaComms.@import_required_backends using OrdinaryDiffEqSSPRK: ODEProblem, solve, SSPRK33 +using ClimaCore.CommonGrids +using ClimaCore.Grids using ClimaTimeSteppers import ClimaCore: @@ -28,7 +30,8 @@ import ClimaCore: # visualization artifacts ENV["GKSwstype"] = "nul" -using ClimaCorePlots, Plots +import ClimaCorePlots +import Plots Plots.GRBackend() dir = "vert_mass_borrow_advection" path = joinpath(@__DIR__, "output", dir) @@ -36,7 +39,17 @@ mkpath(path) function lim!(y, parameters, t, y_ref) (; w, Δt, limiter) = parameters - Limiters.apply_limiter!(y.q, limiter) + Limiters.apply_limiter!(y.q, y.ρ, limiter) + return nothing +end + +function perturb_field!(f::Fields.Field; perturb_radius) + device = ClimaComms.device(f) + ArrayType = ClimaComms.array_type(device) + rand_data = ArrayType(rand(size(parent(f))...)) # [0-1] + rand_data = rand_data .- sum(rand_data) / length(rand_data) # make centered about 0 ([-0.5:0.5]) + rand_data = (rand_data ./ maximum(rand_data)) .* perturb_radius # rand_data now in [-perturb_radius:perturb_radius] + parent(f) .= parent(f) .+ rand_data # f in [f ± perturb_radius] return nothing end @@ -57,10 +70,7 @@ function tendency!(yₜ, y, parameters, t) top = Operators.ThirdOrderOneSided(), ) If = Operators.InterpolateC2F() - @. yₜ.q = - # -divf2c(w * If(y.q)) - -divf2c(upwind1(w, y.q) * If(y.q)) - # -divf2c(upwind3(w, y.q) * If(y.q)) + @. yₜ.q = -divf2c(upwind1(w, y.q) * If(y.q)) return nothing end @@ -76,30 +86,51 @@ z₁ = FT(1.0) speed = FT(-1.0) pulse(z, t, z₀, zₕ, z₁) = abs(z - speed * t) ≤ zₕ ? z₁ : z₀ -n = 2 .^ 6 - -domain = Domains.IntervalDomain( - Geometry.ZPoint{FT}(-π), - Geometry.ZPoint{FT}(π); - boundary_names = (:bottom, :top), -) - -# stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(FT(7.0))) -stretch_fns = (Meshes.Uniform(),) +stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(FT(7.0))) plot_string = ["uniform", "stretched"] @testset "VerticalMassBorrowingLimiter on advection" begin - for (i, stretch_fn) in enumerate(stretch_fns) - mesh = Meshes.IntervalMesh(domain, stretch_fn; nelems = n) - cent_space = Spaces.CenterFiniteDifferenceSpace(mesh) - face_space = Spaces.FaceFiniteDifferenceSpace(cent_space) - z = Fields.coordinate_field(cent_space).z - O = ones(FT, face_space) + for (i, stretch) in enumerate(stretch_fns) + i = 1 + stretch = Meshes.Uniform() + + z_elem = 2^6 + z_min = -π + z_max = π + device = ClimaComms.device() + + # use_column = true + use_column = false + if use_column + grid = ColumnGrid(; z_elem, z_min, z_max, stretch, device) + cspace = Spaces.FiniteDifferenceSpace(grid, Grids.CellCenter()) + fspace = Spaces.FaceFiniteDifferenceSpace(cspace) + else + grid = ExtrudedCubedSphereGrid(; + z_elem, + z_min, + z_max, + stretch, + radius = 10, + h_elem = 10, + n_quad_points = 4, + device, + ) + cspace = + Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) + fspace = Spaces.FaceExtrudedFiniteDifferenceSpace(cspace) + end + + z = Fields.coordinate_field(cspace).z + O = ones(FT, fspace) # Initial condition q_init = pulse.(z, 0.0, z₀, zₕ, z₁) q = q_init - y = Fields.FieldVector(; q) + coords = Fields.coordinate_field(q) + ρ = map(coord -> 1.0, coords) + perturb_field!(ρ; perturb_radius = 0.1) + y = Fields.FieldVector(; q, ρ) limiter = Limiters.VerticalMassBorrowingLimiter(q, (0.0,)) # Unitary, constant advective velocity @@ -127,17 +158,48 @@ plot_string = ["uniform", "stretched"] rel_mass_err = norm((sum(q_final) - sum(q_init)) / sum(q_init)) - p = plot() - Plots.plot!(q_init, label = "init") - Plots.plot!(q_final, label = "computed") - Plots.plot!(q_analytic, label = "analytic") - Plots.plot!(; legend = :topright) - Plots.plot!(; xlabel = "q", title = "VerticalMassBorrowingLimiter") - f = joinpath( - path, - "VerticalMassBorrowingLimiter_comparison_$(plot_string[i]).png", - ) - Plots.png(p, f) + if use_column + p = Plots.plot() + Plots.plot!(q_init, label = "init") + Plots.plot!(q_final, label = "computed") + Plots.plot!(q_analytic, label = "analytic") + Plots.plot!(; legend = :topright) + Plots.plot!(; xlabel = "q", title = "VerticalMassBorrowingLimiter") + f = joinpath( + path, + "VerticalMassBorrowingLimiter_comparison_$(plot_string[i]).png", + ) + Plots.png(p, f) + else + colidx = Fields.ColumnIndex((1, 1), 1) + p = Plots.plot() + Plots.plot!( + vec(parent(z[colidx])), + vec(parent(q_init[colidx])), + label = "init", + ) + Plots.plot!( + vec(parent(z[colidx])), + vec(parent(q_final[colidx])), + label = "computed", + ) + Plots.plot!( + vec(parent(z[colidx])), + vec(parent(q_analytic[colidx])), + label = "analytic", + ) + Plots.plot!(; legend = :topright) + Plots.plot!(; + xlabel = "q", + ylabel = "z", + title = "VerticalMassBorrowingLimiter", + ) + f = joinpath( + path, + "VerticalMassBorrowingLimiter_comparison_$(plot_string[i]).png", + ) + Plots.png(p, f) + end @test err ≤ 0.25 @test rel_mass_err ≤ 10eps() @test minimum(q_final) ≥ 0 From 039712b3c23761ab0a2c891febb78cb5818053d3 Mon Sep 17 00:00:00 2001 From: imreddyTeja Date: Wed, 22 Oct 2025 15:07:51 -0700 Subject: [PATCH 3/7] First draft of gpu support for vertical mass lim. Add gpu support for vertical mass limiter. This addition needs documentation, and also needs to be made more similar to the cpu version. Other fixes: Correct plotting axis Important TODOS: I think the indexing is actually happening bottom to top... Ask about \geq q_min or < qmin Ask about tolerances in the test Probably a good idea to check performance. I suspect this implementation will have poor performance at h_elem < 30 --- ext/cuda/limiters.jl | 122 +++++++++++++++++- .../vertical_mass_borrowing_limiter.jl | 47 ++++++- .../vertical_mass_borrowing_limiter.jl | 14 +- ...rtical_mass_borrowing_limiter_advection.jl | 8 +- 4 files changed, 180 insertions(+), 11 deletions(-) diff --git a/ext/cuda/limiters.jl b/ext/cuda/limiters.jl index e99c713e6a..871e72ff54 100644 --- a/ext/cuda/limiters.jl +++ b/ext/cuda/limiters.jl @@ -2,10 +2,11 @@ import ClimaCore.Limiters: QuasiMonotoneLimiter, compute_element_bounds!, compute_neighbor_bounds_local!, - apply_limiter! + apply_limiter!, + VerticalMassBorrowingLimiter import ClimaCore.Fields import ClimaCore: DataLayouts, Spaces, Topologies, Fields -import ClimaCore.DataLayouts: slab_index +import ClimaCore.DataLayouts: slab_index, getindex_field, setindex_field! using CUDA function config_threadblock(Nv, Nh) @@ -281,3 +282,120 @@ function apply_limiter_kernel!( return nothing end + + +function apply_limiter!( + q::Fields.Field, + ρ::Fields.Field, + space, + limiter::VerticalMassBorrowingLimiter, + dev::ClimaComms.CUDADevice; + warn::Bool = true, +) + q_data = Fields.field_values(q) + Nf = DataLayouts.ncomponents(q_data) + us = DataLayouts.UniversalSize(q_data) + q_min = limiter.q_min + us = DataLayouts.UniversalSize(q_data) + Δz = Fields.Δz_field(q) + (Ni, Nj, _, Nv, Nh) = DataLayouts.universal_size(us) + ncols = Ni * Nj * Nh + # if Ni * Nj * Nf > 64 + nthread_x = Ni * Nj + nthread_y = Nf + nthread_z = cld(64, nthread_x * nthread_y) + nthreads = (nthread_x, nthread_y, nthread_z) + nblocks = cld(ncols * Nf, prod(nthreads)) + + + args = ( + typeof(limiter), + Fields.field_values(Operators.strip_space(q, axes(q))), + Fields.field_values(Operators.strip_space(ρ, axes(ρ))), + Fields.field_values(Operators.strip_space(Δz, axes(Δz))), + q_min, + us, + ) + auto_launch!( + apply_limiter_kernel!, + args; + threads_s = nthreads, + blocks_s = nblocks, + ) + call_post_op_callback() && post_op_callback(q, ρ, limiter, dev) + return nothing +end + +function apply_limiter_kernel!( + ::Type{LM}, + q_data, + ρ_data, + Δz_data, + q_min_tuple, + us::DataLayouts.UniversalSize) where {LM <: VerticalMassBorrowingLimiter} + (Ni, Nj, _, Nv, Nh) = DataLayouts.universal_size(us) + j_idx, i_idx = divrem(CUDA.threadIdx().x - Int32(1), Ni) + j_idx = j_idx + Int32(1) + i_idx = i_idx + Int32(1) + f_idx = CUDA.threadIdx().y + h_idx = CUDA.blockDim().z * (CUDA.blockIdx().x - Int32(1)) + CUDA.threadIdx().z + # tidx = (CUDA.blockIdx().x - Int32(1)) * CUDA.blockDim().x + CUDA.threadIdx().x - Int32(1) + q_min = q_min_tuple[f_idx] + borrowed_mass = zero(eltype(q_data)) + @inbounds if h_idx <= Nh + # TODO: unroll this? + for v in 1:Nv + CI = CartesianIndex(i_idx, j_idx, f_idx, v, h_idx) + ρΔV_lev = getindex_field(ρ_data, CI) * getindex_field(Δz_data, CI) + new_mass = getindex_field(q_data, CI) - (borrowed_mass / ρΔV_lev) + if new_mass > q_min + setindex_field!(q_data, new_mass, CI) + borrowed_mass = zero(borrowed_mass) + else + borrowed_mass = (q_min - new_mass) * ρΔV_lev + setindex_field!(q_data, q_min, CI) + end + end + for i in 0:(Nv - 1) # CUDA.jl recommends avoiding stepranges + v = Nv - i + if borrowed_mass > zero(borrowed_mass) + CI = CartesianIndex(i_idx, j_idx, f_idx, v, h_idx) + ρΔV_lev = getindex_field(ρ_data, CI) * getindex_field(Δz_data, CI) + new_mass = getindex_field(q_data, CI) - (borrowed_mass / ρΔV_lev) + if new_mass > q_min + setindex_field!(q_data, new_mass, CI) + return + else + borrowed_mass = (q_min - new_mass) * ρΔV_lev + setindex_field!(q_data, q_min, CI) + end + end + end + end + return +end + +# PLAN +# Implementation 1: each thread handles a column. This likely will not most of the gpu until h >= 128 +# 1.1: no shmem +# 1.2: shmem +# 1.3 split fields? +# Implementation 2: each block handles a column? +#= + +while sync_threads_and(lev_mass < lim || bottom_lev) + borrowed_from_lower = lim - lev_mass + borrowed_from_me = gotten_from_shuffle + lev_mass = lim - borrowed_from_me +end + +while lev_mass < lim + borrowing_from_above = lev_mass + borrowed_from_me = gotten_from_shuffle + lev_mass = lev_mass = lim - borrowed_from_me +end + + +=# + +# Implementation 3: each warp handles each col diff --git a/src/Limiters/vertical_mass_borrowing_limiter.jl b/src/Limiters/vertical_mass_borrowing_limiter.jl index c33d79d148..64a223ecd2 100644 --- a/src/Limiters/vertical_mass_borrowing_limiter.jl +++ b/src/Limiters/vertical_mass_borrowing_limiter.jl @@ -76,8 +76,9 @@ end # TODO: can we improve the performance? # `bycolumn` on the CPU may be better here since we could multithread it. function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # column fields + # TODO: maybe rm some stuff from struct or cache (; bmass, ic, q_min) = cache - + # TODO: verify index direction Δz = Fields.Δz_field(q) Δz_vals = Fields.field_values(Δz) (; J) = Fields.local_geometry_field(ρ) @@ -88,6 +89,8 @@ function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # c nlevels = Spaces.nlevels(axes(q)) @. ic = 0 @. bmass = 0 + # TODO: Maybe delete ic + # TODO: make bmass positive q_vals = Fields.field_values(q) # top to bottom for f in 1:DataLayouts.ncomponents(q_vals) @@ -97,6 +100,7 @@ function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # c ρΔV_lev = DL.getindex_field(ΔV_vals, CI) * DL.getindex_field(ρ_vals, CI) nmass = DL.getindex_field(q_vals, CI) + bmass[] / ρΔV_lev + # TODO: should this be \geq if nmass > q_min[f] # if new mass in the current layer is positive, don't borrow mass any more DL.setindex_field!(q_vals, nmass, CI) @@ -134,3 +138,44 @@ function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # c return nothing end + + +function basic_lim(q, ρ = ones(Float64, length(q)), v = ones(Float64, length(q))) + ic = 0 + bmass = 0.0 + @show q + println("") + for i in 1:length(q) + @show i + ρΔV_lev = ρ[i] * v[i] + nmass = q[i] + bmass / ρΔV_lev + if nmass > 0.0 + q[i] = nmass + bmass = 0.0 + else + bmass = nmass * ρΔV_lev + q[i] = 0.0 + end + @show q + println("") + end + for i in length(q):-1:1 + @show i + if bmass < 0.0 + ρΔV_lev = ρ[i] * v[i] + nmass = q[i] + bmass / ρΔV_lev + @show nmass + if nmass > 0 + q[i] = nmass + @show q + return + bmass = 0.0 + else + bmass = nmass * ρΔV_lev + q[i] = 0.0 + end + @show q + println("") + end + end +end diff --git a/test/Limiters/vertical_mass_borrowing_limiter.jl b/test/Limiters/vertical_mass_borrowing_limiter.jl index e8c9702ce4..d347bfd038 100644 --- a/test/Limiters/vertical_mass_borrowing_limiter.jl +++ b/test/Limiters/vertical_mass_borrowing_limiter.jl @@ -9,10 +9,11 @@ using ClimaCore.RecursiveApply using ClimaCore.Geometry using ClimaCore.Grids using ClimaCore.CommonGrids +import ClimaCore using Test using Random -#= + import Plots import ClimaCorePlots function plot_results(f, f₀) @@ -22,9 +23,10 @@ function plot_results(f, f₀) p = Plots.plot() Plots.plot(fcol; label = "field") Plots.plot!(f₀col; label = "initial") + Plots.savefig("lim.png") end -plot_results(ρq, ρq_init) -=# +# plot_results(ρq, ρq_init) + function perturb_field!(f::Fields.Field; perturb_radius) device = ClimaComms.device(f) @@ -38,7 +40,7 @@ end @testset "Vertical mass borrowing limiter - column" begin FT = Float64 - Random.seed!(1234) + Random.seed!(1134) z_elem = 10 z_min = 0 z_max = 1 @@ -70,6 +72,7 @@ end ρq = ρ .⊠ q @test isapprox(sum(ρq), sum_ρq_init; atol = 1e-15) @test isapprox(sum(ρq), sum_ρq_init; rtol = 1e-10) + plot_results(ClimaCore.to_cpu(ρq), ClimaCore.to_cpu(ρq_init)) # @show sum(ρq) # 0.07388931313511024 # @show sum_ρq_init # 0.07388931313511025 end @@ -110,8 +113,10 @@ end @test 0.10 ≤ count(x -> x < 0.0001, parent(q)) / 96000 ≤ 1 # ensure 10% of points are negative @test -2 * perturb_q ≤ minimum(q) ≤ -tol + @show q limiter = Limiters.VerticalMassBorrowingLimiter(q, (0.0,)) Limiters.apply_limiter!(q, ρ, limiter) + @show q @test 0 ≤ minimum(q) ρq = ρ .⊠ q @test isapprox(sum(ρq), sum_ρq_init; atol = 0.07) @@ -122,6 +127,7 @@ end @testset "Vertical mass borrowing limiter - deep atmosphere" begin FT = Float64 + Random.seed!(12214) z_elem = 10 z_min = 0 z_max = 1 diff --git a/test/Limiters/vertical_mass_borrowing_limiter_advection.jl b/test/Limiters/vertical_mass_borrowing_limiter_advection.jl index 884d1f2076..8fafcd1a79 100644 --- a/test/Limiters/vertical_mass_borrowing_limiter_advection.jl +++ b/test/Limiters/vertical_mass_borrowing_limiter_advection.jl @@ -148,7 +148,7 @@ plot_string = ["uniform", "stretched"] prob, ExplicitAlgorithm(SSP33ShuOsher()), dt = Δt, - saveat = Δt, + save_everystep = true, ) q_init = sol.u[1].q @@ -174,18 +174,18 @@ plot_string = ["uniform", "stretched"] colidx = Fields.ColumnIndex((1, 1), 1) p = Plots.plot() Plots.plot!( - vec(parent(z[colidx])), vec(parent(q_init[colidx])), + vec(parent(z[colidx])), label = "init", ) Plots.plot!( - vec(parent(z[colidx])), vec(parent(q_final[colidx])), + vec(parent(z[colidx])), label = "computed", ) Plots.plot!( - vec(parent(z[colidx])), vec(parent(q_analytic[colidx])), + vec(parent(z[colidx])), label = "analytic", ) Plots.plot!(; legend = :topright) From 1d18d4131a91f325e117fc4c2733a548046a45cf Mon Sep 17 00:00:00 2001 From: imreddyTeja Date: Wed, 22 Oct 2025 15:17:37 -0700 Subject: [PATCH 4/7] temp add test --- .buildkite/pipeline.yml | 4118 ++++++++++++++++++++------------------- 1 file changed, 2071 insertions(+), 2047 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b6cc3860c9..9b8c3a7943 100755 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -61,2068 +61,2092 @@ steps: key: unit_recursive_apply command: "julia --color=yes --check-bounds=yes --project=.buildkite test/RecursiveApply/unit_recursive_apply.jl" - - group: "Unit: Utilities" - steps: - - - label: "Unit: plushalf" - key: unit_plushalf - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Utilities/unit_plushalf.jl" - - - group: "Unit: DataLayouts" - steps: - - - label: "Unit: data0d" - key: unit_data0d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data0d.jl" - - - label: "Unit: data_fill" - key: unit_data_fill - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_fill.jl" - - - label: "Unit: data_copyto" - key: unit_data_copyto - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_copyto.jl" - - - label: "Unit: cartesian_field_index" - key: unit_data_cartesian_field_index - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_cartesian_field_index.jl" - - - label: "Unit: non_extruded_broadcast" - key: unit_non_extruded_broadcast - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_non_extruded_broadcast.jl" - - - label: "Unit: mapreduce" - key: unit_data_mapreduce - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" - - - label: "Unit: data_opt_similar" - key: data_opt_similar - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/opt_similar.jl" - - - label: "Unit: opt_universal_size" - key: opt_universal_size - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/opt_universal_size.jl" - - - label: "Unit: data_ndims" - key: unit_data_ndims - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_ndims.jl" - - - label: "Unit: unit_data2array" - key: unit_data2array - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_data2array.jl" - - - label: "Unit: data1d" - key: unit_data1d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data1d.jl" - - - label: "Unit: data2d" - key: unit_data2d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data2d.jl" - - - label: "Unit: data1dx" - key: unit_data1dx - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data1dx.jl" - - - label: "Unit: data2dx" - key: unit_data2dx - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data2dx.jl" - - - label: "Unit: data cuda" - key: unit_data_cuda - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/cuda.jl CUDA" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: data cuda threadblocks" - key: unit_data_threadblock - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_cuda_threadblocks.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: data fill" - key: gpu_unit_data_fill - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_fill.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: data mapreduce" - key: gpu_unit_data_mapreduce - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: data mapreduce (2 gpus)" - key: gpu_2_unit_data_mapreduce - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - CLIMACOMMS_CONTEXT: "MPI" - agents: - slum_ntasks: 2 - slurm_gpus: 2 - - - label: "Unit: data copyto" - key: gpu_unit_data_copyto - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_copyto.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Unit: Geometry" - steps: - - - label: "Unit: geometry" - key: unit_geometry - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/geometry.jl" - - - label: "Unit: axistensors" - key: unit_axistensors - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/axistensors.jl" - - - label: "Unit: rmul_with_projection" - key: unit_rmul_with_projection - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/rmul_with_projection.jl" - - - group: "Unit: Meshes" - steps: - - - label: "Unit: interval" - key: unit_interval - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/interval.jl" - - - label: "Unit: meshes rectangle" - key: unit_meshes_rectangle - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/rectangle.jl" - - - label: "Unit: meshes opt" - key: unit_meshes_opt - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/opt_meshes.jl" - - - label: "Unit: meshes cubedsphere" - key: unit_meshes_cubed_sphere - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/cubedsphere.jl" - - - group: "Unit: Topologies" - steps: - - - label: "Unit: topo interval" - key: unit_topo_interval - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/interval.jl" + # - group: "Unit: Utilities" + # steps: + + # - label: "Unit: plushalf" + # key: unit_plushalf + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Utilities/unit_plushalf.jl" + + # - group: "Unit: DataLayouts" + # steps: + + # - label: "Unit: data0d" + # key: unit_data0d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data0d.jl" + + # - label: "Unit: data_fill" + # key: unit_data_fill + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_fill.jl" + + # - label: "Unit: data_copyto" + # key: unit_data_copyto + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_copyto.jl" + + # - label: "Unit: cartesian_field_index" + # key: unit_data_cartesian_field_index + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_cartesian_field_index.jl" + + # - label: "Unit: non_extruded_broadcast" + # key: unit_non_extruded_broadcast + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_non_extruded_broadcast.jl" + + # - label: "Unit: mapreduce" + # key: unit_data_mapreduce + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" + + # - label: "Unit: data_opt_similar" + # key: data_opt_similar + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/opt_similar.jl" + + # - label: "Unit: opt_universal_size" + # key: opt_universal_size + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/opt_universal_size.jl" + + # - label: "Unit: data_ndims" + # key: unit_data_ndims + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_ndims.jl" + + # - label: "Unit: unit_data2array" + # key: unit_data2array + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_data2array.jl" + + # - label: "Unit: data1d" + # key: unit_data1d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data1d.jl" + + # - label: "Unit: data2d" + # key: unit_data2d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data2d.jl" + + # - label: "Unit: data1dx" + # key: unit_data1dx + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data1dx.jl" + + # - label: "Unit: data2dx" + # key: unit_data2dx + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data2dx.jl" + + # - label: "Unit: data cuda" + # key: unit_data_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/cuda.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: data cuda threadblocks" + # key: unit_data_threadblock + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_cuda_threadblocks.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: data fill" + # key: gpu_unit_data_fill + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_fill.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: data mapreduce" + # key: gpu_unit_data_mapreduce + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: data mapreduce (2 gpus)" + # key: gpu_2_unit_data_mapreduce + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slum_ntasks: 2 + # slurm_gpus: 2 + + # - label: "Unit: data copyto" + # key: gpu_unit_data_copyto + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_copyto.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - group: "Unit: Geometry" + # steps: + + # - label: "Unit: geometry" + # key: unit_geometry + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/geometry.jl" + + # - label: "Unit: axistensors" + # key: unit_axistensors + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/axistensors.jl" + + # - label: "Unit: rmul_with_projection" + # key: unit_rmul_with_projection + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/rmul_with_projection.jl" + + # - group: "Unit: Meshes" + # steps: + + # - label: "Unit: interval" + # key: unit_interval + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/interval.jl" + + # - label: "Unit: meshes rectangle" + # key: unit_meshes_rectangle + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/rectangle.jl" + + # - label: "Unit: meshes opt" + # key: unit_meshes_opt + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/opt_meshes.jl" + + # - label: "Unit: meshes cubedsphere" + # key: unit_meshes_cubed_sphere + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/cubedsphere.jl" + + # - group: "Unit: Topologies" + # steps: + + # - label: "Unit: topo interval" + # key: unit_topo_interval + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/interval.jl" + + # - label: "Unit: topo rectangle" + # key: unit_topo_rectangle + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/rectangle.jl" + + # - label: "Unit: rectangle sfc" + # key: unit_rectangle_sfc + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/rectangle_sfc.jl" + + # - label: "Unit: cubedsphere" + # key: unit_cubedsphere + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/cubedsphere.jl" + + # - label: "Unit: cubedsphere sfc" + # key: unit_cubedsphere_sfc + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/cubedsphere_sfc.jl" + + # - label: "Unit: topologies distributed" + # key: unit_topologies_distributed + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/dtopo4.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 4 + + # - group: "Unit: Spaces" + # steps: + + # - label: "Unit: Quadratures" + # key: unit_quadrature + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Quadratures/Quadratures.jl" + + # - label: "Unit: spaces" + # key: unit_spaces + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_spaces.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/opt_spaces.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_high_resolution_space.jl" + + # - label: "Unit: cuda spaces" + # key: "gpu_cuda_spaces" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_spaces.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/opt_spaces.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_high_resolution_space.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: common spaces with CUDA" + # key: "gpu_common_cuda_spaces" + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: common spaces with CUDA and MPI" + # key: "gpu_common_cuda_mpi_spaces" + # command: + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_ntasks: 2 + + # - label: "Unit: common spaces with MPI" + # key: "common_cuda_mpi_spaces" + # command: + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + + # - label: "Unit: distributed cuda spaces" + # key: "gpu_distributed_extruded_cuda_spaces" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/space_construction.jl CUDA" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_gpus_per_task: 1 + # slurm_ntasks: 3 + + # - label: "Unit: ddss1" + # key: unit_ddss1 + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: ddss1 cs" + # key: unit_cuda_ddss1_cs + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1_cs.jl" + + # - label: "Unit: ddss1 cs" + # key: unit_ddss1_cs + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1_cs.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: sphere" + # key: unit_sphere + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/sphere.jl" + + # - label: "Unit: terrain warp" + # key: unit_terrain_warp + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/terrain_warp.jl" + + # - label: "Unit: distributed dss2" + # key: unit_distributed_dss2 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss2.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + + # - label: "Unit: distributed dss3" + # key: unit_distributed_dss3 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss3.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 3 + + # - label: "Unit: distributed dss4" + # key: unit_distributed_dss4 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss4.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 4 + + # - label: "Unit: distributed remapping (1 process)" + # key: distributed_remapping_1proc + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" + # env: + # CLIMACOMMS_DEVICE: "CPU" + + # - label: "Unit: distributed remapping (2 processes)" + # key: distributed_remapping_2procs + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CPU" + # agents: + # slurm_ntasks: 2 + + # - label: "Unit: distributed remapping with CUDA (1 process)" + # key: distributed_remapping_gpu_1proc + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: distributed remapping with CUDA (2 processes)" + # key: distributed_remapping_gpu_2procs + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" + # soft_fail: true + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 2 + # slurm_gpus_per_task: 1 + + # - label: "Unit: distributed gather" + # key: unit_distributed_gather4 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/gather4.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 4 + + # - label: "Unit: cuda extruded spaces" + # key: "extruded_spaces_cuda" + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/extruded_cuda.jl CUDA" + # artifact_paths: + # - output/extruded_spaces_cuda + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: cuda point spaces" + # key: "point_space_cuda" + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/point_cuda.jl CUDA" + # artifact_paths: + # - output/point_cuda + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: cuda dss 2-process test" + # key: "gpu_ddss2_test" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss2.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 2 + # slurm_gpus: 2 + + # - label: "Unit: cuda dss 3-process test" + # key: "gpu_ddss3_test" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss3.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 3 + # slurm_gpus: 3 + + # - label: "Unit: cuda dss 4-process test" + # key: "gpu_ddss4_test" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss4.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 4 + # slurm_gpus: 4 + + # - label: "Unit: cuda Cubed Sphere dss; ne = 32; 2-process test" + # key: "gpu_ddss_ne32_cs_2processes" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 2 + # slurm_gpus: 2 + + # - label: "Unit: cuda Cubed Sphere dss; ne = 32; 3-process test" + # key: "gpu_ddss_ne32_cs_3processes" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 3 + # slurm_gpus: 3 + + # - label: "Unit: cuda Cubed Sphere dss; ne = 32; 4-process test" + # key: "gpu_ddss_ne32_cs_4processes" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 4 + # slurm_gpus: 4 + + # - group: "Unit: Fields" + # steps: + + # - label: "Unit: field" + # key: unit_field + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/unit_field.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_fieldvectors.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_field_multi_broadcast_fusion.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/convergence_field_integrals.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/inference_repro.jl" + # agents: + # slurm_mem: 20GB + + # - label: "Unit: field cuda" + # key: unit_field_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/unit_field.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_fieldvectors.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_field_multi_broadcast_fusion.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/convergence_field_integrals.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/inference_repro.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: reduction cuda" + # key: unit_reduction_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/reduction_cuda.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: distributed reduction cuda" + # key: unit_distributed_reduction_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/reduction_cuda_distributed.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_gpus_per_task: 1 + # slurm_ntasks: 2 + + # - group: "Unit: Operators" + # steps: + + # - label: "Unit: rectilinear" + # key: unit_rectilinear + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear.jl" + + # - label: "Unit: diffusion2d" + # key: unit_diffusion2d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_diffusion2d.jl" + + # - label: "Unit: sphere geometry" + # key: unit_sphere_geometry + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry.jl" + + # - label: "Unit: sphere gradient" + # key: unit_sphere_gradient + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_gradient.jl" + + # - label: "Unit: sphere divergence" + # key: unit_sphere_divergence + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_divergence.jl" + + # - label: "Unit: sphere curl" + # key: unit_sphere_curl + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_curl.jl" + + # - label: "Unit: sphere diffusion" + # key: unit_sphere_diffusion + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_diffusion.jl" + + # - label: "Unit: sphere diffusion vec" + # key: unit_sphere_diffusion_vec + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_diffusion_vec.jl" + + # - label: "Unit: sphere hyperdiffusion" + # key: unit_sphere_hyperdiffusion + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_sphere_hyperdiffusion.jl" + + # - label: "Unit: sphere hyperdiffusion vec" + # key: unit_sphere_hyperdiffusion_vec + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_sphere_hyperdiffusion_vec.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/convergence_sphere_hyperdiffusion_vec.jl" + + # - label: "Unit: spec ops plane" + # key: unit_spec_ops_plane + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/plane.jl" + + # - label: "Unit: FD operator (shmem)" + # key: unit_fd_operator_shmem + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_fd_ops_shared_memory.jl" + # - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_fd_ops_shared_memory.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: gpu columnwise" + # key: unit_gpu_columnwise + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_columnwise.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: columnwise" + # key: unit_columnwise + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_columnwise.jl" + + # - label: "Unit: column" + # key: unit_column + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_column.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/convergence_column.jl" + + # - label: "Unit: fd tensor" + # key: unit_fd_tensor + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/tensor.jl" + + # - label: "Unit: advection operator convergence" + # key: unit_adv_conv + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/convergence_advection_diffusion1d.jl" + # soft_fail: true + + # - label: "Unit: hyb ops 2d" + # key: unit_hyb_ops_2d + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_2d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_2d.jl" + + # - label: "Unit: hyb ops 2d CUDA" + # key: unit_hyb_ops_2d_cuda + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_2d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_2d.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: hyb ops 3d" + # key: unit_hyb_ops_3d + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_3d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_3d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_3d.jl" + + # - label: "Unit: hyb ops 3d CUDA" + # key: unit_hyb_ops_3d_cuda + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_3d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_3d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_3d.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: remapping" + # key: unit_remapping + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/remapping.jl" + + # - label: "Unit: run sphere geometry distributed (2)" + # key: unit_run_sphere_geometry_distributed2 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 20GB + + # - label: "Unit: run sphere geometry distributed (3)" + # key: unit_run_sphere_geometry_distributed3 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 3 + # slurm_mem: 20GB + + # - label: "Unit: run sphere geometry distributed (4)" + # key: unit_run_sphere_geometry_distributed4 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 4 + # slurm_mem: 20GB + + # - label: "Unit: rectilinear cuda" + # key: unit_rectilinear_cuda + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear_cuda.jl" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear_cuda.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: operators on levels and extruded spaces" + # key: unit_cuda_operators_example + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/unit_operators_examples.jl" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/unit_operators_examples.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: velocity grad tensor ops" + # key: unit_spectral_tensor_op_cuda + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/covar_deriv_ops.jl" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/covar_deriv_ops.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: hybrid operators cuda" + # key: unit_ops_cuda + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_cuda.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_cuda.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: extruded sphere cuda" + # key: unit_extruded_sphere_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/extruded_sphere_cuda.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: extruded 3dbox cuda" + # key: unit_extruded_3dbox_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/extruded_3dbox_cuda.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: Integrals (CPU)" + # key: "cpu_integrals" + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/integrals.jl" + + # - label: "Unit: Integrals (GPU)" + # key: "gpu_integrals" + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/integrals.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - group: "Unit: MatrixFields" + # steps: + + # - label: "Unit: BandMatrixRow" + # key: unit_band_matrix_row + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/band_matrix_row.jl" + + # - label: "Unit: field2arrays" + # key: unit_field2arrays + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field2arrays.jl" + + # - label: "Unit: matrix multiplication at boundaries" + # key: unit_matrix_multiplication_at_boundaries + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_multiplication_at_boundaries.jl" + + # - label: "Unit: operator matrices (CPU)" + # key: unit_operator_matrices_cpu + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/operator_matrices.jl" + + # - label: "Unit: operator matrices (GPU)" + # key: unit_operator_matrices_gpu + # command: "julia --color=yes --project=.buildkite test/MatrixFields/operator_matrices.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 40GB + + # - label: "Unit: field names" + # key: unit_field_names + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_names.jl" + + # - label: "Unit: field matrix solvers (CPU)" + # key: unit_field_matrix_solvers_cpu + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_matrix_solvers.jl" + # agents: + # slurm_mem: 40GB + + # - label: "Unit: field matrix solvers (GPU)" + # key: unit_field_matrix_solvers_gpu + # command: "julia --color=yes --project=.buildkite test/MatrixFields/field_matrix_solvers.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 40GB + + # - label: "Unit: bidiag matrix row example (CPU)" + # key: cpu_gpu_compat_bidiag_matrix_row + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/gpu_compat_bidiag_matrix_row.jl" + + # - label: "Unit: bidiag matrix row example (GPU)" + # key: gpu_compat_bidiag_matrix_row + # command: "julia --color=yes --project=.buildkite test/MatrixFields/gpu_compat_bidiag_matrix_row.jl" + # soft_fail: true + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: multiple field solve example (GPU)" + # key: gpu_multiple_field_solve_example + # command: "julia --color=yes --project=.buildkite test/MatrixFields/multiple_field_solve_reproducer_1.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: matrix multiplication recursion example (CPU)" + # key: cpu_matrix_multiplication_recursion + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_multiplication_recursion.jl" + + # - label: "Unit: matrix multiplication recursion example (GPU)" + # key: gpu_matrix_multiplication_recursion + # command: "julia --color=yes --project=.buildkite test/MatrixFields/matrix_multiplication_recursion.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: field_matrix_indexing (CPU)" + # key: cpu_field_matrix_indexing + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_matrix_indexing.jl" + + # - label: "Unit: field_matrix_indexing (GPU)" + # key: gpu_field_matrix_indexing + # command: "julia --color=yes --project=.buildkite test/MatrixFields/field_matrix_indexing.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - group: "Unit: MatrixFields - broadcasting (CPU)" + # steps: + + # # scalar + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_1 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_1.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_2 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_2.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_3 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_3.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_4 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_4.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_5 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_5.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_6 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_6.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_7 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_7.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_8 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_8.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_9 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_9.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_10 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_10.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_11 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_11.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_12 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_12.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_13 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_13.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_14 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_14.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_15 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_15.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_16 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_16.jl" + + # # non-scalar + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_non_scalar_1 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_1.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_non_scalar_2 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_2.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_non_scalar_3 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_3.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_non_scalar_4 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_4.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_non_scalar_5 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_5.jl" + # soft_fail: true + + # - group: "Unit: MatrixFields - broadcasting (GPU)" + # steps: + + # # scalar + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_1 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_1.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_2 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_2.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_3 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_3.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_4 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_4.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_5 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_5.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_6 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_6.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_7 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_7.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_8 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_8.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_9 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_9.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_10 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_10.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_11 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_11.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_12 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_12.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_13 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_13.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_14 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_14.jl" + # soft_fail: true + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_15 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_15.jl" + # soft_fail: true + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_16 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_16.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # # non-scalar + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_non_scalar_1 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_1.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_non_scalar_2 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_2.jl" + # soft_fail: true + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_non_scalar_3 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_3.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_non_scalar_4 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_4.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_non_scalar_5 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_5.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - group: "Unit: Hypsography" + # steps: + + # - label: "Unit: hypsography 2d" + # key: unit_hypsography_2d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Hypsography/2d.jl" + + # - label: "Unit: 3dsphere" + # key: unit_3dsphere + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Hypsography/3dsphere.jl" + + # - group: "Unit: InputOutput" + # steps: + + # - label: "Unit: read_type" + # key: unit_read_type + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_read_type.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 3GB + + # - label: "Unit: spectralelement2d" + # key: unit_spectralelement2d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_spectralelement2d.jl" + + # - label: "Unit: spectralelement2d" + # key: unit_spectralelement2d_gpu + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_spectralelement2d.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: hybrid2dbox" + # key: unit_hybrid2dbox + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox.jl" + + # - label: "Unit: hybrid2dbox topography" + # key: unit_hybrid2dbox_topography + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox_topography.jl" + + # - label: "Unit: hybrid2dbox stretched" + # key: unit_hybrid2dbox_stretched + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox_stretched.jl" + + # - label: "Unit: hybrid3dbox" + # key: unit_hybrid3dbox + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dbox.jl" + + # - label: "Unit: hybrid3dcubedsphere" + # key: unit_hybrid3dcubedsphere + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere.jl" + + # - label: "Unit: hybrid3dcubedsphere topography" + # key: unit_hybrid3dcubedsphere_topography + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere_topography.jl" + + # - label: "Unit: hybrid3dcubedsphere topography" + # key: unit_hybrid3dcubedsphere_topography_gpu + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere_topography.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: finitedifference" + # key: unit_finitedifference + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_finitedifference.jl" + + # - label: "Unit: Parallel HDF5 IO tests" + # key: "cpu_parallel_hdf5" + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere.jl" + # timeout_in_minutes: 5 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # retry: + # automatic: true + # agents: + # slurm_nodes: 3 + # slurm_tasks_per_node: 1 + + # - group: "Unit: Remapping" + # steps: + + # - label: "Unit: interpolate array" + # key: unit_interpolate_array + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/interpolate_array.jl" - - label: "Unit: topo rectangle" - key: unit_topo_rectangle - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/rectangle.jl" - - - label: "Unit: rectangle sfc" - key: unit_rectangle_sfc - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/rectangle_sfc.jl" - - - label: "Unit: cubedsphere" - key: unit_cubedsphere - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/cubedsphere.jl" - - - label: "Unit: cubedsphere sfc" - key: unit_cubedsphere_sfc - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/cubedsphere_sfc.jl" - - - label: "Unit: topologies distributed" - key: unit_topologies_distributed - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/dtopo4.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 4 - - - group: "Unit: Spaces" + - group: "Unit: Limiters" steps: - - label: "Unit: Quadratures" - key: unit_quadrature - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Quadratures/Quadratures.jl" - - - label: "Unit: spaces" - key: unit_spaces - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_spaces.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/opt_spaces.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_high_resolution_space.jl" - - - label: "Unit: cuda spaces" - key: "gpu_cuda_spaces" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_spaces.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/opt_spaces.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_high_resolution_space.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: common spaces with CUDA" - key: "gpu_common_cuda_spaces" - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: common spaces with CUDA and MPI" - key: "gpu_common_cuda_mpi_spaces" - command: - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_ntasks: 2 - - - label: "Unit: common spaces with MPI" - key: "common_cuda_mpi_spaces" - command: - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - - - label: "Unit: distributed cuda spaces" - key: "gpu_distributed_extruded_cuda_spaces" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/space_construction.jl CUDA" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_gpus_per_task: 1 - slurm_ntasks: 3 - - - label: "Unit: ddss1" - key: unit_ddss1 - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: ddss1 cs" - key: unit_cuda_ddss1_cs - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1_cs.jl" - - - label: "Unit: ddss1 cs" - key: unit_ddss1_cs - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1_cs.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: sphere" - key: unit_sphere - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/sphere.jl" - - - label: "Unit: terrain warp" - key: unit_terrain_warp - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/terrain_warp.jl" - - - label: "Unit: distributed dss2" - key: unit_distributed_dss2 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss2.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - - - label: "Unit: distributed dss3" - key: unit_distributed_dss3 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss3.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 3 - - - label: "Unit: distributed dss4" - key: unit_distributed_dss4 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss4.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 4 - - - label: "Unit: distributed remapping (1 process)" - key: distributed_remapping_1proc - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" - env: - CLIMACOMMS_DEVICE: "CPU" - - - label: "Unit: distributed remapping (2 processes)" - key: distributed_remapping_2procs - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CPU" - agents: - slurm_ntasks: 2 - - - label: "Unit: distributed remapping with CUDA (1 process)" - key: distributed_remapping_gpu_1proc - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: distributed remapping with CUDA (2 processes)" - key: distributed_remapping_gpu_2procs - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" - soft_fail: true - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 2 - slurm_gpus_per_task: 1 - - - label: "Unit: distributed gather" - key: unit_distributed_gather4 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/gather4.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 4 - - - label: "Unit: cuda extruded spaces" - key: "extruded_spaces_cuda" - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/extruded_cuda.jl CUDA" - artifact_paths: - - output/extruded_spaces_cuda - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: cuda point spaces" - key: "point_space_cuda" - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/point_cuda.jl CUDA" - artifact_paths: - - output/point_cuda - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: cuda dss 2-process test" - key: "gpu_ddss2_test" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss2.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 2 - slurm_gpus: 2 - - - label: "Unit: cuda dss 3-process test" - key: "gpu_ddss3_test" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss3.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 3 - slurm_gpus: 3 - - - label: "Unit: cuda dss 4-process test" - key: "gpu_ddss4_test" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss4.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 4 - slurm_gpus: 4 - - - label: "Unit: cuda Cubed Sphere dss; ne = 32; 2-process test" - key: "gpu_ddss_ne32_cs_2processes" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 2 - slurm_gpus: 2 - - - label: "Unit: cuda Cubed Sphere dss; ne = 32; 3-process test" - key: "gpu_ddss_ne32_cs_3processes" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 3 - slurm_gpus: 3 - - - label: "Unit: cuda Cubed Sphere dss; ne = 32; 4-process test" - key: "gpu_ddss_ne32_cs_4processes" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 4 - slurm_gpus: 4 - - - group: "Unit: Fields" - steps: + # - label: "Unit: limiter" + # key: unit_limiter + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Limiters/limiter.jl" - - label: "Unit: field" - key: unit_field - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/unit_field.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_fieldvectors.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_field_multi_broadcast_fusion.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/convergence_field_integrals.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/inference_repro.jl" - agents: - slurm_mem: 20GB + # - label: "Unit: limiter cuda" + # key: unit_limiter_gpu + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --project=.buildkite test/Limiters/limiter.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 - - label: "Unit: field cuda" - key: unit_field_cuda - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/unit_field.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_fieldvectors.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_field_multi_broadcast_fusion.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/convergence_field_integrals.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/inference_repro.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - label: "Unit: limiter" + # key: unit_limiter + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Limiters/limiter.jl" - - label: "Unit: reduction cuda" - key: unit_reduction_cuda + - label: "Unit: limiter cuda" + key: mass_borrow command: - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/reduction_cuda.jl CUDA" + - "julia --color=yes --project=.buildkite test/Limiters/vertical_mass_borrowing_limiter.jl" env: CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 - - label: "Unit: distributed reduction cuda" - key: unit_distributed_reduction_cuda + - label: "Unit: limiter cuda" + key: mass_borrow_advection command: - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/reduction_cuda_distributed.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_gpus_per_task: 1 - slurm_ntasks: 2 - - - group: "Unit: Operators" - steps: - - - label: "Unit: rectilinear" - key: unit_rectilinear - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear.jl" - - - label: "Unit: diffusion2d" - key: unit_diffusion2d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_diffusion2d.jl" - - - label: "Unit: sphere geometry" - key: unit_sphere_geometry - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry.jl" - - - label: "Unit: sphere gradient" - key: unit_sphere_gradient - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_gradient.jl" - - - label: "Unit: sphere divergence" - key: unit_sphere_divergence - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_divergence.jl" - - - label: "Unit: sphere curl" - key: unit_sphere_curl - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_curl.jl" - - - label: "Unit: sphere diffusion" - key: unit_sphere_diffusion - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_diffusion.jl" - - - label: "Unit: sphere diffusion vec" - key: unit_sphere_diffusion_vec - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_diffusion_vec.jl" - - - label: "Unit: sphere hyperdiffusion" - key: unit_sphere_hyperdiffusion - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_sphere_hyperdiffusion.jl" - - - label: "Unit: sphere hyperdiffusion vec" - key: unit_sphere_hyperdiffusion_vec - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_sphere_hyperdiffusion_vec.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/convergence_sphere_hyperdiffusion_vec.jl" - - - label: "Unit: spec ops plane" - key: unit_spec_ops_plane - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/plane.jl" - - - label: "Unit: FD operator (shmem)" - key: unit_fd_operator_shmem - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_fd_ops_shared_memory.jl" - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_fd_ops_shared_memory.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: gpu columnwise" - key: unit_gpu_columnwise - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_columnwise.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: columnwise" - key: unit_columnwise - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_columnwise.jl" - - - label: "Unit: column" - key: unit_column - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_column.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/convergence_column.jl" - - - label: "Unit: fd tensor" - key: unit_fd_tensor - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/tensor.jl" - - - label: "Unit: advection operator convergence" - key: unit_adv_conv - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/convergence_advection_diffusion1d.jl" - soft_fail: true - - - label: "Unit: hyb ops 2d" - key: unit_hyb_ops_2d - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_2d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_2d.jl" - - - label: "Unit: hyb ops 2d CUDA" - key: unit_hyb_ops_2d_cuda - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_2d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_2d.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: hyb ops 3d" - key: unit_hyb_ops_3d - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_3d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_3d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_3d.jl" - - - label: "Unit: hyb ops 3d CUDA" - key: unit_hyb_ops_3d_cuda - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_3d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_3d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_3d.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: remapping" - key: unit_remapping - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/remapping.jl" - - - label: "Unit: run sphere geometry distributed (2)" - key: unit_run_sphere_geometry_distributed2 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 20GB - - - label: "Unit: run sphere geometry distributed (3)" - key: unit_run_sphere_geometry_distributed3 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 3 - slurm_mem: 20GB - - - label: "Unit: run sphere geometry distributed (4)" - key: unit_run_sphere_geometry_distributed4 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 4 - slurm_mem: 20GB - - - label: "Unit: rectilinear cuda" - key: unit_rectilinear_cuda - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear_cuda.jl" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear_cuda.jl CUDA" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: operators on levels and extruded spaces" - key: unit_cuda_operators_example - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/unit_operators_examples.jl" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/unit_operators_examples.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: velocity grad tensor ops" - key: unit_spectral_tensor_op_cuda - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/covar_deriv_ops.jl" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/covar_deriv_ops.jl CUDA" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: hybrid operators cuda" - key: unit_ops_cuda - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_cuda.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_cuda.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: extruded sphere cuda" - key: unit_extruded_sphere_cuda - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/extruded_sphere_cuda.jl CUDA" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: extruded 3dbox cuda" - key: unit_extruded_3dbox_cuda - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/extruded_3dbox_cuda.jl CUDA" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: Integrals (CPU)" - key: "cpu_integrals" - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/integrals.jl" - - - label: "Unit: Integrals (GPU)" - key: "gpu_integrals" - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/integrals.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Unit: MatrixFields" - steps: - - - label: "Unit: BandMatrixRow" - key: unit_band_matrix_row - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/band_matrix_row.jl" - - - label: "Unit: field2arrays" - key: unit_field2arrays - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field2arrays.jl" - - - label: "Unit: matrix multiplication at boundaries" - key: unit_matrix_multiplication_at_boundaries - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_multiplication_at_boundaries.jl" - - - label: "Unit: operator matrices (CPU)" - key: unit_operator_matrices_cpu - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/operator_matrices.jl" - - - label: "Unit: operator matrices (GPU)" - key: unit_operator_matrices_gpu - command: "julia --color=yes --project=.buildkite test/MatrixFields/operator_matrices.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 40GB - - - label: "Unit: field names" - key: unit_field_names - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_names.jl" - - - label: "Unit: field matrix solvers (CPU)" - key: unit_field_matrix_solvers_cpu - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_matrix_solvers.jl" - agents: - slurm_mem: 40GB - - - label: "Unit: field matrix solvers (GPU)" - key: unit_field_matrix_solvers_gpu - command: "julia --color=yes --project=.buildkite test/MatrixFields/field_matrix_solvers.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 40GB - - - label: "Unit: bidiag matrix row example (CPU)" - key: cpu_gpu_compat_bidiag_matrix_row - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/gpu_compat_bidiag_matrix_row.jl" - - - label: "Unit: bidiag matrix row example (GPU)" - key: gpu_compat_bidiag_matrix_row - command: "julia --color=yes --project=.buildkite test/MatrixFields/gpu_compat_bidiag_matrix_row.jl" - soft_fail: true - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: multiple field solve example (GPU)" - key: gpu_multiple_field_solve_example - command: "julia --color=yes --project=.buildkite test/MatrixFields/multiple_field_solve_reproducer_1.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: matrix multiplication recursion example (CPU)" - key: cpu_matrix_multiplication_recursion - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_multiplication_recursion.jl" - - - label: "Unit: matrix multiplication recursion example (GPU)" - key: gpu_matrix_multiplication_recursion - command: "julia --color=yes --project=.buildkite test/MatrixFields/matrix_multiplication_recursion.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: field_matrix_indexing (CPU)" - key: cpu_field_matrix_indexing - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_matrix_indexing.jl" - - - label: "Unit: field_matrix_indexing (GPU)" - key: gpu_field_matrix_indexing - command: "julia --color=yes --project=.buildkite test/MatrixFields/field_matrix_indexing.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Unit: MatrixFields - broadcasting (CPU)" - steps: - - # scalar - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_1 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_1.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_2 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_2.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_3 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_3.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_4 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_4.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_5 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_5.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_6 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_6.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_7 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_7.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_8 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_8.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_9 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_9.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_10 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_10.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_11 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_11.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_12 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_12.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_13 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_13.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_14 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_14.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_15 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_15.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_16 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_16.jl" - - # non-scalar - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_non_scalar_1 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_1.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_non_scalar_2 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_2.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_non_scalar_3 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_3.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_non_scalar_4 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_4.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_non_scalar_5 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_5.jl" - soft_fail: true - - - group: "Unit: MatrixFields - broadcasting (GPU)" - steps: - - # scalar - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_1 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_1.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_2 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_2.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_3 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_3.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_4 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_4.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_5 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_5.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_6 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_6.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_7 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_7.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_8 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_8.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_9 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_9.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_10 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_10.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_11 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_11.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_12 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_12.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_13 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_13.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_14 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_14.jl" - soft_fail: true - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_15 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_15.jl" - soft_fail: true - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_16 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_16.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - # non-scalar - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_non_scalar_1 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_1.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_non_scalar_2 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_2.jl" - soft_fail: true - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_non_scalar_3 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_3.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_non_scalar_4 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_4.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_non_scalar_5 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_5.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - group: "Unit: Hypsography" - steps: - - - label: "Unit: hypsography 2d" - key: unit_hypsography_2d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Hypsography/2d.jl" - - - label: "Unit: 3dsphere" - key: unit_3dsphere - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Hypsography/3dsphere.jl" - - - group: "Unit: InputOutput" - steps: - - - label: "Unit: read_type" - key: unit_read_type - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_read_type.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 3GB - - - label: "Unit: spectralelement2d" - key: unit_spectralelement2d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_spectralelement2d.jl" - - - label: "Unit: spectralelement2d" - key: unit_spectralelement2d_gpu - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_spectralelement2d.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: hybrid2dbox" - key: unit_hybrid2dbox - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox.jl" - - - label: "Unit: hybrid2dbox topography" - key: unit_hybrid2dbox_topography - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox_topography.jl" - - - label: "Unit: hybrid2dbox stretched" - key: unit_hybrid2dbox_stretched - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox_stretched.jl" - - - label: "Unit: hybrid3dbox" - key: unit_hybrid3dbox - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dbox.jl" - - - label: "Unit: hybrid3dcubedsphere" - key: unit_hybrid3dcubedsphere - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere.jl" - - - label: "Unit: hybrid3dcubedsphere topography" - key: unit_hybrid3dcubedsphere_topography - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere_topography.jl" - - - label: "Unit: hybrid3dcubedsphere topography" - key: unit_hybrid3dcubedsphere_topography_gpu - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere_topography.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: finitedifference" - key: unit_finitedifference - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_finitedifference.jl" - - - label: "Unit: Parallel HDF5 IO tests" - key: "cpu_parallel_hdf5" - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere.jl" - timeout_in_minutes: 5 - env: - CLIMACOMMS_CONTEXT: "MPI" - retry: - automatic: true - agents: - slurm_nodes: 3 - slurm_tasks_per_node: 1 - - - group: "Unit: Remapping" - steps: - - - label: "Unit: interpolate array" - key: unit_interpolate_array - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/interpolate_array.jl" - - - group: "Unit: Limiters" - steps: - - - label: "Unit: limiter" - key: unit_limiter - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Limiters/limiter.jl" - - - label: "Unit: limiter cuda" - key: unit_limiter_gpu - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Limiters/limiter.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: distributed limiters" - key: unit_limiters_distributed - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Limiters/distributed/dlimiter.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 3 - - # TODO: improve performance label: [inference, allocs, flops, latency, benchmark, boundscheck] - # TODO: use perf env for perf jobs, or merge test/perf envs - - group: "Perf: Geometry" - steps: - - - label: "Perf: Axis tensor conversion benchmarks" - key: "cpu_axis_tensor_conversion_perf_bm" - command: "julia --color=yes --project=.buildkite test/Geometry/axistensor_conversion_benchmarks.jl" - - - group: "Perf: DataLayouts" - steps: - - - label: "Perf: DataLayouts fill" - key: "cpu_datalayouts_fill" - command: "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_fill.jl" - - - label: "Perf: DataLayouts copyto!" - key: "cpu_datalayouts_copyto" - command: "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_copyto.jl" - - - label: "Perf: DataLayouts fill" - key: "gpu_datalayouts_fill" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_fill.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: DataLayouts copyto" - key: "gpu_datalayouts_copyto" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_copyto.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Perf: Fields" - steps: - - - label: "Perf: Field broadcast" - key: "cpu_field_perf" - command: "julia --color=yes --project=.buildkite test/Fields/field_opt.jl" - - - group: "Perf: Benchmark scripts" - steps: - - - label: "Perf: benchmark scripts index_swapping" - key: perf_index_swapping - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite benchmarks/scripts/index_swapping.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: benchmark scripts indexing_and_static_ndranges" - key: indexing_and_static_ndranges - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite benchmarks/scripts/indexing_and_static_ndranges.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: benchmark scripts thermo_bench_bw" - key: thermo_bench_bw - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite benchmarks/scripts/thermo_bench_bw.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: benchmark scripts benchmark_offset" - key: benchmark_offset - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite benchmarks/scripts/benchmark_offset.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: benchmark scripts benchmark_field_last" - key: benchmark_field_last - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite benchmarks/scripts/benchmark_field_last.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Perf: Operators" - steps: - - - label: "Perf: SEM operators" - key: perf_SEM - command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/opt.jl" - - - label: "Perf: FD operators" - key: perf_FD - command: "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt.jl" - - # TODO: combine this with FD operators above - - label: "Perf: FD operators from the wild" - key: perf_FD_ops_examples - command: - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt_examples.jl" - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_examples.jl" - - - label: "Perf: FD operators from the wild (gpu)" - key: perf_FD_ops_examples_gpu - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt_examples.jl" - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_examples.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: dss" - key: perf_dss - command: "julia --color=yes --project=.buildkite test/Operators/hybrid/dss_opt.jl" - - - label: "Perf: hybrid operators" - key: perf_hybrid_ops - command: "julia --color=yes --project=.buildkite test/Operators/hybrid/opt.jl" - - - label: "Perf: FD operator stencil benchmarks" - key: "perf_fd_ops" - command: "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_stencils.jl" - agents: - slurm_mem: 20GB - - - label: "Perf: GPU FD operator stencil benchmarks" - key: "gpu_perf_fd_ops" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_stencils.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: SEM operator benchmarks (cuda Float32)" - key: "perf_gpu_spectral_ops_cuda_float32" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float32" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: SEM operator benchmarks (CPU Float32)" - key: "perf_gpu_spectral_ops_cpu_float32" - command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float32" - - - label: "Perf: SEM operator benchmarks (cuda Float64)" - key: "perf_gpu_spectral_ops_cuda_float64" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: SEM operator benchmarks (CPU Float64)" - key: "perf_gpu_spectral_ops_cpu_float64" - command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64" - - - label: "Perf: SEM operator benchmarks (extruded CPU Float64)" - key: "perf_gpu_spectral_ops_extruded_cpu_float64" - command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64 --space-type ExtrudedFiniteDifferenceSpace" - - - label: "Perf: SEM operator benchmarks" - key: "perf_gpu_spectral_ops" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: Integrals (CPU)" - key: "cpu_integrals_perf" - command: - - "julia --color=yes --project=.buildkite test/Operators/integrals.jl" - - - group: "Examples: Column" - steps: - - - label: ":computer: Column Heat Diffusion Eq" - key: "cpu_column_heat" - command: - - "julia --color=yes --project=.buildkite examples/column/heat.jl" - artifact_paths: - - "examples/column/output/heat/*" - - - label: ":computer: Column Advection Step Eq" - key: "cpu_column_step_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/step.jl" - artifact_paths: - - "examples/column/output/advect_step_function/*" - - - label: ":computer: Column Advection Eq" - key: "cpu_column_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/advect.jl" - artifact_paths: - - "examples/column/output/advect/*" - - - label: ":computer: Column FCT Advection Eq" - key: "cpu_fct_column_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/fct_advection.jl" - artifact_paths: - - "examples/column/output/fct_advection/*" - - - label: ":computer: Column TVD Slope-limited Advection Eq" - key: "cpu_tvd_column_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/tvd_advection.jl" - artifact_paths: - - "examples/column/output/tvd_advection/*" - - - label: ":computer: Column Lin vanLeer Limiter Advection Eq" - key: "cpu_lvl_column_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/vanleer_advection.jl" - artifact_paths: - - "examples/column/output/vanleer_advection/*" - - - label: ":computer: Column Lin vanLeer Limiter Advection Eq cuda" - key: "gpu_lvl_column_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/vanleer_advection.jl" - artifact_paths: - - "examples/column/output/vanleer_advection/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":computer: Column BB FCT Advection Eq" - key: "cpu_bb_fct_column_advect" - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite examples/column/bb_fct_advection.jl" - artifact_paths: - - "examples/column/output/bb_fct_advection/*" - - - label: ":computer: Column Zalesak FCT Advection Eq" - key: "cpu_zalesak_fct_column_advect" - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite examples/column/zalesak_fct_advection.jl" - artifact_paths: - - "examples/column/output/zalesak_fct_advection/*" - - - label: ":computer: Column Advection Diffusion Eq" - key: "cpu_column_advect_diff" - command: - - "julia --color=yes --project=.buildkite examples/column/advect_diffusion.jl" - artifact_paths: - - "examples/column/output/advect_diffusion/*" - - - label: ":computer: Column Ekman Eq" - key: "cpu_column_ekman" - command: - - "julia --color=yes --project=.buildkite examples/column/ekman.jl" - artifact_paths: - - "examples/column/output/ekman/*" - - - label: ":computer: Column Hydrostatic Ekman Eq" - key: "cpu_column_hydrostatic_ekman" - command: - - "julia --color=yes --project=.buildkite examples/column/hydrostatic_ekman.jl" - artifact_paths: - - "examples/column/output/hydrostatic_ekman/*" - - - label: ":computer: Column Wave Eq" - key: "cpu_column_wave" - command: - - "julia --color=yes --project=.buildkite examples/column/wave.jl" - artifact_paths: - - "examples/column/output/wave/*" - - - label: ":computer: Column Hydrostatic Balance Eq" - key: "cpu_column_hydrostatic" - command: - - "julia --color=yes --project=.buildkite examples/column/hydrostatic.jl" - artifact_paths: - - "examples/column/output/hydrostatic/*" - - - label: ":computer: Column Hydrostatic Balance Eq with discretely balanced initial condition" - key: "cpu_column_hydrostatic_discrete" - command: - - "julia --color=yes --project=.buildkite examples/column/hydrostatic_discrete.jl" - artifact_paths: - - "examples/column/output/hydrostatic_discrete/*" - - - group: "Examples: Spectral element" - steps: - - label: ":computer: Bickley jet CG" - key: "cpu_bickleyjet_cg" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg.jl" - artifact_paths: - - "examples/bickleyjet/output/cg/*" - - - label: ":computer: Bickley jet CG unstructured mesh" - key: "cpu_bickleyjet_cg_unsmesh" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_unsmesh.jl" - artifact_paths: - - "examples/bickleyjet/output/cg_unsmesh/*" - - - label: ":computer: Bickley jet CG vector invariant hyperviscosity" - key: "cpu_bickleyjet_cg_invariant_hypervisc" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_invariant_hypervisc.jl" - artifact_paths: - - "examples/bickleyjet/output/cg_invariant_hypervisc/*" - - - label: ":computer: MPI Bickley jet CG vector invariant hyperviscosity" - key: "cpu_mpi_bickleyjet_cg_invariant_hypervisc" - command: - - "srun julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_invariant_hypervisc.jl" - artifact_paths: - - "examples/bickleyjet/output/cg_invariant_hypervisc/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 4 - - - label: ":computer: Bickley jet DG rusanov" - key: "cpu_bickleyjet_dg_rusanov" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl rusanov" - artifact_paths: - - "examples/bickleyjet/output/dg_rusanov/*" - - - label: ":computer: Bickley jet DG roe" - key: "cpu_bickleyjet_dg_roe" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl roe" - artifact_paths: - - "examples/bickleyjet/output/dg_roe/*" - - - label: ":computer: Bickley jet DG roe noslip" - key: "cpu_bickleyjet_dg_roe_noslip" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl roe noslip" - artifact_paths: - - "examples/bickleyjet/output/dg_roe_noslip/*" - - - label: ":computer: Plane limiters advection cosine bells" - key: "cpu_cg_plane_advection_limiter_cosine_bells" - command: - - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl" - artifact_paths: - - "examples/plane/output/plane_advection_limiter_cosine_bells_D0/*" - - - label: ":computer: Plane limiters advection Gaussian bells" - key: "cpu_cg_plane_advection_limiter_gaussian_bells" - command: - - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl gaussian_bells" - artifact_paths: - - "examples/plane/output/plane_advection_limiter_gaussian_bells_D0/*" - - - label: ":computer: Plane limiters advection cylinders" - key: "cpu_cg_plane_advection_limiter_cylinders" - command: - - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl cylinders" - artifact_paths: - - "examples/plane/output/plane_advection_limiter_cylinders_D0/*" - - - group: "Examples: Hybrid" - steps: - - - label: ":computer: 3D Box limiters advection cosine bells" - key: "cpu_box_advection_limiter_cosine_bells" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl" - artifact_paths: - - "examples/hybrid/box/output/box_advection_limiter_cosine_bells_D0/*" - - - label: ":computer: 3D Box limiters advection Gaussian bells" - key: "cpu_box_advection_limiter_gaussian_bells" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl gaussian_bells" - artifact_paths: - - "examples/hybrid/box/output/box_advection_limiter_gaussian_bells_D0/*" - - - label: ":computer: 3D Box limiters advection Gaussian bells GPU" - key: "gpu_box_advection_limiter_gaussian_bells" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl gaussian_bells" - artifact_paths: - - "examples/hybrid/box/output/box_advection_limiter_gaussian_bells_D0/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":computer: Float 32 3D sphere baroclinic wave (ρe) HF datalayout GPU" - key: "gpu_baroclinic_wave_rho_e_float32_hf" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhoe_hf/Float32/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhoe_hf" - FLOAT_TYPE: "Float32" - horizontal_layout_type: "IJHF" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":computer: 3D Box limiters advection slotted spheres" - key: "cpu_box_advection_limiter_slotted_spheres" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl slotted_spheres" - artifact_paths: - - "examples/hybrid/box/output/box_advection_limiter_slotted_spheres_D0/*" - - - label: ":computer: Isothermal channel flow 2D hybrid (ρe)" - key: "cpu_isothermal_channel_2d_hybrid_rhoe" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/isothermal_channel.jl" - artifact_paths: - - "examples/hybrid/plane/output/iso_channel_2d/*" - - - label: ":computer: Rising Bubble 3D hybrid (ρθ)" - key: "cpu_rising_bubble_3d_hybrid_rhotheta" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_rhotheta.jl" - artifact_paths: - - "examples/hybrid/box/output/bubble_3d_rhotheta/*" - - - label: ":computer: Rising Bubble 2D hybrid invariant (ρe)" - key: "cpu_rising_bubble_2d_hybrid_invariant_rhoe" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/bubble_2d_invariant_rhoe.jl" - artifact_paths: - - "examples/hybrid/plane/output/bubble_2d_invariant_rhoe/*" - - - label: ":computer: Rising Bubble 3D hybrid invariant (ρθ)" - key: "cpu_rising_bubble_3d_hybrid_invariant_rhotheta" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhotheta.jl" - artifact_paths: - - "examples/hybrid/box/output/bubble_3d_invariant_rhotheta/*" - - - label: ":computer: Rising Bubble 3D hybrid invariant (ρe)" - key: "cpu_rising_bubble_3d_hybrid_invariant_rhoe" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" - artifact_paths: - - "examples/hybrid/box/output/bubble_3d_invariant_rhoe/*" - - - label: ":flower_playing_cards: Rising Bubble 3D hybrid invariant (ρe)" - key: "gpu_rising_bubble_3d_hybrid_invariant_rhoe" - command: -# - "nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" - - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" - artifact_paths: - - "examples/hybrid/box/output/gpu_bubble_3d_invariant_rhoe/*_low_*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":flower_playing_cards: Rising Bubble 3D hybrid invariant (ρe), custom resolution" - key: "gpu_rising_bubble_3d_hybrid_invariant_rhoe_custom" - command: -# - "nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl Float64 custom 1000 1000 4 16 3 0.05 700.0" - - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl Float64 custom 1000 1000 4 16 3 0.05 700.0" - artifact_paths: - - "examples/hybrid/box/output/gpu_bubble_3d_invariant_rhoe/*_custom_*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":computer: Density current 2D hybrid invariant total energy" - key: "cpu_density_current_2d_hybrid_invariant_total_energy" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/density_current_2dinvariant_rhoe.jl" - artifact_paths: - - "examples/hybrid/plane/output/dc_invariant_etot/*" - - - label: ":computer: Nonhydrostatic Agnesi Mountain total energy (topography mesh interface)" - key: "cpu_agnesi_mtn_2d_hybrid_invariant_total_energy_topography" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/topo_agnesi_nh.jl" - artifact_paths: - - "examples/hybrid/plane/output/agnesi_etot_nh/*" - - - label: ":computer: Schar Mountain total energy (topography mesh interface)" - key: "cpu_schaer_mtn_2d_hybrid_invariant_total_energy_topography" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/topo_schar_nh.jl" - artifact_paths: - - "examples/hybrid/plane/output/schar_etot_nh/*" - - - label: ":computer: Density current 2D hybrid conservative form potential temperature" - key: "cpu_density_current_2d_hybrid_conservative_potential_temperature" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/density_current_2d_flux_form.jl" - artifact_paths: - - "examples/hybrid/plane/output/dc_fluxform/*" - - - label: ":computer: MPI Rising Bubble 3D hybrid invariant (ρe)" - key: "cpu_mpi_rising_bubble_3d_hybrid_invariant_rhoe" - command: - - "srun julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" - artifact_paths: - - "examples/hybrid/box/output/bubble_3d_invariant_rhoe/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - - - group: "Examples: Sphere" - steps: - - - label: ":computer: Solid body sphere cosine bell alpha0" - key: "cpu_solidbody_cg_sphere_cosine_bell_alpha0" - command: - - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl" - artifact_paths: - - "examples/sphere/output/cg_sphere_solidbody_cosine_bell_alpha0/*" - - - label: ":computer: Solid body sphere cosine bell alpha45" - key: "cpu_solidbody_cg_sphere_cosine_bell_alpha45" - command: - - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl cosine_bell alpha45" - artifact_paths: - - "examples/sphere/output/cg_sphere_solidbody_cosine_bell_alpha45/*" - - - label: ":computer: Solid body sphere Gaussian bell alpha0" - key: "cpu_solidbody_cg_sphere_gaussian_bell_alpha0" - command: - - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl gaussian_bell" - artifact_paths: - - "examples/sphere/output/cg_sphere_solidbody_gaussian_bell_alpha0/*" - - - label: ":computer: Solid body sphere Gaussian bell alpha45" - key: "cpu_solidbody_cg_sphere_gaussian_bell_alpha45" - command: - - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl gaussian_bell alpha45" - artifact_paths: - - "examples/sphere/output/cg_sphere_solidbody_gaussian_bell_alpha45/*" - - - label: ":computer: Sphere limiters advection cosine bells" - key: "cpu_cg_sphere_advection_limiter_cosine_bells" - command: - - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl" - artifact_paths: - - "examples/sphere/output/cg_sphere_advection_limiter_cosine_bells/*" - - - label: ":computer: Sphere limiters advection Gaussian bells" - key: "cpu_cg_advection_limiter_gaussian_bells" - command: - - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl gaussian_bells" - artifact_paths: - - "examples/sphere/output/cg_sphere_advection_limiter_gaussian_bells/*" - - - label: ":computer: Sphere limiters advection cylinders" - key: "cpu_cg_advection_limiter_cylinders" - command: - - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl cylinders" - artifact_paths: - - "examples/sphere/output/cg_sphere_advection_limiter_cylinders/*" - - - label: ":computer: Steady-state shallow water 2D sphere alpha0" - key: "cpu_shallowwater_2d_cg_sphere_alpha0" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_steady_state_alpha0/*" - - - label: ":computer: Shallow-water 2D sphere steady-state alpha45" - key: "cpu_shallowwater_2d_cg_sphere_alpha45" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state alpha45" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_steady_state_alpha45/*" - - - label: ":computer: Shallow-water 2D sphere steady-state with compact support alpha0" - key: "cpu_shallowwater_2d_cg_sphere_compact_alpha0" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state_compact" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_steady_state_compact_alpha0/*" - - - label: ":computer: Shallow-water 2D sphere steady-state with compact support alpha60" - key: "cpu_shallowwater_2d_cg_sphere_compact_alpha60" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state_compact alpha60" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_steady_state_compact_alpha60/*" - - - label: ":computer: Shallow-water 2D sphere barotropic instability alpha0" - key: "cpu_shallowwater_2d_cg_sphere_barotropic_alpha0" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/*" - - - label: ":computer: MPI Shallow-water 2D sphere barotropic instability alpha0" - key: "cpu_mpi_shallowwater_2d_cg_sphere_barotropic_alpha0" - command: -# - "nsys profile --trace=nvtx,mpi --mpi-impl=openmpi --output=examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/report.%q{NPROCS} mpiexec julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" - - "mpiexec julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - NPROCS: 2 - agents: - slurm_nodes: 1 - slurm_tasks_per_node: 2 - - - label: ":computer: Shallow-water 2D sphere barotropic instability alpha30" - key: "cpu_shallowwater_2d_cg_sphere_barotropic_alpha30" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability alpha30" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha30/*" - - - label: ":computer: Shallow-water 2D sphere mountain alpha0" - key: "cpu_nonuniform_shallowwater_2d_cg_sphere" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl mountain" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_mountain_alpha0/*" - - - label: ":computer: Shallow-water 2D sphere Rossby Haurwitz" - key: "cpu_rossbyhaurwitz_2d_cg_sphere" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl rossby_haurwitz" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_rossby_haurwitz_alpha0/*" - - - label: ":flower_playing_cards: CUDA Shallow-water 2D sphere" - key: "cuda_shallowwater_2d_cg_sphere" - command: - - mkdir -p output/$$BUILDKITE_STEP_KEY -# - nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/sphere/shallow_water_cuda.jl - - julia --color=yes --project=.buildkite examples/sphere/shallow_water_cuda.jl - artifact_paths: - - output/cuda_shallowwater_2d_cg_sphere - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Examples: hybrid sphere" - steps: - - - label: ":computer: 3D sphere deformation flow w/ limiter & FCT" - key: "cpu_3d_deformation_flow" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/sphere/deformation_flow.jl" - artifact_paths: - - "examples/hybrid/sphere/output/deformation_flow/*" - - - label: ":computer: 3D sphere Hadley circulation" - key: "cpu_3d_hadley_circulation" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/sphere/hadley_circulation.jl" - artifact_paths: - - "examples/hybrid/sphere/output/hadley_circulation/*" - agents: - slurm_mem: 20GB - - - label: ":computer: Float 64 3D sphere baroclinic wave (ρe)" - key: "cpu_baroclinic_wave_rho_e_float64" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float64/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhoe" - FLOAT_TYPE: "Float64" - - - label: ":computer: Float 64 3D sphere baroclinic wave (ρe) HF datalayout" - key: "cpu_baroclinic_wave_rho_e_float64_hf" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhoe_hf/Float64/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhoe_hf" - FLOAT_TYPE: "Float64" - horizontal_layout_type: "IJHF" - - - label: ":computer: 3D sphere baroclinic wave (ρe)" - key: "cpu_baroclinic_wave_rho_e" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float32/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhoe" - - - label: ":computer: MPI 3D sphere baroclinic wave (ρe)" - key: "cpu_mpi_baroclinic_wave_rho_e" - command: - - "srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float32/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhoe" - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - - - - label: ":computer: 3D sphere baroclinic wave (ρθ)" - key: "cpu_baroclinic_wave_rho_theta" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhotheta/Float32/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhotheta" - - - label: ":computer: 3D sphere nonhydrostatic gravity wave" - key: "cpu_nonhydrostatic_gravity_wave" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/sphere/nonhydrostatic_gravity_wave.jl" - artifact_paths: - - "examples/hybrid/sphere/output/nonhydrostatic_gravity_wave/*" - - - label: ":computer: 3D sphere solid-body rotation" - key: "cpu_solid_body_rotation" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/sphere/solid_body_rotation_3d.jl" - - - label: ":computer: 3D sphere hydrostatically and geostrophically balanced flow (ρe)" - key: "cpu_balanced_flow_rho_e" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/balanced_flow_rhoe/Float32/*" - env: - TEST_NAME: "sphere/balanced_flow_rhoe" - - - label: ":computer: 3D sphere hydrostatically and geostrophically balanced flow (ρθ)" - key: "cpu_balanced_flow_rho_theta" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/balanced_flow_rhotheta/Float32/*" - env: - TEST_NAME: "sphere/balanced_flow_rhotheta" - - - label: ":computer: 3D sphere dry Held-Suarez (ρe)" - key: "cpu_held_suarez_rho_e" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/held_suarez_rhoe/Float32/*" - env: - TEST_NAME: "sphere/held_suarez_rhoe" - - - label: ":computer: Float64 3D sphere dry Held-Suarez (ρθ)" - key: "cpu_held_suarez_rho_theta_float64" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/held_suarez_rhotheta/Float64/*" - env: - TEST_NAME: "sphere/held_suarez_rhotheta" - FLOAT_TYPE: "Float64" - - - label: ":computer: 3D sphere dry Held-Suarez (ρθ)" - key: "cpu_held_suarez_rho_theta" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/held_suarez_rhotheta/Float32/*" - env: - TEST_NAME: "sphere/held_suarez_rhotheta" - - - label: ":computer: 3D sphere dry Held-Suarez (ρe_int)" - key: "cpu_held_suarez_rho_e_int" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/held_suarez_rhoe_int/Float32/*" - env: - TEST_NAME: "sphere/held_suarez_rhoe_int" - - - group: "Examples: hybrid plane" - steps: - - - label: ":computer: 2D plane inertial gravity wave" - key: "cpu_inertial_gravity_wave" - command: - - "julia --threads 8 --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/plane/output/inertial_gravity_wave/Float32/*" - env: - TEST_NAME: "plane/inertial_gravity_wave" - agents: - slurm_cpus_per_task: 8 - slurm_mem: 20GB - - - label: ":computer: stretched 2D plane inertial gravity wave" - key: "cpu_stretch_inertial_gravity_wave" - command: - - "julia --threads 8 --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/plane/output/stretched_inertial_gravity_wave/Float32/*" - env: - TEST_NAME: "plane/inertial_gravity_wave" - Z_STRETCH: "true" - agents: - slurm_cpus_per_task: 8 - slurm_mem: 20GB + - "julia --color=yes --project=.buildkite test/Limiters/vertical_mass_borrowing_limiter_advection.jl" + env: + CLIMACOMMS_DEVICE: "CUDA" + agents: + slurm_gpus: 1 + +# - label: "Unit: distributed limiters" +# key: unit_limiters_distributed +# command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Limiters/distributed/dlimiter.jl" +# env: +# CLIMACOMMS_CONTEXT: "MPI" +# agents: +# slurm_ntasks: 3 + +# # TODO: improve performance label: [inference, allocs, flops, latency, benchmark, boundscheck] +# # TODO: use perf env for perf jobs, or merge test/perf envs +# - group: "Perf: Geometry" +# steps: + +# - label: "Perf: Axis tensor conversion benchmarks" +# key: "cpu_axis_tensor_conversion_perf_bm" +# command: "julia --color=yes --project=.buildkite test/Geometry/axistensor_conversion_benchmarks.jl" + +# - group: "Perf: DataLayouts" +# steps: + +# - label: "Perf: DataLayouts fill" +# key: "cpu_datalayouts_fill" +# command: "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_fill.jl" + +# - label: "Perf: DataLayouts copyto!" +# key: "cpu_datalayouts_copyto" +# command: "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_copyto.jl" + +# - label: "Perf: DataLayouts fill" +# key: "gpu_datalayouts_fill" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_fill.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: DataLayouts copyto" +# key: "gpu_datalayouts_copyto" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_copyto.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - group: "Perf: Fields" +# steps: + +# - label: "Perf: Field broadcast" +# key: "cpu_field_perf" +# command: "julia --color=yes --project=.buildkite test/Fields/field_opt.jl" + +# - group: "Perf: Benchmark scripts" +# steps: + +# - label: "Perf: benchmark scripts index_swapping" +# key: perf_index_swapping +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite benchmarks/scripts/index_swapping.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: benchmark scripts indexing_and_static_ndranges" +# key: indexing_and_static_ndranges +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite benchmarks/scripts/indexing_and_static_ndranges.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: benchmark scripts thermo_bench_bw" +# key: thermo_bench_bw +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite benchmarks/scripts/thermo_bench_bw.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: benchmark scripts benchmark_offset" +# key: benchmark_offset +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite benchmarks/scripts/benchmark_offset.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: benchmark scripts benchmark_field_last" +# key: benchmark_field_last +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite benchmarks/scripts/benchmark_field_last.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - group: "Perf: Operators" +# steps: + +# - label: "Perf: SEM operators" +# key: perf_SEM +# command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/opt.jl" + +# - label: "Perf: FD operators" +# key: perf_FD +# command: "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt.jl" + +# # TODO: combine this with FD operators above +# - label: "Perf: FD operators from the wild" +# key: perf_FD_ops_examples +# command: +# - "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt_examples.jl" +# - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_examples.jl" + +# - label: "Perf: FD operators from the wild (gpu)" +# key: perf_FD_ops_examples_gpu +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt_examples.jl" +# - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_examples.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: dss" +# key: perf_dss +# command: "julia --color=yes --project=.buildkite test/Operators/hybrid/dss_opt.jl" + +# - label: "Perf: hybrid operators" +# key: perf_hybrid_ops +# command: "julia --color=yes --project=.buildkite test/Operators/hybrid/opt.jl" + +# - label: "Perf: FD operator stencil benchmarks" +# key: "perf_fd_ops" +# command: "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_stencils.jl" +# agents: +# slurm_mem: 20GB + +# - label: "Perf: GPU FD operator stencil benchmarks" +# key: "gpu_perf_fd_ops" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_stencils.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: SEM operator benchmarks (cuda Float32)" +# key: "perf_gpu_spectral_ops_cuda_float32" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float32" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: SEM operator benchmarks (CPU Float32)" +# key: "perf_gpu_spectral_ops_cpu_float32" +# command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float32" + +# - label: "Perf: SEM operator benchmarks (cuda Float64)" +# key: "perf_gpu_spectral_ops_cuda_float64" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: SEM operator benchmarks (CPU Float64)" +# key: "perf_gpu_spectral_ops_cpu_float64" +# command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64" + +# - label: "Perf: SEM operator benchmarks (extruded CPU Float64)" +# key: "perf_gpu_spectral_ops_extruded_cpu_float64" +# command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64 --space-type ExtrudedFiniteDifferenceSpace" + +# - label: "Perf: SEM operator benchmarks" +# key: "perf_gpu_spectral_ops" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: Integrals (CPU)" +# key: "cpu_integrals_perf" +# command: +# - "julia --color=yes --project=.buildkite test/Operators/integrals.jl" + +# - group: "Examples: Column" +# steps: + +# - label: ":computer: Column Heat Diffusion Eq" +# key: "cpu_column_heat" +# command: +# - "julia --color=yes --project=.buildkite examples/column/heat.jl" +# artifact_paths: +# - "examples/column/output/heat/*" + +# - label: ":computer: Column Advection Step Eq" +# key: "cpu_column_step_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/step.jl" +# artifact_paths: +# - "examples/column/output/advect_step_function/*" + +# - label: ":computer: Column Advection Eq" +# key: "cpu_column_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/advect.jl" +# artifact_paths: +# - "examples/column/output/advect/*" + +# - label: ":computer: Column FCT Advection Eq" +# key: "cpu_fct_column_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/fct_advection.jl" +# artifact_paths: +# - "examples/column/output/fct_advection/*" + +# - label: ":computer: Column TVD Slope-limited Advection Eq" +# key: "cpu_tvd_column_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/tvd_advection.jl" +# artifact_paths: +# - "examples/column/output/tvd_advection/*" + +# - label: ":computer: Column Lin vanLeer Limiter Advection Eq" +# key: "cpu_lvl_column_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/vanleer_advection.jl" +# artifact_paths: +# - "examples/column/output/vanleer_advection/*" + +# - label: ":computer: Column Lin vanLeer Limiter Advection Eq cuda" +# key: "gpu_lvl_column_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/vanleer_advection.jl" +# artifact_paths: +# - "examples/column/output/vanleer_advection/*" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: ":computer: Column BB FCT Advection Eq" +# key: "cpu_bb_fct_column_advect" +# command: +# - "julia --color=yes --check-bounds=yes --project=.buildkite examples/column/bb_fct_advection.jl" +# artifact_paths: +# - "examples/column/output/bb_fct_advection/*" + +# - label: ":computer: Column Zalesak FCT Advection Eq" +# key: "cpu_zalesak_fct_column_advect" +# command: +# - "julia --color=yes --check-bounds=yes --project=.buildkite examples/column/zalesak_fct_advection.jl" +# artifact_paths: +# - "examples/column/output/zalesak_fct_advection/*" + +# - label: ":computer: Column Advection Diffusion Eq" +# key: "cpu_column_advect_diff" +# command: +# - "julia --color=yes --project=.buildkite examples/column/advect_diffusion.jl" +# artifact_paths: +# - "examples/column/output/advect_diffusion/*" + +# - label: ":computer: Column Ekman Eq" +# key: "cpu_column_ekman" +# command: +# - "julia --color=yes --project=.buildkite examples/column/ekman.jl" +# artifact_paths: +# - "examples/column/output/ekman/*" + +# - label: ":computer: Column Hydrostatic Ekman Eq" +# key: "cpu_column_hydrostatic_ekman" +# command: +# - "julia --color=yes --project=.buildkite examples/column/hydrostatic_ekman.jl" +# artifact_paths: +# - "examples/column/output/hydrostatic_ekman/*" + +# - label: ":computer: Column Wave Eq" +# key: "cpu_column_wave" +# command: +# - "julia --color=yes --project=.buildkite examples/column/wave.jl" +# artifact_paths: +# - "examples/column/output/wave/*" + +# - label: ":computer: Column Hydrostatic Balance Eq" +# key: "cpu_column_hydrostatic" +# command: +# - "julia --color=yes --project=.buildkite examples/column/hydrostatic.jl" +# artifact_paths: +# - "examples/column/output/hydrostatic/*" + +# - label: ":computer: Column Hydrostatic Balance Eq with discretely balanced initial condition" +# key: "cpu_column_hydrostatic_discrete" +# command: +# - "julia --color=yes --project=.buildkite examples/column/hydrostatic_discrete.jl" +# artifact_paths: +# - "examples/column/output/hydrostatic_discrete/*" + +# - group: "Examples: Spectral element" +# steps: +# - label: ":computer: Bickley jet CG" +# key: "cpu_bickleyjet_cg" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg.jl" +# artifact_paths: +# - "examples/bickleyjet/output/cg/*" + +# - label: ":computer: Bickley jet CG unstructured mesh" +# key: "cpu_bickleyjet_cg_unsmesh" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_unsmesh.jl" +# artifact_paths: +# - "examples/bickleyjet/output/cg_unsmesh/*" + +# - label: ":computer: Bickley jet CG vector invariant hyperviscosity" +# key: "cpu_bickleyjet_cg_invariant_hypervisc" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_invariant_hypervisc.jl" +# artifact_paths: +# - "examples/bickleyjet/output/cg_invariant_hypervisc/*" + +# - label: ":computer: MPI Bickley jet CG vector invariant hyperviscosity" +# key: "cpu_mpi_bickleyjet_cg_invariant_hypervisc" +# command: +# - "srun julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_invariant_hypervisc.jl" +# artifact_paths: +# - "examples/bickleyjet/output/cg_invariant_hypervisc/*" +# env: +# CLIMACOMMS_CONTEXT: "MPI" +# agents: +# slurm_ntasks: 4 + +# - label: ":computer: Bickley jet DG rusanov" +# key: "cpu_bickleyjet_dg_rusanov" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl rusanov" +# artifact_paths: +# - "examples/bickleyjet/output/dg_rusanov/*" + +# - label: ":computer: Bickley jet DG roe" +# key: "cpu_bickleyjet_dg_roe" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl roe" +# artifact_paths: +# - "examples/bickleyjet/output/dg_roe/*" + +# - label: ":computer: Bickley jet DG roe noslip" +# key: "cpu_bickleyjet_dg_roe_noslip" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl roe noslip" +# artifact_paths: +# - "examples/bickleyjet/output/dg_roe_noslip/*" + +# - label: ":computer: Plane limiters advection cosine bells" +# key: "cpu_cg_plane_advection_limiter_cosine_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl" +# artifact_paths: +# - "examples/plane/output/plane_advection_limiter_cosine_bells_D0/*" + +# - label: ":computer: Plane limiters advection Gaussian bells" +# key: "cpu_cg_plane_advection_limiter_gaussian_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl gaussian_bells" +# artifact_paths: +# - "examples/plane/output/plane_advection_limiter_gaussian_bells_D0/*" + +# - label: ":computer: Plane limiters advection cylinders" +# key: "cpu_cg_plane_advection_limiter_cylinders" +# command: +# - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl cylinders" +# artifact_paths: +# - "examples/plane/output/plane_advection_limiter_cylinders_D0/*" + +# - group: "Examples: Hybrid" +# steps: + +# - label: ":computer: 3D Box limiters advection cosine bells" +# key: "cpu_box_advection_limiter_cosine_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl" +# artifact_paths: +# - "examples/hybrid/box/output/box_advection_limiter_cosine_bells_D0/*" + +# - label: ":computer: 3D Box limiters advection Gaussian bells" +# key: "cpu_box_advection_limiter_gaussian_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl gaussian_bells" +# artifact_paths: +# - "examples/hybrid/box/output/box_advection_limiter_gaussian_bells_D0/*" + +# - label: ":computer: 3D Box limiters advection Gaussian bells GPU" +# key: "gpu_box_advection_limiter_gaussian_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl gaussian_bells" +# artifact_paths: +# - "examples/hybrid/box/output/box_advection_limiter_gaussian_bells_D0/*" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: ":computer: Float 32 3D sphere baroclinic wave (ρe) HF datalayout GPU" +# key: "gpu_baroclinic_wave_rho_e_float32_hf" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhoe_hf/Float32/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhoe_hf" +# FLOAT_TYPE: "Float32" +# horizontal_layout_type: "IJHF" +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: ":computer: 3D Box limiters advection slotted spheres" +# key: "cpu_box_advection_limiter_slotted_spheres" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl slotted_spheres" +# artifact_paths: +# - "examples/hybrid/box/output/box_advection_limiter_slotted_spheres_D0/*" + +# - label: ":computer: Isothermal channel flow 2D hybrid (ρe)" +# key: "cpu_isothermal_channel_2d_hybrid_rhoe" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/isothermal_channel.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/iso_channel_2d/*" + +# - label: ":computer: Rising Bubble 3D hybrid (ρθ)" +# key: "cpu_rising_bubble_3d_hybrid_rhotheta" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_rhotheta.jl" +# artifact_paths: +# - "examples/hybrid/box/output/bubble_3d_rhotheta/*" + +# - label: ":computer: Rising Bubble 2D hybrid invariant (ρe)" +# key: "cpu_rising_bubble_2d_hybrid_invariant_rhoe" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/bubble_2d_invariant_rhoe.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/bubble_2d_invariant_rhoe/*" + +# - label: ":computer: Rising Bubble 3D hybrid invariant (ρθ)" +# key: "cpu_rising_bubble_3d_hybrid_invariant_rhotheta" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhotheta.jl" +# artifact_paths: +# - "examples/hybrid/box/output/bubble_3d_invariant_rhotheta/*" + +# - label: ":computer: Rising Bubble 3D hybrid invariant (ρe)" +# key: "cpu_rising_bubble_3d_hybrid_invariant_rhoe" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" +# artifact_paths: +# - "examples/hybrid/box/output/bubble_3d_invariant_rhoe/*" + +# - label: ":flower_playing_cards: Rising Bubble 3D hybrid invariant (ρe)" +# key: "gpu_rising_bubble_3d_hybrid_invariant_rhoe" +# command: +# # - "nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" +# - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" +# artifact_paths: +# - "examples/hybrid/box/output/gpu_bubble_3d_invariant_rhoe/*_low_*" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: ":flower_playing_cards: Rising Bubble 3D hybrid invariant (ρe), custom resolution" +# key: "gpu_rising_bubble_3d_hybrid_invariant_rhoe_custom" +# command: +# # - "nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl Float64 custom 1000 1000 4 16 3 0.05 700.0" +# - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl Float64 custom 1000 1000 4 16 3 0.05 700.0" +# artifact_paths: +# - "examples/hybrid/box/output/gpu_bubble_3d_invariant_rhoe/*_custom_*" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: ":computer: Density current 2D hybrid invariant total energy" +# key: "cpu_density_current_2d_hybrid_invariant_total_energy" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/density_current_2dinvariant_rhoe.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/dc_invariant_etot/*" + +# - label: ":computer: Nonhydrostatic Agnesi Mountain total energy (topography mesh interface)" +# key: "cpu_agnesi_mtn_2d_hybrid_invariant_total_energy_topography" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/topo_agnesi_nh.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/agnesi_etot_nh/*" + +# - label: ":computer: Schar Mountain total energy (topography mesh interface)" +# key: "cpu_schaer_mtn_2d_hybrid_invariant_total_energy_topography" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/topo_schar_nh.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/schar_etot_nh/*" + +# - label: ":computer: Density current 2D hybrid conservative form potential temperature" +# key: "cpu_density_current_2d_hybrid_conservative_potential_temperature" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/density_current_2d_flux_form.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/dc_fluxform/*" + +# - label: ":computer: MPI Rising Bubble 3D hybrid invariant (ρe)" +# key: "cpu_mpi_rising_bubble_3d_hybrid_invariant_rhoe" +# command: +# - "srun julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" +# artifact_paths: +# - "examples/hybrid/box/output/bubble_3d_invariant_rhoe/*" +# env: +# CLIMACOMMS_CONTEXT: "MPI" +# agents: +# slurm_ntasks: 2 + +# - group: "Examples: Sphere" +# steps: + +# - label: ":computer: Solid body sphere cosine bell alpha0" +# key: "cpu_solidbody_cg_sphere_cosine_bell_alpha0" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_solidbody_cosine_bell_alpha0/*" + +# - label: ":computer: Solid body sphere cosine bell alpha45" +# key: "cpu_solidbody_cg_sphere_cosine_bell_alpha45" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl cosine_bell alpha45" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_solidbody_cosine_bell_alpha45/*" + +# - label: ":computer: Solid body sphere Gaussian bell alpha0" +# key: "cpu_solidbody_cg_sphere_gaussian_bell_alpha0" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl gaussian_bell" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_solidbody_gaussian_bell_alpha0/*" + +# - label: ":computer: Solid body sphere Gaussian bell alpha45" +# key: "cpu_solidbody_cg_sphere_gaussian_bell_alpha45" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl gaussian_bell alpha45" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_solidbody_gaussian_bell_alpha45/*" + +# - label: ":computer: Sphere limiters advection cosine bells" +# key: "cpu_cg_sphere_advection_limiter_cosine_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_advection_limiter_cosine_bells/*" + +# - label: ":computer: Sphere limiters advection Gaussian bells" +# key: "cpu_cg_advection_limiter_gaussian_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl gaussian_bells" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_advection_limiter_gaussian_bells/*" + +# - label: ":computer: Sphere limiters advection cylinders" +# key: "cpu_cg_advection_limiter_cylinders" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl cylinders" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_advection_limiter_cylinders/*" + +# - label: ":computer: Steady-state shallow water 2D sphere alpha0" +# key: "cpu_shallowwater_2d_cg_sphere_alpha0" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_steady_state_alpha0/*" + +# - label: ":computer: Shallow-water 2D sphere steady-state alpha45" +# key: "cpu_shallowwater_2d_cg_sphere_alpha45" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state alpha45" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_steady_state_alpha45/*" + +# - label: ":computer: Shallow-water 2D sphere steady-state with compact support alpha0" +# key: "cpu_shallowwater_2d_cg_sphere_compact_alpha0" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state_compact" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_steady_state_compact_alpha0/*" + +# - label: ":computer: Shallow-water 2D sphere steady-state with compact support alpha60" +# key: "cpu_shallowwater_2d_cg_sphere_compact_alpha60" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state_compact alpha60" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_steady_state_compact_alpha60/*" + +# - label: ":computer: Shallow-water 2D sphere barotropic instability alpha0" +# key: "cpu_shallowwater_2d_cg_sphere_barotropic_alpha0" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/*" + +# - label: ":computer: MPI Shallow-water 2D sphere barotropic instability alpha0" +# key: "cpu_mpi_shallowwater_2d_cg_sphere_barotropic_alpha0" +# command: +# # - "nsys profile --trace=nvtx,mpi --mpi-impl=openmpi --output=examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/report.%q{NPROCS} mpiexec julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" +# - "mpiexec julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/*" +# env: +# CLIMACOMMS_CONTEXT: "MPI" +# NPROCS: 2 +# agents: +# slurm_nodes: 1 +# slurm_tasks_per_node: 2 + +# - label: ":computer: Shallow-water 2D sphere barotropic instability alpha30" +# key: "cpu_shallowwater_2d_cg_sphere_barotropic_alpha30" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability alpha30" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha30/*" + +# - label: ":computer: Shallow-water 2D sphere mountain alpha0" +# key: "cpu_nonuniform_shallowwater_2d_cg_sphere" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl mountain" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_mountain_alpha0/*" + +# - label: ":computer: Shallow-water 2D sphere Rossby Haurwitz" +# key: "cpu_rossbyhaurwitz_2d_cg_sphere" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl rossby_haurwitz" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_rossby_haurwitz_alpha0/*" + +# - label: ":flower_playing_cards: CUDA Shallow-water 2D sphere" +# key: "cuda_shallowwater_2d_cg_sphere" +# command: +# - mkdir -p output/$$BUILDKITE_STEP_KEY +# # - nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/sphere/shallow_water_cuda.jl +# - julia --color=yes --project=.buildkite examples/sphere/shallow_water_cuda.jl +# artifact_paths: +# - output/cuda_shallowwater_2d_cg_sphere +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - group: "Examples: hybrid sphere" +# steps: + +# - label: ":computer: 3D sphere deformation flow w/ limiter & FCT" +# key: "cpu_3d_deformation_flow" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/sphere/deformation_flow.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/deformation_flow/*" + +# - label: ":computer: 3D sphere Hadley circulation" +# key: "cpu_3d_hadley_circulation" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/sphere/hadley_circulation.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/hadley_circulation/*" +# agents: +# slurm_mem: 20GB + +# - label: ":computer: Float 64 3D sphere baroclinic wave (ρe)" +# key: "cpu_baroclinic_wave_rho_e_float64" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float64/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhoe" +# FLOAT_TYPE: "Float64" + +# - label: ":computer: Float 64 3D sphere baroclinic wave (ρe) HF datalayout" +# key: "cpu_baroclinic_wave_rho_e_float64_hf" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhoe_hf/Float64/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhoe_hf" +# FLOAT_TYPE: "Float64" +# horizontal_layout_type: "IJHF" + +# - label: ":computer: 3D sphere baroclinic wave (ρe)" +# key: "cpu_baroclinic_wave_rho_e" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float32/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhoe" + +# - label: ":computer: MPI 3D sphere baroclinic wave (ρe)" +# key: "cpu_mpi_baroclinic_wave_rho_e" +# command: +# - "srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float32/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhoe" +# CLIMACOMMS_CONTEXT: "MPI" +# agents: +# slurm_ntasks: 2 + + +# - label: ":computer: 3D sphere baroclinic wave (ρθ)" +# key: "cpu_baroclinic_wave_rho_theta" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhotheta/Float32/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhotheta" + +# - label: ":computer: 3D sphere nonhydrostatic gravity wave" +# key: "cpu_nonhydrostatic_gravity_wave" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/sphere/nonhydrostatic_gravity_wave.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/nonhydrostatic_gravity_wave/*" + +# - label: ":computer: 3D sphere solid-body rotation" +# key: "cpu_solid_body_rotation" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/sphere/solid_body_rotation_3d.jl" + +# - label: ":computer: 3D sphere hydrostatically and geostrophically balanced flow (ρe)" +# key: "cpu_balanced_flow_rho_e" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/balanced_flow_rhoe/Float32/*" +# env: +# TEST_NAME: "sphere/balanced_flow_rhoe" + +# - label: ":computer: 3D sphere hydrostatically and geostrophically balanced flow (ρθ)" +# key: "cpu_balanced_flow_rho_theta" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/balanced_flow_rhotheta/Float32/*" +# env: +# TEST_NAME: "sphere/balanced_flow_rhotheta" + +# - label: ":computer: 3D sphere dry Held-Suarez (ρe)" +# key: "cpu_held_suarez_rho_e" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/held_suarez_rhoe/Float32/*" +# env: +# TEST_NAME: "sphere/held_suarez_rhoe" + +# - label: ":computer: Float64 3D sphere dry Held-Suarez (ρθ)" +# key: "cpu_held_suarez_rho_theta_float64" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/held_suarez_rhotheta/Float64/*" +# env: +# TEST_NAME: "sphere/held_suarez_rhotheta" +# FLOAT_TYPE: "Float64" + +# - label: ":computer: 3D sphere dry Held-Suarez (ρθ)" +# key: "cpu_held_suarez_rho_theta" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/held_suarez_rhotheta/Float32/*" +# env: +# TEST_NAME: "sphere/held_suarez_rhotheta" + +# - label: ":computer: 3D sphere dry Held-Suarez (ρe_int)" +# key: "cpu_held_suarez_rho_e_int" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/held_suarez_rhoe_int/Float32/*" +# env: +# TEST_NAME: "sphere/held_suarez_rhoe_int" + +# - group: "Examples: hybrid plane" +# steps: + +# - label: ":computer: 2D plane inertial gravity wave" +# key: "cpu_inertial_gravity_wave" +# command: +# - "julia --threads 8 --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/inertial_gravity_wave/Float32/*" +# env: +# TEST_NAME: "plane/inertial_gravity_wave" +# agents: +# slurm_cpus_per_task: 8 +# slurm_mem: 20GB + +# - label: ":computer: stretched 2D plane inertial gravity wave" +# key: "cpu_stretch_inertial_gravity_wave" +# command: +# - "julia --threads 8 --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/stretched_inertial_gravity_wave/Float32/*" +# env: +# TEST_NAME: "plane/inertial_gravity_wave" +# Z_STRETCH: "true" +# agents: +# slurm_cpus_per_task: 8 +# slurm_mem: 20GB - group: "Analysis" steps: From 331b10b2e27225362868d1b372ff277d403e4570 Mon Sep 17 00:00:00 2001 From: imreddyTeja Date: Wed, 22 Oct 2025 16:05:03 -0700 Subject: [PATCH 5/7] Fix pulse func type inf for gpu --- ...rtical_mass_borrowing_limiter_advection.jl | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/test/Limiters/vertical_mass_borrowing_limiter_advection.jl b/test/Limiters/vertical_mass_borrowing_limiter_advection.jl index 8fafcd1a79..45c6d88046 100644 --- a/test/Limiters/vertical_mass_borrowing_limiter_advection.jl +++ b/test/Limiters/vertical_mass_borrowing_limiter_advection.jl @@ -10,6 +10,7 @@ using OrdinaryDiffEqSSPRK: ODEProblem, solve, SSPRK33 using ClimaCore.CommonGrids using ClimaCore.Grids using ClimaTimeSteppers +import ClimaCore import ClimaCore: Fields, @@ -56,7 +57,7 @@ end function tendency!(yₜ, y, parameters, t) (; w, Δt, limiter) = parameters FT = Spaces.undertype(axes(y.q)) - bcvel = pulse(-π, t, z₀, zₕ, z₁) + bcvel = pulse(-π, t, z₀, zₕ, z₁, speed) divf2c = Operators.DivergenceF2C( bottom = Operators.SetValue(Geometry.WVector(FT(bcvel))), top = Operators.SetValue(Geometry.WVector(FT(0))), @@ -84,7 +85,7 @@ z₀ = FT(0.0) zₕ = FT(1.0) z₁ = FT(1.0) speed = FT(-1.0) -pulse(z, t, z₀, zₕ, z₁) = abs(z - speed * t) ≤ zₕ ? z₁ : z₀ +pulse(z, t, z₀, zₕ, z₁, speed) = abs(z - speed * t) ≤ zₕ ? z₁ : z₀ stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(FT(7.0))) plot_string = ["uniform", "stretched"] @@ -125,7 +126,7 @@ plot_string = ["uniform", "stretched"] O = ones(FT, fspace) # Initial condition - q_init = pulse.(z, 0.0, z₀, zₕ, z₁) + q_init = pulse.(z, 0.0, z₀, zₕ, z₁, speed) q = q_init coords = Fields.coordinate_field(q) ρ = map(coord -> 1.0, coords) @@ -153,16 +154,16 @@ plot_string = ["uniform", "stretched"] q_init = sol.u[1].q q_final = sol.u[end].q - q_analytic = pulse.(z, t₁, z₀, zₕ, z₁) + q_analytic = pulse.(z, t₁, z₀, zₕ, z₁, speed) err = norm(q_final .- q_analytic) rel_mass_err = norm((sum(q_final) - sum(q_init)) / sum(q_init)) if use_column p = Plots.plot() - Plots.plot!(q_init, label = "init") - Plots.plot!(q_final, label = "computed") - Plots.plot!(q_analytic, label = "analytic") + Plots.plot!(q_init |> ClimaCore.to_cpu, label = "init") + Plots.plot!(q_final |> ClimaCore.to_cpu, label = "computed") + Plots.plot!(q_analytic |> ClimaCore.to_cpu, label = "analytic") Plots.plot!(; legend = :topright) Plots.plot!(; xlabel = "q", title = "VerticalMassBorrowingLimiter") f = joinpath( @@ -174,18 +175,18 @@ plot_string = ["uniform", "stretched"] colidx = Fields.ColumnIndex((1, 1), 1) p = Plots.plot() Plots.plot!( - vec(parent(q_init[colidx])), - vec(parent(z[colidx])), + vec(parent(q_init[colidx] |> ClimaCore.to_cpu)), + vec(parent(z[colidx] |> ClimaCore.to_cpu)), label = "init", ) Plots.plot!( - vec(parent(q_final[colidx])), - vec(parent(z[colidx])), + vec(parent(q_final[colidx] |> ClimaCore.to_cpu)), + vec(parent(z[colidx] |> ClimaCore.to_cpu)), label = "computed", ) Plots.plot!( - vec(parent(q_analytic[colidx])), - vec(parent(z[colidx])), + vec(parent(q_analytic[colidx] |> ClimaCore.to_cpu)), + vec(parent(z[colidx] |> ClimaCore.to_cpu)), label = "analytic", ) Plots.plot!(; legend = :topright) From df84b163c3882aece68409febff3d79084c408ef Mon Sep 17 00:00:00 2001 From: imreddyTeja Date: Fri, 24 Oct 2025 09:35:09 -0700 Subject: [PATCH 6/7] Flip vertical indexing --- ext/cuda/limiters.jl | 6 +-- .../vertical_mass_borrowing_limiter.jl | 46 +------------------ 2 files changed, 5 insertions(+), 47 deletions(-) diff --git a/ext/cuda/limiters.jl b/ext/cuda/limiters.jl index 871e72ff54..ebc797b62e 100644 --- a/ext/cuda/limiters.jl +++ b/ext/cuda/limiters.jl @@ -344,7 +344,8 @@ function apply_limiter_kernel!( borrowed_mass = zero(eltype(q_data)) @inbounds if h_idx <= Nh # TODO: unroll this? - for v in 1:Nv + for i in 0:(Nv - 1) # CUDA.jl recommends avoiding stepranges + v = Nv - i CI = CartesianIndex(i_idx, j_idx, f_idx, v, h_idx) ρΔV_lev = getindex_field(ρ_data, CI) * getindex_field(Δz_data, CI) new_mass = getindex_field(q_data, CI) - (borrowed_mass / ρΔV_lev) @@ -356,8 +357,7 @@ function apply_limiter_kernel!( setindex_field!(q_data, q_min, CI) end end - for i in 0:(Nv - 1) # CUDA.jl recommends avoiding stepranges - v = Nv - i + for v in 1:Nv if borrowed_mass > zero(borrowed_mass) CI = CartesianIndex(i_idx, j_idx, f_idx, v, h_idx) ρΔV_lev = getindex_field(ρ_data, CI) * getindex_field(Δz_data, CI) diff --git a/src/Limiters/vertical_mass_borrowing_limiter.jl b/src/Limiters/vertical_mass_borrowing_limiter.jl index 64a223ecd2..fe5a2cec69 100644 --- a/src/Limiters/vertical_mass_borrowing_limiter.jl +++ b/src/Limiters/vertical_mass_borrowing_limiter.jl @@ -78,7 +78,6 @@ end function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # column fields # TODO: maybe rm some stuff from struct or cache (; bmass, ic, q_min) = cache - # TODO: verify index direction Δz = Fields.Δz_field(q) Δz_vals = Fields.field_values(Δz) (; J) = Fields.local_geometry_field(ρ) @@ -94,7 +93,7 @@ function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # c q_vals = Fields.field_values(q) # top to bottom for f in 1:DataLayouts.ncomponents(q_vals) - for v in 1:nlevels + for v in nlevels:-1:1 CI = CartesianIndex(1, 1, f, v, 1) # new mass in the current layer ρΔV_lev = @@ -114,7 +113,7 @@ function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # c end # bottom to top - for v in nlevels:-1:1 + for v in 1:nlevels CI = CartesianIndex(1, 1, f, v, 1) # if the surface layer still needs to borrow mass if bmass[] < 0 @@ -138,44 +137,3 @@ function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # c return nothing end - - -function basic_lim(q, ρ = ones(Float64, length(q)), v = ones(Float64, length(q))) - ic = 0 - bmass = 0.0 - @show q - println("") - for i in 1:length(q) - @show i - ρΔV_lev = ρ[i] * v[i] - nmass = q[i] + bmass / ρΔV_lev - if nmass > 0.0 - q[i] = nmass - bmass = 0.0 - else - bmass = nmass * ρΔV_lev - q[i] = 0.0 - end - @show q - println("") - end - for i in length(q):-1:1 - @show i - if bmass < 0.0 - ρΔV_lev = ρ[i] * v[i] - nmass = q[i] + bmass / ρΔV_lev - @show nmass - if nmass > 0 - q[i] = nmass - @show q - return - bmass = 0.0 - else - bmass = nmass * ρΔV_lev - q[i] = 0.0 - end - @show q - println("") - end - end -end From 1c98cca9ba0df38aef217f1e940d1d833c92f96a Mon Sep 17 00:00:00 2001 From: imreddyTeja Date: Mon, 17 Nov 2025 14:30:34 -0800 Subject: [PATCH 7/7] clean up comments --- ext/cuda/limiters.jl | 30 ++----------------- .../vertical_mass_borrowing_limiter.jl | 1 + 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/ext/cuda/limiters.jl b/ext/cuda/limiters.jl index ebc797b62e..f18c51a7b6 100644 --- a/ext/cuda/limiters.jl +++ b/ext/cuda/limiters.jl @@ -303,7 +303,7 @@ function apply_limiter!( # if Ni * Nj * Nf > 64 nthread_x = Ni * Nj nthread_y = Nf - nthread_z = cld(64, nthread_x * nthread_y) + nthread_z = cld(64, nthread_x * nthread_y) # ensure block is at least 64 threads nthreads = (nthread_x, nthread_y, nthread_z) nblocks = cld(ncols * Nf, prod(nthreads)) @@ -338,12 +338,11 @@ function apply_limiter_kernel!( j_idx = j_idx + Int32(1) i_idx = i_idx + Int32(1) f_idx = CUDA.threadIdx().y + # each z in a block is a different element h_idx = CUDA.blockDim().z * (CUDA.blockIdx().x - Int32(1)) + CUDA.threadIdx().z - # tidx = (CUDA.blockIdx().x - Int32(1)) * CUDA.blockDim().x + CUDA.threadIdx().x - Int32(1) q_min = q_min_tuple[f_idx] borrowed_mass = zero(eltype(q_data)) @inbounds if h_idx <= Nh - # TODO: unroll this? for i in 0:(Nv - 1) # CUDA.jl recommends avoiding stepranges v = Nv - i CI = CartesianIndex(i_idx, j_idx, f_idx, v, h_idx) @@ -374,28 +373,3 @@ function apply_limiter_kernel!( end return end - -# PLAN -# Implementation 1: each thread handles a column. This likely will not most of the gpu until h >= 128 -# 1.1: no shmem -# 1.2: shmem -# 1.3 split fields? -# Implementation 2: each block handles a column? -#= - -while sync_threads_and(lev_mass < lim || bottom_lev) - borrowed_from_lower = lim - lev_mass - borrowed_from_me = gotten_from_shuffle - lev_mass = lim - borrowed_from_me -end - -while lev_mass < lim - borrowing_from_above = lev_mass - borrowed_from_me = gotten_from_shuffle - lev_mass = lev_mass = lim - borrowed_from_me -end - - -=# - -# Implementation 3: each warp handles each col diff --git a/src/Limiters/vertical_mass_borrowing_limiter.jl b/src/Limiters/vertical_mass_borrowing_limiter.jl index fe5a2cec69..95e4f6c60f 100644 --- a/src/Limiters/vertical_mass_borrowing_limiter.jl +++ b/src/Limiters/vertical_mass_borrowing_limiter.jl @@ -30,6 +30,7 @@ function VerticalMassBorrowingLimiter(f::Fields.Field, q_min) ic = similar(Spaces.level(f, 1)) return VerticalMassBorrowingLimiter(bmass, ic, q_min) end +# TODO: should q_min be a field that varies in space? """