diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b6cc3860c9..9b8c3a7943 100755 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -61,2068 +61,2092 @@ steps: key: unit_recursive_apply command: "julia --color=yes --check-bounds=yes --project=.buildkite test/RecursiveApply/unit_recursive_apply.jl" - - group: "Unit: Utilities" - steps: - - - label: "Unit: plushalf" - key: unit_plushalf - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Utilities/unit_plushalf.jl" - - - group: "Unit: DataLayouts" - steps: - - - label: "Unit: data0d" - key: unit_data0d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data0d.jl" - - - label: "Unit: data_fill" - key: unit_data_fill - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_fill.jl" - - - label: "Unit: data_copyto" - key: unit_data_copyto - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_copyto.jl" - - - label: "Unit: cartesian_field_index" - key: unit_data_cartesian_field_index - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_cartesian_field_index.jl" - - - label: "Unit: non_extruded_broadcast" - key: unit_non_extruded_broadcast - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_non_extruded_broadcast.jl" - - - label: "Unit: mapreduce" - key: unit_data_mapreduce - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" - - - label: "Unit: data_opt_similar" - key: data_opt_similar - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/opt_similar.jl" - - - label: "Unit: opt_universal_size" - key: opt_universal_size - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/opt_universal_size.jl" - - - label: "Unit: data_ndims" - key: unit_data_ndims - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_ndims.jl" - - - label: "Unit: unit_data2array" - key: unit_data2array - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_data2array.jl" - - - label: "Unit: data1d" - key: unit_data1d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data1d.jl" - - - label: "Unit: data2d" - key: unit_data2d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data2d.jl" - - - label: "Unit: data1dx" - key: unit_data1dx - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data1dx.jl" - - - label: "Unit: data2dx" - key: unit_data2dx - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data2dx.jl" - - - label: "Unit: data cuda" - key: unit_data_cuda - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/cuda.jl CUDA" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: data cuda threadblocks" - key: unit_data_threadblock - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_cuda_threadblocks.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: data fill" - key: gpu_unit_data_fill - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_fill.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: data mapreduce" - key: gpu_unit_data_mapreduce - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: data mapreduce (2 gpus)" - key: gpu_2_unit_data_mapreduce - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - CLIMACOMMS_CONTEXT: "MPI" - agents: - slum_ntasks: 2 - slurm_gpus: 2 - - - label: "Unit: data copyto" - key: gpu_unit_data_copyto - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_copyto.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Unit: Geometry" - steps: - - - label: "Unit: geometry" - key: unit_geometry - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/geometry.jl" - - - label: "Unit: axistensors" - key: unit_axistensors - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/axistensors.jl" - - - label: "Unit: rmul_with_projection" - key: unit_rmul_with_projection - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/rmul_with_projection.jl" - - - group: "Unit: Meshes" - steps: - - - label: "Unit: interval" - key: unit_interval - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/interval.jl" - - - label: "Unit: meshes rectangle" - key: unit_meshes_rectangle - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/rectangle.jl" - - - label: "Unit: meshes opt" - key: unit_meshes_opt - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/opt_meshes.jl" - - - label: "Unit: meshes cubedsphere" - key: unit_meshes_cubed_sphere - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/cubedsphere.jl" - - - group: "Unit: Topologies" - steps: - - - label: "Unit: topo interval" - key: unit_topo_interval - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/interval.jl" + # - group: "Unit: Utilities" + # steps: + + # - label: "Unit: plushalf" + # key: unit_plushalf + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Utilities/unit_plushalf.jl" + + # - group: "Unit: DataLayouts" + # steps: + + # - label: "Unit: data0d" + # key: unit_data0d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data0d.jl" + + # - label: "Unit: data_fill" + # key: unit_data_fill + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_fill.jl" + + # - label: "Unit: data_copyto" + # key: unit_data_copyto + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_copyto.jl" + + # - label: "Unit: cartesian_field_index" + # key: unit_data_cartesian_field_index + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_cartesian_field_index.jl" + + # - label: "Unit: non_extruded_broadcast" + # key: unit_non_extruded_broadcast + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_non_extruded_broadcast.jl" + + # - label: "Unit: mapreduce" + # key: unit_data_mapreduce + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" + + # - label: "Unit: data_opt_similar" + # key: data_opt_similar + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/opt_similar.jl" + + # - label: "Unit: opt_universal_size" + # key: opt_universal_size + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/opt_universal_size.jl" + + # - label: "Unit: data_ndims" + # key: unit_data_ndims + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_ndims.jl" + + # - label: "Unit: unit_data2array" + # key: unit_data2array + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_data2array.jl" + + # - label: "Unit: data1d" + # key: unit_data1d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data1d.jl" + + # - label: "Unit: data2d" + # key: unit_data2d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data2d.jl" + + # - label: "Unit: data1dx" + # key: unit_data1dx + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data1dx.jl" + + # - label: "Unit: data2dx" + # key: unit_data2dx + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/data2dx.jl" + + # - label: "Unit: data cuda" + # key: unit_data_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/cuda.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: data cuda threadblocks" + # key: unit_data_threadblock + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_cuda_threadblocks.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: data fill" + # key: gpu_unit_data_fill + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_fill.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: data mapreduce" + # key: gpu_unit_data_mapreduce + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: data mapreduce (2 gpus)" + # key: gpu_2_unit_data_mapreduce + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_mapreduce.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slum_ntasks: 2 + # slurm_gpus: 2 + + # - label: "Unit: data copyto" + # key: gpu_unit_data_copyto + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/DataLayouts/unit_copyto.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - group: "Unit: Geometry" + # steps: + + # - label: "Unit: geometry" + # key: unit_geometry + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/geometry.jl" + + # - label: "Unit: axistensors" + # key: unit_axistensors + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/axistensors.jl" + + # - label: "Unit: rmul_with_projection" + # key: unit_rmul_with_projection + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Geometry/rmul_with_projection.jl" + + # - group: "Unit: Meshes" + # steps: + + # - label: "Unit: interval" + # key: unit_interval + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/interval.jl" + + # - label: "Unit: meshes rectangle" + # key: unit_meshes_rectangle + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/rectangle.jl" + + # - label: "Unit: meshes opt" + # key: unit_meshes_opt + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/opt_meshes.jl" + + # - label: "Unit: meshes cubedsphere" + # key: unit_meshes_cubed_sphere + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Meshes/cubedsphere.jl" + + # - group: "Unit: Topologies" + # steps: + + # - label: "Unit: topo interval" + # key: unit_topo_interval + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/interval.jl" + + # - label: "Unit: topo rectangle" + # key: unit_topo_rectangle + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/rectangle.jl" + + # - label: "Unit: rectangle sfc" + # key: unit_rectangle_sfc + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/rectangle_sfc.jl" + + # - label: "Unit: cubedsphere" + # key: unit_cubedsphere + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/cubedsphere.jl" + + # - label: "Unit: cubedsphere sfc" + # key: unit_cubedsphere_sfc + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/cubedsphere_sfc.jl" + + # - label: "Unit: topologies distributed" + # key: unit_topologies_distributed + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/dtopo4.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 4 + + # - group: "Unit: Spaces" + # steps: + + # - label: "Unit: Quadratures" + # key: unit_quadrature + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Quadratures/Quadratures.jl" + + # - label: "Unit: spaces" + # key: unit_spaces + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_spaces.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/opt_spaces.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_high_resolution_space.jl" + + # - label: "Unit: cuda spaces" + # key: "gpu_cuda_spaces" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_spaces.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/opt_spaces.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_high_resolution_space.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: common spaces with CUDA" + # key: "gpu_common_cuda_spaces" + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: common spaces with CUDA and MPI" + # key: "gpu_common_cuda_mpi_spaces" + # command: + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_ntasks: 2 + + # - label: "Unit: common spaces with MPI" + # key: "common_cuda_mpi_spaces" + # command: + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + + # - label: "Unit: distributed cuda spaces" + # key: "gpu_distributed_extruded_cuda_spaces" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/space_construction.jl CUDA" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_gpus_per_task: 1 + # slurm_ntasks: 3 + + # - label: "Unit: ddss1" + # key: unit_ddss1 + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: ddss1 cs" + # key: unit_cuda_ddss1_cs + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1_cs.jl" + + # - label: "Unit: ddss1 cs" + # key: unit_ddss1_cs + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1_cs.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: sphere" + # key: unit_sphere + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/sphere.jl" + + # - label: "Unit: terrain warp" + # key: unit_terrain_warp + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/terrain_warp.jl" + + # - label: "Unit: distributed dss2" + # key: unit_distributed_dss2 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss2.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + + # - label: "Unit: distributed dss3" + # key: unit_distributed_dss3 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss3.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 3 + + # - label: "Unit: distributed dss4" + # key: unit_distributed_dss4 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss4.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 4 + + # - label: "Unit: distributed remapping (1 process)" + # key: distributed_remapping_1proc + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" + # env: + # CLIMACOMMS_DEVICE: "CPU" + + # - label: "Unit: distributed remapping (2 processes)" + # key: distributed_remapping_2procs + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CPU" + # agents: + # slurm_ntasks: 2 + + # - label: "Unit: distributed remapping with CUDA (1 process)" + # key: distributed_remapping_gpu_1proc + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: distributed remapping with CUDA (2 processes)" + # key: distributed_remapping_gpu_2procs + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" + # soft_fail: true + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 2 + # slurm_gpus_per_task: 1 + + # - label: "Unit: distributed gather" + # key: unit_distributed_gather4 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/gather4.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 4 + + # - label: "Unit: cuda extruded spaces" + # key: "extruded_spaces_cuda" + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/extruded_cuda.jl CUDA" + # artifact_paths: + # - output/extruded_spaces_cuda + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: cuda point spaces" + # key: "point_space_cuda" + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/point_cuda.jl CUDA" + # artifact_paths: + # - output/point_cuda + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: cuda dss 2-process test" + # key: "gpu_ddss2_test" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss2.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 2 + # slurm_gpus: 2 + + # - label: "Unit: cuda dss 3-process test" + # key: "gpu_ddss3_test" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss3.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 3 + # slurm_gpus: 3 + + # - label: "Unit: cuda dss 4-process test" + # key: "gpu_ddss4_test" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss4.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 4 + # slurm_gpus: 4 + + # - label: "Unit: cuda Cubed Sphere dss; ne = 32; 2-process test" + # key: "gpu_ddss_ne32_cs_2processes" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 2 + # slurm_gpus: 2 + + # - label: "Unit: cuda Cubed Sphere dss; ne = 32; 3-process test" + # key: "gpu_ddss_ne32_cs_3processes" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 3 + # slurm_gpus: 3 + + # - label: "Unit: cuda Cubed Sphere dss; ne = 32; 4-process test" + # key: "gpu_ddss_ne32_cs_4processes" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" + # timeout_in_minutes: 15 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_ntasks: 4 + # slurm_gpus: 4 + + # - group: "Unit: Fields" + # steps: + + # - label: "Unit: field" + # key: unit_field + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/unit_field.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_fieldvectors.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_field_multi_broadcast_fusion.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/convergence_field_integrals.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/inference_repro.jl" + # agents: + # slurm_mem: 20GB + + # - label: "Unit: field cuda" + # key: unit_field_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/unit_field.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_fieldvectors.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_field_multi_broadcast_fusion.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/convergence_field_integrals.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/inference_repro.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: reduction cuda" + # key: unit_reduction_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/reduction_cuda.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: distributed reduction cuda" + # key: unit_distributed_reduction_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/reduction_cuda_distributed.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_gpus_per_task: 1 + # slurm_ntasks: 2 + + # - group: "Unit: Operators" + # steps: + + # - label: "Unit: rectilinear" + # key: unit_rectilinear + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear.jl" + + # - label: "Unit: diffusion2d" + # key: unit_diffusion2d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_diffusion2d.jl" + + # - label: "Unit: sphere geometry" + # key: unit_sphere_geometry + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry.jl" + + # - label: "Unit: sphere gradient" + # key: unit_sphere_gradient + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_gradient.jl" + + # - label: "Unit: sphere divergence" + # key: unit_sphere_divergence + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_divergence.jl" + + # - label: "Unit: sphere curl" + # key: unit_sphere_curl + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_curl.jl" + + # - label: "Unit: sphere diffusion" + # key: unit_sphere_diffusion + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_diffusion.jl" + + # - label: "Unit: sphere diffusion vec" + # key: unit_sphere_diffusion_vec + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_diffusion_vec.jl" + + # - label: "Unit: sphere hyperdiffusion" + # key: unit_sphere_hyperdiffusion + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_sphere_hyperdiffusion.jl" + + # - label: "Unit: sphere hyperdiffusion vec" + # key: unit_sphere_hyperdiffusion_vec + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_sphere_hyperdiffusion_vec.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/convergence_sphere_hyperdiffusion_vec.jl" + + # - label: "Unit: spec ops plane" + # key: unit_spec_ops_plane + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/plane.jl" + + # - label: "Unit: FD operator (shmem)" + # key: unit_fd_operator_shmem + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_fd_ops_shared_memory.jl" + # - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_fd_ops_shared_memory.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: gpu columnwise" + # key: unit_gpu_columnwise + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_columnwise.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: columnwise" + # key: unit_columnwise + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_columnwise.jl" + + # - label: "Unit: column" + # key: unit_column + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_column.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/convergence_column.jl" + + # - label: "Unit: fd tensor" + # key: unit_fd_tensor + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/tensor.jl" + + # - label: "Unit: advection operator convergence" + # key: unit_adv_conv + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/convergence_advection_diffusion1d.jl" + # soft_fail: true + + # - label: "Unit: hyb ops 2d" + # key: unit_hyb_ops_2d + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_2d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_2d.jl" + + # - label: "Unit: hyb ops 2d CUDA" + # key: unit_hyb_ops_2d_cuda + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_2d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_2d.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: hyb ops 3d" + # key: unit_hyb_ops_3d + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_3d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_3d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_3d.jl" + + # - label: "Unit: hyb ops 3d CUDA" + # key: unit_hyb_ops_3d_cuda + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_3d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_3d.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_3d.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: remapping" + # key: unit_remapping + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/remapping.jl" + + # - label: "Unit: run sphere geometry distributed (2)" + # key: unit_run_sphere_geometry_distributed2 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 2 + # slurm_mem: 20GB + + # - label: "Unit: run sphere geometry distributed (3)" + # key: unit_run_sphere_geometry_distributed3 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 3 + # slurm_mem: 20GB + + # - label: "Unit: run sphere geometry distributed (4)" + # key: unit_run_sphere_geometry_distributed4 + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # agents: + # slurm_ntasks: 4 + # slurm_mem: 20GB + + # - label: "Unit: rectilinear cuda" + # key: unit_rectilinear_cuda + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear_cuda.jl" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear_cuda.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: operators on levels and extruded spaces" + # key: unit_cuda_operators_example + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/unit_operators_examples.jl" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/unit_operators_examples.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: velocity grad tensor ops" + # key: unit_spectral_tensor_op_cuda + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/covar_deriv_ops.jl" + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/covar_deriv_ops.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: hybrid operators cuda" + # key: unit_ops_cuda + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_cuda.jl" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_cuda.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: extruded sphere cuda" + # key: unit_extruded_sphere_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/extruded_sphere_cuda.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: extruded 3dbox cuda" + # key: unit_extruded_3dbox_cuda + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/extruded_3dbox_cuda.jl CUDA" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: Integrals (CPU)" + # key: "cpu_integrals" + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/integrals.jl" + + # - label: "Unit: Integrals (GPU)" + # key: "gpu_integrals" + # command: + # - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/integrals.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - group: "Unit: MatrixFields" + # steps: + + # - label: "Unit: BandMatrixRow" + # key: unit_band_matrix_row + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/band_matrix_row.jl" + + # - label: "Unit: field2arrays" + # key: unit_field2arrays + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field2arrays.jl" + + # - label: "Unit: matrix multiplication at boundaries" + # key: unit_matrix_multiplication_at_boundaries + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_multiplication_at_boundaries.jl" + + # - label: "Unit: operator matrices (CPU)" + # key: unit_operator_matrices_cpu + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/operator_matrices.jl" + + # - label: "Unit: operator matrices (GPU)" + # key: unit_operator_matrices_gpu + # command: "julia --color=yes --project=.buildkite test/MatrixFields/operator_matrices.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 40GB + + # - label: "Unit: field names" + # key: unit_field_names + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_names.jl" + + # - label: "Unit: field matrix solvers (CPU)" + # key: unit_field_matrix_solvers_cpu + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_matrix_solvers.jl" + # agents: + # slurm_mem: 40GB + + # - label: "Unit: field matrix solvers (GPU)" + # key: unit_field_matrix_solvers_gpu + # command: "julia --color=yes --project=.buildkite test/MatrixFields/field_matrix_solvers.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 40GB + + # - label: "Unit: bidiag matrix row example (CPU)" + # key: cpu_gpu_compat_bidiag_matrix_row + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/gpu_compat_bidiag_matrix_row.jl" + + # - label: "Unit: bidiag matrix row example (GPU)" + # key: gpu_compat_bidiag_matrix_row + # command: "julia --color=yes --project=.buildkite test/MatrixFields/gpu_compat_bidiag_matrix_row.jl" + # soft_fail: true + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: multiple field solve example (GPU)" + # key: gpu_multiple_field_solve_example + # command: "julia --color=yes --project=.buildkite test/MatrixFields/multiple_field_solve_reproducer_1.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: matrix multiplication recursion example (CPU)" + # key: cpu_matrix_multiplication_recursion + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_multiplication_recursion.jl" + + # - label: "Unit: matrix multiplication recursion example (GPU)" + # key: gpu_matrix_multiplication_recursion + # command: "julia --color=yes --project=.buildkite test/MatrixFields/matrix_multiplication_recursion.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: field_matrix_indexing (CPU)" + # key: cpu_field_matrix_indexing + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_matrix_indexing.jl" + + # - label: "Unit: field_matrix_indexing (GPU)" + # key: gpu_field_matrix_indexing + # command: "julia --color=yes --project=.buildkite test/MatrixFields/field_matrix_indexing.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - group: "Unit: MatrixFields - broadcasting (CPU)" + # steps: + + # # scalar + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_1 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_1.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_2 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_2.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_3 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_3.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_4 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_4.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_5 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_5.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_6 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_6.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_7 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_7.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_8 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_8.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_9 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_9.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_10 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_10.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_11 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_11.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_12 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_12.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_13 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_13.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_14 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_14.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_15 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_15.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_scalar_16 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_16.jl" + + # # non-scalar + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_non_scalar_1 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_1.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_non_scalar_2 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_2.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_non_scalar_3 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_3.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_non_scalar_4 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_4.jl" + + # - label: "Unit: matrix field broadcasting (CPU)" + # key: unit_matrix_field_broadcasting_cpu_non_scalar_5 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_5.jl" + # soft_fail: true + + # - group: "Unit: MatrixFields - broadcasting (GPU)" + # steps: + + # # scalar + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_1 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_1.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_2 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_2.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_3 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_3.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_4 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_4.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_5 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_5.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_6 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_6.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_7 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_7.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_8 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_8.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_9 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_9.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_10 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_10.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_11 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_11.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_12 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_12.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_13 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_13.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_14 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_14.jl" + # soft_fail: true + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_15 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_15.jl" + # soft_fail: true + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_scalar_16 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_16.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # # non-scalar + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_non_scalar_1 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_1.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_non_scalar_2 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_2.jl" + # soft_fail: true + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_non_scalar_3 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_3.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_non_scalar_4 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_4.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - label: "Unit: matrix field broadcasting (GPU)" + # key: unit_matrix_field_broadcasting_gpu_non_scalar_5 + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_5.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 10GB + + # - group: "Unit: Hypsography" + # steps: + + # - label: "Unit: hypsography 2d" + # key: unit_hypsography_2d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Hypsography/2d.jl" + + # - label: "Unit: 3dsphere" + # key: unit_3dsphere + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Hypsography/3dsphere.jl" + + # - group: "Unit: InputOutput" + # steps: + + # - label: "Unit: read_type" + # key: unit_read_type + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_read_type.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + # slurm_mem: 3GB + + # - label: "Unit: spectralelement2d" + # key: unit_spectralelement2d + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_spectralelement2d.jl" + + # - label: "Unit: spectralelement2d" + # key: unit_spectralelement2d_gpu + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_spectralelement2d.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: hybrid2dbox" + # key: unit_hybrid2dbox + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox.jl" + + # - label: "Unit: hybrid2dbox topography" + # key: unit_hybrid2dbox_topography + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox_topography.jl" + + # - label: "Unit: hybrid2dbox stretched" + # key: unit_hybrid2dbox_stretched + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox_stretched.jl" + + # - label: "Unit: hybrid3dbox" + # key: unit_hybrid3dbox + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dbox.jl" + + # - label: "Unit: hybrid3dcubedsphere" + # key: unit_hybrid3dcubedsphere + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere.jl" + + # - label: "Unit: hybrid3dcubedsphere topography" + # key: unit_hybrid3dcubedsphere_topography + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere_topography.jl" + + # - label: "Unit: hybrid3dcubedsphere topography" + # key: unit_hybrid3dcubedsphere_topography_gpu + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere_topography.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 + + # - label: "Unit: finitedifference" + # key: unit_finitedifference + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_finitedifference.jl" + + # - label: "Unit: Parallel HDF5 IO tests" + # key: "cpu_parallel_hdf5" + # command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere.jl" + # timeout_in_minutes: 5 + # env: + # CLIMACOMMS_CONTEXT: "MPI" + # retry: + # automatic: true + # agents: + # slurm_nodes: 3 + # slurm_tasks_per_node: 1 + + # - group: "Unit: Remapping" + # steps: + + # - label: "Unit: interpolate array" + # key: unit_interpolate_array + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/interpolate_array.jl" - - label: "Unit: topo rectangle" - key: unit_topo_rectangle - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/rectangle.jl" - - - label: "Unit: rectangle sfc" - key: unit_rectangle_sfc - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/rectangle_sfc.jl" - - - label: "Unit: cubedsphere" - key: unit_cubedsphere - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/cubedsphere.jl" - - - label: "Unit: cubedsphere sfc" - key: unit_cubedsphere_sfc - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/cubedsphere_sfc.jl" - - - label: "Unit: topologies distributed" - key: unit_topologies_distributed - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Topologies/dtopo4.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 4 - - - group: "Unit: Spaces" + - group: "Unit: Limiters" steps: - - label: "Unit: Quadratures" - key: unit_quadrature - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Quadratures/Quadratures.jl" - - - label: "Unit: spaces" - key: unit_spaces - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_spaces.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/opt_spaces.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_high_resolution_space.jl" - - - label: "Unit: cuda spaces" - key: "gpu_cuda_spaces" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_spaces.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/opt_spaces.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/unit_high_resolution_space.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: common spaces with CUDA" - key: "gpu_common_cuda_spaces" - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: common spaces with CUDA and MPI" - key: "gpu_common_cuda_mpi_spaces" - command: - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_ntasks: 2 - - - label: "Unit: common spaces with MPI" - key: "common_cuda_mpi_spaces" - command: - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/CommonSpaces/unit_common_spaces.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - - - label: "Unit: distributed cuda spaces" - key: "gpu_distributed_extruded_cuda_spaces" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/space_construction.jl CUDA" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_gpus_per_task: 1 - slurm_ntasks: 3 - - - label: "Unit: ddss1" - key: unit_ddss1 - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: ddss1 cs" - key: unit_cuda_ddss1_cs - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1_cs.jl" - - - label: "Unit: ddss1 cs" - key: unit_ddss1_cs - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/ddss1_cs.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: sphere" - key: unit_sphere - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/sphere.jl" - - - label: "Unit: terrain warp" - key: unit_terrain_warp - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/terrain_warp.jl" - - - label: "Unit: distributed dss2" - key: unit_distributed_dss2 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss2.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - - - label: "Unit: distributed dss3" - key: unit_distributed_dss3 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss3.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 3 - - - label: "Unit: distributed dss4" - key: unit_distributed_dss4 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/ddss4.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 4 - - - label: "Unit: distributed remapping (1 process)" - key: distributed_remapping_1proc - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" - env: - CLIMACOMMS_DEVICE: "CPU" - - - label: "Unit: distributed remapping (2 processes)" - key: distributed_remapping_2procs - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CPU" - agents: - slurm_ntasks: 2 - - - label: "Unit: distributed remapping with CUDA (1 process)" - key: distributed_remapping_gpu_1proc - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: distributed remapping with CUDA (2 processes)" - key: distributed_remapping_gpu_2procs - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/distributed_remapping.jl" - soft_fail: true - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 2 - slurm_gpus_per_task: 1 - - - label: "Unit: distributed gather" - key: unit_distributed_gather4 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed/gather4.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 4 - - - label: "Unit: cuda extruded spaces" - key: "extruded_spaces_cuda" - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/extruded_cuda.jl CUDA" - artifact_paths: - - output/extruded_spaces_cuda - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: cuda point spaces" - key: "point_space_cuda" - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/point_cuda.jl CUDA" - artifact_paths: - - output/point_cuda - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: cuda dss 2-process test" - key: "gpu_ddss2_test" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss2.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 2 - slurm_gpus: 2 - - - label: "Unit: cuda dss 3-process test" - key: "gpu_ddss3_test" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss3.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 3 - slurm_gpus: 3 - - - label: "Unit: cuda dss 4-process test" - key: "gpu_ddss4_test" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss4.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 4 - slurm_gpus: 4 - - - label: "Unit: cuda Cubed Sphere dss; ne = 32; 2-process test" - key: "gpu_ddss_ne32_cs_2processes" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 2 - slurm_gpus: 2 - - - label: "Unit: cuda Cubed Sphere dss; ne = 32; 3-process test" - key: "gpu_ddss_ne32_cs_3processes" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 3 - slurm_gpus: 3 - - - label: "Unit: cuda Cubed Sphere dss; ne = 32; 4-process test" - key: "gpu_ddss_ne32_cs_4processes" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Spaces/distributed_cuda/ddss_ne32_cs.jl" - timeout_in_minutes: 15 - env: - CLIMACOMMS_CONTEXT: "MPI" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_ntasks: 4 - slurm_gpus: 4 - - - group: "Unit: Fields" - steps: + # - label: "Unit: limiter" + # key: unit_limiter + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Limiters/limiter.jl" - - label: "Unit: field" - key: unit_field - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/unit_field.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_fieldvectors.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_field_multi_broadcast_fusion.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/convergence_field_integrals.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/inference_repro.jl" - agents: - slurm_mem: 20GB + # - label: "Unit: limiter cuda" + # key: unit_limiter_gpu + # command: + # - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" + # - "julia --color=yes --project=.buildkite test/Limiters/limiter.jl" + # env: + # CLIMACOMMS_DEVICE: "CUDA" + # agents: + # slurm_gpus: 1 - - label: "Unit: field cuda" - key: unit_field_cuda - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/unit_field.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_fieldvectors.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/benchmark_field_multi_broadcast_fusion.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/convergence_field_integrals.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/inference_repro.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 + # - label: "Unit: limiter" + # key: unit_limiter + # command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Limiters/limiter.jl" - - label: "Unit: reduction cuda" - key: unit_reduction_cuda + - label: "Unit: limiter cuda" + key: mass_borrow command: - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/reduction_cuda.jl CUDA" + - "julia --color=yes --project=.buildkite test/Limiters/vertical_mass_borrowing_limiter.jl" env: CLIMACOMMS_DEVICE: "CUDA" agents: slurm_gpus: 1 - - label: "Unit: distributed reduction cuda" - key: unit_distributed_reduction_cuda + - label: "Unit: limiter cuda" + key: mass_borrow_advection command: - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Fields/reduction_cuda_distributed.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_gpus_per_task: 1 - slurm_ntasks: 2 - - - group: "Unit: Operators" - steps: - - - label: "Unit: rectilinear" - key: unit_rectilinear - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear.jl" - - - label: "Unit: diffusion2d" - key: unit_diffusion2d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_diffusion2d.jl" - - - label: "Unit: sphere geometry" - key: unit_sphere_geometry - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry.jl" - - - label: "Unit: sphere gradient" - key: unit_sphere_gradient - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_gradient.jl" - - - label: "Unit: sphere divergence" - key: unit_sphere_divergence - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_divergence.jl" - - - label: "Unit: sphere curl" - key: unit_sphere_curl - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_curl.jl" - - - label: "Unit: sphere diffusion" - key: unit_sphere_diffusion - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_diffusion.jl" - - - label: "Unit: sphere diffusion vec" - key: unit_sphere_diffusion_vec - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_diffusion_vec.jl" - - - label: "Unit: sphere hyperdiffusion" - key: unit_sphere_hyperdiffusion - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_sphere_hyperdiffusion.jl" - - - label: "Unit: sphere hyperdiffusion vec" - key: unit_sphere_hyperdiffusion_vec - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/unit_sphere_hyperdiffusion_vec.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/convergence_sphere_hyperdiffusion_vec.jl" - - - label: "Unit: spec ops plane" - key: unit_spec_ops_plane - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/plane.jl" - - - label: "Unit: FD operator (shmem)" - key: unit_fd_operator_shmem - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_fd_ops_shared_memory.jl" - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_fd_ops_shared_memory.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: gpu columnwise" - key: unit_gpu_columnwise - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_columnwise.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: columnwise" - key: unit_columnwise - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_columnwise.jl" - - - label: "Unit: column" - key: unit_column - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/unit_column.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/convergence_column.jl" - - - label: "Unit: fd tensor" - key: unit_fd_tensor - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/tensor.jl" - - - label: "Unit: advection operator convergence" - key: unit_adv_conv - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/finitedifference/convergence_advection_diffusion1d.jl" - soft_fail: true - - - label: "Unit: hyb ops 2d" - key: unit_hyb_ops_2d - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_2d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_2d.jl" - - - label: "Unit: hyb ops 2d CUDA" - key: unit_hyb_ops_2d_cuda - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_2d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_2d.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: hyb ops 3d" - key: unit_hyb_ops_3d - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_3d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_3d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_3d.jl" - - - label: "Unit: hyb ops 3d CUDA" - key: unit_hyb_ops_3d_cuda - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_3d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/convergence_3d.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_3d.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: remapping" - key: unit_remapping - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/remapping.jl" - - - label: "Unit: run sphere geometry distributed (2)" - key: unit_run_sphere_geometry_distributed2 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - slurm_mem: 20GB - - - label: "Unit: run sphere geometry distributed (3)" - key: unit_run_sphere_geometry_distributed3 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 3 - slurm_mem: 20GB - - - label: "Unit: run sphere geometry distributed (4)" - key: unit_run_sphere_geometry_distributed4 - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/sphere_geometry_distributed.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 4 - slurm_mem: 20GB - - - label: "Unit: rectilinear cuda" - key: unit_rectilinear_cuda - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear_cuda.jl" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/rectilinear_cuda.jl CUDA" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: operators on levels and extruded spaces" - key: unit_cuda_operators_example - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/unit_operators_examples.jl" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/unit_operators_examples.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: velocity grad tensor ops" - key: unit_spectral_tensor_op_cuda - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/covar_deriv_ops.jl" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/spectralelement/covar_deriv_ops.jl CUDA" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: hybrid operators cuda" - key: unit_ops_cuda - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/unit_cuda.jl" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/simulation_cuda.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: extruded sphere cuda" - key: unit_extruded_sphere_cuda - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/extruded_sphere_cuda.jl CUDA" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: extruded 3dbox cuda" - key: unit_extruded_3dbox_cuda - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/hybrid/extruded_3dbox_cuda.jl CUDA" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: Integrals (CPU)" - key: "cpu_integrals" - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/integrals.jl" - - - label: "Unit: Integrals (GPU)" - key: "gpu_integrals" - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite test/Operators/integrals.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Unit: MatrixFields" - steps: - - - label: "Unit: BandMatrixRow" - key: unit_band_matrix_row - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/band_matrix_row.jl" - - - label: "Unit: field2arrays" - key: unit_field2arrays - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field2arrays.jl" - - - label: "Unit: matrix multiplication at boundaries" - key: unit_matrix_multiplication_at_boundaries - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_multiplication_at_boundaries.jl" - - - label: "Unit: operator matrices (CPU)" - key: unit_operator_matrices_cpu - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/operator_matrices.jl" - - - label: "Unit: operator matrices (GPU)" - key: unit_operator_matrices_gpu - command: "julia --color=yes --project=.buildkite test/MatrixFields/operator_matrices.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 40GB - - - label: "Unit: field names" - key: unit_field_names - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_names.jl" - - - label: "Unit: field matrix solvers (CPU)" - key: unit_field_matrix_solvers_cpu - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_matrix_solvers.jl" - agents: - slurm_mem: 40GB - - - label: "Unit: field matrix solvers (GPU)" - key: unit_field_matrix_solvers_gpu - command: "julia --color=yes --project=.buildkite test/MatrixFields/field_matrix_solvers.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 40GB - - - label: "Unit: bidiag matrix row example (CPU)" - key: cpu_gpu_compat_bidiag_matrix_row - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/gpu_compat_bidiag_matrix_row.jl" - - - label: "Unit: bidiag matrix row example (GPU)" - key: gpu_compat_bidiag_matrix_row - command: "julia --color=yes --project=.buildkite test/MatrixFields/gpu_compat_bidiag_matrix_row.jl" - soft_fail: true - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: multiple field solve example (GPU)" - key: gpu_multiple_field_solve_example - command: "julia --color=yes --project=.buildkite test/MatrixFields/multiple_field_solve_reproducer_1.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: matrix multiplication recursion example (CPU)" - key: cpu_matrix_multiplication_recursion - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_multiplication_recursion.jl" - - - label: "Unit: matrix multiplication recursion example (GPU)" - key: gpu_matrix_multiplication_recursion - command: "julia --color=yes --project=.buildkite test/MatrixFields/matrix_multiplication_recursion.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: field_matrix_indexing (CPU)" - key: cpu_field_matrix_indexing - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/field_matrix_indexing.jl" - - - label: "Unit: field_matrix_indexing (GPU)" - key: gpu_field_matrix_indexing - command: "julia --color=yes --project=.buildkite test/MatrixFields/field_matrix_indexing.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Unit: MatrixFields - broadcasting (CPU)" - steps: - - # scalar - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_1 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_1.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_2 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_2.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_3 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_3.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_4 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_4.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_5 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_5.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_6 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_6.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_7 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_7.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_8 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_8.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_9 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_9.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_10 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_10.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_11 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_11.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_12 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_12.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_13 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_13.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_14 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_14.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_15 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_15.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_scalar_16 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_16.jl" - - # non-scalar - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_non_scalar_1 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_1.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_non_scalar_2 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_2.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_non_scalar_3 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_3.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_non_scalar_4 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_4.jl" - - - label: "Unit: matrix field broadcasting (CPU)" - key: unit_matrix_field_broadcasting_cpu_non_scalar_5 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_5.jl" - soft_fail: true - - - group: "Unit: MatrixFields - broadcasting (GPU)" - steps: - - # scalar - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_1 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_1.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_2 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_2.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_3 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_3.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_4 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_4.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_5 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_5.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_6 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_6.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_7 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_7.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_8 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_8.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_9 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_9.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_10 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_10.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_11 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_11.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_12 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_12.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_13 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_13.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_14 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_14.jl" - soft_fail: true - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_15 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_15.jl" - soft_fail: true - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_scalar_16 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_scalar_16.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - # non-scalar - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_non_scalar_1 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_1.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_non_scalar_2 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_2.jl" - soft_fail: true - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_non_scalar_3 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_3.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_non_scalar_4 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_4.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - label: "Unit: matrix field broadcasting (GPU)" - key: unit_matrix_field_broadcasting_gpu_non_scalar_5 - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/MatrixFields/matrix_fields_broadcasting/test_non_scalar_5.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 10GB - - - group: "Unit: Hypsography" - steps: - - - label: "Unit: hypsography 2d" - key: unit_hypsography_2d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Hypsography/2d.jl" - - - label: "Unit: 3dsphere" - key: unit_3dsphere - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Hypsography/3dsphere.jl" - - - group: "Unit: InputOutput" - steps: - - - label: "Unit: read_type" - key: unit_read_type - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_read_type.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - slurm_mem: 3GB - - - label: "Unit: spectralelement2d" - key: unit_spectralelement2d - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_spectralelement2d.jl" - - - label: "Unit: spectralelement2d" - key: unit_spectralelement2d_gpu - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_spectralelement2d.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: hybrid2dbox" - key: unit_hybrid2dbox - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox.jl" - - - label: "Unit: hybrid2dbox topography" - key: unit_hybrid2dbox_topography - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox_topography.jl" - - - label: "Unit: hybrid2dbox stretched" - key: unit_hybrid2dbox_stretched - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid2dbox_stretched.jl" - - - label: "Unit: hybrid3dbox" - key: unit_hybrid3dbox - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dbox.jl" - - - label: "Unit: hybrid3dcubedsphere" - key: unit_hybrid3dcubedsphere - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere.jl" - - - label: "Unit: hybrid3dcubedsphere topography" - key: unit_hybrid3dcubedsphere_topography - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere_topography.jl" - - - label: "Unit: hybrid3dcubedsphere topography" - key: unit_hybrid3dcubedsphere_topography_gpu - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere_topography.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: finitedifference" - key: unit_finitedifference - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_finitedifference.jl" - - - label: "Unit: Parallel HDF5 IO tests" - key: "cpu_parallel_hdf5" - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/InputOutput/unit_hybrid3dcubedsphere.jl" - timeout_in_minutes: 5 - env: - CLIMACOMMS_CONTEXT: "MPI" - retry: - automatic: true - agents: - slurm_nodes: 3 - slurm_tasks_per_node: 1 - - - group: "Unit: Remapping" - steps: - - - label: "Unit: interpolate array" - key: unit_interpolate_array - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Remapping/interpolate_array.jl" - - - group: "Unit: Limiters" - steps: - - - label: "Unit: limiter" - key: unit_limiter - command: "julia --color=yes --check-bounds=yes --project=.buildkite test/Limiters/limiter.jl" - - - label: "Unit: limiter cuda" - key: unit_limiter_gpu - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Limiters/limiter.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Unit: distributed limiters" - key: unit_limiters_distributed - command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Limiters/distributed/dlimiter.jl" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 3 - - # TODO: improve performance label: [inference, allocs, flops, latency, benchmark, boundscheck] - # TODO: use perf env for perf jobs, or merge test/perf envs - - group: "Perf: Geometry" - steps: - - - label: "Perf: Axis tensor conversion benchmarks" - key: "cpu_axis_tensor_conversion_perf_bm" - command: "julia --color=yes --project=.buildkite test/Geometry/axistensor_conversion_benchmarks.jl" - - - group: "Perf: DataLayouts" - steps: - - - label: "Perf: DataLayouts fill" - key: "cpu_datalayouts_fill" - command: "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_fill.jl" - - - label: "Perf: DataLayouts copyto!" - key: "cpu_datalayouts_copyto" - command: "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_copyto.jl" - - - label: "Perf: DataLayouts fill" - key: "gpu_datalayouts_fill" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_fill.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: DataLayouts copyto" - key: "gpu_datalayouts_copyto" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_copyto.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Perf: Fields" - steps: - - - label: "Perf: Field broadcast" - key: "cpu_field_perf" - command: "julia --color=yes --project=.buildkite test/Fields/field_opt.jl" - - - group: "Perf: Benchmark scripts" - steps: - - - label: "Perf: benchmark scripts index_swapping" - key: perf_index_swapping - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite benchmarks/scripts/index_swapping.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: benchmark scripts indexing_and_static_ndranges" - key: indexing_and_static_ndranges - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite benchmarks/scripts/indexing_and_static_ndranges.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: benchmark scripts thermo_bench_bw" - key: thermo_bench_bw - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite benchmarks/scripts/thermo_bench_bw.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: benchmark scripts benchmark_offset" - key: benchmark_offset - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite benchmarks/scripts/benchmark_offset.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: benchmark scripts benchmark_field_last" - key: benchmark_field_last - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite benchmarks/scripts/benchmark_field_last.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Perf: Operators" - steps: - - - label: "Perf: SEM operators" - key: perf_SEM - command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/opt.jl" - - - label: "Perf: FD operators" - key: perf_FD - command: "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt.jl" - - # TODO: combine this with FD operators above - - label: "Perf: FD operators from the wild" - key: perf_FD_ops_examples - command: - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt_examples.jl" - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_examples.jl" - - - label: "Perf: FD operators from the wild (gpu)" - key: perf_FD_ops_examples_gpu - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt_examples.jl" - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_examples.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: dss" - key: perf_dss - command: "julia --color=yes --project=.buildkite test/Operators/hybrid/dss_opt.jl" - - - label: "Perf: hybrid operators" - key: perf_hybrid_ops - command: "julia --color=yes --project=.buildkite test/Operators/hybrid/opt.jl" - - - label: "Perf: FD operator stencil benchmarks" - key: "perf_fd_ops" - command: "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_stencils.jl" - agents: - slurm_mem: 20GB - - - label: "Perf: GPU FD operator stencil benchmarks" - key: "gpu_perf_fd_ops" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_stencils.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: SEM operator benchmarks (cuda Float32)" - key: "perf_gpu_spectral_ops_cuda_float32" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float32" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: SEM operator benchmarks (CPU Float32)" - key: "perf_gpu_spectral_ops_cpu_float32" - command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float32" - - - label: "Perf: SEM operator benchmarks (cuda Float64)" - key: "perf_gpu_spectral_ops_cuda_float64" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: SEM operator benchmarks (CPU Float64)" - key: "perf_gpu_spectral_ops_cpu_float64" - command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64" - - - label: "Perf: SEM operator benchmarks (extruded CPU Float64)" - key: "perf_gpu_spectral_ops_extruded_cpu_float64" - command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64 --space-type ExtrudedFiniteDifferenceSpace" - - - label: "Perf: SEM operator benchmarks" - key: "perf_gpu_spectral_ops" - command: - - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" - - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: "Perf: Integrals (CPU)" - key: "cpu_integrals_perf" - command: - - "julia --color=yes --project=.buildkite test/Operators/integrals.jl" - - - group: "Examples: Column" - steps: - - - label: ":computer: Column Heat Diffusion Eq" - key: "cpu_column_heat" - command: - - "julia --color=yes --project=.buildkite examples/column/heat.jl" - artifact_paths: - - "examples/column/output/heat/*" - - - label: ":computer: Column Advection Step Eq" - key: "cpu_column_step_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/step.jl" - artifact_paths: - - "examples/column/output/advect_step_function/*" - - - label: ":computer: Column Advection Eq" - key: "cpu_column_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/advect.jl" - artifact_paths: - - "examples/column/output/advect/*" - - - label: ":computer: Column FCT Advection Eq" - key: "cpu_fct_column_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/fct_advection.jl" - artifact_paths: - - "examples/column/output/fct_advection/*" - - - label: ":computer: Column TVD Slope-limited Advection Eq" - key: "cpu_tvd_column_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/tvd_advection.jl" - artifact_paths: - - "examples/column/output/tvd_advection/*" - - - label: ":computer: Column Lin vanLeer Limiter Advection Eq" - key: "cpu_lvl_column_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/vanleer_advection.jl" - artifact_paths: - - "examples/column/output/vanleer_advection/*" - - - label: ":computer: Column Lin vanLeer Limiter Advection Eq cuda" - key: "gpu_lvl_column_advect" - command: - - "julia --color=yes --project=.buildkite examples/column/vanleer_advection.jl" - artifact_paths: - - "examples/column/output/vanleer_advection/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":computer: Column BB FCT Advection Eq" - key: "cpu_bb_fct_column_advect" - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite examples/column/bb_fct_advection.jl" - artifact_paths: - - "examples/column/output/bb_fct_advection/*" - - - label: ":computer: Column Zalesak FCT Advection Eq" - key: "cpu_zalesak_fct_column_advect" - command: - - "julia --color=yes --check-bounds=yes --project=.buildkite examples/column/zalesak_fct_advection.jl" - artifact_paths: - - "examples/column/output/zalesak_fct_advection/*" - - - label: ":computer: Column Advection Diffusion Eq" - key: "cpu_column_advect_diff" - command: - - "julia --color=yes --project=.buildkite examples/column/advect_diffusion.jl" - artifact_paths: - - "examples/column/output/advect_diffusion/*" - - - label: ":computer: Column Ekman Eq" - key: "cpu_column_ekman" - command: - - "julia --color=yes --project=.buildkite examples/column/ekman.jl" - artifact_paths: - - "examples/column/output/ekman/*" - - - label: ":computer: Column Hydrostatic Ekman Eq" - key: "cpu_column_hydrostatic_ekman" - command: - - "julia --color=yes --project=.buildkite examples/column/hydrostatic_ekman.jl" - artifact_paths: - - "examples/column/output/hydrostatic_ekman/*" - - - label: ":computer: Column Wave Eq" - key: "cpu_column_wave" - command: - - "julia --color=yes --project=.buildkite examples/column/wave.jl" - artifact_paths: - - "examples/column/output/wave/*" - - - label: ":computer: Column Hydrostatic Balance Eq" - key: "cpu_column_hydrostatic" - command: - - "julia --color=yes --project=.buildkite examples/column/hydrostatic.jl" - artifact_paths: - - "examples/column/output/hydrostatic/*" - - - label: ":computer: Column Hydrostatic Balance Eq with discretely balanced initial condition" - key: "cpu_column_hydrostatic_discrete" - command: - - "julia --color=yes --project=.buildkite examples/column/hydrostatic_discrete.jl" - artifact_paths: - - "examples/column/output/hydrostatic_discrete/*" - - - group: "Examples: Spectral element" - steps: - - label: ":computer: Bickley jet CG" - key: "cpu_bickleyjet_cg" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg.jl" - artifact_paths: - - "examples/bickleyjet/output/cg/*" - - - label: ":computer: Bickley jet CG unstructured mesh" - key: "cpu_bickleyjet_cg_unsmesh" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_unsmesh.jl" - artifact_paths: - - "examples/bickleyjet/output/cg_unsmesh/*" - - - label: ":computer: Bickley jet CG vector invariant hyperviscosity" - key: "cpu_bickleyjet_cg_invariant_hypervisc" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_invariant_hypervisc.jl" - artifact_paths: - - "examples/bickleyjet/output/cg_invariant_hypervisc/*" - - - label: ":computer: MPI Bickley jet CG vector invariant hyperviscosity" - key: "cpu_mpi_bickleyjet_cg_invariant_hypervisc" - command: - - "srun julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_invariant_hypervisc.jl" - artifact_paths: - - "examples/bickleyjet/output/cg_invariant_hypervisc/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 4 - - - label: ":computer: Bickley jet DG rusanov" - key: "cpu_bickleyjet_dg_rusanov" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl rusanov" - artifact_paths: - - "examples/bickleyjet/output/dg_rusanov/*" - - - label: ":computer: Bickley jet DG roe" - key: "cpu_bickleyjet_dg_roe" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl roe" - artifact_paths: - - "examples/bickleyjet/output/dg_roe/*" - - - label: ":computer: Bickley jet DG roe noslip" - key: "cpu_bickleyjet_dg_roe_noslip" - command: - - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl roe noslip" - artifact_paths: - - "examples/bickleyjet/output/dg_roe_noslip/*" - - - label: ":computer: Plane limiters advection cosine bells" - key: "cpu_cg_plane_advection_limiter_cosine_bells" - command: - - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl" - artifact_paths: - - "examples/plane/output/plane_advection_limiter_cosine_bells_D0/*" - - - label: ":computer: Plane limiters advection Gaussian bells" - key: "cpu_cg_plane_advection_limiter_gaussian_bells" - command: - - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl gaussian_bells" - artifact_paths: - - "examples/plane/output/plane_advection_limiter_gaussian_bells_D0/*" - - - label: ":computer: Plane limiters advection cylinders" - key: "cpu_cg_plane_advection_limiter_cylinders" - command: - - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl cylinders" - artifact_paths: - - "examples/plane/output/plane_advection_limiter_cylinders_D0/*" - - - group: "Examples: Hybrid" - steps: - - - label: ":computer: 3D Box limiters advection cosine bells" - key: "cpu_box_advection_limiter_cosine_bells" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl" - artifact_paths: - - "examples/hybrid/box/output/box_advection_limiter_cosine_bells_D0/*" - - - label: ":computer: 3D Box limiters advection Gaussian bells" - key: "cpu_box_advection_limiter_gaussian_bells" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl gaussian_bells" - artifact_paths: - - "examples/hybrid/box/output/box_advection_limiter_gaussian_bells_D0/*" - - - label: ":computer: 3D Box limiters advection Gaussian bells GPU" - key: "gpu_box_advection_limiter_gaussian_bells" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl gaussian_bells" - artifact_paths: - - "examples/hybrid/box/output/box_advection_limiter_gaussian_bells_D0/*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":computer: Float 32 3D sphere baroclinic wave (ρe) HF datalayout GPU" - key: "gpu_baroclinic_wave_rho_e_float32_hf" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhoe_hf/Float32/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhoe_hf" - FLOAT_TYPE: "Float32" - horizontal_layout_type: "IJHF" - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":computer: 3D Box limiters advection slotted spheres" - key: "cpu_box_advection_limiter_slotted_spheres" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl slotted_spheres" - artifact_paths: - - "examples/hybrid/box/output/box_advection_limiter_slotted_spheres_D0/*" - - - label: ":computer: Isothermal channel flow 2D hybrid (ρe)" - key: "cpu_isothermal_channel_2d_hybrid_rhoe" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/isothermal_channel.jl" - artifact_paths: - - "examples/hybrid/plane/output/iso_channel_2d/*" - - - label: ":computer: Rising Bubble 3D hybrid (ρθ)" - key: "cpu_rising_bubble_3d_hybrid_rhotheta" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_rhotheta.jl" - artifact_paths: - - "examples/hybrid/box/output/bubble_3d_rhotheta/*" - - - label: ":computer: Rising Bubble 2D hybrid invariant (ρe)" - key: "cpu_rising_bubble_2d_hybrid_invariant_rhoe" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/bubble_2d_invariant_rhoe.jl" - artifact_paths: - - "examples/hybrid/plane/output/bubble_2d_invariant_rhoe/*" - - - label: ":computer: Rising Bubble 3D hybrid invariant (ρθ)" - key: "cpu_rising_bubble_3d_hybrid_invariant_rhotheta" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhotheta.jl" - artifact_paths: - - "examples/hybrid/box/output/bubble_3d_invariant_rhotheta/*" - - - label: ":computer: Rising Bubble 3D hybrid invariant (ρe)" - key: "cpu_rising_bubble_3d_hybrid_invariant_rhoe" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" - artifact_paths: - - "examples/hybrid/box/output/bubble_3d_invariant_rhoe/*" - - - label: ":flower_playing_cards: Rising Bubble 3D hybrid invariant (ρe)" - key: "gpu_rising_bubble_3d_hybrid_invariant_rhoe" - command: -# - "nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" - - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" - artifact_paths: - - "examples/hybrid/box/output/gpu_bubble_3d_invariant_rhoe/*_low_*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":flower_playing_cards: Rising Bubble 3D hybrid invariant (ρe), custom resolution" - key: "gpu_rising_bubble_3d_hybrid_invariant_rhoe_custom" - command: -# - "nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl Float64 custom 1000 1000 4 16 3 0.05 700.0" - - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl Float64 custom 1000 1000 4 16 3 0.05 700.0" - artifact_paths: - - "examples/hybrid/box/output/gpu_bubble_3d_invariant_rhoe/*_custom_*" - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - label: ":computer: Density current 2D hybrid invariant total energy" - key: "cpu_density_current_2d_hybrid_invariant_total_energy" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/density_current_2dinvariant_rhoe.jl" - artifact_paths: - - "examples/hybrid/plane/output/dc_invariant_etot/*" - - - label: ":computer: Nonhydrostatic Agnesi Mountain total energy (topography mesh interface)" - key: "cpu_agnesi_mtn_2d_hybrid_invariant_total_energy_topography" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/topo_agnesi_nh.jl" - artifact_paths: - - "examples/hybrid/plane/output/agnesi_etot_nh/*" - - - label: ":computer: Schar Mountain total energy (topography mesh interface)" - key: "cpu_schaer_mtn_2d_hybrid_invariant_total_energy_topography" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/topo_schar_nh.jl" - artifact_paths: - - "examples/hybrid/plane/output/schar_etot_nh/*" - - - label: ":computer: Density current 2D hybrid conservative form potential temperature" - key: "cpu_density_current_2d_hybrid_conservative_potential_temperature" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/plane/density_current_2d_flux_form.jl" - artifact_paths: - - "examples/hybrid/plane/output/dc_fluxform/*" - - - label: ":computer: MPI Rising Bubble 3D hybrid invariant (ρe)" - key: "cpu_mpi_rising_bubble_3d_hybrid_invariant_rhoe" - command: - - "srun julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" - artifact_paths: - - "examples/hybrid/box/output/bubble_3d_invariant_rhoe/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - - - group: "Examples: Sphere" - steps: - - - label: ":computer: Solid body sphere cosine bell alpha0" - key: "cpu_solidbody_cg_sphere_cosine_bell_alpha0" - command: - - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl" - artifact_paths: - - "examples/sphere/output/cg_sphere_solidbody_cosine_bell_alpha0/*" - - - label: ":computer: Solid body sphere cosine bell alpha45" - key: "cpu_solidbody_cg_sphere_cosine_bell_alpha45" - command: - - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl cosine_bell alpha45" - artifact_paths: - - "examples/sphere/output/cg_sphere_solidbody_cosine_bell_alpha45/*" - - - label: ":computer: Solid body sphere Gaussian bell alpha0" - key: "cpu_solidbody_cg_sphere_gaussian_bell_alpha0" - command: - - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl gaussian_bell" - artifact_paths: - - "examples/sphere/output/cg_sphere_solidbody_gaussian_bell_alpha0/*" - - - label: ":computer: Solid body sphere Gaussian bell alpha45" - key: "cpu_solidbody_cg_sphere_gaussian_bell_alpha45" - command: - - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl gaussian_bell alpha45" - artifact_paths: - - "examples/sphere/output/cg_sphere_solidbody_gaussian_bell_alpha45/*" - - - label: ":computer: Sphere limiters advection cosine bells" - key: "cpu_cg_sphere_advection_limiter_cosine_bells" - command: - - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl" - artifact_paths: - - "examples/sphere/output/cg_sphere_advection_limiter_cosine_bells/*" - - - label: ":computer: Sphere limiters advection Gaussian bells" - key: "cpu_cg_advection_limiter_gaussian_bells" - command: - - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl gaussian_bells" - artifact_paths: - - "examples/sphere/output/cg_sphere_advection_limiter_gaussian_bells/*" - - - label: ":computer: Sphere limiters advection cylinders" - key: "cpu_cg_advection_limiter_cylinders" - command: - - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl cylinders" - artifact_paths: - - "examples/sphere/output/cg_sphere_advection_limiter_cylinders/*" - - - label: ":computer: Steady-state shallow water 2D sphere alpha0" - key: "cpu_shallowwater_2d_cg_sphere_alpha0" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_steady_state_alpha0/*" - - - label: ":computer: Shallow-water 2D sphere steady-state alpha45" - key: "cpu_shallowwater_2d_cg_sphere_alpha45" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state alpha45" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_steady_state_alpha45/*" - - - label: ":computer: Shallow-water 2D sphere steady-state with compact support alpha0" - key: "cpu_shallowwater_2d_cg_sphere_compact_alpha0" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state_compact" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_steady_state_compact_alpha0/*" - - - label: ":computer: Shallow-water 2D sphere steady-state with compact support alpha60" - key: "cpu_shallowwater_2d_cg_sphere_compact_alpha60" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state_compact alpha60" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_steady_state_compact_alpha60/*" - - - label: ":computer: Shallow-water 2D sphere barotropic instability alpha0" - key: "cpu_shallowwater_2d_cg_sphere_barotropic_alpha0" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/*" - - - label: ":computer: MPI Shallow-water 2D sphere barotropic instability alpha0" - key: "cpu_mpi_shallowwater_2d_cg_sphere_barotropic_alpha0" - command: -# - "nsys profile --trace=nvtx,mpi --mpi-impl=openmpi --output=examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/report.%q{NPROCS} mpiexec julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" - - "mpiexec julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/*" - env: - CLIMACOMMS_CONTEXT: "MPI" - NPROCS: 2 - agents: - slurm_nodes: 1 - slurm_tasks_per_node: 2 - - - label: ":computer: Shallow-water 2D sphere barotropic instability alpha30" - key: "cpu_shallowwater_2d_cg_sphere_barotropic_alpha30" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability alpha30" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha30/*" - - - label: ":computer: Shallow-water 2D sphere mountain alpha0" - key: "cpu_nonuniform_shallowwater_2d_cg_sphere" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl mountain" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_mountain_alpha0/*" - - - label: ":computer: Shallow-water 2D sphere Rossby Haurwitz" - key: "cpu_rossbyhaurwitz_2d_cg_sphere" - command: - - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl rossby_haurwitz" - artifact_paths: - - "examples/sphere/output/cg_sphere_shallowwater_rossby_haurwitz_alpha0/*" - - - label: ":flower_playing_cards: CUDA Shallow-water 2D sphere" - key: "cuda_shallowwater_2d_cg_sphere" - command: - - mkdir -p output/$$BUILDKITE_STEP_KEY -# - nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/sphere/shallow_water_cuda.jl - - julia --color=yes --project=.buildkite examples/sphere/shallow_water_cuda.jl - artifact_paths: - - output/cuda_shallowwater_2d_cg_sphere - env: - CLIMACOMMS_DEVICE: "CUDA" - agents: - slurm_gpus: 1 - - - group: "Examples: hybrid sphere" - steps: - - - label: ":computer: 3D sphere deformation flow w/ limiter & FCT" - key: "cpu_3d_deformation_flow" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/sphere/deformation_flow.jl" - artifact_paths: - - "examples/hybrid/sphere/output/deformation_flow/*" - - - label: ":computer: 3D sphere Hadley circulation" - key: "cpu_3d_hadley_circulation" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/sphere/hadley_circulation.jl" - artifact_paths: - - "examples/hybrid/sphere/output/hadley_circulation/*" - agents: - slurm_mem: 20GB - - - label: ":computer: Float 64 3D sphere baroclinic wave (ρe)" - key: "cpu_baroclinic_wave_rho_e_float64" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float64/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhoe" - FLOAT_TYPE: "Float64" - - - label: ":computer: Float 64 3D sphere baroclinic wave (ρe) HF datalayout" - key: "cpu_baroclinic_wave_rho_e_float64_hf" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhoe_hf/Float64/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhoe_hf" - FLOAT_TYPE: "Float64" - horizontal_layout_type: "IJHF" - - - label: ":computer: 3D sphere baroclinic wave (ρe)" - key: "cpu_baroclinic_wave_rho_e" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float32/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhoe" - - - label: ":computer: MPI 3D sphere baroclinic wave (ρe)" - key: "cpu_mpi_baroclinic_wave_rho_e" - command: - - "srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float32/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhoe" - CLIMACOMMS_CONTEXT: "MPI" - agents: - slurm_ntasks: 2 - - - - label: ":computer: 3D sphere baroclinic wave (ρθ)" - key: "cpu_baroclinic_wave_rho_theta" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/baroclinic_wave_rhotheta/Float32/*" - env: - TEST_NAME: "sphere/baroclinic_wave_rhotheta" - - - label: ":computer: 3D sphere nonhydrostatic gravity wave" - key: "cpu_nonhydrostatic_gravity_wave" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/sphere/nonhydrostatic_gravity_wave.jl" - artifact_paths: - - "examples/hybrid/sphere/output/nonhydrostatic_gravity_wave/*" - - - label: ":computer: 3D sphere solid-body rotation" - key: "cpu_solid_body_rotation" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/sphere/solid_body_rotation_3d.jl" - - - label: ":computer: 3D sphere hydrostatically and geostrophically balanced flow (ρe)" - key: "cpu_balanced_flow_rho_e" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/balanced_flow_rhoe/Float32/*" - env: - TEST_NAME: "sphere/balanced_flow_rhoe" - - - label: ":computer: 3D sphere hydrostatically and geostrophically balanced flow (ρθ)" - key: "cpu_balanced_flow_rho_theta" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/balanced_flow_rhotheta/Float32/*" - env: - TEST_NAME: "sphere/balanced_flow_rhotheta" - - - label: ":computer: 3D sphere dry Held-Suarez (ρe)" - key: "cpu_held_suarez_rho_e" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/held_suarez_rhoe/Float32/*" - env: - TEST_NAME: "sphere/held_suarez_rhoe" - - - label: ":computer: Float64 3D sphere dry Held-Suarez (ρθ)" - key: "cpu_held_suarez_rho_theta_float64" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/held_suarez_rhotheta/Float64/*" - env: - TEST_NAME: "sphere/held_suarez_rhotheta" - FLOAT_TYPE: "Float64" - - - label: ":computer: 3D sphere dry Held-Suarez (ρθ)" - key: "cpu_held_suarez_rho_theta" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/held_suarez_rhotheta/Float32/*" - env: - TEST_NAME: "sphere/held_suarez_rhotheta" - - - label: ":computer: 3D sphere dry Held-Suarez (ρe_int)" - key: "cpu_held_suarez_rho_e_int" - command: - - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/sphere/output/held_suarez_rhoe_int/Float32/*" - env: - TEST_NAME: "sphere/held_suarez_rhoe_int" - - - group: "Examples: hybrid plane" - steps: - - - label: ":computer: 2D plane inertial gravity wave" - key: "cpu_inertial_gravity_wave" - command: - - "julia --threads 8 --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/plane/output/inertial_gravity_wave/Float32/*" - env: - TEST_NAME: "plane/inertial_gravity_wave" - agents: - slurm_cpus_per_task: 8 - slurm_mem: 20GB - - - label: ":computer: stretched 2D plane inertial gravity wave" - key: "cpu_stretch_inertial_gravity_wave" - command: - - "julia --threads 8 --color=yes --project=.buildkite examples/hybrid/driver.jl" - artifact_paths: - - "examples/hybrid/plane/output/stretched_inertial_gravity_wave/Float32/*" - env: - TEST_NAME: "plane/inertial_gravity_wave" - Z_STRETCH: "true" - agents: - slurm_cpus_per_task: 8 - slurm_mem: 20GB + - "julia --color=yes --project=.buildkite test/Limiters/vertical_mass_borrowing_limiter_advection.jl" + env: + CLIMACOMMS_DEVICE: "CUDA" + agents: + slurm_gpus: 1 + +# - label: "Unit: distributed limiters" +# key: unit_limiters_distributed +# command: "srun julia --color=yes --check-bounds=yes --project=.buildkite test/Limiters/distributed/dlimiter.jl" +# env: +# CLIMACOMMS_CONTEXT: "MPI" +# agents: +# slurm_ntasks: 3 + +# # TODO: improve performance label: [inference, allocs, flops, latency, benchmark, boundscheck] +# # TODO: use perf env for perf jobs, or merge test/perf envs +# - group: "Perf: Geometry" +# steps: + +# - label: "Perf: Axis tensor conversion benchmarks" +# key: "cpu_axis_tensor_conversion_perf_bm" +# command: "julia --color=yes --project=.buildkite test/Geometry/axistensor_conversion_benchmarks.jl" + +# - group: "Perf: DataLayouts" +# steps: + +# - label: "Perf: DataLayouts fill" +# key: "cpu_datalayouts_fill" +# command: "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_fill.jl" + +# - label: "Perf: DataLayouts copyto!" +# key: "cpu_datalayouts_copyto" +# command: "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_copyto.jl" + +# - label: "Perf: DataLayouts fill" +# key: "gpu_datalayouts_fill" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_fill.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: DataLayouts copyto" +# key: "gpu_datalayouts_copyto" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/DataLayouts/benchmark_copyto.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - group: "Perf: Fields" +# steps: + +# - label: "Perf: Field broadcast" +# key: "cpu_field_perf" +# command: "julia --color=yes --project=.buildkite test/Fields/field_opt.jl" + +# - group: "Perf: Benchmark scripts" +# steps: + +# - label: "Perf: benchmark scripts index_swapping" +# key: perf_index_swapping +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite benchmarks/scripts/index_swapping.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: benchmark scripts indexing_and_static_ndranges" +# key: indexing_and_static_ndranges +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite benchmarks/scripts/indexing_and_static_ndranges.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: benchmark scripts thermo_bench_bw" +# key: thermo_bench_bw +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite benchmarks/scripts/thermo_bench_bw.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: benchmark scripts benchmark_offset" +# key: benchmark_offset +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite benchmarks/scripts/benchmark_offset.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: benchmark scripts benchmark_field_last" +# key: benchmark_field_last +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite benchmarks/scripts/benchmark_field_last.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - group: "Perf: Operators" +# steps: + +# - label: "Perf: SEM operators" +# key: perf_SEM +# command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/opt.jl" + +# - label: "Perf: FD operators" +# key: perf_FD +# command: "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt.jl" + +# # TODO: combine this with FD operators above +# - label: "Perf: FD operators from the wild" +# key: perf_FD_ops_examples +# command: +# - "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt_examples.jl" +# - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_examples.jl" + +# - label: "Perf: FD operators from the wild (gpu)" +# key: perf_FD_ops_examples_gpu +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/Operators/finitedifference/opt_examples.jl" +# - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_examples.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: dss" +# key: perf_dss +# command: "julia --color=yes --project=.buildkite test/Operators/hybrid/dss_opt.jl" + +# - label: "Perf: hybrid operators" +# key: perf_hybrid_ops +# command: "julia --color=yes --project=.buildkite test/Operators/hybrid/opt.jl" + +# - label: "Perf: FD operator stencil benchmarks" +# key: "perf_fd_ops" +# command: "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_stencils.jl" +# agents: +# slurm_mem: 20GB + +# - label: "Perf: GPU FD operator stencil benchmarks" +# key: "gpu_perf_fd_ops" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/Operators/finitedifference/benchmark_stencils.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: SEM operator benchmarks (cuda Float32)" +# key: "perf_gpu_spectral_ops_cuda_float32" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float32" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: SEM operator benchmarks (CPU Float32)" +# key: "perf_gpu_spectral_ops_cpu_float32" +# command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float32" + +# - label: "Perf: SEM operator benchmarks (cuda Float64)" +# key: "perf_gpu_spectral_ops_cuda_float64" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: SEM operator benchmarks (CPU Float64)" +# key: "perf_gpu_spectral_ops_cpu_float64" +# command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64" + +# - label: "Perf: SEM operator benchmarks (extruded CPU Float64)" +# key: "perf_gpu_spectral_ops_extruded_cpu_float64" +# command: "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl --float-type Float64 --space-type ExtrudedFiniteDifferenceSpace" + +# - label: "Perf: SEM operator benchmarks" +# key: "perf_gpu_spectral_ops" +# command: +# - "julia --project=.buildkite -e 'using CUDA; CUDA.versioninfo()'" +# - "julia --color=yes --project=.buildkite test/Operators/spectralelement/benchmark_ops.jl" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: "Perf: Integrals (CPU)" +# key: "cpu_integrals_perf" +# command: +# - "julia --color=yes --project=.buildkite test/Operators/integrals.jl" + +# - group: "Examples: Column" +# steps: + +# - label: ":computer: Column Heat Diffusion Eq" +# key: "cpu_column_heat" +# command: +# - "julia --color=yes --project=.buildkite examples/column/heat.jl" +# artifact_paths: +# - "examples/column/output/heat/*" + +# - label: ":computer: Column Advection Step Eq" +# key: "cpu_column_step_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/step.jl" +# artifact_paths: +# - "examples/column/output/advect_step_function/*" + +# - label: ":computer: Column Advection Eq" +# key: "cpu_column_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/advect.jl" +# artifact_paths: +# - "examples/column/output/advect/*" + +# - label: ":computer: Column FCT Advection Eq" +# key: "cpu_fct_column_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/fct_advection.jl" +# artifact_paths: +# - "examples/column/output/fct_advection/*" + +# - label: ":computer: Column TVD Slope-limited Advection Eq" +# key: "cpu_tvd_column_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/tvd_advection.jl" +# artifact_paths: +# - "examples/column/output/tvd_advection/*" + +# - label: ":computer: Column Lin vanLeer Limiter Advection Eq" +# key: "cpu_lvl_column_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/vanleer_advection.jl" +# artifact_paths: +# - "examples/column/output/vanleer_advection/*" + +# - label: ":computer: Column Lin vanLeer Limiter Advection Eq cuda" +# key: "gpu_lvl_column_advect" +# command: +# - "julia --color=yes --project=.buildkite examples/column/vanleer_advection.jl" +# artifact_paths: +# - "examples/column/output/vanleer_advection/*" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: ":computer: Column BB FCT Advection Eq" +# key: "cpu_bb_fct_column_advect" +# command: +# - "julia --color=yes --check-bounds=yes --project=.buildkite examples/column/bb_fct_advection.jl" +# artifact_paths: +# - "examples/column/output/bb_fct_advection/*" + +# - label: ":computer: Column Zalesak FCT Advection Eq" +# key: "cpu_zalesak_fct_column_advect" +# command: +# - "julia --color=yes --check-bounds=yes --project=.buildkite examples/column/zalesak_fct_advection.jl" +# artifact_paths: +# - "examples/column/output/zalesak_fct_advection/*" + +# - label: ":computer: Column Advection Diffusion Eq" +# key: "cpu_column_advect_diff" +# command: +# - "julia --color=yes --project=.buildkite examples/column/advect_diffusion.jl" +# artifact_paths: +# - "examples/column/output/advect_diffusion/*" + +# - label: ":computer: Column Ekman Eq" +# key: "cpu_column_ekman" +# command: +# - "julia --color=yes --project=.buildkite examples/column/ekman.jl" +# artifact_paths: +# - "examples/column/output/ekman/*" + +# - label: ":computer: Column Hydrostatic Ekman Eq" +# key: "cpu_column_hydrostatic_ekman" +# command: +# - "julia --color=yes --project=.buildkite examples/column/hydrostatic_ekman.jl" +# artifact_paths: +# - "examples/column/output/hydrostatic_ekman/*" + +# - label: ":computer: Column Wave Eq" +# key: "cpu_column_wave" +# command: +# - "julia --color=yes --project=.buildkite examples/column/wave.jl" +# artifact_paths: +# - "examples/column/output/wave/*" + +# - label: ":computer: Column Hydrostatic Balance Eq" +# key: "cpu_column_hydrostatic" +# command: +# - "julia --color=yes --project=.buildkite examples/column/hydrostatic.jl" +# artifact_paths: +# - "examples/column/output/hydrostatic/*" + +# - label: ":computer: Column Hydrostatic Balance Eq with discretely balanced initial condition" +# key: "cpu_column_hydrostatic_discrete" +# command: +# - "julia --color=yes --project=.buildkite examples/column/hydrostatic_discrete.jl" +# artifact_paths: +# - "examples/column/output/hydrostatic_discrete/*" + +# - group: "Examples: Spectral element" +# steps: +# - label: ":computer: Bickley jet CG" +# key: "cpu_bickleyjet_cg" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg.jl" +# artifact_paths: +# - "examples/bickleyjet/output/cg/*" + +# - label: ":computer: Bickley jet CG unstructured mesh" +# key: "cpu_bickleyjet_cg_unsmesh" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_unsmesh.jl" +# artifact_paths: +# - "examples/bickleyjet/output/cg_unsmesh/*" + +# - label: ":computer: Bickley jet CG vector invariant hyperviscosity" +# key: "cpu_bickleyjet_cg_invariant_hypervisc" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_invariant_hypervisc.jl" +# artifact_paths: +# - "examples/bickleyjet/output/cg_invariant_hypervisc/*" + +# - label: ":computer: MPI Bickley jet CG vector invariant hyperviscosity" +# key: "cpu_mpi_bickleyjet_cg_invariant_hypervisc" +# command: +# - "srun julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_cg_invariant_hypervisc.jl" +# artifact_paths: +# - "examples/bickleyjet/output/cg_invariant_hypervisc/*" +# env: +# CLIMACOMMS_CONTEXT: "MPI" +# agents: +# slurm_ntasks: 4 + +# - label: ":computer: Bickley jet DG rusanov" +# key: "cpu_bickleyjet_dg_rusanov" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl rusanov" +# artifact_paths: +# - "examples/bickleyjet/output/dg_rusanov/*" + +# - label: ":computer: Bickley jet DG roe" +# key: "cpu_bickleyjet_dg_roe" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl roe" +# artifact_paths: +# - "examples/bickleyjet/output/dg_roe/*" + +# - label: ":computer: Bickley jet DG roe noslip" +# key: "cpu_bickleyjet_dg_roe_noslip" +# command: +# - "julia --color=yes --project=.buildkite examples/bickleyjet/bickleyjet_dg.jl roe noslip" +# artifact_paths: +# - "examples/bickleyjet/output/dg_roe_noslip/*" + +# - label: ":computer: Plane limiters advection cosine bells" +# key: "cpu_cg_plane_advection_limiter_cosine_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl" +# artifact_paths: +# - "examples/plane/output/plane_advection_limiter_cosine_bells_D0/*" + +# - label: ":computer: Plane limiters advection Gaussian bells" +# key: "cpu_cg_plane_advection_limiter_gaussian_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl gaussian_bells" +# artifact_paths: +# - "examples/plane/output/plane_advection_limiter_gaussian_bells_D0/*" + +# - label: ":computer: Plane limiters advection cylinders" +# key: "cpu_cg_plane_advection_limiter_cylinders" +# command: +# - "julia --color=yes --project=.buildkite examples/plane/limiters_advection.jl cylinders" +# artifact_paths: +# - "examples/plane/output/plane_advection_limiter_cylinders_D0/*" + +# - group: "Examples: Hybrid" +# steps: + +# - label: ":computer: 3D Box limiters advection cosine bells" +# key: "cpu_box_advection_limiter_cosine_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl" +# artifact_paths: +# - "examples/hybrid/box/output/box_advection_limiter_cosine_bells_D0/*" + +# - label: ":computer: 3D Box limiters advection Gaussian bells" +# key: "cpu_box_advection_limiter_gaussian_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl gaussian_bells" +# artifact_paths: +# - "examples/hybrid/box/output/box_advection_limiter_gaussian_bells_D0/*" + +# - label: ":computer: 3D Box limiters advection Gaussian bells GPU" +# key: "gpu_box_advection_limiter_gaussian_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl gaussian_bells" +# artifact_paths: +# - "examples/hybrid/box/output/box_advection_limiter_gaussian_bells_D0/*" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: ":computer: Float 32 3D sphere baroclinic wave (ρe) HF datalayout GPU" +# key: "gpu_baroclinic_wave_rho_e_float32_hf" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhoe_hf/Float32/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhoe_hf" +# FLOAT_TYPE: "Float32" +# horizontal_layout_type: "IJHF" +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: ":computer: 3D Box limiters advection slotted spheres" +# key: "cpu_box_advection_limiter_slotted_spheres" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/limiters_advection.jl slotted_spheres" +# artifact_paths: +# - "examples/hybrid/box/output/box_advection_limiter_slotted_spheres_D0/*" + +# - label: ":computer: Isothermal channel flow 2D hybrid (ρe)" +# key: "cpu_isothermal_channel_2d_hybrid_rhoe" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/isothermal_channel.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/iso_channel_2d/*" + +# - label: ":computer: Rising Bubble 3D hybrid (ρθ)" +# key: "cpu_rising_bubble_3d_hybrid_rhotheta" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_rhotheta.jl" +# artifact_paths: +# - "examples/hybrid/box/output/bubble_3d_rhotheta/*" + +# - label: ":computer: Rising Bubble 2D hybrid invariant (ρe)" +# key: "cpu_rising_bubble_2d_hybrid_invariant_rhoe" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/bubble_2d_invariant_rhoe.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/bubble_2d_invariant_rhoe/*" + +# - label: ":computer: Rising Bubble 3D hybrid invariant (ρθ)" +# key: "cpu_rising_bubble_3d_hybrid_invariant_rhotheta" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhotheta.jl" +# artifact_paths: +# - "examples/hybrid/box/output/bubble_3d_invariant_rhotheta/*" + +# - label: ":computer: Rising Bubble 3D hybrid invariant (ρe)" +# key: "cpu_rising_bubble_3d_hybrid_invariant_rhoe" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" +# artifact_paths: +# - "examples/hybrid/box/output/bubble_3d_invariant_rhoe/*" + +# - label: ":flower_playing_cards: Rising Bubble 3D hybrid invariant (ρe)" +# key: "gpu_rising_bubble_3d_hybrid_invariant_rhoe" +# command: +# # - "nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" +# - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" +# artifact_paths: +# - "examples/hybrid/box/output/gpu_bubble_3d_invariant_rhoe/*_low_*" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: ":flower_playing_cards: Rising Bubble 3D hybrid invariant (ρe), custom resolution" +# key: "gpu_rising_bubble_3d_hybrid_invariant_rhoe_custom" +# command: +# # - "nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl Float64 custom 1000 1000 4 16 3 0.05 700.0" +# - "julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl Float64 custom 1000 1000 4 16 3 0.05 700.0" +# artifact_paths: +# - "examples/hybrid/box/output/gpu_bubble_3d_invariant_rhoe/*_custom_*" +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - label: ":computer: Density current 2D hybrid invariant total energy" +# key: "cpu_density_current_2d_hybrid_invariant_total_energy" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/density_current_2dinvariant_rhoe.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/dc_invariant_etot/*" + +# - label: ":computer: Nonhydrostatic Agnesi Mountain total energy (topography mesh interface)" +# key: "cpu_agnesi_mtn_2d_hybrid_invariant_total_energy_topography" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/topo_agnesi_nh.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/agnesi_etot_nh/*" + +# - label: ":computer: Schar Mountain total energy (topography mesh interface)" +# key: "cpu_schaer_mtn_2d_hybrid_invariant_total_energy_topography" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/topo_schar_nh.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/schar_etot_nh/*" + +# - label: ":computer: Density current 2D hybrid conservative form potential temperature" +# key: "cpu_density_current_2d_hybrid_conservative_potential_temperature" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/plane/density_current_2d_flux_form.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/dc_fluxform/*" + +# - label: ":computer: MPI Rising Bubble 3D hybrid invariant (ρe)" +# key: "cpu_mpi_rising_bubble_3d_hybrid_invariant_rhoe" +# command: +# - "srun julia --color=yes --project=.buildkite examples/hybrid/box/bubble_3d_invariant_rhoe.jl" +# artifact_paths: +# - "examples/hybrid/box/output/bubble_3d_invariant_rhoe/*" +# env: +# CLIMACOMMS_CONTEXT: "MPI" +# agents: +# slurm_ntasks: 2 + +# - group: "Examples: Sphere" +# steps: + +# - label: ":computer: Solid body sphere cosine bell alpha0" +# key: "cpu_solidbody_cg_sphere_cosine_bell_alpha0" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_solidbody_cosine_bell_alpha0/*" + +# - label: ":computer: Solid body sphere cosine bell alpha45" +# key: "cpu_solidbody_cg_sphere_cosine_bell_alpha45" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl cosine_bell alpha45" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_solidbody_cosine_bell_alpha45/*" + +# - label: ":computer: Solid body sphere Gaussian bell alpha0" +# key: "cpu_solidbody_cg_sphere_gaussian_bell_alpha0" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl gaussian_bell" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_solidbody_gaussian_bell_alpha0/*" + +# - label: ":computer: Solid body sphere Gaussian bell alpha45" +# key: "cpu_solidbody_cg_sphere_gaussian_bell_alpha45" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/solidbody.jl gaussian_bell alpha45" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_solidbody_gaussian_bell_alpha45/*" + +# - label: ":computer: Sphere limiters advection cosine bells" +# key: "cpu_cg_sphere_advection_limiter_cosine_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_advection_limiter_cosine_bells/*" + +# - label: ":computer: Sphere limiters advection Gaussian bells" +# key: "cpu_cg_advection_limiter_gaussian_bells" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl gaussian_bells" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_advection_limiter_gaussian_bells/*" + +# - label: ":computer: Sphere limiters advection cylinders" +# key: "cpu_cg_advection_limiter_cylinders" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/limiters_advection.jl cylinders" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_advection_limiter_cylinders/*" + +# - label: ":computer: Steady-state shallow water 2D sphere alpha0" +# key: "cpu_shallowwater_2d_cg_sphere_alpha0" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_steady_state_alpha0/*" + +# - label: ":computer: Shallow-water 2D sphere steady-state alpha45" +# key: "cpu_shallowwater_2d_cg_sphere_alpha45" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state alpha45" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_steady_state_alpha45/*" + +# - label: ":computer: Shallow-water 2D sphere steady-state with compact support alpha0" +# key: "cpu_shallowwater_2d_cg_sphere_compact_alpha0" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state_compact" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_steady_state_compact_alpha0/*" + +# - label: ":computer: Shallow-water 2D sphere steady-state with compact support alpha60" +# key: "cpu_shallowwater_2d_cg_sphere_compact_alpha60" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl steady_state_compact alpha60" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_steady_state_compact_alpha60/*" + +# - label: ":computer: Shallow-water 2D sphere barotropic instability alpha0" +# key: "cpu_shallowwater_2d_cg_sphere_barotropic_alpha0" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/*" + +# - label: ":computer: MPI Shallow-water 2D sphere barotropic instability alpha0" +# key: "cpu_mpi_shallowwater_2d_cg_sphere_barotropic_alpha0" +# command: +# # - "nsys profile --trace=nvtx,mpi --mpi-impl=openmpi --output=examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/report.%q{NPROCS} mpiexec julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" +# - "mpiexec julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha0/*" +# env: +# CLIMACOMMS_CONTEXT: "MPI" +# NPROCS: 2 +# agents: +# slurm_nodes: 1 +# slurm_tasks_per_node: 2 + +# - label: ":computer: Shallow-water 2D sphere barotropic instability alpha30" +# key: "cpu_shallowwater_2d_cg_sphere_barotropic_alpha30" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl barotropic_instability alpha30" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_barotropic_instability_alpha30/*" + +# - label: ":computer: Shallow-water 2D sphere mountain alpha0" +# key: "cpu_nonuniform_shallowwater_2d_cg_sphere" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl mountain" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_mountain_alpha0/*" + +# - label: ":computer: Shallow-water 2D sphere Rossby Haurwitz" +# key: "cpu_rossbyhaurwitz_2d_cg_sphere" +# command: +# - "julia --color=yes --project=.buildkite examples/sphere/shallow_water.jl rossby_haurwitz" +# artifact_paths: +# - "examples/sphere/output/cg_sphere_shallowwater_rossby_haurwitz_alpha0/*" + +# - label: ":flower_playing_cards: CUDA Shallow-water 2D sphere" +# key: "cuda_shallowwater_2d_cg_sphere" +# command: +# - mkdir -p output/$$BUILDKITE_STEP_KEY +# # - nsys profile --trace=nvtx,cuda --output=output/$$BUILDKITE_STEP_KEY/report julia --color=yes --project=.buildkite examples/sphere/shallow_water_cuda.jl +# - julia --color=yes --project=.buildkite examples/sphere/shallow_water_cuda.jl +# artifact_paths: +# - output/cuda_shallowwater_2d_cg_sphere +# env: +# CLIMACOMMS_DEVICE: "CUDA" +# agents: +# slurm_gpus: 1 + +# - group: "Examples: hybrid sphere" +# steps: + +# - label: ":computer: 3D sphere deformation flow w/ limiter & FCT" +# key: "cpu_3d_deformation_flow" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/sphere/deformation_flow.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/deformation_flow/*" + +# - label: ":computer: 3D sphere Hadley circulation" +# key: "cpu_3d_hadley_circulation" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/sphere/hadley_circulation.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/hadley_circulation/*" +# agents: +# slurm_mem: 20GB + +# - label: ":computer: Float 64 3D sphere baroclinic wave (ρe)" +# key: "cpu_baroclinic_wave_rho_e_float64" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float64/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhoe" +# FLOAT_TYPE: "Float64" + +# - label: ":computer: Float 64 3D sphere baroclinic wave (ρe) HF datalayout" +# key: "cpu_baroclinic_wave_rho_e_float64_hf" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhoe_hf/Float64/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhoe_hf" +# FLOAT_TYPE: "Float64" +# horizontal_layout_type: "IJHF" + +# - label: ":computer: 3D sphere baroclinic wave (ρe)" +# key: "cpu_baroclinic_wave_rho_e" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float32/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhoe" + +# - label: ":computer: MPI 3D sphere baroclinic wave (ρe)" +# key: "cpu_mpi_baroclinic_wave_rho_e" +# command: +# - "srun julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhoe/Float32/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhoe" +# CLIMACOMMS_CONTEXT: "MPI" +# agents: +# slurm_ntasks: 2 + + +# - label: ":computer: 3D sphere baroclinic wave (ρθ)" +# key: "cpu_baroclinic_wave_rho_theta" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/baroclinic_wave_rhotheta/Float32/*" +# env: +# TEST_NAME: "sphere/baroclinic_wave_rhotheta" + +# - label: ":computer: 3D sphere nonhydrostatic gravity wave" +# key: "cpu_nonhydrostatic_gravity_wave" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/sphere/nonhydrostatic_gravity_wave.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/nonhydrostatic_gravity_wave/*" + +# - label: ":computer: 3D sphere solid-body rotation" +# key: "cpu_solid_body_rotation" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/sphere/solid_body_rotation_3d.jl" + +# - label: ":computer: 3D sphere hydrostatically and geostrophically balanced flow (ρe)" +# key: "cpu_balanced_flow_rho_e" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/balanced_flow_rhoe/Float32/*" +# env: +# TEST_NAME: "sphere/balanced_flow_rhoe" + +# - label: ":computer: 3D sphere hydrostatically and geostrophically balanced flow (ρθ)" +# key: "cpu_balanced_flow_rho_theta" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/balanced_flow_rhotheta/Float32/*" +# env: +# TEST_NAME: "sphere/balanced_flow_rhotheta" + +# - label: ":computer: 3D sphere dry Held-Suarez (ρe)" +# key: "cpu_held_suarez_rho_e" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/held_suarez_rhoe/Float32/*" +# env: +# TEST_NAME: "sphere/held_suarez_rhoe" + +# - label: ":computer: Float64 3D sphere dry Held-Suarez (ρθ)" +# key: "cpu_held_suarez_rho_theta_float64" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/held_suarez_rhotheta/Float64/*" +# env: +# TEST_NAME: "sphere/held_suarez_rhotheta" +# FLOAT_TYPE: "Float64" + +# - label: ":computer: 3D sphere dry Held-Suarez (ρθ)" +# key: "cpu_held_suarez_rho_theta" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/held_suarez_rhotheta/Float32/*" +# env: +# TEST_NAME: "sphere/held_suarez_rhotheta" + +# - label: ":computer: 3D sphere dry Held-Suarez (ρe_int)" +# key: "cpu_held_suarez_rho_e_int" +# command: +# - "julia --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/sphere/output/held_suarez_rhoe_int/Float32/*" +# env: +# TEST_NAME: "sphere/held_suarez_rhoe_int" + +# - group: "Examples: hybrid plane" +# steps: + +# - label: ":computer: 2D plane inertial gravity wave" +# key: "cpu_inertial_gravity_wave" +# command: +# - "julia --threads 8 --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/inertial_gravity_wave/Float32/*" +# env: +# TEST_NAME: "plane/inertial_gravity_wave" +# agents: +# slurm_cpus_per_task: 8 +# slurm_mem: 20GB + +# - label: ":computer: stretched 2D plane inertial gravity wave" +# key: "cpu_stretch_inertial_gravity_wave" +# command: +# - "julia --threads 8 --color=yes --project=.buildkite examples/hybrid/driver.jl" +# artifact_paths: +# - "examples/hybrid/plane/output/stretched_inertial_gravity_wave/Float32/*" +# env: +# TEST_NAME: "plane/inertial_gravity_wave" +# Z_STRETCH: "true" +# agents: +# slurm_cpus_per_task: 8 +# slurm_mem: 20GB - group: "Analysis" steps: diff --git a/docs/refs.bib b/docs/refs.bib index fb5bab7d2c..3700198cd9 100644 --- a/docs/refs.bib +++ b/docs/refs.bib @@ -271,3 +271,14 @@ @article{Wiin1967 year = {1967}, publisher = {Wiley Online Library} } + +@article{zhang2018impact, + title={Impact of numerical choices on water conservation in the E3SM Atmosphere Model version 1 (EAMv1)}, + author={Zhang, Kai and Rasch, Philip J and Taylor, Mark A and Wan, Hui and Leung, Ruby and Ma, Po-Lun and Golaz, Jean-Christophe and Wolfe, Jon and Lin, Wuyin and Singh, Balwinder and others}, + journal={Geoscientific Model Development}, + volume={11}, + number={5}, + pages={1971--1988}, + year={2018}, + publisher={Copernicus Publications G{\"o}ttingen, Germany} +} diff --git a/ext/cuda/limiters.jl b/ext/cuda/limiters.jl index e99c713e6a..f18c51a7b6 100644 --- a/ext/cuda/limiters.jl +++ b/ext/cuda/limiters.jl @@ -2,10 +2,11 @@ import ClimaCore.Limiters: QuasiMonotoneLimiter, compute_element_bounds!, compute_neighbor_bounds_local!, - apply_limiter! + apply_limiter!, + VerticalMassBorrowingLimiter import ClimaCore.Fields import ClimaCore: DataLayouts, Spaces, Topologies, Fields -import ClimaCore.DataLayouts: slab_index +import ClimaCore.DataLayouts: slab_index, getindex_field, setindex_field! using CUDA function config_threadblock(Nv, Nh) @@ -281,3 +282,94 @@ function apply_limiter_kernel!( return nothing end + + +function apply_limiter!( + q::Fields.Field, + ρ::Fields.Field, + space, + limiter::VerticalMassBorrowingLimiter, + dev::ClimaComms.CUDADevice; + warn::Bool = true, +) + q_data = Fields.field_values(q) + Nf = DataLayouts.ncomponents(q_data) + us = DataLayouts.UniversalSize(q_data) + q_min = limiter.q_min + us = DataLayouts.UniversalSize(q_data) + Δz = Fields.Δz_field(q) + (Ni, Nj, _, Nv, Nh) = DataLayouts.universal_size(us) + ncols = Ni * Nj * Nh + # if Ni * Nj * Nf > 64 + nthread_x = Ni * Nj + nthread_y = Nf + nthread_z = cld(64, nthread_x * nthread_y) # ensure block is at least 64 threads + nthreads = (nthread_x, nthread_y, nthread_z) + nblocks = cld(ncols * Nf, prod(nthreads)) + + + args = ( + typeof(limiter), + Fields.field_values(Operators.strip_space(q, axes(q))), + Fields.field_values(Operators.strip_space(ρ, axes(ρ))), + Fields.field_values(Operators.strip_space(Δz, axes(Δz))), + q_min, + us, + ) + auto_launch!( + apply_limiter_kernel!, + args; + threads_s = nthreads, + blocks_s = nblocks, + ) + call_post_op_callback() && post_op_callback(q, ρ, limiter, dev) + return nothing +end + +function apply_limiter_kernel!( + ::Type{LM}, + q_data, + ρ_data, + Δz_data, + q_min_tuple, + us::DataLayouts.UniversalSize) where {LM <: VerticalMassBorrowingLimiter} + (Ni, Nj, _, Nv, Nh) = DataLayouts.universal_size(us) + j_idx, i_idx = divrem(CUDA.threadIdx().x - Int32(1), Ni) + j_idx = j_idx + Int32(1) + i_idx = i_idx + Int32(1) + f_idx = CUDA.threadIdx().y + # each z in a block is a different element + h_idx = CUDA.blockDim().z * (CUDA.blockIdx().x - Int32(1)) + CUDA.threadIdx().z + q_min = q_min_tuple[f_idx] + borrowed_mass = zero(eltype(q_data)) + @inbounds if h_idx <= Nh + for i in 0:(Nv - 1) # CUDA.jl recommends avoiding stepranges + v = Nv - i + CI = CartesianIndex(i_idx, j_idx, f_idx, v, h_idx) + ρΔV_lev = getindex_field(ρ_data, CI) * getindex_field(Δz_data, CI) + new_mass = getindex_field(q_data, CI) - (borrowed_mass / ρΔV_lev) + if new_mass > q_min + setindex_field!(q_data, new_mass, CI) + borrowed_mass = zero(borrowed_mass) + else + borrowed_mass = (q_min - new_mass) * ρΔV_lev + setindex_field!(q_data, q_min, CI) + end + end + for v in 1:Nv + if borrowed_mass > zero(borrowed_mass) + CI = CartesianIndex(i_idx, j_idx, f_idx, v, h_idx) + ρΔV_lev = getindex_field(ρ_data, CI) * getindex_field(Δz_data, CI) + new_mass = getindex_field(q_data, CI) - (borrowed_mass / ρΔV_lev) + if new_mass > q_min + setindex_field!(q_data, new_mass, CI) + return + else + borrowed_mass = (q_min - new_mass) * ρΔV_lev + setindex_field!(q_data, q_min, CI) + end + end + end + end + return +end diff --git a/src/Limiters/Limiters.jl b/src/Limiters/Limiters.jl index bd9116cf0d..a731f38fcc 100644 --- a/src/Limiters/Limiters.jl +++ b/src/Limiters/Limiters.jl @@ -20,5 +20,6 @@ abstract type AbstractLimiter end # implementations include("quasimonotone.jl") +include("vertical_mass_borrowing_limiter.jl") end # end module diff --git a/src/Limiters/vertical_mass_borrowing_limiter.jl b/src/Limiters/vertical_mass_borrowing_limiter.jl new file mode 100644 index 0000000000..95e4f6c60f --- /dev/null +++ b/src/Limiters/vertical_mass_borrowing_limiter.jl @@ -0,0 +1,140 @@ +import .DataLayouts as DL + +""" + VerticalMassBorrowingLimiter(f::Fields.Field, q_min) + +A vertical-only mass borrowing limiter. + +The mass borrower borrows tracer mass from an adjacent, lower layer. +It conserves the total tracer mass and can avoid negative tracers. + +At level k, it will first borrow the mass from the layer k+1 (lower level). +If the mass is not sufficient in layer k+1, it will borrow mass from +layer k+2. The borrower will proceed this process until the bottom layer. +If the tracer mass in the bottom layer goes negative, it will repeat the +process from the bottom to the top. In this way, the borrower works for +any shape of mass profiles. + +This code was adapted from [E3SM](https://github.com/E3SM-Project/E3SM/blob/2c377c5ec9a5585170524b366ad85074ab1b1a5c/components/eam/src/physics/cam/massborrow.F90) + +References: + - [zhang2018impact](@cite) +""" +struct VerticalMassBorrowingLimiter{F, T} + bmass::F + ic::F + q_min::T +end +function VerticalMassBorrowingLimiter(f::Fields.Field, q_min) + bmass = similar(Spaces.level(f, 1)) + ic = similar(Spaces.level(f, 1)) + return VerticalMassBorrowingLimiter(bmass, ic, q_min) +end +# TODO: should q_min be a field that varies in space? + + +""" + apply_limiter!(q::Fields.Field, ρ::Fields.Field, lim::VerticalMassBorrowingLimiter) + +Apply the vertical mass borrowing +limiter `lim` to field `q`, given +density `ρ`. +""" +apply_limiter!( + q::Fields.Field, + ρ::Fields.Field, + lim::VerticalMassBorrowingLimiter, +) = apply_limiter!(q, ρ, axes(q), lim, ClimaComms.device(axes(q))) + +function apply_limiter!( + q::Fields.Field, + ρ::Fields.Field, + space::Spaces.FiniteDifferenceSpace, + lim::VerticalMassBorrowingLimiter, + device::ClimaComms.AbstractCPUDevice, +) + cache = (; bmass = lim.bmass, ic = lim.ic, q_min = lim.q_min) + columnwise_massborrow_cpu(q, ρ, cache) +end + +function apply_limiter!( + q::Fields.Field, + ρ::Fields.Field, + space::Spaces.ExtrudedFiniteDifferenceSpace, + lim::VerticalMassBorrowingLimiter, + device::ClimaComms.AbstractCPUDevice, +) + Fields.bycolumn(axes(q)) do colidx + cache = (; + bmass = lim.bmass[colidx], + ic = lim.ic[colidx], + q_min = lim.q_min, + ) + columnwise_massborrow_cpu(q[colidx], ρ[colidx], cache) + end +end + +# TODO: can we improve the performance? +# `bycolumn` on the CPU may be better here since we could multithread it. +function columnwise_massborrow_cpu(q::Fields.Field, ρ::Fields.Field, cache) # column fields + # TODO: maybe rm some stuff from struct or cache + (; bmass, ic, q_min) = cache + Δz = Fields.Δz_field(q) + Δz_vals = Fields.field_values(Δz) + (; J) = Fields.local_geometry_field(ρ) + # ΔV_vals = Fields.field_values(J) + ΔV_vals = Δz_vals + ρ_vals = Fields.field_values(ρ) + # loop over tracers + nlevels = Spaces.nlevels(axes(q)) + @. ic = 0 + @. bmass = 0 + # TODO: Maybe delete ic + # TODO: make bmass positive + q_vals = Fields.field_values(q) + # top to bottom + for f in 1:DataLayouts.ncomponents(q_vals) + for v in nlevels:-1:1 + CI = CartesianIndex(1, 1, f, v, 1) + # new mass in the current layer + ρΔV_lev = + DL.getindex_field(ΔV_vals, CI) * DL.getindex_field(ρ_vals, CI) + nmass = DL.getindex_field(q_vals, CI) + bmass[] / ρΔV_lev + # TODO: should this be \geq + if nmass > q_min[f] + # if new mass in the current layer is positive, don't borrow mass any more + DL.setindex_field!(q_vals, nmass, CI) + bmass[] = 0 + else + # set mass to q_min in the current layer, and save bmass + bmass[] = (nmass - q_min[f]) * ρΔV_lev + DL.setindex_field!(q_vals, q_min[f], CI) + ic[] = ic[] + 1 + end + end + + # bottom to top + for v in 1:nlevels + CI = CartesianIndex(1, 1, f, v, 1) + # if the surface layer still needs to borrow mass + if bmass[] < 0 + ρΔV_lev = + DL.getindex_field(ΔV_vals, CI) * + DL.getindex_field(ρ_vals, CI) + # new mass in the current layer + nmass = DL.getindex_field(q_vals, CI) + bmass[] / ρΔV_lev + if nmass > q_min[f] + # if new mass in the current layer is positive, don't borrow mass any more + DL.setindex_field!(q_vals, nmass, CI) + bmass[] = 0 + else + # if new mass in the current layer is negative, continue to borrow mass + bmass[] = (nmass - q_min[f]) * ρΔV_lev + DL.setindex_field!(q_vals, q_min[f], CI) + end + end + end + end + + return nothing +end diff --git a/test/Limiters/vertical_mass_borrowing_limiter.jl b/test/Limiters/vertical_mass_borrowing_limiter.jl new file mode 100644 index 0000000000..d347bfd038 --- /dev/null +++ b/test/Limiters/vertical_mass_borrowing_limiter.jl @@ -0,0 +1,174 @@ +#= +julia --project=.buildkite +using Revise; include(joinpath("test", "Limiters", "vertical_mass_borrowing_limiter.jl")) +=# +using ClimaComms +ClimaComms.@import_required_backends +using ClimaCore: Fields, Spaces, Limiters +using ClimaCore.RecursiveApply +using ClimaCore.Geometry +using ClimaCore.Grids +using ClimaCore.CommonGrids +import ClimaCore +using Test +using Random + + +import Plots +import ClimaCorePlots +function plot_results(f, f₀) + col = Fields.ColumnIndex((1, 1), 1) + fcol = f[col] + f₀col = f₀[col] + p = Plots.plot() + Plots.plot(fcol; label = "field") + Plots.plot!(f₀col; label = "initial") + Plots.savefig("lim.png") +end +# plot_results(ρq, ρq_init) + + +function perturb_field!(f::Fields.Field; perturb_radius) + device = ClimaComms.device(f) + ArrayType = ClimaComms.array_type(device) + rand_data = ArrayType(rand(size(parent(f))...)) # [0-1] + rand_data = rand_data .- sum(rand_data) / length(rand_data) # make centered about 0 ([-0.5:0.5]) + rand_data = (rand_data ./ maximum(rand_data)) .* perturb_radius # rand_data now in [-perturb_radius:perturb_radius] + parent(f) .= parent(f) .+ rand_data # f in [f ± perturb_radius] + return nothing +end + +@testset "Vertical mass borrowing limiter - column" begin + FT = Float64 + Random.seed!(1134) + z_elem = 10 + z_min = 0 + z_max = 1 + device = ClimaComms.device() + grid = ColumnGrid(FT; z_elem, z_min, z_max, device) + cspace = Spaces.FiniteDifferenceSpace(grid, Grids.CellCenter()) + tol = 0.01 + perturb_q = 0.3 + perturb_ρ = 0.2 + + # Initialize fields + coords = Fields.coordinate_field(cspace) + ρ = map(coord -> 1.0, coords) + q = map(coord -> 0.1, coords) + (; z) = coords + perturb_field!(q; perturb_radius = perturb_q) + perturb_field!(ρ; perturb_radius = perturb_ρ) + ρq_init = ρ .⊠ q + sum_ρq_init = sum(ρq_init) + + # Test that the minimum is below 0 + @test length(parent(q)) == z_elem == 10 + @test 0.3 ≤ count(x -> x < 0, parent(q)) / 10 ≤ 0.5 # ensure reasonable percentage of points are negative + + @test -2 * perturb_q ≤ minimum(q) ≤ -tol + limiter = Limiters.VerticalMassBorrowingLimiter(q, (0.0,)) + Limiters.apply_limiter!(q, ρ, limiter) + @test 0 ≤ minimum(q) + ρq = ρ .⊠ q + @test isapprox(sum(ρq), sum_ρq_init; atol = 1e-15) + @test isapprox(sum(ρq), sum_ρq_init; rtol = 1e-10) + plot_results(ClimaCore.to_cpu(ρq), ClimaCore.to_cpu(ρq_init)) + # @show sum(ρq) # 0.07388931313511024 + # @show sum_ρq_init # 0.07388931313511025 +end + +@testset "Vertical mass borrowing limiter - sphere" begin + FT = Float64 + z_elem = 10 + z_min = 0 + z_max = 1 + radius = 10 + h_elem = 10 + n_quad_points = 4 + grid = ExtrudedCubedSphereGrid(FT; + z_elem, + z_min, + z_max, + radius, + h_elem, + n_quad_points, + ) + cspace = Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) + tol = 0.01 + perturb_q = 0.2 + perturb_ρ = 0.1 + + # Initialize fields + coords = Fields.coordinate_field(cspace) + ρ = map(coord -> 1.0, coords) + q = map(coord -> 0.1, coords) + + perturb_field!(q; perturb_radius = perturb_q) + perturb_field!(ρ; perturb_radius = perturb_ρ) + ρq_init = ρ .⊠ q + sum_ρq_init = sum(ρq_init) + + # Test that the minimum is below 0 + @test length(parent(q)) == z_elem * h_elem^2 * 6 * n_quad_points^2 == 96000 + @test 0.10 ≤ count(x -> x < 0.0001, parent(q)) / 96000 ≤ 1 # ensure 10% of points are negative + + @test -2 * perturb_q ≤ minimum(q) ≤ -tol + @show q + limiter = Limiters.VerticalMassBorrowingLimiter(q, (0.0,)) + Limiters.apply_limiter!(q, ρ, limiter) + @show q + @test 0 ≤ minimum(q) + ρq = ρ .⊠ q + @test isapprox(sum(ρq), sum_ρq_init; atol = 0.07) + @test isapprox(sum(ρq), sum_ρq_init; rtol = 0.07) + # @show sum(ρq) # 125.5483524237572 + # @show sum_ρq_init # 125.52052986152977 +end + +@testset "Vertical mass borrowing limiter - deep atmosphere" begin + FT = Float64 + Random.seed!(12214) + z_elem = 10 + z_min = 0 + z_max = 1 + radius = 10 + h_elem = 10 + n_quad_points = 4 + grid = ExtrudedCubedSphereGrid(FT; + z_elem, + z_min, + z_max, + radius, + global_geometry = Geometry.DeepSphericalGlobalGeometry{FT}(radius), + h_elem, + n_quad_points, + ) + cspace = Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) + tol = 0.01 + perturb_q = 0.2 + perturb_ρ = 0.1 + + # Initialize fields + coords = Fields.coordinate_field(cspace) + ρ = map(coord -> 1.0, coords) + q = map(coord -> 0.1, coords) + + perturb_field!(q; perturb_radius = perturb_q) + perturb_field!(ρ; perturb_radius = perturb_ρ) + ρq_init = ρ .⊠ q + sum_ρq_init = sum(ρq_init) + + # Test that the minimum is below 0 + @test length(parent(q)) == z_elem * h_elem^2 * 6 * n_quad_points^2 == 96000 + @test 0.10 ≤ count(x -> x < 0.0001, parent(q)) / 96000 ≤ 1 # ensure 10% of points are negative + + @test -2 * perturb_q ≤ minimum(q) ≤ -tol + limiter = Limiters.VerticalMassBorrowingLimiter(q, (0.0,)) + Limiters.apply_limiter!(q, ρ, limiter) + @test 0 ≤ minimum(q) + ρq = ρ .⊠ q + @test isapprox(sum(ρq), sum_ρq_init; atol = 0.269) + @test isapprox(sum(ρq), sum_ρq_init; rtol = 0.00199) + # @show sum(ρq) # 138.90494931641584 + # @show sum_ρq_init # 139.1735731377394 +end diff --git a/test/Limiters/vertical_mass_borrowing_limiter_advection.jl b/test/Limiters/vertical_mass_borrowing_limiter_advection.jl new file mode 100644 index 0000000000..45c6d88046 --- /dev/null +++ b/test/Limiters/vertical_mass_borrowing_limiter_advection.jl @@ -0,0 +1,208 @@ +#= +julia --project=.buildkite +using Revise; include(joinpath("test", "Limiters", "vertical_mass_borrowing_limiter_advection.jl")) +=# +using Test +using LinearAlgebra +import ClimaComms +ClimaComms.@import_required_backends +using OrdinaryDiffEqSSPRK: ODEProblem, solve, SSPRK33 +using ClimaCore.CommonGrids +using ClimaCore.Grids +using ClimaTimeSteppers +import ClimaCore + +import ClimaCore: + Fields, + Domains, + Limiters, + Topologies, + Meshes, + DataLayouts, + Operators, + Geometry, + Spaces + + +# Advection Equation, with constant advective velocity (so advection form = flux form) +# ∂_t y + w ∂_z y = 0 +# the solution translates to the right at speed w, +# so at time t, the solution is y(z - w * t) + +# visualization artifacts +ENV["GKSwstype"] = "nul" +import ClimaCorePlots +import Plots +Plots.GRBackend() +dir = "vert_mass_borrow_advection" +path = joinpath(@__DIR__, "output", dir) +mkpath(path) + +function lim!(y, parameters, t, y_ref) + (; w, Δt, limiter) = parameters + Limiters.apply_limiter!(y.q, y.ρ, limiter) + return nothing +end + +function perturb_field!(f::Fields.Field; perturb_radius) + device = ClimaComms.device(f) + ArrayType = ClimaComms.array_type(device) + rand_data = ArrayType(rand(size(parent(f))...)) # [0-1] + rand_data = rand_data .- sum(rand_data) / length(rand_data) # make centered about 0 ([-0.5:0.5]) + rand_data = (rand_data ./ maximum(rand_data)) .* perturb_radius # rand_data now in [-perturb_radius:perturb_radius] + parent(f) .= parent(f) .+ rand_data # f in [f ± perturb_radius] + return nothing +end + +function tendency!(yₜ, y, parameters, t) + (; w, Δt, limiter) = parameters + FT = Spaces.undertype(axes(y.q)) + bcvel = pulse(-π, t, z₀, zₕ, z₁, speed) + divf2c = Operators.DivergenceF2C( + bottom = Operators.SetValue(Geometry.WVector(FT(bcvel))), + top = Operators.SetValue(Geometry.WVector(FT(0))), + ) + upwind1 = Operators.UpwindBiasedProductC2F( + bottom = Operators.Extrapolate(), + top = Operators.Extrapolate(), + ) + upwind3 = Operators.Upwind3rdOrderBiasedProductC2F( + bottom = Operators.ThirdOrderOneSided(), + top = Operators.ThirdOrderOneSided(), + ) + If = Operators.InterpolateC2F() + @. yₜ.q = -divf2c(upwind1(w, y.q) * If(y.q)) + return nothing +end + +# Define a pulse wave or square wave + +FT = Float64 +t₀ = FT(0.0) +Δt = 0.0001 * 25 +t₁ = 200Δt +z₀ = FT(0.0) +zₕ = FT(1.0) +z₁ = FT(1.0) +speed = FT(-1.0) +pulse(z, t, z₀, zₕ, z₁, speed) = abs(z - speed * t) ≤ zₕ ? z₁ : z₀ + +stretch_fns = (Meshes.Uniform(), Meshes.ExponentialStretching(FT(7.0))) +plot_string = ["uniform", "stretched"] + +@testset "VerticalMassBorrowingLimiter on advection" begin + for (i, stretch) in enumerate(stretch_fns) + i = 1 + stretch = Meshes.Uniform() + + z_elem = 2^6 + z_min = -π + z_max = π + device = ClimaComms.device() + + # use_column = true + use_column = false + if use_column + grid = ColumnGrid(; z_elem, z_min, z_max, stretch, device) + cspace = Spaces.FiniteDifferenceSpace(grid, Grids.CellCenter()) + fspace = Spaces.FaceFiniteDifferenceSpace(cspace) + else + grid = ExtrudedCubedSphereGrid(; + z_elem, + z_min, + z_max, + stretch, + radius = 10, + h_elem = 10, + n_quad_points = 4, + device, + ) + cspace = + Spaces.ExtrudedFiniteDifferenceSpace(grid, Grids.CellCenter()) + fspace = Spaces.FaceExtrudedFiniteDifferenceSpace(cspace) + end + + z = Fields.coordinate_field(cspace).z + O = ones(FT, fspace) + + # Initial condition + q_init = pulse.(z, 0.0, z₀, zₕ, z₁, speed) + q = q_init + coords = Fields.coordinate_field(q) + ρ = map(coord -> 1.0, coords) + perturb_field!(ρ; perturb_radius = 0.1) + y = Fields.FieldVector(; q, ρ) + limiter = Limiters.VerticalMassBorrowingLimiter(q, (0.0,)) + + # Unitary, constant advective velocity + w = Geometry.WVector.(speed .* O) + + # Solve the ODE + parameters = (; w, Δt, limiter) + prob = ODEProblem( + ClimaODEFunction(; lim!, T_lim! = tendency!), + y, + (t₀, t₁), + parameters, + ) + sol = solve( + prob, + ExplicitAlgorithm(SSP33ShuOsher()), + dt = Δt, + save_everystep = true, + ) + + q_init = sol.u[1].q + q_final = sol.u[end].q + q_analytic = pulse.(z, t₁, z₀, zₕ, z₁, speed) + err = norm(q_final .- q_analytic) + rel_mass_err = norm((sum(q_final) - sum(q_init)) / sum(q_init)) + + + if use_column + p = Plots.plot() + Plots.plot!(q_init |> ClimaCore.to_cpu, label = "init") + Plots.plot!(q_final |> ClimaCore.to_cpu, label = "computed") + Plots.plot!(q_analytic |> ClimaCore.to_cpu, label = "analytic") + Plots.plot!(; legend = :topright) + Plots.plot!(; xlabel = "q", title = "VerticalMassBorrowingLimiter") + f = joinpath( + path, + "VerticalMassBorrowingLimiter_comparison_$(plot_string[i]).png", + ) + Plots.png(p, f) + else + colidx = Fields.ColumnIndex((1, 1), 1) + p = Plots.plot() + Plots.plot!( + vec(parent(q_init[colidx] |> ClimaCore.to_cpu)), + vec(parent(z[colidx] |> ClimaCore.to_cpu)), + label = "init", + ) + Plots.plot!( + vec(parent(q_final[colidx] |> ClimaCore.to_cpu)), + vec(parent(z[colidx] |> ClimaCore.to_cpu)), + label = "computed", + ) + Plots.plot!( + vec(parent(q_analytic[colidx] |> ClimaCore.to_cpu)), + vec(parent(z[colidx] |> ClimaCore.to_cpu)), + label = "analytic", + ) + Plots.plot!(; legend = :topright) + Plots.plot!(; + xlabel = "q", + ylabel = "z", + title = "VerticalMassBorrowingLimiter", + ) + f = joinpath( + path, + "VerticalMassBorrowingLimiter_comparison_$(plot_string[i]).png", + ) + Plots.png(p, f) + end + @test err ≤ 0.25 + @test rel_mass_err ≤ 10eps() + @test minimum(q_final) ≥ 0 + end +end