From 64b2e721b890f19be37d5bf71c44cda53b6703d1 Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Fri, 30 Jan 2026 00:07:57 -0800 Subject: [PATCH 01/10] Enable CHAI only when running Kripke on GPU --- repo/kripke/package.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/repo/kripke/package.py b/repo/kripke/package.py index a34715d14..5834b2835 100644 --- a/repo/kripke/package.py +++ b/repo/kripke/package.py @@ -114,6 +114,11 @@ def cmake_args(self): spec = self.spec args = [] + if "+rocm" in spec or "+cuda" in spec: + enable_chai = "ON" + else: + enable_chai = "OFF" + args.extend( [ "-DCAMP_DIR=%s" % self.spec["camp"].prefix, @@ -121,7 +126,7 @@ def cmake_args(self): "-Dumpire_DIR=%s" % self.spec["umpire"].prefix, "-DRAJA_DIR=%s" % self.spec["raja"].prefix, "-Dchai_DIR=%s" % self.spec["chai"].prefix, - "-DENABLE_CHAI=ON", + "-DENABLE_CHAI=%s" % enable_chai, "-DMPI_CXX_LINK_FLAGS='%s'" % self.spec['mpi'].libs.ld_flags, ] ) From 6a7fe0bca2f07f9e87194522a68d94692cb8eeff Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Mon, 23 Feb 2026 11:03:53 -0800 Subject: [PATCH 02/10] Add apu support for El Capitan --- repos/spack_repo/benchpark/packages/sparta_snl/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repos/spack_repo/benchpark/packages/sparta_snl/package.py b/repos/spack_repo/benchpark/packages/sparta_snl/package.py index a96fc4d5c..24a240a47 100644 --- a/repos/spack_repo/benchpark/packages/sparta_snl/package.py +++ b/repos/spack_repo/benchpark/packages/sparta_snl/package.py @@ -23,6 +23,7 @@ class SpartaSnl(CMakePackage, CudaPackage, ROCmPackage): variant("openmp", default=False, description="Enable OpenMP support") variant("jpeg", default=False, description="Build with jpeg support") variant("png", default=False, description="Build with png support") + variant("apu", default=False, description="Enable APU support", when="@4.5: +rocm") variant( "kokkos", @@ -56,6 +57,7 @@ class SpartaSnl(CMakePackage, CudaPackage, ROCmPackage): depends_on("kokkos+openmp cxxstd=17", when="+openmp") depends_on("kokkos+rocm", when="+rocm") depends_on("kokkos+wrapper+cuda cxxstd=17", when="+cuda") + depends_on("kokkos+apu", when="+apu") # Kokkos 5 not building depends_on("kokkos@:4", when="+kokkos") From aec46163957103d18ec15383a6e9eb54069b141a Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Tue, 24 Feb 2026 08:55:34 -0800 Subject: [PATCH 03/10] Changes for ElCapitan build --- experiments/sparta-snl/experiment.py | 28 ++++++++++++++++++- .../benchpark/packages/kokkos/package.py | 9 ++++++ .../benchpark/packages/sparta_snl/package.py | 18 ++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/experiments/sparta-snl/experiment.py b/experiments/sparta-snl/experiment.py index 3a58d863b..d8cf940a1 100644 --- a/experiments/sparta-snl/experiment.py +++ b/experiments/sparta-snl/experiment.py @@ -55,6 +55,20 @@ class SpartaSnl( description="Enable GPU-aware MPI", ) + variant( + "apu", + default=False, + values=(True, False), + description="Enable APU support", + ) + + variant( + "other", + default=False, + values=(True, False), + description="Set other input/environment variables", + ) + maintainers("rfhaque") def compute_applications_section(self): @@ -88,6 +102,14 @@ def compute_applications_section(self): self.add_experiment_variable("stats", stats, True) self.add_experiment_variable("run", run, True) + if self.spec.satisfies("+rocm"): + if self.spec.satisfies("+other"): + self.set_environment_variable("MPICH_OFI_NIC_POLICY", "GPU") + self.set_environment_variable("FI_HMEM", "rocr") + self.set_environment_variable("HUGETLB_MORECORE", "yes") + self.set_environment_variable("HUGETLB_RESTRICT_EXE", "defrag:lmp") + self.set_environment_variable("HSA_XNACK", 1) + if self.spec.satisfies("+rocm") or self.spec.satisfies("+cuda"): kokkos_mode = "g 1" kokkos_gpu_aware = "yes" if self.spec.satisfies("+gpu-aware-mpi") else "no" @@ -126,10 +148,14 @@ def compute_package_section(self): fft_kokkos = "cufft" if self.spec.satisfies("+rocm"): fft_kokkos = "hipfft" + if self.spec.satisfies("+apu"): + apu = "+apu" + else: + apu = "~apu" self.add_package_spec( self.name, [ - f"sparta-snl{self.determine_version()} +mpi+kokkos fft_kokkos={fft_kokkos} fft={fft} " + f"sparta-snl{self.determine_version()} +mpi+kokkos fft_kokkos={fft_kokkos} fft={fft} {apu} " ], ) diff --git a/repos/spack_repo/benchpark/packages/kokkos/package.py b/repos/spack_repo/benchpark/packages/kokkos/package.py index fe1054438..1ea93b64b 100644 --- a/repos/spack_repo/benchpark/packages/kokkos/package.py +++ b/repos/spack_repo/benchpark/packages/kokkos/package.py @@ -13,3 +13,12 @@ class Kokkos(BuiltinKokkos): def setup_build_environment(self, env): if "+cuda" in self.spec: env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") + + def flag_handler(self, name, flags): + if spec.satisfies("+apu"): + if name == "cxxflags": + flags.append("-munsafe-fp-atomics") + + return flags, None, None + + diff --git a/repos/spack_repo/benchpark/packages/sparta_snl/package.py b/repos/spack_repo/benchpark/packages/sparta_snl/package.py index 24a240a47..896829000 100644 --- a/repos/spack_repo/benchpark/packages/sparta_snl/package.py +++ b/repos/spack_repo/benchpark/packages/sparta_snl/package.py @@ -114,6 +114,14 @@ def flag_handler(self, name, flags): return (wrapper_flags, [], build_system_flags) + def flag_handler(self, name, flags): + if spec.satisfies("+apu"): + if name == "cxxflags": + flags.append("-fdenormal-fp-math=ieee") + flags.append("-fgpu-flush-denormals-to-zero") + + return flags, None, None + def cmake_args(self): spec = self.spec @@ -150,6 +158,16 @@ def cmake_args(self): else: args.append(self.define("PKG_KOKKOS", False)) + if spec.satisfies("+apu"): + existing_ldflags = self.spec.compiler_flags.get("ldflags", []) + existing = " ".join(existing_ldflags) + extra = " -lxpmem -lhugetlbfs" + if existing: + combined = f"{existing} {extra}" + else: + combined = extra + args.append(f"-DCMAKE_EXE_LINKER_FLAGS={combined}") + return args def install(self, spec, prefix): From f424d1c6795f09572389db5615e1b45f7c904a94 Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Wed, 25 Feb 2026 23:28:29 -0800 Subject: [PATCH 04/10] Changes --- experiments/sparta-snl/experiment.py | 12 ++++----- .../sparta_snl/application.py | 26 ++++++++++++++----- .../benchpark/packages/kokkos/package.py | 9 ------- .../benchpark/packages/sparta_snl/package.py | 11 +++----- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/experiments/sparta-snl/experiment.py b/experiments/sparta-snl/experiment.py index 2e2f89368..7c5c19f65 100644 --- a/experiments/sparta-snl/experiment.py +++ b/experiments/sparta-snl/experiment.py @@ -83,12 +83,12 @@ def compute_applications_section(self): else: L = 2 # will increase problem size by 4X ppc = 64 - stats = 10 - run = 100 - xmin = -1.0 - xmax = 1.1 - ymin = -1.1 - ymax = 1.1 + stats = 100 + run = 10000000 + xmin = -5.0 + xmax = 5.1 + ymin = -5.1 + ymax = 5.1 if self.spec.satisfies("workload=cylinder"): self.add_experiment_variable("L", L, True) diff --git a/repos/ramble_applications/sparta_snl/application.py b/repos/ramble_applications/sparta_snl/application.py index 58b5bbd67..ce97de109 100644 --- a/repos/ramble_applications/sparta_snl/application.py +++ b/repos/ramble_applications/sparta_snl/application.py @@ -26,12 +26,16 @@ class SpartaSnl(ExecutableApplication): executable( "set-size", template=[ - "sed -i -e 's/xmin equal .*/xmin equal {xmin}*$L/g' -i {in}", - "sed -i -e 's/xmax equal .*/xmax equal {xmax}*$L/g' -i {in}", - "sed -i -e 's/ymin equal .*/ymin equal {ymin}*$L/g' -i {in}", - "sed -i -e 's/ymax equal .*/ymax equal {ymax}*$L/g' -i {in}", + "sed -i -e 's/xmin equal .*/xmin equal {xmin}*$\{L\}/g' -i {in}", + "sed -i -e 's/xmax equal .*/xmax equal {xmax}*$\{L\}/g' -i {in}", + "sed -i -e 's/ymin equal .*/ymin equal {ymin}*$\{L\}/g' -i {in}", + "sed -i -e 's/ymax equal .*/ymax equal {ymax}*$\{L\}/g' -i {in}", "sed -i -e 's/ppc equal .*/ppc equal {ppc}/g' -i {in}", - "sed -i -e 's/L equal .*/L equal {L}./g' -i {in}", + "sed -i -e 's/L equal .*/L index {L}./g' -i {in}", + "sed -i -e 's/variable[[:space:]]\+tstep.*/# The default time step is too small\\\n" + "# variable tstep equal (-$\{xmin\}+$\{xmax\})\/$\{Vx\}\/$\{xncells\}\/10\/4\\\n" + "variable tstep equal (-$\{xmin\}+$\{xmax\})\/$\{Vx\}\/$\{xncells\}\/4/' -i {in}", + "sed -i -e 's/circle_R0.5_P10000.surf .*/circle_R0.5_P10000.surf group 1 invert scale $\{L\} $\{L\} 1.0/g' -i {in}", "sed -i -e 's/collide_modify .*/collide_modify {collide_modify}/g' -i {in}", ], use_mpi=False, @@ -39,7 +43,17 @@ class SpartaSnl(ExecutableApplication): executable( "set-timesteps", template=[ - "sed 's/stats.*[0-9]\+/stats {stats}/g' -i {in}", + "sed -i -e 's/stats.*[0-9]\+/stats {stats}\\\n" + "\\\n" + "# Some systems buffer extensively\\\n" + "stats_modify flush yes\\\n" + "\\\n" + "# Stop after 11 minutes\\\n" + "fix 1 halt 10 tlimit > 660.0 message no\\\n" + "\\\n" + "# Print out the value of L for parsing ease\\\n" + "print \"The value of L is \$L\" /' -i {in}", + "sed 's/run.*[0-9]\+/run {run}/g' -i {in}", ], use_mpi=False, diff --git a/repos/spack_repo/benchpark/packages/kokkos/package.py b/repos/spack_repo/benchpark/packages/kokkos/package.py index 1ea93b64b..fe1054438 100644 --- a/repos/spack_repo/benchpark/packages/kokkos/package.py +++ b/repos/spack_repo/benchpark/packages/kokkos/package.py @@ -13,12 +13,3 @@ class Kokkos(BuiltinKokkos): def setup_build_environment(self, env): if "+cuda" in self.spec: env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") - - def flag_handler(self, name, flags): - if spec.satisfies("+apu"): - if name == "cxxflags": - flags.append("-munsafe-fp-atomics") - - return flags, None, None - - diff --git a/repos/spack_repo/benchpark/packages/sparta_snl/package.py b/repos/spack_repo/benchpark/packages/sparta_snl/package.py index 896829000..9552a3aee 100644 --- a/repos/spack_repo/benchpark/packages/sparta_snl/package.py +++ b/repos/spack_repo/benchpark/packages/sparta_snl/package.py @@ -112,15 +112,12 @@ def flag_handler(self, name, flags): else: wrapper_flags.extend(flags) - return (wrapper_flags, [], build_system_flags) - - def flag_handler(self, name, flags): - if spec.satisfies("+apu"): + if self.spec.satisfies("+apu"): if name == "cxxflags": - flags.append("-fdenormal-fp-math=ieee") - flags.append("-fgpu-flush-denormals-to-zero") + build_system_flags.append("-fdenormal-fp-math=ieee") + build_system_flags.append("-fgpu-flush-denormals-to-zero") - return flags, None, None + return (wrapper_flags, [], build_system_flags) def cmake_args(self): spec = self.spec From a077d499d385b50b724b6f9671ddb0518b740ca8 Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Tue, 3 Mar 2026 16:19:45 -0800 Subject: [PATCH 05/10] Fix input parameters --- experiments/sparta-snl/experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/sparta-snl/experiment.py b/experiments/sparta-snl/experiment.py index 7c5c19f65..61312bf42 100644 --- a/experiments/sparta-snl/experiment.py +++ b/experiments/sparta-snl/experiment.py @@ -82,7 +82,7 @@ def compute_applications_section(self): ymax = 1.1 else: L = 2 # will increase problem size by 4X - ppc = 64 + ppc = 47 stats = 100 run = 10000000 xmin = -5.0 From 276a29ea0248b0239292f81e646151eaadf25943 Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Wed, 25 Mar 2026 22:35:55 -0700 Subject: [PATCH 06/10] Add kokkos 5 support --- experiments/sparta-snl/experiment.py | 4 ++-- .../benchpark/packages/kokkos/package.py | 16 +++++++++++++++- .../benchpark/packages/sparta_snl/package.py | 8 ++++++-- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/experiments/sparta-snl/experiment.py b/experiments/sparta-snl/experiment.py index 61312bf42..2293e8bf4 100644 --- a/experiments/sparta-snl/experiment.py +++ b/experiments/sparta-snl/experiment.py @@ -26,8 +26,8 @@ class SpartaSnl( variant( "version", - default="master", - values=("master",), + default="20260102", + values=("master","20260102"), description="app version", ) diff --git a/repos/spack_repo/benchpark/packages/kokkos/package.py b/repos/spack_repo/benchpark/packages/kokkos/package.py index fe1054438..383a98ca9 100644 --- a/repos/spack_repo/benchpark/packages/kokkos/package.py +++ b/repos/spack_repo/benchpark/packages/kokkos/package.py @@ -8,8 +8,22 @@ class Kokkos(BuiltinKokkos): - flag_handler = build_system_flags + variant( + "deprecated_code_4", + default=False, + when="@5:", + description="Whether to enable deprecated code in Kokkos 4", + ) def setup_build_environment(self, env): if "+cuda" in self.spec: env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") + + def cmake_args(self): + options = [opt for opt in super().cmake_args()] + if self.spec.satisfies("+deprecated_code_4"): + options.append( + self.define_from_variant("Kokkos_ENABLE_DEPRECATED_CODE_4", "deprecated_code_4") + ) + + return options diff --git a/repos/spack_repo/benchpark/packages/sparta_snl/package.py b/repos/spack_repo/benchpark/packages/sparta_snl/package.py index 9552a3aee..bb7996218 100644 --- a/repos/spack_repo/benchpark/packages/sparta_snl/package.py +++ b/repos/spack_repo/benchpark/packages/sparta_snl/package.py @@ -18,6 +18,7 @@ class SpartaSnl(CMakePackage, CudaPackage, ROCmPackage): version("master", branch="master") version("20Jan2025", tag="20Jan2025") + version("20260102", commit="478143bcc766083a100480a0a6e8a0c42c85e7e4") variant("mpi", default=True, description="Build with mpi") variant("openmp", default=False, description="Enable OpenMP support") @@ -53,13 +54,12 @@ class SpartaSnl(CMakePackage, CudaPackage, ROCmPackage): depends_on("mpi", when="+mpi") + depends_on("kokkos@5.0.0: +deprecated_code_4", when="@20260102: +kokkos") depends_on("kokkos", when="+kokkos") depends_on("kokkos+openmp cxxstd=17", when="+openmp") depends_on("kokkos+rocm", when="+rocm") depends_on("kokkos+wrapper+cuda cxxstd=17", when="+cuda") depends_on("kokkos+apu", when="+apu") - # Kokkos 5 not building - depends_on("kokkos@:4", when="+kokkos") depends_on("jpeg", when="+jpeg") depends_on("libpng", when="+png") @@ -87,6 +87,10 @@ def setup_run_environment(self, env): if self.spec.satisfies("+kokkos+rocm fft_kokkos=hipfft"): env.prepend_path("LD_LIBRARY_PATH", self.spec["hipfft"].prefix.lib) + if self.compiler.extra_rpaths: + for rpath in self.compiler.extra_rpaths: + env.prepend_path("LD_LIBRARY_PATH", rpath) + def setup_build_environment(self, env): if "+cuda" in self.spec: env.set("NVCC_APPEND_FLAGS", "-allow-unsupported-compiler") From 171bc980c48b0bef620bd4a39d5642421bdffa30 Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Thu, 26 Mar 2026 01:29:29 -0700 Subject: [PATCH 07/10] Fixes --- experiments/sparta-snl/experiment.py | 3 +-- repos/spack_repo/benchpark/packages/sparta_snl/package.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/experiments/sparta-snl/experiment.py b/experiments/sparta-snl/experiment.py index 2293e8bf4..c7487b7d1 100644 --- a/experiments/sparta-snl/experiment.py +++ b/experiments/sparta-snl/experiment.py @@ -145,12 +145,11 @@ def compute_package_section(self): fft_kokkos = self.spec.variants["fft_kokkos"][0] if self.spec.satisfies("+cuda"): fft_kokkos = "cufft" + apu = "" if self.spec.satisfies("+rocm"): fft_kokkos = "hipfft" if self.spec.satisfies("+apu"): apu = "+apu" - else: - apu = "~apu" self.add_package_spec( self.name, diff --git a/repos/spack_repo/benchpark/packages/sparta_snl/package.py b/repos/spack_repo/benchpark/packages/sparta_snl/package.py index bb7996218..f9e115a4b 100644 --- a/repos/spack_repo/benchpark/packages/sparta_snl/package.py +++ b/repos/spack_repo/benchpark/packages/sparta_snl/package.py @@ -56,9 +56,9 @@ class SpartaSnl(CMakePackage, CudaPackage, ROCmPackage): depends_on("kokkos@5.0.0: +deprecated_code_4", when="@20260102: +kokkos") depends_on("kokkos", when="+kokkos") - depends_on("kokkos+openmp cxxstd=17", when="+openmp") + depends_on("kokkos+openmp", when="+openmp") depends_on("kokkos+rocm", when="+rocm") - depends_on("kokkos+wrapper+cuda cxxstd=17", when="+cuda") + depends_on("kokkos+wrapper", when="+cuda") depends_on("kokkos+apu", when="+apu") depends_on("jpeg", when="+jpeg") From 9b4df02e639db6b9c0131af08fa859b761df760b Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Thu, 26 Mar 2026 01:56:35 -0700 Subject: [PATCH 08/10] Change default value for L --- experiments/sparta-snl/experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/sparta-snl/experiment.py b/experiments/sparta-snl/experiment.py index c7487b7d1..47eee25b4 100644 --- a/experiments/sparta-snl/experiment.py +++ b/experiments/sparta-snl/experiment.py @@ -81,7 +81,7 @@ def compute_applications_section(self): ymin = -1.1 ymax = 1.1 else: - L = 2 # will increase problem size by 4X + L = 1 # will increase problem size by 4X ppc = 47 stats = 100 run = 10000000 From 5a18c7ae93413b907f081a7b74665f9a6d8fb619 Mon Sep 17 00:00:00 2001 From: Riyaz Haque <5333387+rfhaque@users.noreply.github.com> Date: Thu, 26 Mar 2026 13:18:51 -0700 Subject: [PATCH 09/10] lint --- experiments/sparta-snl/experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/sparta-snl/experiment.py b/experiments/sparta-snl/experiment.py index 47eee25b4..275280603 100644 --- a/experiments/sparta-snl/experiment.py +++ b/experiments/sparta-snl/experiment.py @@ -27,7 +27,7 @@ class SpartaSnl( variant( "version", default="20260102", - values=("master","20260102"), + values=("master", "20260102"), description="app version", ) From 28946303444dceb9dfad8304c023de6dcd49bc1e Mon Sep 17 00:00:00 2001 From: Riyaz Haque Date: Tue, 9 Jun 2026 01:26:53 -0700 Subject: [PATCH 10/10] Remove kokkos v4 dependency for sparta --- .../benchpark/packages/kokkos/package.py | 25 ------------------- .../benchpark/packages/sparta_snl/package.py | 2 +- 2 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 repos/spack_repo/benchpark/packages/kokkos/package.py diff --git a/repos/spack_repo/benchpark/packages/kokkos/package.py b/repos/spack_repo/benchpark/packages/kokkos/package.py deleted file mode 100644 index 89cf5c38c..000000000 --- a/repos/spack_repo/benchpark/packages/kokkos/package.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -from spack.package import * -from spack_repo.builtin.packages.kokkos.package import Kokkos as BuiltinKokkos - - -class Kokkos(BuiltinKokkos): - variant( - "deprecated_code_4", - default=False, - when="@5:", - description="Whether to enable deprecated code in Kokkos 4", - ) - - def cmake_args(self): - options = [opt for opt in super().cmake_args()] - if self.spec.satisfies("+deprecated_code_4"): - options.append( - self.define_from_variant("Kokkos_ENABLE_DEPRECATED_CODE_4", "deprecated_code_4") - ) - - return options diff --git a/repos/spack_repo/benchpark/packages/sparta_snl/package.py b/repos/spack_repo/benchpark/packages/sparta_snl/package.py index 3380743d7..0d7706616 100644 --- a/repos/spack_repo/benchpark/packages/sparta_snl/package.py +++ b/repos/spack_repo/benchpark/packages/sparta_snl/package.py @@ -54,7 +54,7 @@ class SpartaSnl(CMakePackage, CudaPackage, ROCmPackage): depends_on("mpi", when="+mpi") - depends_on("kokkos@5.0.0: +deprecated_code_4", when="@20260102: +kokkos") + depends_on("kokkos@5.0.0:", when="@20260102: +kokkos") depends_on("kokkos", when="+kokkos") depends_on("kokkos+openmp", when="+openmp") depends_on("kokkos+rocm", when="+rocm")