Sparta changes

experiments/sparta-snl/experiment.py

@@ -26,8 +26,8 @@ class SpartaSnl(
 
     variant(
         "version",
-        default="master",
-        values=("master",),
+        default="20260102",
+        values=("master", "20260102"),
         description="app version",
     )
 
@@ -54,6 +54,20 @@ class SpartaSnl(
         description="Enable GPU-aware MPI",
     )
 
+    variant(
+        "apu",
+        default=False,
+        values=(True, False),
+        description="Enable APU support",
+    )
+
+    variant(
+        "other",
+        default=False,
+        values=(True, False),
+        description="Set other input/environment variables",
+    )
+
     maintainers("rfhaque")
 
     def compute_applications_section(self):
@@ -67,14 +81,14 @@ def compute_applications_section(self):
             ymin = -1.1
             ymax = 1.1
         else:
-            L = 2  # will increase problem size by 4X
-            ppc = 64
-            stats = 10
-            run = 100
-            xmin = -1.0
-            xmax = 1.1
-            ymin = -1.1
-            ymax = 1.1
+            L = 1  # will increase problem size by 4X
+            ppc = 47
+            stats = 100
+            run = 10000000
+            xmin = -5.0
+            xmax = 5.1
+            ymin = -5.1
+            ymax = 5.1
 
         if self.spec.satisfies("workload=cylinder"):
             self.add_experiment_variable("L", L, True)
@@ -87,6 +101,14 @@ def compute_applications_section(self):
         self.add_experiment_variable("stats", stats, True)
         self.add_experiment_variable("run", run, True)
 
+        if self.spec.satisfies("+rocm"):
+            if self.spec.satisfies("+other"):
+                self.set_environment_variable("MPICH_OFI_NIC_POLICY", "GPU")
+                self.set_environment_variable("FI_HMEM", "rocr")
+                self.set_environment_variable("HUGETLB_MORECORE", "yes")
+                self.set_environment_variable("HUGETLB_RESTRICT_EXE", "defrag:lmp")
+                self.set_environment_variable("HSA_XNACK", 1)
+
         if self.spec.satisfies("+rocm") or self.spec.satisfies("+cuda"):
             kokkos_mode = "g 1"
             kokkos_gpu_aware = "yes" if self.spec.satisfies("+gpu-aware-mpi") else "no"
@@ -123,12 +145,15 @@ def compute_package_section(self):
         fft_kokkos = self.spec.variants["fft_kokkos"][0]
         if self.spec.satisfies("+cuda"):
             fft_kokkos = "cufft"
+        apu = ""
         if self.spec.satisfies("+rocm"):
             fft_kokkos = "hipfft"
+            if self.spec.satisfies("+apu"):
+                apu = "+apu"
 
         self.add_package_spec(
             self.name,
             [
-                f"sparta-snl{self.determine_version()} +mpi+kokkos fft_kokkos={fft_kokkos} fft={fft} "
+                f"sparta-snl{self.determine_version()} +mpi+kokkos fft_kokkos={fft_kokkos} fft={fft} {apu} "
             ],
         )

repos/ramble_applications/sparta_snl/application.py

@@ -26,20 +26,34 @@ class SpartaSnl(ExecutableApplication):
     executable(
         "set-size",
         template=[
-            "sed -i -e 's/xmin equal .*/xmin equal {xmin}*$L/g' -i {in}",
-            "sed -i -e 's/xmax equal .*/xmax equal {xmax}*$L/g' -i {in}",
-            "sed -i -e 's/ymin equal .*/ymin equal {ymin}*$L/g' -i {in}",
-            "sed -i -e 's/ymax equal .*/ymax equal {ymax}*$L/g' -i {in}",
+            "sed -i -e 's/xmin equal .*/xmin equal {xmin}*$\{L\}/g' -i {in}",
+            "sed -i -e 's/xmax equal .*/xmax equal {xmax}*$\{L\}/g' -i {in}",
+            "sed -i -e 's/ymin equal .*/ymin equal {ymin}*$\{L\}/g' -i {in}",
+            "sed -i -e 's/ymax equal .*/ymax equal {ymax}*$\{L\}/g' -i {in}",
             "sed -i -e 's/ppc equal .*/ppc equal {ppc}/g' -i {in}",
-            "sed -i -e 's/L equal .*/L equal {L}./g' -i {in}",
+            "sed -i -e 's/L equal .*/L index {L}./g' -i {in}",
+            "sed -i -e 's/variable[[:space:]]\+tstep.*/# The default time step is too small\\\n"
+            "# variable            tstep equal (-$\{xmin\}+$\{xmax\})\/$\{Vx\}\/$\{xncells\}\/10\/4\\\n"
+            "variable            tstep equal (-$\{xmin\}+$\{xmax\})\/$\{Vx\}\/$\{xncells\}\/4/' -i {in}",
+            "sed -i -e 's/circle_R0.5_P10000.surf .*/circle_R0.5_P10000.surf group 1 invert scale $\{L\} $\{L\} 1.0/g' -i {in}",
             "sed -i -e 's/collide_modify .*/collide_modify      {collide_modify}/g' -i {in}",
         ],
         use_mpi=False,
     )
     executable(
         "set-timesteps",
         template=[
-            "sed 's/stats.*[0-9]\+/stats               {stats}/g' -i {in}",
+            "sed -i -e 's/stats.*[0-9]\+/stats               {stats}\\\n"
+            "\\\n"
+            "# Some systems buffer extensively\\\n"
+            "stats_modify        flush yes\\\n"
+            "\\\n"
+            "# Stop after 11 minutes\\\n"
+            "fix 1 halt 10 tlimit > 660.0 message no\\\n"
+            "\\\n"
+            "# Print out the value of L for parsing ease\\\n"
+            "print \"The value of L is \$L\" /' -i {in}",
+
             "sed 's/run.*[0-9]\+/run                 {run}/g' -i {in}",
         ],
         use_mpi=False,

repos/spack_repo/benchpark/packages/sparta_snl/package.py

@@ -18,11 +18,13 @@ class SpartaSnl(CMakePackage, CudaPackage, ROCmPackage):
 
     version("master", branch="master")
     version("20Jan2025", tag="20Jan2025")
+    version("20260102", commit="478143bcc766083a100480a0a6e8a0c42c85e7e4")
 
     variant("mpi", default=True, description="Build with mpi")
     variant("openmp", default=False, description="Enable OpenMP support")
     variant("jpeg", default=False, description="Build with jpeg support")
     variant("png", default=False, description="Build with png support")
+    variant("apu", default=False, description="Enable APU support", when="@4.5: +rocm")
 
     variant(
         "kokkos",
@@ -52,12 +54,12 @@ class SpartaSnl(CMakePackage, CudaPackage, ROCmPackage):
 
     depends_on("mpi", when="+mpi")
 
+    depends_on("kokkos@5.0.0:", when="@20260102: +kokkos")
     depends_on("kokkos", when="+kokkos")
-    depends_on("kokkos+openmp cxxstd=17", when="+openmp")
+    depends_on("kokkos+openmp", when="+openmp")
     depends_on("kokkos+rocm", when="+rocm")
-    depends_on("kokkos+wrapper+cuda cxxstd=17", when="+cuda")
-    # Kokkos 5 not building
-    depends_on("kokkos@:4", when="+kokkos")
+    depends_on("kokkos+wrapper", when="+cuda")
+    depends_on("kokkos+apu", when="+apu")
 
     depends_on("jpeg", when="+jpeg")
     depends_on("libpng", when="+png")
@@ -85,6 +87,10 @@ def setup_run_environment(self, env):
         if self.spec.satisfies("+kokkos+rocm fft_kokkos=hipfft"):
             env.prepend_path("LD_LIBRARY_PATH", self.spec["hipfft"].prefix.lib)
 
+        if self.compiler.extra_rpaths:
+            for rpath in self.compiler.extra_rpaths:
+                env.prepend_path("LD_LIBRARY_PATH", rpath)
+
     def setup_build_environment(self, env):
         if self.spec.satisfies("+kokkos+rocm fft_kokkos=hipfft"):
             env.prepend_path("LD_LIBRARY_PATH", self.spec["hipfft"].prefix.lib)
@@ -108,6 +114,11 @@ def flag_handler(self, name, flags):
         else:
             wrapper_flags.extend(flags)
 
+        if self.spec.satisfies("+apu"):
+            if name == "cxxflags":
+                build_system_flags.append("-fdenormal-fp-math=ieee")
+                build_system_flags.append("-fgpu-flush-denormals-to-zero")
+
         return (wrapper_flags, [], build_system_flags)
 
     def cmake_args(self):
@@ -146,6 +157,16 @@ def cmake_args(self):
         else:
             args.append(self.define("PKG_KOKKOS", False))
 
+        if spec.satisfies("+apu"):
+            existing_ldflags = self.spec.compiler_flags.get("ldflags", [])
+            existing = " ".join(existing_ldflags)
+            extra = " -lxpmem -lhugetlbfs"
+            if existing:
+                combined = f"{existing} {extra}"
+            else:
+                combined = extra
+            args.append(f"-DCMAKE_EXE_LINKER_FLAGS={combined}")
+
         return args
 
     def install(self, spec, prefix):