Adjustments for (upcoming) LLVM-23 and Clang's new offload driver

This patch updates chipStar to work on non-forked, upstream
LLVM-23. The recent Clang has switched to use its new offload driver
by default for HIP compilation and chipStar is adjusted to it. There
is a still an option to use the old while it still around in Clang.

That said, chipStar doesn't yet work on the recent Clang until the
following upstream patch lands:
llvm/llvm-project#187655.

Author: linehill

CMakeLists.txt

@@ -235,6 +235,22 @@ set(SHARE_INSTALL_DIR ${CMAKE_INSTALL_PREFIX}/share CACHE PATH "share install di
 set(CONFIG_PACKAGE_INSTALL_DIR ${LIB_INSTALL_DIR}/cmake/hip CACHE PATH "cmake config install dir")
 set(CONFIG_LANG_PACKAGE_INSTALL_DIR ${LIB_INSTALL_DIR}/cmake/hip-lang CACHE PATH "cmake lang install dir")
 
+set(HAVE_CLANG_NEW_OFFLOAD_DRIVER OFF)
+if(LLVM_VERSION_MAJOR GREATER_EQUAL 23)
+  # LLVM-22: The new offload driver exists on this version but HIPSPV
+  # toolchain doesn't support it.
+  #
+  # LLVM-23: HIPSPV toolchain supports the new offload driver. The new
+  # offload driver is enabled by default. To disable it
+  # --no-offload-new-driver needs to be issued in HIP compilation and
+  # link steps.
+  #
+  # LLVM-23+: the old offload driver may be removed.
+  set(HAVE_CLANG_NEW_OFFLOAD_DRIVER ON)
+endif()
+
+include(CMakeDependentOption)
+
 # HIP options
 option(STANDALONE_TESTS "Create a separate executable for each test instead of combining tests into a shared lib by category" ON)
 
@@ -281,6 +297,10 @@ option(OCML_BASIC_ROUNDED_OPERATIONS "Use OCML implementations for devicelib fun
 option(CHIP_MALI_GPU_WORKAROUNDS "Apply work-arounds for avoiding SPIR-V \
 consumption issues in ARM Mali GPU driver." OFF)
 
+cmake_dependent_option(
+  USE_NEW_OFFLOAD_DRIVER "Use Clang's new offload driver."
+  ON HAVE_CLANG_NEW_OFFLOAD_DRIVER OFF)
+
 if(CHIP_EXT_FLOAT_ATOMICS)
   message(DEPRECATION "-DCHIP_EXT_FLOAT_ATOMICS is no longer effective.")
 endif()
@@ -404,7 +424,15 @@ endif()
 
 message(STATUS "chipStar will be installed to: ${CMAKE_INSTALL_PREFIX}")
 
-set(OFFLOAD_TRIPLE spirv64)
+if(LLVM_VERSION_MAJOR GREATER_EQUAL 23)
+  # On LLVM-23+ the emitted SPIR-V version can be controlled with
+  # subarch string (v#.#). The new "chipstar" OS component sets
+  # allowed SPIR-V extensions tailored for chipStar.
+  set(OFFLOAD_TRIPLE spirv64v1.2-unknown-chipstar)
+else()
+  set(OFFLOAD_TRIPLE spirv64)
+endif()
+
 # =============================================================================
 # chipStar BINARIES & TESTS
 add_subdirectory(llvm_passes)
@@ -415,6 +443,7 @@ add_subdirectory(bitcode)
 target_sources(CHIP PRIVATE $<TARGET_OBJECTS:rtdevlib>)
 
 set(HIPCC_BUILD_PATH "${CMAKE_BINARY_DIR}/bin")
+set(HIPCC_SPIRV_TRIPLE ${OFFLOAD_TRIPLE})
 add_subdirectory(HIPCC)
 
 # Override stale version set by HIPCC subdirectory with values from HIP/VERSION
@@ -595,22 +624,39 @@ set(HIP_OFFLOAD_COMPILE_ONLY_OPTIONS_
   # excludes the wrappers.
   -nohipwrapperinc)
 
+if(LLVM_VERSION_MAJOR GREATER_EQUAL 23 AND NOT USE_NEW_OFFLOAD_DRIVER)
+  list(APPEND HIP_OFFLOAD_COMPILE_ONLY_OPTIONS_ --no-offload-new-driver)
+endif()
+
 # Include a header for applying fixups before any user or system includes.
 set(HIP_FIXUPS_HEADER_BUILD
   -include ${CMAKE_SOURCE_DIR}/include/hip/spirv_fixups.h)
 set(HIP_FIXUPS_HEADER_INSTALL
   -include ${CMAKE_INSTALL_PREFIX}/include/hip/spirv_fixups.h)
 
 # Flags needed additionally for linking phase with -fgpu-rdc.
-set(HIP_RDC_SUPPLEMENT_LINK_FLAGS_
-  # Infors clang the type of the code object inputs (which are different than
-  # in regular host C/C++ linking)
-  --hip-link
-  # Required for selecting HIPSPV toolchain instead of AMD's one in clang.
-  --offload=spirv64
-  # --hip-path is also needed but not included here (different option
-  # value is needed for build and installation).
-)
+#
+# --hip-path is also needed in RDC linking stage but on purpose it is
+# not added to HIP_RDC_SUPPLEMENT_LINK_FLAGS_ (different option value
+# is needed for build and installation). The option is used to
+# implicitly load and run chipStar's LLVM passes during this stage.
+#
+# On the new offload driver, the --hip-path needs to be wrapped in
+# -Xoffload-compiler-${OFFLOAD_TRIPLE}.
+if(USE_NEW_OFFLOAD_DRIVER)
+  set(HIP_RDC_SUPPLEMENT_LINK_FLAGS_ --offload-link)
+else()
+  set(HIP_RDC_SUPPLEMENT_LINK_FLAGS_
+    # Infors clang the type of the code object inputs (which are different than
+    # in regular host C/C++ linking).
+    --hip-link
+    # Required for selecting HIPSPV toolchain instead of AMD's one in clang.
+    --offload=${OFFLOAD_TRIPLE})
+
+  if(LLVM_VERSION_MAJOR GREATER_EQUAL 23)
+    list(APPEND HIP_RDC_SUPPLEMENT_LINK_FLAGS_ --no-offload-new-driver)
+  endif()
+endif()
 
 if (NOT CLANG_VERSION_LESS_15)
   list(APPEND HIP_RDC_SUPPLEMENT_LINK_FLAGS_
@@ -728,13 +774,16 @@ target_link_libraries(host INTERFACE CHIP)
 add_library(deviceRDCInternal INTERFACE)
 target_compile_options(deviceRDCInternal INTERFACE -fgpu-rdc)
 target_link_options(deviceRDCInternal INTERFACE
-  -fgpu-rdc ${HIP_RDC_SUPPLEMENT_LINK_FLAGS_} --hip-path=${CMAKE_BINARY_DIR})
+  -fgpu-rdc ${HIP_RDC_SUPPLEMENT_LINK_FLAGS_}
+  $<$<BOOL:${USE_NEW_OFFLOAD_DRIVER}>:-Xoffload-compiler-${OFFLOAD_TRIPLE}>
+  --hip-path=${CMAKE_BINARY_DIR})
 target_link_libraries(deviceRDCInternal INTERFACE deviceInternal)
 
 add_library(deviceRDC INTERFACE)
 target_compile_options(deviceRDC INTERFACE -fgpu-rdc)
 target_link_options(deviceRDC INTERFACE
   -fgpu-rdc ${HIP_RDC_SUPPLEMENT_LINK_FLAGS_}
+  $<$<BOOL:${USE_NEW_OFFLOAD_DRIVER}>:-Xoffload-compiler-${OFFLOAD_TRIPLE}>
   --hip-path=${CMAKE_INSTALL_PREFIX})
 target_link_libraries(deviceRDC INTERFACE device)
 
@@ -822,9 +871,15 @@ add_to_config(_hipInfo_install
   HIP_OFFLOAD_COMPILE_OPTIONS "${HIP_OFFLOAD_COMPILE_OPTIONS_INSTALL}")
 add_to_config(_hipInfo_install
   HIP_OFFLOAD_LINK_OPTIONS "${HIP_OFFLOAD_LINK_OPTIONS_INSTALL}")
-add_to_config(_hipInfo_install
-  HIP_OFFLOAD_RDC_SUPPLEMENT_LINK_OPTIONS
-  "${HIP_RDC_SUPPLEMENT_LINK_FLAGS} --hip-path=${CMAKE_INSTALL_PREFIX}")
+if(USE_NEW_OFFLOAD_DRIVER)
+  add_to_config(_hipInfo_install
+    HIP_OFFLOAD_RDC_SUPPLEMENT_LINK_OPTIONS
+    "${HIP_RDC_SUPPLEMENT_LINK_FLAGS} -Xoffload-compiler-${OFFLOAD_TRIPLE} --hip-path=${CMAKE_INSTALL_PREFIX}")
+else()
+  add_to_config(_hipInfo_install
+    HIP_OFFLOAD_RDC_SUPPLEMENT_LINK_OPTIONS
+    "${HIP_RDC_SUPPLEMENT_LINK_FLAGS} --hip-path=${CMAKE_INSTALL_PREFIX}")
+endif()
 
 # Build version
 set(_hipInfo_build "# Auto-generated by cmake on ${_timestamp} UTC\n")
@@ -837,9 +892,15 @@ add_to_config(_hipInfo_build
   HIP_OFFLOAD_COMPILE_OPTIONS "${HIP_OFFLOAD_COMPILE_OPTIONS_BUILD}")
 add_to_config(_hipInfo_build
   HIP_OFFLOAD_LINK_OPTIONS "${HIP_OFFLOAD_LINK_OPTIONS_BUILD}")
-add_to_config(_hipInfo_build
-  HIP_OFFLOAD_RDC_SUPPLEMENT_LINK_OPTIONS
-  "${HIP_RDC_SUPPLEMENT_LINK_FLAGS} --hip-path=${CMAKE_BINARY_DIR}")
+if(USE_NEW_OFFLOAD_DRIVER)
+  add_to_config(_hipInfo_build
+    HIP_OFFLOAD_RDC_SUPPLEMENT_LINK_OPTIONS
+    "${HIP_RDC_SUPPLEMENT_LINK_FLAGS} -Xoffload-compiler-${OFFLOAD_TRIPLE} --hip-path=${CMAKE_BINARY_DIR}")
+else()
+  add_to_config(_hipInfo_build
+    HIP_OFFLOAD_RDC_SUPPLEMENT_LINK_OPTIONS
+    "${HIP_RDC_SUPPLEMENT_LINK_FLAGS} --hip-path=${CMAKE_BINARY_DIR}")
+endif()
 
 add_library(hip::host ALIAS host)
 add_library(hip::device ALIAS device)

llvm_passes/HipPrintf.cpp

@@ -133,7 +133,11 @@ getFormatStringPieces(Value *FmtStrArg, unsigned &NumberOfFormatSpecs) {
       dyn_cast<ConstantDataSequential>(OrigFmtStr->getInitializer());
 
   if (FmtStrData == nullptr) {
+#if LLVM_VERSION_MAJOR >= 23
+    assert(OrigFmtStr->getInitializer()->isNullValue());
+#else
     assert(OrigFmtStr->getInitializer()->isZeroValue());
+#endif
     FmtStrPieces.push_back("");
     NumberOfFormatSpecs = 0;
     return FmtStrPieces;
@@ -218,9 +222,16 @@ std::string getCDSAsString(GlobalVariable *OrigStr, bool &IsEmpty) {
   ConstantDataSequential *CDSInitializer =
       dyn_cast<ConstantDataSequential>(OrigStr->getInitializer());
 
+#if LLVM_VERSION_MAJOR >= 23
+  assert(OrigStr->getInitializer()->isNullValue() || CDSInitializer != nullptr);
+
+  IsEmpty = OrigStr->getInitializer()->isNullValue();
+#else
   assert(OrigStr->getInitializer()->isZeroValue() || CDSInitializer != nullptr);
 
   IsEmpty = OrigStr->getInitializer()->isZeroValue();
+#endif
+
   return (IsEmpty ? "" : std::string(CDSInitializer->getAsCString()));
 }
 

tests/compiler/TestHipcc692Regression.bash

@@ -15,10 +15,7 @@ mkdir -p ${OUT_DIR}/CMakeFiles/test_hipcub_basic.dir
 export HIPCC_VERBOSE=7
 
 ${HIPCC} \
--DGTEST_LINKED_AS_SHARED_LIBRARY=1  \
--O3 -DNDEBUG  \
--x hip -D__HIP_PLATFORM_SPIRV__= --offload=spirv64 -nohipwrapperinc --hip-path=${BIN_DIR} --target=x86_64-unknown-linux-gnu  \
--include ${SRC_DIR}/include/hip/spirv_fixups.h -std=c++14  \
+-O3 -DNDEBUG -std=c++14  \
 -c ${OUT_DIR}/test_hipcub_basic.cpp \
 -MD -MT ${OUT_DIR}/CMakeFiles/test_hipcub_basic.dir/test_hipcub_basic.cpp.o  \
 -MF ${OUT_DIR}/CMakeFiles/test_hipcub_basic.dir/test_hipcub_basic.cpp.o.d \

tests/compiler/rdcLink/TestStaticLibRDC.bash

@@ -27,7 +27,8 @@ ar rcs "${OUT_DIR}/libk.a" "${OUT_DIR}/k.o" "${OUT_DIR}/k1.o"
 ${HIPCC} -fgpu-rdc -I"${SRC_DIR}" -c "${SRC_DIR}/t.cpp" -o "${OUT_DIR}/t.o"
 
 # Link the main file and the static library
-${HIPCC} --save-temps -v -fgpu-rdc --hip-link "${OUT_DIR}/t.o" "${OUT_DIR}/libk.a" -o "${OUT_DIR}/TestStaticLibRDC" 
+${HIPCC} --save-temps -v -fgpu-rdc "${OUT_DIR}/t.o" "${OUT_DIR}/libk.a" \
+         -o "${OUT_DIR}/TestStaticLibRDC"
 echo "TestStaticLibRDC.log: ${OUT_DIR}/TestStaticLibRDC.log"
 ${OUT_DIR}/TestStaticLibRDC
 

tests/runtime/CMakeLists.txt

@@ -185,7 +185,11 @@ add_hip_runtime_test(TestHipLaunchHostFunc.cpp)
 
 add_hip_runtime_test(TestBoolKernelParam.hip)
 find_program(SPIRV_DIS spirv-dis)
-if(SPIRV_DIS)
+find_program(
+  CLANG_OFFLOAD_BUNDLER
+  NAMES clang-offload-bundler-${LLVM_VERSION_MAJOR} clang-offload-bundler
+  PATHS ${HIP_CLANG_PATH})
+if(SPIRV_DIS AND (NOT USE_NEW_OFFLOAD_DRIVER OR CLANG_OFFLOAD_BUNDLER))
   add_shell_test(TestBoolKernelParamSPIRV.bash)
 endif()
 # Test hipLaunchHostFunc with multi-stream and events (from PR #1071)

tests/runtime/TestBoolKernelParamSPIRV.bash

@@ -22,10 +22,20 @@ fi
 
 # Compile with --save-temps to get the .out SPIR-V file
 cd "$WORKDIR"
-"$HIPCC" --save-temps "$SRC" -o test_bool 2>/dev/null
 
-# Find the SPIR-V binary
-SPV_FILE=$(ls *.out 2>/dev/null | head -1)
+if [ "@USE_NEW_OFFLOAD_DRIVER@" = "ON" ]; then
+    "$HIPCC" --offload-device-only -c "$SRC" -o device.o 2>/dev/null
+    "@CLANG_OFFLOAD_BUNDLER@" -unbundle --type=o \
+                              --targets=hip-@OFFLOAD_TRIPLE@--generic \
+                              --inputs=device.o --output=spv_binary.out
+    SPV_FILE=spv_binary.out
+else
+    "$HIPCC" --save-temps "$SRC" -o test_bool 2>/dev/null
+
+    # Find the SPIR-V binary
+    SPV_FILE=$(ls *.out 2>/dev/null | head -1)
+fi
+
 if [ -z "$SPV_FILE" ]; then
   echo "FAIL: No .out SPIR-V file produced by hipcc --save-temps"
   exit 1