Merge pull request #2911 from ROCm/r2.18-rocm-enhanced-clang-19

Support clang19 as host compiler

Author: draganmladjenovic

third_party/gpus/compiler_common_tools.bzl

@@ -72,8 +72,14 @@ def _get_cxx_inc_directories_impl(repository_ctx, cc, lang_is_cpp, tf_sys_root):
     sysroot = []
     if tf_sys_root:
         sysroot += ["--sysroot", tf_sys_root]
+    no_canonical_prefixes_supported = _is_compiler_option_supported(
+        repository_ctx,
+        cc,
+        "-no-canonical-prefixes",
+    )
+    no_canonical_prefixes = (["-no-canonical-prefixes"] if no_canonical_prefixes_supported else [])
     result = raw_exec(repository_ctx, [cc, "-E", "-x" + lang, "-", "-v"] +
-                                      sysroot)
+                                      sysroot + no_canonical_prefixes)
     stderr = err_out(result)
     index1 = stderr.find(_INC_DIR_MARKER_BEGIN)
     if index1 == -1:
@@ -148,7 +154,7 @@ def _get_cxx_inc_directories_impl(repository_ctx, cc, lang_is_cpp, tf_sys_root):
         compiler_includes = compiler_includes + unresolved_compiler_includes
     return compiler_includes
 
-def get_cxx_inc_directories(repository_ctx, cc, tf_sys_root):
+def get_cxx_inc_directories(repository_ctx, cc, tf_sys_root = None):
     """Compute the list of default C and C++ include directories."""
 
     # For some reason `clang -xc` sometimes returns include paths that are

third_party/gpus/rocm_configure.bzl

@@ -20,10 +20,12 @@ load(
     "get_python_bin",
     "raw_exec",
     "realpath",
+    "relative_to",
     "which",
 )
 load(
     ":compiler_common_tools.bzl",
+    "get_cxx_inc_directories",
     "to_list_of_strings",
 )
 load(
@@ -101,68 +103,6 @@ def find_cc(repository_ctx):
               " environment variable").format(target_cc_name, cc_path_envvar))
     return cc
 
-_INC_DIR_MARKER_BEGIN = "#include <...>"
-
-def _cxx_inc_convert(path):
-    """Convert path returned by cc -E xc++ in a complete path."""
-    path = path.strip()
-    return path
-
-def _get_cxx_inc_directories_impl(repository_ctx, cc, lang_is_cpp):
-    """Compute the list of default C or C++ include directories."""
-    if lang_is_cpp:
-        lang = "c++"
-    else:
-        lang = "c"
-
-    # TODO: We pass -no-canonical-prefixes here to match the compiler flags,
-    #       but in rocm_clang CROSSTOOL file that is a `feature` and we should
-    #       handle the case when it's disabled and no flag is passed
-    result = raw_exec(repository_ctx, [
-        cc,
-        "-no-canonical-prefixes",
-        "-E",
-        "-x" + lang,
-        "-",
-        "-v",
-    ])
-    stderr = err_out(result)
-    index1 = stderr.find(_INC_DIR_MARKER_BEGIN)
-    if index1 == -1:
-        return []
-    index1 = stderr.find("\n", index1)
-    if index1 == -1:
-        return []
-    index2 = stderr.rfind("\n ")
-    if index2 == -1 or index2 < index1:
-        return []
-    index2 = stderr.find("\n", index2 + 1)
-    if index2 == -1:
-        inc_dirs = stderr[index1 + 1:]
-    else:
-        inc_dirs = stderr[index1 + 1:index2].strip()
-
-    return [
-        str(repository_ctx.path(_cxx_inc_convert(p)))
-        for p in inc_dirs.split("\n")
-    ]
-
-def get_cxx_inc_directories(repository_ctx, cc):
-    """Compute the list of default C and C++ include directories."""
-
-    # For some reason `clang -xc` sometimes returns include paths that are
-    # different from the ones from `clang -xc++`. (Symlink and a dir)
-    # So we run the compiler with both `-xc` and `-xc++` and merge resulting lists
-    includes_cpp = _get_cxx_inc_directories_impl(repository_ctx, cc, True)
-    includes_c = _get_cxx_inc_directories_impl(repository_ctx, cc, False)
-
-    includes_cpp_set = depset(includes_cpp)
-    return includes_cpp + [
-        inc
-        for inc in includes_c
-        if inc not in includes_cpp_set.to_list()
-    ]
-
 def auto_configure_fail(msg):
     """Output failure message when rocm configuration fails."""
     red = "\033[0;31m"
@@ -178,54 +118,36 @@ def auto_configure_warning(msg):
 # END cc_configure common functions (see TODO above).
 
 def _rocm_include_path(repository_ctx, rocm_config, bash_bin):
-    """Generates the cxx_builtin_include_directory entries for rocm inc dirs.
+    """Generates the entries for rocm inc dirs based on rocm_config.
 
     Args:
       repository_ctx: The repository context.
       rocm_config: The path to the gcc host compiler.
+      bash_bin: path to the bash interpreter.
 
     Returns:
-      A string containing the Starlark string for each of the gcc
-      host compiler include directories, which can be added to the CROSSTOOL
+      A string containing the Starlark string for each of the hipcc
+      compiler include directories, which can be added to the CROSSTOOL
       file.
     """
     inc_dirs = []
 
-    # Add HSA headers (needs to match $HSA_PATH)
-    inc_dirs.append(rocm_config.rocm_toolkit_path + "/hsa/include")
-
-    # Add HIP headers (needs to match $HIP_PATH)
-    inc_dirs.append(rocm_config.rocm_toolkit_path + "/hip/include")
-    if int(rocm_config.rocm_version_number) >= 50200:
-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include")
-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/hip")
-        inc_dirs.append(rocm_config.rocm_paths["ROCPRIM"] + "/include/rocprim")
-        inc_dirs.append(rocm_config.rocm_paths["ROCSOLVER"] + "/include/rocsolver")
-        inc_dirs.append(rocm_config.rocm_paths["ROCBLAS"] + "/include/rocblas")
-
-    # Add HIP-Clang headers (realpath relative to compiler binary)
-    inc_dirs.append(rocm_config.llvm_path + "/lib/clang/12.0.0/include")
-    inc_dirs.append(rocm_config.llvm_path + "/lib/clang/13.0.0/include")
-    inc_dirs.append(rocm_config.llvm_path + "/lib/clang/14.0.0/include")
-    inc_dirs.append(rocm_config.llvm_path + "/lib/clang/15.0.0/include")
-    inc_dirs.append(rocm_config.llvm_path + "/lib/clang/16.0.0/include")
-    inc_dirs.append(rocm_config.llvm_path + "/lib/clang/17.0.0/include/")
-    inc_dirs.append(rocm_config.llvm_path + "/lib/clang/17/include")
-    inc_dirs.append(rocm_config.llvm_path + "/lib/clang/18/include")
-    inc_dirs.append(rocm_config.llvm_path + "/lib/clang/19/include")
-    rocm_toolkit_path = realpath(repository_ctx, rocm_config.rocm_toolkit_path, bash_bin)
-    if int(rocm_config.rocm_version_number) >= 60200:
-        inc_dirs.append(rocm_toolkit_path + "/lib/llvm/lib/clang/17/include")
-        inc_dirs.append(rocm_toolkit_path + "/lib/llvm/lib/clang/18/include")
-        inc_dirs.append(rocm_toolkit_path + "/lib/llvm/lib/clang/19/include")
-        inc_dirs.append(rocm_toolkit_path + "/lib/llvm/lib/clang/20/include")
-
-    # Support hcc based off clang 10.0.0 (for ROCm 3.3)
-    inc_dirs.append(rocm_toolkit_path + "/hcc/compiler/lib/clang/10.0.0/include/")
-    inc_dirs.append(rocm_toolkit_path + "/hcc/lib/clang/10.0.0/include")
-
-    # Add hcc headers
-    inc_dirs.append(rocm_toolkit_path + "/hcc/include")
+    # Add HIP-Clang headers (relative to rocm root)
+    rocm_path = repository_ctx.path(rocm_config.rocm_toolkit_path)
+    clang_path = rocm_path.get_child("llvm/bin/clang")
+    resource_dir_result = execute(repository_ctx, [str(clang_path), "-print-resource-dir"])
+
+    if resource_dir_result.return_code:
+        auto_configure_fail("Failed to run hipcc -print-resource-dir: %s" % err_out(resource_dir_result))
+
+    resource_dir_abs = resource_dir_result.stdout.strip()
+
+    resource_dir_rel = relative_to(repository_ctx, str(rocm_path.realpath), resource_dir_abs, bash_bin)
+
+    resource_dir = str(rocm_path.get_child(resource_dir_rel))
+
+    inc_dirs.append(resource_dir + "/include")
+    inc_dirs.append(resource_dir + "/share")
 
     return inc_dirs
 

third_party/remote_config/common.bzl

@@ -289,6 +289,23 @@ def realpath(repository_ctx, path, bash_bin = None):
 
     return execute(repository_ctx, [bash_bin, "-c", "realpath \"%s\"" % path]).stdout.strip()
 
+def relative_to(repository_ctx, base, path, bash_bin = None):
+    """Returns the result of "realpath --relative-to".
+
+    Args:
+      repository_ctx: the repository_ctx
+      base: a path on the file system
+      path: a path on the file system
+      bash_bin: path to the bash interpreter
+
+    Returns:
+      Returns the result of "realpath --relative-to"
+    """
+    if bash_bin == None:
+        bash_bin = get_bash_bin(repository_ctx)
+
+    return execute(repository_ctx, [bash_bin, "-c", "realpath --relative-to \"%s\" \"%s\"" % (base, path)]).stdout.strip()
+
 def err_out(result):
     """Returns stderr if set, else stdout.
 

third_party/xla/third_party/tsl/third_party/gpus/compiler_common_tools.bzl

@@ -72,8 +72,14 @@ def _get_cxx_inc_directories_impl(repository_ctx, cc, lang_is_cpp, tf_sys_root):
     sysroot = []
     if tf_sys_root:
         sysroot += ["--sysroot", tf_sys_root]
+    no_canonical_prefixes_supported = _is_compiler_option_supported(
+        repository_ctx,
+        cc,
+        "-no-canonical-prefixes",
+    )
+    no_canonical_prefixes = (["-no-canonical-prefixes"] if no_canonical_prefixes_supported else [])
     result = raw_exec(repository_ctx, [cc, "-E", "-x" + lang, "-", "-v"] +
-                                      sysroot)
+                                      sysroot + no_canonical_prefixes)
     stderr = err_out(result)
     index1 = stderr.find(_INC_DIR_MARKER_BEGIN)
     if index1 == -1:
@@ -148,7 +154,7 @@ def _get_cxx_inc_directories_impl(repository_ctx, cc, lang_is_cpp, tf_sys_root):
         compiler_includes = compiler_includes + unresolved_compiler_includes
     return compiler_includes
 
-def get_cxx_inc_directories(repository_ctx, cc, tf_sys_root):
+def get_cxx_inc_directories(repository_ctx, cc, tf_sys_root = None):
     """Compute the list of default C and C++ include directories."""
 
     # For some reason `clang -xc` sometimes returns include paths that are

third_party/xla/third_party/tsl/third_party/gpus/rocm_configure.bzl

@@ -20,10 +20,12 @@ load(
     "get_python_bin",
     "raw_exec",
     "realpath",
+    "relative_to",
     "which",
 )
 load(
     ":compiler_common_tools.bzl",
+    "get_cxx_inc_directories",
     "to_list_of_strings",
 )
 load(
@@ -92,68 +94,6 @@ def find_cc(repository_ctx):
               " environment variable").format(target_cc_name, cc_path_envvar))
     return cc
 
-_INC_DIR_MARKER_BEGIN = "#include <...>"
-
-def _cxx_inc_convert(path):
-    """Convert path returned by cc -E xc++ in a complete path."""
-    path = path.strip()
-    return path
-
-def _get_cxx_inc_directories_impl(repository_ctx, cc, lang_is_cpp):
-    """Compute the list of default C or C++ include directories."""
-    if lang_is_cpp:
-        lang = "c++"
-    else:
-        lang = "c"
-
-    # TODO: We pass -no-canonical-prefixes here to match the compiler flags,
-    #       but in rocm_clang CROSSTOOL file that is a `feature` and we should
-    #       handle the case when it's disabled and no flag is passed
-    result = raw_exec(repository_ctx, [
-        cc,
-        "-no-canonical-prefixes",
-        "-E",
-        "-x" + lang,
-        "-",
-        "-v",
-    ])
-    stderr = err_out(result)
-    index1 = stderr.find(_INC_DIR_MARKER_BEGIN)
-    if index1 == -1:
-        return []
-    index1 = stderr.find("\n", index1)
-    if index1 == -1:
-        return []
-    index2 = stderr.rfind("\n ")
-    if index2 == -1 or index2 < index1:
-        return []
-    index2 = stderr.find("\n", index2 + 1)
-    if index2 == -1:
-        inc_dirs = stderr[index1 + 1:]
-    else:
-        inc_dirs = stderr[index1 + 1:index2].strip()
-
-    return [
-        str(repository_ctx.path(_cxx_inc_convert(p)))
-        for p in inc_dirs.split("\n")
-    ]
-
-def get_cxx_inc_directories(repository_ctx, cc):
-    """Compute the list of default C and C++ include directories."""
-
-    # For some reason `clang -xc` sometimes returns include paths that are
-    # different from the ones from `clang -xc++`. (Symlink and a dir)
-    # So we run the compiler with both `-xc` and `-xc++` and merge resulting lists
-    includes_cpp = _get_cxx_inc_directories_impl(repository_ctx, cc, True)
-    includes_c = _get_cxx_inc_directories_impl(repository_ctx, cc, False)
-
-    includes_cpp_set = depset(includes_cpp)
-    return includes_cpp + [
-        inc
-        for inc in includes_c
-        if inc not in includes_cpp_set.to_list()
-    ]
-
 def auto_configure_fail(msg):
     """Output failure message when rocm configuration fails."""
     red = "\033[0;31m"
@@ -169,54 +109,36 @@ def auto_configure_warning(msg):
 # END cc_configure common functions (see TODO above).
 
 def _rocm_include_path(repository_ctx, rocm_config, bash_bin):
-    """Generates the cxx_builtin_include_directory entries for rocm inc dirs.
+    """Generates the entries for rocm inc dirs based on rocm_config.
 
     Args:
       repository_ctx: The repository context.
       rocm_config: The path to the gcc host compiler.
+      bash_bin: path to the bash interpreter.
 
     Returns:
-      A string containing the Starlark string for each of the gcc
-      host compiler include directories, which can be added to the CROSSTOOL
+      A string containing the Starlark string for each of the hipcc
+      compiler include directories, which can be added to the CROSSTOOL
       file.
     """
     inc_dirs = []
 
-    # Add HSA headers (needs to match $HSA_PATH)
-    inc_dirs.append(rocm_config.rocm_toolkit_path + "/hsa/include")
-
-    # Add HIP headers (needs to match $HIP_PATH)
-    inc_dirs.append(rocm_config.rocm_toolkit_path + "/hip/include")
-    if int(rocm_config.rocm_version_number) >= 50200:
-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include")
-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/hip")
-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocprim")
-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocsolver")
-        inc_dirs.append(rocm_config.rocm_toolkit_path + "/include/rocblas")
-
-    # Add HIP-Clang headers (realpath relative to compiler binary)
-    rocm_toolkit_path = realpath(repository_ctx, rocm_config.rocm_toolkit_path, bash_bin)
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/8.0/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/9.0.0/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/10.0.0/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/11.0.0/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/12.0.0/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/13.0.0/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/14.0.0/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/15.0.0/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/16.0.0/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/17.0.0/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/17/include")
-    inc_dirs.append(rocm_toolkit_path + "/llvm/lib/clang/18/include")
-    if int(rocm_config.rocm_version_number) >= 60200:
-        inc_dirs.append(rocm_toolkit_path + "/lib/llvm/lib/clang/18/include")
-
-    # Support hcc based off clang 10.0.0 (for ROCm 3.3)
-    inc_dirs.append(rocm_toolkit_path + "/hcc/compiler/lib/clang/10.0.0/include/")
-    inc_dirs.append(rocm_toolkit_path + "/hcc/lib/clang/10.0.0/include")
-
-    # Add hcc headers
-    inc_dirs.append(rocm_toolkit_path + "/hcc/include")
+    # Add HIP-Clang headers (relative to rocm root)
+    rocm_path = repository_ctx.path(rocm_config.rocm_toolkit_path)
+    clang_path = rocm_path.get_child("llvm/bin/clang")
+    resource_dir_result = execute(repository_ctx, [str(clang_path), "-print-resource-dir"])
+
+    if resource_dir_result.return_code:
+        auto_configure_fail("Failed to run hipcc -print-resource-dir: %s" % err_out(resource_dir_result))
+
+    resource_dir_abs = resource_dir_result.stdout.strip()
+
+    resource_dir_rel = relative_to(repository_ctx, str(rocm_path.realpath), resource_dir_abs, bash_bin)
+
+    resource_dir = str(rocm_path.get_child(resource_dir_rel))
+
+    inc_dirs.append(resource_dir + "/include")
+    inc_dirs.append(resource_dir + "/share")
 
     return inc_dirs