Level zero codegen (#250)

* WIP level zero support.

* Add support to build a SYCL program from SPIR-V.

* Ignore the generated config file.

* Skip test on Windows.

* Black formatting.

* Skip test properly.

* Fix comment.

* Fix black error.

* Fix comments.

* Fix potential issue on Windows.

* Make level zero a dynamically loaded library.

* Do not install helper headers and run tests during linux build.

* Cleanups...

Author: diptorupd

dpctl-capi/CMakeLists.txt

@@ -94,9 +94,6 @@ if(DPCTL_ENABLE_LO_PROGRAM_CREATION)
             PRIVATE
             ${LEVEL_ZERO_INCLUDE_DIR}
         )
-        target_link_libraries(DPCTLSyclInterface
-            PRIVATE ${LEVEL_ZERO_LIBRARY}
-        )
     else()
         message(WARNING
             "DPCTL support Level Zero program creation not supported "
@@ -124,12 +121,6 @@ foreach(HEADER ${HEADERS})
   install(FILES "${HEADER}" DESTINATION include/Support)
 endforeach()
 
-# Install all headers in helper/include
-file(GLOB HEADERS "${CMAKE_SOURCE_DIR}/helper/include/*.h")
-foreach(HEADER ${HEADERS})
-  install(FILES "${HEADER}" DESTINATION helper/include)
-endforeach()
-
 # Install all headers in include/Config
 file(GLOB HEADERS "${CMAKE_SOURCE_DIR}/include/Config/*.h")
 foreach(HEADER ${HEADERS})

dpctl-capi/helper/include/dpctl_dynamic_lib_helper.h

@@ -0,0 +1,108 @@
+//===--------------- dpctl_dynamic_lib_helper.h - dpctl-C_API     -*-C++-*-===//
+//
+//               Data Parallel Control Library (dpCtl)
+//
+// Copyright 2020 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Helper for dynamic libs management.
+//===----------------------------------------------------------------------===//
+
+#ifndef __DPCTL_DYNAMIC_LIB_HELPER_H__
+#define __DPCTL_DYNAMIC_LIB_HELPER_H__
+
+#if defined(__linux__) || defined(_WIN32) || defined(_WIN64)
+
+    #ifdef __linux__
+
+        #include <dlfcn.h>
+
+    #elif defined(_WIN32) || defined(_WIN64)
+
+        #define NOMINMAX
+        #include <windows.h>
+
+    #endif // __linux__
+
+#include <cstdint>
+
+namespace dpctl
+{
+
+class DynamicLibHelper final
+{
+public:
+    DynamicLibHelper()                         = delete;
+    DynamicLibHelper(const DynamicLibHelper &) = delete;
+    DynamicLibHelper(const char * libName, int flag)
+    {
+
+    #ifdef __linux__
+        _handle = dlopen(libName, flag);
+    #elif defined(_WIN32) || defined(_WIN64)
+        _handle = LoadLibraryA(libName);
+    #endif
+    }
+
+    ~DynamicLibHelper()
+    {
+    #ifdef __linux__
+        dlclose(_handle);
+    #elif defined(_WIN32) || defined(_WIN64)
+        FreeLibrary((HMODULE)_handle);
+    #endif
+    };
+
+    template <typename T>
+    T getSymbol(const char * symName)
+    {
+    #ifdef __linux__
+        void * sym   = dlsym(_handle, symName);
+        char * error = dlerror();
+
+        if (NULL != error)
+        {
+            return nullptr;
+        }
+    #elif defined(_WIN32) || defined(_WIN64)
+        void * sym = (void *)GetProcAddress((HMODULE)_handle, symName);
+
+        if (NULL == sym)
+        {
+            return nullptr;
+        }
+    #endif
+
+        return (T)sym;
+    }
+
+    bool opened () const
+    {
+        if (!_handle)
+            return false;
+        else
+            return true;
+    }
+
+private:
+    void * _handle = nullptr;
+};
+
+} // namespace dpctl
+
+#endif // #if defined(__linux__) || defined(_WIN32) || defined(_WIN64)
+#endif // __DPCTL_DYNAMIC_LIB_HELPER_H__

dpctl-capi/source/dpctl_sycl_program_interface.cpp

@@ -33,12 +33,32 @@
 #ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION
 #include <level_zero/zet_api.h> /* Level Zero headers */
 #include <CL/sycl/backend/level_zero.hpp>
+#include "../helper/include/dpctl_dynamic_lib_helper.h"
 #endif
 
 using namespace cl::sycl;
 
 namespace
 {
+#ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION
+
+#ifdef __linux__
+static const char * zeLoaderName = "libze_loader.so";
+static const int libLoadFlags    = RTLD_NOLOAD | RTLD_NOW | RTLD_LOCAL;
+#else
+    #error "Level Zero program compilation is unavailable for this platform"
+#endif
+
+typedef ze_result_t (*zeModuleCreateFT)(ze_context_handle_t,
+                                        ze_device_handle_t,
+                                        const ze_module_desc_t *,
+                                        ze_module_handle_t *,
+                                        ze_module_build_log_handle_t *);
+
+const char * zeModuleCreateFuncName  = "zeModuleCreate";
+
+#endif // #ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION
+
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(context, DPCTLSyclContextRef)
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(program, DPCTLSyclProgramRef)
 DEFINE_SIMPLE_CONVERSION_FUNCTIONS(kernel, DPCTLSyclKernelRef)
@@ -90,6 +110,23 @@ createOpenCLInterOpProgram (const context &SyclCtx,
 }
 
 #ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION
+
+zeModuleCreateFT getZeModuleCreateFn ()
+{
+    static dpctl::DynamicLibHelper zeLib(zeLoaderName, libLoadFlags);
+    if(!zeLib.opened()) {
+        // TODO: handle error
+        std::cerr << "The level zero loader dynamic library could not "
+                     "be opened.\n";
+        return nullptr;
+    }
+    static auto stZeModuleCreateF = zeLib.getSymbol<zeModuleCreateFT>(
+                                        zeModuleCreateFuncName
+                                    );
+
+    return stZeModuleCreateF;
+}
+
 __dpctl_give DPCTLSyclProgramRef
 createLevelZeroInterOpProgram (const context &SyclCtx,
                                const void *IL,
@@ -99,8 +136,8 @@ createLevelZeroInterOpProgram (const context &SyclCtx,
     auto ZeCtx   = SyclCtx.get_native<backend::level_zero>();
     auto SyclDevices = SyclCtx.get_devices();
     if(SyclDevices.size() > 1) {
-        // We only support build to one device with Level Zero now.
-        // TODO: log error
+        std::cerr << "Level zero program can be created for only one device.\n";
+        // TODO: handle error
         return nullptr;
     }
 
@@ -119,8 +156,14 @@ createLevelZeroInterOpProgram (const context &SyclCtx,
 
     auto ZeDevice = SyclDevices[0].get_native<backend::level_zero>();
     ze_module_handle_t ZeModule;
-    auto ret = zeModuleCreate(ZeCtx, ZeDevice, &ZeModuleDesc, &ZeModule,
-                              nullptr);
+
+    auto stZeModuleCreateF = getZeModuleCreateFn();
+
+    if(!stZeModuleCreateF)
+        return nullptr;
+
+    auto ret = stZeModuleCreateF(ZeCtx, ZeDevice, &ZeModuleDesc, &ZeModule,
+                                 nullptr);
     if(ret != ZE_RESULT_SUCCESS) {
         // TODO: handle error
         return nullptr;
@@ -138,7 +181,8 @@ createLevelZeroInterOpProgram (const context &SyclCtx,
         return nullptr;
     }
 }
-#endif
+#endif /* #ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION */
+
 } /* end of anonymous namespace */
 
 __dpctl_give DPCTLSyclProgramRef
@@ -156,7 +200,7 @@ DPCTLProgram_CreateFromSpirv (__dpctl_keep const DPCTLSyclContextRef CtxRef,
     SyclCtx = unwrap(CtxRef);
     // get the backend type
     auto BE = SyclCtx->get_platform().get_backend();
-    switch (BE)
+    switch(BE)
     {
     case backend::opencl:
         Pref = createOpenCLInterOpProgram(*SyclCtx, IL, length, CompileOpts);

dpctl-capi/tests/test_sycl_program_interface.cpp

@@ -127,16 +127,12 @@ struct TestDPCTLSyclProgramInterface : public ::testing::Test
     size_t spirvFileSize = 0;
     std::vector<char> spirvBuffer;
     size_t nOpenCLGpuQ = 0;
-#ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION
-    size_t nL0GpuQ = 0;
-#endif
 
     TestDPCTLSyclProgramInterface () :
         spirvFile{"./multi_kernel.spv", std::ios::binary | std::ios::ate},
         spirvFileSize(std::filesystem::file_size("./multi_kernel.spv")),
         spirvBuffer(spirvFileSize),
-        nOpenCLGpuQ(DPCTLQueueMgr_GetNumQueues(DPCTL_OPENCL, DPCTL_GPU)),
-        nL0GpuQ(DPCTLQueueMgr_GetNumQueues(DPCTL_LEVEL_ZERO, DPCTL_GPU))
+        nOpenCLGpuQ(DPCTLQueueMgr_GetNumQueues(DPCTL_OPENCL, DPCTL_GPU))
     {
         spirvFile.seekg(0, std::ios::beg);
         spirvFile.read(spirvBuffer.data(), spirvFileSize);
@@ -188,6 +184,7 @@ TEST_F (TestDPCTLSyclProgramInterface, CheckCreateFromSpirvOCL)
 #ifdef DPCTL_ENABLE_LO_PROGRAM_CREATION
 TEST_F (TestDPCTLSyclProgramInterface, CheckCreateFromSpirvL0)
 {
+    auto nL0GpuQ = DPCTLQueueMgr_GetNumQueues(DPCTL_LEVEL_ZERO, DPCTL_GPU);
     if(!nL0GpuQ)
         GTEST_SKIP_("Skipping as no OpenCL GPU device found.\n");