From 5ae18e5f81a1bc183e9e5b7b2915d3abdf9f77d1 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Mon, 20 Jul 2026 13:24:08 -0400 Subject: [PATCH 01/57] Define runtime transport layer interface for backline backends --- runtime/include/Transport.hpp | 190 ++++++++++++++++++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 runtime/include/Transport.hpp diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp new file mode 100644 index 0000000000..a601bc2a02 --- /dev/null +++ b/runtime/include/Transport.hpp @@ -0,0 +1,190 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include +#include + +namespace catalyst::transport { + +/** + * @brief Data-plane strategy: which engine issues the transfer. + */ +enum class DataPath : std::uint8_t { + CpuVerbs, // Plain ibverbs on CPU. + GpuEngine, // Gpu-initiated comms. + Other, +}; + +/** + * @brief Memory kind: selects the allocation and registration path. + */ +enum class MemKind : std::uint8_t { + CpuRam, + GpuHbm, + Ddr, + Other, +}; + +/** + * @brief Out-of-band connection parameters for bringing up a session. + */ +struct ConnectInfo { + std::string peer; + std::uint16_t oob_port; +}; + +/** + * @brief Locally allocated and registered memory region. + */ +struct MemRegion { + void *addr = nullptr; + std::uint64_t size = 0; + std::uint32_t lkey = 0; + std::uint32_t rkey = 0; + MemKind kind = MemKind::CpuRam; +}; + +/** + * @brief Handle to a peer's memory region, exchanged over the out-of-band channel. + */ +struct PeerRef { + std::uint32_t rkey = 0; + std::uint64_t remote_addr = 0; + std::uint64_t size = 0; +}; + +/** + * @brief Configuration for the data-movement channel a session uses. + */ +struct ChannelDesc { + DataPath data_path = DataPath::CpuVerbs; + bool persistent = true; +}; + +/** + * @brief Stateful transport session shared by the controller and coprocessor roles. + * + * Methods must be called in this order: + * 1. connect - bring up QPs + the out-of-band channel + * 2. alloc_memory - register the region (needs the connected context) + * 3. exchange_keys - swap region handles over the out-of-band channel + * 4. establish_channel - program the channel from the local + peer regions + * 5. (coprocessor) set_coprocessor_fn / (controller) set_max_in_flight - before start() + * 6. start / collect / stop + */ +class TransportSession { + public: + virtual ~TransportSession() = default; + + /** + * @brief Bring up the connection (out-of-band handshake and QP transition to RTS). + * + * @param info Peer address and out-of-band port. + * + * @return `int` + */ + virtual int connect(const ConnectInfo &info) = 0; + + /** + * @brief Allocate and register a memory region on the device. + * + * @param size Size of the region in bytes. + * @param kind Memory kind selecting the allocation and registration path. + * @param access Access flags for the registration. + * + * @return `MemRegion` The allocated and registered region. + */ + virtual MemRegion alloc_memory(std::size_t size, MemKind kind, std::uint32_t access) = 0; + + /** + * @brief Advertise a local region and receive the peer's region over the out-of-band channel. + * + * @param local The local region to advertise. + * + * @return `PeerRef` The peer's advertised region. + */ + virtual PeerRef exchange_keys(const MemRegion &local) = 0; + + /** + * @brief Program the data movement this session will run (single channel per session). + * + * @param desc Channel configuration. + * @param local The local memory region. + * @param peer The peer's memory region. + */ + virtual void establish_channel(const ChannelDesc &desc, const MemRegion &local, + const PeerRef &peer) = 0; + + /** + * @brief Launch the engine (non-blocking; runs until stop()). + */ + virtual void start() = 0; + + /** + * @brief Wait for a result and write it out. + * + * @param outputs Array of output buffers to write into. + * @param n Number of output buffers. + * + * @return `int` + */ + virtual int collect(void *const *outputs, std::size_t n) = 0; + + /** + * @brief Stop the engine and join. Idempotent. + */ + virtual void stop() = 0; +}; + +/** + * @brief Controller role: writes syndromes out and receives corrections. + */ +class ControllerSession : public TransportSession { + public: + /** + * @brief Set the sliding-window depth: how many syndromes to keep in flight. + * + * Call before start(). A value of 1 means strict one-in-flight. + * + * @param n Maximum number of syndromes in flight. + */ + virtual void set_max_in_flight(std::uint32_t n) = 0; +}; + +/** + * @brief Opaque function to run on the coprocessor. May include a persistent kernel on the GPU. + */ +using CoprocessorFn = std::size_t (*)(const void *in, std::size_t in_len, void *out, + std::size_t out_cap, void *ctx); + +/** + * @brief Coprocessor role: receives syndromes, decodes, and returns corrections. + */ +class CoprocessorSession : public TransportSession { + public: + /** + * @brief Bind the coprocessor function this session runs. + * + * Call before start(). `fn` is a local function pointer; `ctx` is passed + * back to `fn` on every invocation and may be null. + * + * @param fn The coprocessor function to run per received syndrome. + * @param ctx Opaque context passed to `fn` on each invocation; may be null. + */ + virtual void set_coprocessor_fn(CoprocessorFn fn, void *ctx) = 0; +}; + +} // namespace catalyst::transport From 5e7633379073f71d8ef7370f29a46a4711abef30 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Mon, 20 Jul 2026 13:28:01 -0400 Subject: [PATCH 02/57] update changelog --- doc/releases/changelog-dev.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md index 532835a10e..3ccd5df3c2 100644 --- a/doc/releases/changelog-dev.md +++ b/doc/releases/changelog-dev.md @@ -12,6 +12,9 @@

Improvements 🛠

+* A new runtime transport layer for remote/local executors is introduced. + [(#3043)](https://github.com/PennyLaneAI/catalyst/pull/3043) + * A `BufferizableOpInterface` implementation is now added for `catalyst.launch_kernel` operation and it is now bufferizable. [(#3024)](https://github.com/PennyLaneAI/catalyst/pull/3024) From 279a2b4962f17582b83b933761aa1dcbb98ff467 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Mon, 20 Jul 2026 14:35:20 -0400 Subject: [PATCH 03/57] comments --- runtime/include/Transport.hpp | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index a601bc2a02..d8813d22cc 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -71,7 +71,6 @@ struct PeerRef { */ struct ChannelDesc { DataPath data_path = DataPath::CpuVerbs; - bool persistent = true; }; /** @@ -82,7 +81,7 @@ struct ChannelDesc { * 2. alloc_memory - register the region (needs the connected context) * 3. exchange_keys - swap region handles over the out-of-band channel * 4. establish_channel - program the channel from the local + peer regions - * 5. (coprocessor) set_coprocessor_fn / (controller) set_max_in_flight - before start() + * 5. (coprocessor) set_coprocessor_fn before start() * 6. start / collect / stop */ class TransportSession { @@ -150,19 +149,9 @@ class TransportSession { }; /** - * @brief Controller role: writes syndromes out and receives corrections. + * @brief Controller role: writes messages out and receives corrections. */ -class ControllerSession : public TransportSession { - public: - /** - * @brief Set the sliding-window depth: how many syndromes to keep in flight. - * - * Call before start(). A value of 1 means strict one-in-flight. - * - * @param n Maximum number of syndromes in flight. - */ - virtual void set_max_in_flight(std::uint32_t n) = 0; -}; +class ControllerSession : public TransportSession {}; /** * @brief Opaque function to run on the coprocessor. May include a persistent kernel on the GPU. @@ -171,7 +160,7 @@ using CoprocessorFn = std::size_t (*)(const void *in, std::size_t in_len, void * std::size_t out_cap, void *ctx); /** - * @brief Coprocessor role: receives syndromes, decodes, and returns corrections. + * @brief Coprocessor role: receives messages, process, and returns corrections. */ class CoprocessorSession : public TransportSession { public: @@ -181,7 +170,7 @@ class CoprocessorSession : public TransportSession { * Call before start(). `fn` is a local function pointer; `ctx` is passed * back to `fn` on every invocation and may be null. * - * @param fn The coprocessor function to run per received syndrome. + * @param fn The coprocessor function to run per received message. * @param ctx Opaque context passed to `fn` on each invocation; may be null. */ virtual void set_coprocessor_fn(CoprocessorFn fn, void *ctx) = 0; From a319b39d7fd8961170530a8856f4ed7c6a406de0 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Mon, 20 Jul 2026 15:15:12 -0400 Subject: [PATCH 04/57] update changelog --- doc/releases/changelog-dev.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md index 3ccd5df3c2..42490dbf5d 100644 --- a/doc/releases/changelog-dev.md +++ b/doc/releases/changelog-dev.md @@ -461,6 +461,7 @@ JiaRung Jian, Jacob Kitchen, Korbinian Kottmann, Christina Lee, +Joseph Lee, Rylan Malarchick, Mehrdad Malekmohammadi, River McCubbin, From c523f677106b61d6236606df8bb68c25fe97a2c0 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 15:41:04 -0400 Subject: [PATCH 05/57] Transport Layer Loader is added --- runtime/CMakeLists.txt | 1 + runtime/include/Transport.hpp | 24 +- runtime/include/TransportBackend.h | 55 +++++ runtime/include/TransportCAPI.h | 107 +++++++++ runtime/lib/CMakeLists.txt | 4 + runtime/lib/transport/CMakeLists.txt | 19 ++ runtime/lib/transport/TransportCAPI.cpp | 281 ++++++++++++++++++++++++ 7 files changed, 489 insertions(+), 2 deletions(-) create mode 100644 runtime/include/TransportBackend.h create mode 100644 runtime/include/TransportCAPI.h create mode 100644 runtime/lib/transport/CMakeLists.txt create mode 100644 runtime/lib/transport/TransportCAPI.cpp diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 9fe951ed2a..8cf8c3a201 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -15,6 +15,7 @@ option(RUNTIME_ENABLE_WARNINGS "Enable -Wall and -Werror" ON) option(ENABLE_OPENQASM "Build OpenQasm backend device" OFF) option(ENABLE_OQD "Build OQD backend device" OFF) +option(ENABLE_TRANSPORT "Build the backend-agnostic transport loader (rt_transport)" OFF) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index d8813d22cc..241ec5bdf4 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -136,22 +136,42 @@ class TransportSession { * @brief Wait for a result and write it out. * * @param outputs Array of output buffers to write into. + * @param output_bytes Array of output buffer sizes. * @param n Number of output buffers. * * @return `int` */ - virtual int collect(void *const *outputs, std::size_t n) = 0; + virtual int collect(void *const *outputs, const std::size_t *output_bytes, std::size_t n) = 0; /** * @brief Stop the engine and join. Idempotent. */ virtual void stop() = 0; + + /** + * @brief Last round-trip time, in nanoseconds (for testing purposes). + * + * @return `std::uint64_t` + */ + virtual std::uint64_t last_rtt_ns() const { return 0; } }; /** * @brief Controller role: writes messages out and receives corrections. */ -class ControllerSession : public TransportSession {}; +class ControllerSession : public TransportSession { + public: + // Build the work item in slot `work_item_idx` from `schema`. + virtual void commit_work_item(std::uint32_t work_item_idx, std::uint64_t in_bytes, + std::uint64_t out_bytes) = 0; + + // Fire one round using work item `work_item_idx` and whatever payload is currently in + // data_slot(). Pairs with a subsequent collect(). Returns 0 on success. + virtual int kick(std::uint32_t work_item_idx = 0) = 0; + + // Current round's outbound slot in the transport-owned ring. + virtual void *data_slot() = 0; +}; /** * @brief Opaque function to run on the coprocessor. May include a persistent kernel on the GPU. diff --git a/runtime/include/TransportBackend.h b/runtime/include/TransportBackend.h new file mode 100644 index 0000000000..87974aba7d --- /dev/null +++ b/runtime/include/TransportBackend.h @@ -0,0 +1,55 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The plugin ABI for out-of-tree transport backends. +// +// A transport backend is a shared library that implements a TransportSession role and exports a +// factory symbol. + +#pragma once +#ifndef TRANSPORTBACKEND_H +#define TRANSPORTBACKEND_H + +#include +#include + +#include "Transport.hpp" + +#define CATALYST_TRANSPORT_CONTROLLER_FACTORY_SYMBOL "CatalystTransportControllerFactory" + +// The factory signature backends must export with C linkage. +extern "C" { +using CatalystTransportControllerFactoryFn = catalyst::transport::ControllerSession *(const char *); +} + +// A helper template macro to generate the Factory function. +// e.g. GENERATE_TRANSPORT_CONTROLLER_FACTORY(CatalystTransportController, make_controller) +// where `make_controller(const std::string &config) -> ControllerSession*`. +#define GENERATE_TRANSPORT_CONTROLLER_FACTORY(IDENTIFIER, BUILDER) \ + extern "C" catalyst::transport::ControllerSession *IDENTIFIER##Factory(const char *config) \ + { \ + try { \ + return (BUILDER)(config ? std::string(config) : std::string()); \ + } \ + catch (const std::exception &e) { \ + std::fprintf(stderr, "[transport] controller factory failed: %s\n", e.what()); \ + return nullptr; \ + } \ + catch (...) { \ + std::fprintf(stderr, "[transport] controller factory failed: unknown exception\n"); \ + return nullptr; \ + } \ + } + +#endif // TRANSPORTBACKEND_H diff --git a/runtime/include/TransportCAPI.h b/runtime/include/TransportCAPI.h new file mode 100644 index 0000000000..2f0a4aa312 --- /dev/null +++ b/runtime/include/TransportCAPI.h @@ -0,0 +1,107 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// TransportCAPI.h - C entry points the Catalyst compiler emits to drive a transport session. +// +// The backend is a separate plugin `.so` the runtime dlopen's at session create (see +// TransportBackend.h) + +#pragma once +#ifndef TRANSPORTCAPI_H +#define TRANSPORTCAPI_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Opaque transport session handle +typedef struct CatalystTransportSession CatalystTransportSession; + +// Return codes: 0 == success; negative == error +enum { + CATALYST_TRANSPORT_OK = 0, + CATALYST_TRANSPORT_ERR = -1, // Generic exception + CATALYST_TRANSPORT_ERR_MEMORY = -2, // Memory error + CATALYST_TRANSPORT_ERR_TIMEOUT = -3, // Timeout error + CATALYST_TRANSPORT_ERR_STUCK = -4, // Something got stuck +}; + +// DataPath enum (mirrors catalyst::transport::DataPath) +enum { + CATALYST_TRANSPORT_PATH_CPU_VERBS = 0, + CATALYST_TRANSPORT_PATH_GPU_ENGINE = 1, + CATALYST_TRANSPORT_PATH_OTHER = 2, +}; + +// MemKind enum (mirrors catalyst::transport::MemKind) +enum { + CATALYST_TRANSPORT_MEM_CPU_RAM = 0, + CATALYST_TRANSPORT_MEM_GPU_HBM = 1, + CATALYST_TRANSPORT_MEM_DDR = 2, + CATALYST_TRANSPORT_MEM_OTHER = 3, +}; + +// ibverbs access flags for the advertised reply region +enum { + CATALYST_TRANSPORT_ACCESS_REPLY = 7, +}; + +// Registered memory region handed back to the caller +typedef struct { + void *addr; + uint64_t size; + uint32_t lkey; + uint32_t rkey; + int32_t kind; // one of MemKind enum values +} CatalystTransportMemRegion; + +// Remote peer region descriptor +typedef struct { + uint32_t rkey; + uint64_t remote_addr; + uint64_t size; +} CatalystTransportPeerRef; + +// Create a controller session from a named backend plugin `.so` (dlopen'd by the runtime). +// `config` is the backend's "key=value;..." string. Returns NULL on failure. +CatalystTransportSession *__catalyst__transport__controller_create(const char *backend_lib, + const char *config); + +void __catalyst__transport__close(CatalystTransportSession *s); +int __catalyst__transport__connect(CatalystTransportSession *s, const char *peer, + uint16_t oob_port); +int __catalyst__transport__alloc_reply(CatalystTransportSession *s, uint64_t size, int32_t mem_kind, + uint32_t access, CatalystTransportMemRegion *out); + +int __catalyst__transport__exchange_keys(CatalystTransportSession *s, + CatalystTransportPeerRef *out); +int __catalyst__transport__establish_channel(CatalystTransportSession *s, int32_t data_path, + const CatalystTransportPeerRef *peer); +int __catalyst__transport__commit_work_item(CatalystTransportSession *s, uint32_t work_item_idx, + uint64_t in_bytes, uint64_t out_bytes); +void *__catalyst__transport__data_slot(CatalystTransportSession *s); +int __catalyst__transport__kick(CatalystTransportSession *s, uint32_t work_item_idx); +int __catalyst__transport__collect(CatalystTransportSession *s, void *correction, uint64_t bytes); +uint64_t __catalyst__transport__last_rtt_ns(CatalystTransportSession *s); +void __catalyst__transport__stop(CatalystTransportSession *s); +void __catalyst__transport__destroy(CatalystTransportSession *s); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // TRANSPORTCAPI_H diff --git a/runtime/lib/CMakeLists.txt b/runtime/lib/CMakeLists.txt index eb7a77b63d..64d07dcd95 100644 --- a/runtime/lib/CMakeLists.txt +++ b/runtime/lib/CMakeLists.txt @@ -68,3 +68,7 @@ add_subdirectory(QEC) if(ENABLE_OQD) add_subdirectory(OQDcapi) endif() + +if(ENABLE_TRANSPORT) +add_subdirectory(transport) +endif() diff --git a/runtime/lib/transport/CMakeLists.txt b/runtime/lib/transport/CMakeLists.txt new file mode 100644 index 0000000000..b2cb3a38c1 --- /dev/null +++ b/runtime/lib/transport/CMakeLists.txt @@ -0,0 +1,19 @@ +############################################### +# library rt_transport # +############################################### + +add_library(rt_transport SHARED TransportCAPI.cpp) + +target_include_directories(rt_transport + PUBLIC + ${runtime_includes} + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} + ${PROJECT_SOURCE_DIR}/lib/backend/common # DynamicLibraryLoader.hpp +) + +target_link_libraries(rt_transport PRIVATE + ${CMAKE_DL_LIBS} # dlopen/dlsym for backend plugins +) + +set_property(TARGET rt_transport PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/runtime/lib/transport/TransportCAPI.cpp b/runtime/lib/transport/TransportCAPI.cpp new file mode 100644 index 0000000000..617f05e6c3 --- /dev/null +++ b/runtime/lib/transport/TransportCAPI.cpp @@ -0,0 +1,281 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "TransportCAPI.h" + +#include +#include +#include +#include +#include +#include + +#include "DynamicLibraryLoader.hpp" +#include "Transport.hpp" +#include "TransportBackend.h" + +using catalyst::transport::ChannelDesc; +using catalyst::transport::ConnectInfo; +using catalyst::transport::ControllerSession; +using catalyst::transport::DataPath; +using catalyst::transport::MemKind; +using catalyst::transport::MemRegion; +using catalyst::transport::PeerRef; + +// The opaque handle +struct CatalystTransportSession { + std::unique_ptr backend; + ControllerSession *sess = nullptr; // heap-allocated by the backend factory + MemRegion reply = {}; + bool have_reply = false; +}; + +namespace { + +MemKind to_mem_kind(std::int32_t k) +{ + switch (k) { + case CATALYST_TRANSPORT_MEM_CPU_RAM: + return MemKind::CpuRam; + case CATALYST_TRANSPORT_MEM_GPU_HBM: + return MemKind::GpuHbm; + case CATALYST_TRANSPORT_MEM_DDR: + return MemKind::Ddr; + case CATALYST_TRANSPORT_MEM_OTHER: + return MemKind::Other; + default: + return MemKind::Ddr; + } +} + +DataPath to_data_path(std::int32_t p) +{ + switch (p) { + case CATALYST_TRANSPORT_PATH_CPU_VERBS: + return DataPath::CpuVerbs; + case CATALYST_TRANSPORT_PATH_GPU_ENGINE: + return DataPath::GpuEngine; + case CATALYST_TRANSPORT_PATH_OTHER: + default: + return DataPath::Other; + } +} + +template int guard(Fn &&fn) +{ + try { + return fn(); + } + catch (const std::exception &e) { + std::cerr << "[transport] " << e.what() << "\n"; + return CATALYST_TRANSPORT_ERR; + } + catch (...) { + return CATALYST_TRANSPORT_ERR; + } +} + +} // namespace + +extern "C" { + +CatalystTransportSession *__catalyst__transport__controller_create(const char *backend_lib, + const char *config) +{ + try { + if (!backend_lib || !*backend_lib) { + std::cerr << "[transport] no backend library given\n"; + return nullptr; + } + auto h = std::make_unique(); + h->backend = std::make_unique(backend_lib); + auto *factory = h->backend->getSymbol( + CATALYST_TRANSPORT_CONTROLLER_FACTORY_SYMBOL); + h->sess = factory(config ? config : ""); + if (!h->sess) { + std::cerr << "[transport] backend factory returned null for config: " + << (config ? config : "") << "\n"; + return nullptr; + } + return h.release(); + } + catch (const std::exception &e) { + std::cerr << "[transport] controller_create: " << e.what() << "\n"; + return nullptr; + } + catch (...) { + return nullptr; + } +} + +int __catalyst__transport__connect(CatalystTransportSession *s, const char *peer, + std::uint16_t oob_port) +{ + if (!s || !s->sess) { + return CATALYST_TRANSPORT_ERR; + } + return guard([&] { + ConnectInfo info; + info.peer = peer ? peer : ""; + info.oob_port = oob_port; + return s->sess->connect(info); + }); +} + +int __catalyst__transport__alloc_reply(CatalystTransportSession *s, std::uint64_t size, + std::int32_t mem_kind, std::uint32_t access, + CatalystTransportMemRegion *out) +{ + if (!s || !s->sess) { + return CATALYST_TRANSPORT_ERR; + } + return guard([&] { + MemRegion r = s->sess->alloc_memory(size, to_mem_kind(mem_kind), access); + s->reply = r; + s->have_reply = true; + if (out) { + out->addr = r.addr; + out->size = r.size; + out->lkey = r.lkey; + out->rkey = r.rkey; + out->kind = mem_kind; + } + return CATALYST_TRANSPORT_OK; + }); +} + +int __catalyst__transport__exchange_keys(CatalystTransportSession *s, CatalystTransportPeerRef *out) +{ + if (!s || !s->sess) { + return CATALYST_TRANSPORT_ERR; + } + return guard([&] { + PeerRef p = s->sess->exchange_keys(s->have_reply ? s->reply : MemRegion{}); + if (out) { + out->rkey = p.rkey; + out->remote_addr = p.remote_addr; + out->size = p.size; + } + return CATALYST_TRANSPORT_OK; + }); +} + +int __catalyst__transport__establish_channel(CatalystTransportSession *s, std::int32_t data_path, + const CatalystTransportPeerRef *peer) +{ + if (!s || !s->sess || !peer) { + return CATALYST_TRANSPORT_ERR; + } + return guard([&] { + ChannelDesc desc; + desc.data_path = to_data_path(data_path); + PeerRef p; + p.rkey = peer->rkey; + p.remote_addr = peer->remote_addr; + p.size = peer->size; + s->sess->establish_channel(desc, s->have_reply ? s->reply : MemRegion{}, p); + return CATALYST_TRANSPORT_OK; + }); +} + +int __catalyst__transport__commit_work_item(CatalystTransportSession *s, + std::uint32_t work_item_idx, std::uint64_t in_bytes, + std::uint64_t out_bytes) +{ + if (!s || !s->sess) { + return CATALYST_TRANSPORT_ERR; + } + return guard([&] { + s->sess->commit_work_item(work_item_idx, in_bytes, out_bytes); + return CATALYST_TRANSPORT_OK; + }); +} + +void *__catalyst__transport__data_slot(CatalystTransportSession *s) +{ + if (!s || !s->sess) { + return nullptr; + } + + void *slot = nullptr; + try { + slot = s->sess->data_slot(); + } + catch (const std::exception &e) { + std::cerr << "[transport] data_slot: " << e.what() << "\n"; + return nullptr; + } + catch (...) { + return nullptr; + } + return slot; +} + +int __catalyst__transport__kick(CatalystTransportSession *s, std::uint32_t work_item_idx) +{ + if (!s || !s->sess) { + return CATALYST_TRANSPORT_ERR; + } + return guard([&] { return s->sess->kick(work_item_idx); }); +} + +int __catalyst__transport__collect(CatalystTransportSession *s, void *correction, + std::uint64_t bytes) +{ + if (!s || !s->sess) { + return CATALYST_TRANSPORT_ERR; + } + return guard([&] { + void *outputs[1] = {correction}; + std::size_t caps[1] = {static_cast(bytes)}; + return s->sess->collect(outputs, caps, 1); + }); +} + +std::uint64_t __catalyst__transport__last_rtt_ns(CatalystTransportSession *s) +{ + if (!s || !s->sess) { + return 0; + } + return s->sess->last_rtt_ns(); +} + +void __catalyst__transport__stop(CatalystTransportSession *s) +{ + if (s && s->sess) { + try { + s->sess->stop(); + } + catch (...) { + } + } +} + +void __catalyst__transport__destroy(CatalystTransportSession *s) +{ + if (!s) { + return; + } + delete s->sess; // owned by the backend factory + s->backend.reset(); + delete s; +} + +void __catalyst__transport__close(CatalystTransportSession *s) +{ + __catalyst__transport__stop(s); + __catalyst__transport__destroy(s); +} + +} // extern "C" From 8dccc6201341549f6eab86fbf504625e1af92826 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 16:19:33 -0400 Subject: [PATCH 06/57] update interface --- runtime/include/Transport.hpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index d8813d22cc..241ec5bdf4 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -136,22 +136,42 @@ class TransportSession { * @brief Wait for a result and write it out. * * @param outputs Array of output buffers to write into. + * @param output_bytes Array of output buffer sizes. * @param n Number of output buffers. * * @return `int` */ - virtual int collect(void *const *outputs, std::size_t n) = 0; + virtual int collect(void *const *outputs, const std::size_t *output_bytes, std::size_t n) = 0; /** * @brief Stop the engine and join. Idempotent. */ virtual void stop() = 0; + + /** + * @brief Last round-trip time, in nanoseconds (for testing purposes). + * + * @return `std::uint64_t` + */ + virtual std::uint64_t last_rtt_ns() const { return 0; } }; /** * @brief Controller role: writes messages out and receives corrections. */ -class ControllerSession : public TransportSession {}; +class ControllerSession : public TransportSession { + public: + // Build the work item in slot `work_item_idx` from `schema`. + virtual void commit_work_item(std::uint32_t work_item_idx, std::uint64_t in_bytes, + std::uint64_t out_bytes) = 0; + + // Fire one round using work item `work_item_idx` and whatever payload is currently in + // data_slot(). Pairs with a subsequent collect(). Returns 0 on success. + virtual int kick(std::uint32_t work_item_idx = 0) = 0; + + // Current round's outbound slot in the transport-owned ring. + virtual void *data_slot() = 0; +}; /** * @brief Opaque function to run on the coprocessor. May include a persistent kernel on the GPU. From cdee60bc9d1099eeb026c0b27b1eb199b65bcf62 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 16:23:40 -0400 Subject: [PATCH 07/57] coprocessor backend interface is added --- runtime/include/TransportBackend.h | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/runtime/include/TransportBackend.h b/runtime/include/TransportBackend.h index 87974aba7d..734362510c 100644 --- a/runtime/include/TransportBackend.h +++ b/runtime/include/TransportBackend.h @@ -14,8 +14,8 @@ // The plugin ABI for out-of-tree transport backends. // -// A transport backend is a shared library that implements a TransportSession role and exports a -// factory symbol. +// A transport backend is a shared library that implements a TransportSession role (controller or +// coprocessor) and exports the matching factory symbol. #pragma once #ifndef TRANSPORTBACKEND_H @@ -27,10 +27,13 @@ #include "Transport.hpp" #define CATALYST_TRANSPORT_CONTROLLER_FACTORY_SYMBOL "CatalystTransportControllerFactory" +#define CATALYST_TRANSPORT_COPROCESSOR_FACTORY_SYMBOL "CatalystTransportCoprocessorFactory" -// The factory signature backends must export with C linkage. +// The factory signatures backends must export with C linkage. extern "C" { using CatalystTransportControllerFactoryFn = catalyst::transport::ControllerSession *(const char *); +using CatalystTransportCoprocessorFactoryFn = + catalyst::transport::CoprocessorSession *(const char *); } // A helper template macro to generate the Factory function. @@ -52,4 +55,22 @@ using CatalystTransportControllerFactoryFn = catalyst::transport::ControllerSess } \ } +// e.g. GENERATE_TRANSPORT_COPROCESSOR_FACTORY(CatalystTransportCoprocessor, make_coprocessor) +// where `make_coprocessor(const std::string &config) -> CoprocessorSession*`. +#define GENERATE_TRANSPORT_COPROCESSOR_FACTORY(IDENTIFIER, BUILDER) \ + extern "C" catalyst::transport::CoprocessorSession *IDENTIFIER##Factory(const char *config) \ + { \ + try { \ + return (BUILDER)(config ? std::string(config) : std::string()); \ + } \ + catch (const std::exception &e) { \ + std::fprintf(stderr, "[transport] coprocessor factory failed: %s\n", e.what()); \ + return nullptr; \ + } \ + catch (...) { \ + std::fprintf(stderr, "[transport] coprocessor factory failed: unknown exception\n"); \ + return nullptr; \ + } \ + } + #endif // TRANSPORTBACKEND_H From da41e922e25b95094421775cd8acfcef1f905bc5 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 16:24:55 -0400 Subject: [PATCH 08/57] chagne type --- runtime/include/Transport.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index 241ec5bdf4..3a5ed0ce8d 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -141,7 +141,7 @@ class TransportSession { * * @return `int` */ - virtual int collect(void *const *outputs, const std::size_t *output_bytes, std::size_t n) = 0; + virtual int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) = 0; /** * @brief Stop the engine and join. Idempotent. From 6e942a9fc3c257fafb5214c24efbc82067e56575 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 16:26:04 -0400 Subject: [PATCH 09/57] remove redundancy --- runtime/include/TransportCAPI.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/runtime/include/TransportCAPI.h b/runtime/include/TransportCAPI.h index 2f0a4aa312..4c806b4918 100644 --- a/runtime/include/TransportCAPI.h +++ b/runtime/include/TransportCAPI.h @@ -84,9 +84,6 @@ CatalystTransportSession *__catalyst__transport__controller_create(const char *b void __catalyst__transport__close(CatalystTransportSession *s); int __catalyst__transport__connect(CatalystTransportSession *s, const char *peer, uint16_t oob_port); -int __catalyst__transport__alloc_reply(CatalystTransportSession *s, uint64_t size, int32_t mem_kind, - uint32_t access, CatalystTransportMemRegion *out); - int __catalyst__transport__exchange_keys(CatalystTransportSession *s, CatalystTransportPeerRef *out); int __catalyst__transport__establish_channel(CatalystTransportSession *s, int32_t data_path, From 4408322703086dfd95173ec82f4e960281ef86f9 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 16:32:14 -0400 Subject: [PATCH 10/57] add start --- runtime/include/TransportCAPI.h | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/include/TransportCAPI.h b/runtime/include/TransportCAPI.h index 4c806b4918..e4a7b7e357 100644 --- a/runtime/include/TransportCAPI.h +++ b/runtime/include/TransportCAPI.h @@ -91,6 +91,7 @@ int __catalyst__transport__establish_channel(CatalystTransportSession *s, int32_ int __catalyst__transport__commit_work_item(CatalystTransportSession *s, uint32_t work_item_idx, uint64_t in_bytes, uint64_t out_bytes); void *__catalyst__transport__data_slot(CatalystTransportSession *s); +void __catalyst__transport__start(CatalystTransportSession *s); int __catalyst__transport__kick(CatalystTransportSession *s, uint32_t work_item_idx); int __catalyst__transport__collect(CatalystTransportSession *s, void *correction, uint64_t bytes); uint64_t __catalyst__transport__last_rtt_ns(CatalystTransportSession *s); From 5282f5a1d099dd02762d1cb4092def25f49a144d Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 16:42:09 -0400 Subject: [PATCH 11/57] update comment --- runtime/include/Transport.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index 3a5ed0ce8d..4ed87cbe32 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -161,7 +161,7 @@ class TransportSession { */ class ControllerSession : public TransportSession { public: - // Build the work item in slot `work_item_idx` from `schema`. + // Build the work item in slot `work_item_idx` from in_bytes and out_bytes. virtual void commit_work_item(std::uint32_t work_item_idx, std::uint64_t in_bytes, std::uint64_t out_bytes) = 0; From 32fc6f26d76efdda70104f22644617d97803cba7 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 16:47:25 -0400 Subject: [PATCH 12/57] update changelog --- doc/releases/changelog-dev.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md index 5f66384adf..f037e1bcbc 100644 --- a/doc/releases/changelog-dev.md +++ b/doc/releases/changelog-dev.md @@ -14,6 +14,7 @@ * A new runtime transport layer for remote/local executors is introduced. [(#3043)](https://github.com/PennyLaneAI/catalyst/pull/3043) + [(#3045)](https://github.com/PennyLaneAI/catalyst/pull/3045) * A `BufferizableOpInterface` implementation is now added for `catalyst.launch_kernel` operation and it is now bufferizable. [(#3024)](https://github.com/PennyLaneAI/catalyst/pull/3024) @@ -472,4 +473,5 @@ Mehrdad Malekmohammadi, River McCubbin, Shuli Shu, Paul Haochen Wang, -Jake Zaia. +Jake Zaia, +Hongsheng Zheng. From 2fc50c3cc371647c6a08847a04af96bfed3f0be9 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 17:15:48 -0400 Subject: [PATCH 13/57] update --- runtime/include/Transport.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index 4ed87cbe32..ff5ee22dd5 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -167,7 +167,7 @@ class ControllerSession : public TransportSession { // Fire one round using work item `work_item_idx` and whatever payload is currently in // data_slot(). Pairs with a subsequent collect(). Returns 0 on success. - virtual int kick(std::uint32_t work_item_idx = 0) = 0; + virtual int kick(std::uint32_t work_item_idx) = 0; // Current round's outbound slot in the transport-owned ring. virtual void *data_slot() = 0; From 4d211cefd98b20d3eb441c6211d41074f4cea909 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 17:21:19 -0400 Subject: [PATCH 14/57] update --- runtime/lib/transport/TransportCAPI.cpp | 60 +++++++------------------ 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/runtime/lib/transport/TransportCAPI.cpp b/runtime/lib/transport/TransportCAPI.cpp index 617f05e6c3..311957985d 100644 --- a/runtime/lib/transport/TransportCAPI.cpp +++ b/runtime/lib/transport/TransportCAPI.cpp @@ -29,7 +29,6 @@ using catalyst::transport::ChannelDesc; using catalyst::transport::ConnectInfo; using catalyst::transport::ControllerSession; using catalyst::transport::DataPath; -using catalyst::transport::MemKind; using catalyst::transport::MemRegion; using catalyst::transport::PeerRef; @@ -37,28 +36,10 @@ using catalyst::transport::PeerRef; struct CatalystTransportSession { std::unique_ptr backend; ControllerSession *sess = nullptr; // heap-allocated by the backend factory - MemRegion reply = {}; - bool have_reply = false; }; namespace { -MemKind to_mem_kind(std::int32_t k) -{ - switch (k) { - case CATALYST_TRANSPORT_MEM_CPU_RAM: - return MemKind::CpuRam; - case CATALYST_TRANSPORT_MEM_GPU_HBM: - return MemKind::GpuHbm; - case CATALYST_TRANSPORT_MEM_DDR: - return MemKind::Ddr; - case CATALYST_TRANSPORT_MEM_OTHER: - return MemKind::Other; - default: - return MemKind::Ddr; - } -} - DataPath to_data_path(std::int32_t p) { switch (p) { @@ -133,35 +114,13 @@ int __catalyst__transport__connect(CatalystTransportSession *s, const char *peer }); } -int __catalyst__transport__alloc_reply(CatalystTransportSession *s, std::uint64_t size, - std::int32_t mem_kind, std::uint32_t access, - CatalystTransportMemRegion *out) -{ - if (!s || !s->sess) { - return CATALYST_TRANSPORT_ERR; - } - return guard([&] { - MemRegion r = s->sess->alloc_memory(size, to_mem_kind(mem_kind), access); - s->reply = r; - s->have_reply = true; - if (out) { - out->addr = r.addr; - out->size = r.size; - out->lkey = r.lkey; - out->rkey = r.rkey; - out->kind = mem_kind; - } - return CATALYST_TRANSPORT_OK; - }); -} - int __catalyst__transport__exchange_keys(CatalystTransportSession *s, CatalystTransportPeerRef *out) { if (!s || !s->sess) { return CATALYST_TRANSPORT_ERR; } return guard([&] { - PeerRef p = s->sess->exchange_keys(s->have_reply ? s->reply : MemRegion{}); + PeerRef p = s->sess->exchange_keys(MemRegion{}); if (out) { out->rkey = p.rkey; out->remote_addr = p.remote_addr; @@ -184,7 +143,7 @@ int __catalyst__transport__establish_channel(CatalystTransportSession *s, std::i p.rkey = peer->rkey; p.remote_addr = peer->remote_addr; p.size = peer->size; - s->sess->establish_channel(desc, s->have_reply ? s->reply : MemRegion{}, p); + s->sess->establish_channel(desc, MemRegion{}, p); return CATALYST_TRANSPORT_OK; }); } @@ -251,6 +210,21 @@ std::uint64_t __catalyst__transport__last_rtt_ns(CatalystTransportSession *s) return s->sess->last_rtt_ns(); } +void __catalyst__transport__start(CatalystTransportSession *s) +{ + if (!s || !s->sess) { + return; + } + try { + s->sess->start(); + } + catch (const std::exception &e) { + std::cerr << "[transport] start: " << e.what() << "\n"; + } + catch (...) { + } +} + void __catalyst__transport__stop(CatalystTransportSession *s) { if (s && s->sess) { From 060458b4319033542fdea882a3bb3ec07177732b Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 17:33:30 -0400 Subject: [PATCH 15/57] remove redundancy --- runtime/include/TransportCAPI.h | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/runtime/include/TransportCAPI.h b/runtime/include/TransportCAPI.h index e4a7b7e357..c1fb6817a3 100644 --- a/runtime/include/TransportCAPI.h +++ b/runtime/include/TransportCAPI.h @@ -55,20 +55,6 @@ enum { CATALYST_TRANSPORT_MEM_OTHER = 3, }; -// ibverbs access flags for the advertised reply region -enum { - CATALYST_TRANSPORT_ACCESS_REPLY = 7, -}; - -// Registered memory region handed back to the caller -typedef struct { - void *addr; - uint64_t size; - uint32_t lkey; - uint32_t rkey; - int32_t kind; // one of MemKind enum values -} CatalystTransportMemRegion; - // Remote peer region descriptor typedef struct { uint32_t rkey; From 246a4b7b2e3df32296f85c87c72f1c5a7f0e2ae1 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 18:29:14 -0400 Subject: [PATCH 16/57] number of collection is always 1 --- runtime/include/Transport.hpp | 7 +++---- runtime/lib/transport/TransportCAPI.cpp | 6 +----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index ff5ee22dd5..29c4cc81f1 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -135,13 +135,12 @@ class TransportSession { /** * @brief Wait for a result and write it out. * - * @param outputs Array of output buffers to write into. - * @param output_bytes Array of output buffer sizes. - * @param n Number of output buffers. + * @param correction Output buffer to write the result into. + * @param bytes Capacity of the output buffer, in bytes. * * @return `int` */ - virtual int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) = 0; + virtual int collect(void *correction, std::uint64_t bytes) = 0; /** * @brief Stop the engine and join. Idempotent. diff --git a/runtime/lib/transport/TransportCAPI.cpp b/runtime/lib/transport/TransportCAPI.cpp index 311957985d..429541508a 100644 --- a/runtime/lib/transport/TransportCAPI.cpp +++ b/runtime/lib/transport/TransportCAPI.cpp @@ -195,11 +195,7 @@ int __catalyst__transport__collect(CatalystTransportSession *s, void *correction if (!s || !s->sess) { return CATALYST_TRANSPORT_ERR; } - return guard([&] { - void *outputs[1] = {correction}; - std::size_t caps[1] = {static_cast(bytes)}; - return s->sess->collect(outputs, caps, 1); - }); + return guard([&] { return s->sess->collect(correction, bytes); }); } std::uint64_t __catalyst__transport__last_rtt_ns(CatalystTransportSession *s) From 22cd621cba1e34fb6b67717587d48626849abbc1 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Mon, 20 Jul 2026 18:32:57 -0400 Subject: [PATCH 17/57] update interface --- runtime/include/Transport.hpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index 4ed87cbe32..29c4cc81f1 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -135,13 +135,12 @@ class TransportSession { /** * @brief Wait for a result and write it out. * - * @param outputs Array of output buffers to write into. - * @param output_bytes Array of output buffer sizes. - * @param n Number of output buffers. + * @param correction Output buffer to write the result into. + * @param bytes Capacity of the output buffer, in bytes. * * @return `int` */ - virtual int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) = 0; + virtual int collect(void *correction, std::uint64_t bytes) = 0; /** * @brief Stop the engine and join. Idempotent. @@ -167,7 +166,7 @@ class ControllerSession : public TransportSession { // Fire one round using work item `work_item_idx` and whatever payload is currently in // data_slot(). Pairs with a subsequent collect(). Returns 0 on success. - virtual int kick(std::uint32_t work_item_idx = 0) = 0; + virtual int kick(std::uint32_t work_item_idx) = 0; // Current round's outbound slot in the transport-owned ring. virtual void *data_slot() = 0; From 18c26840ba23d43fdcf02a107e3f7a033b8ab55a Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Tue, 21 Jul 2026 07:10:23 -0400 Subject: [PATCH 18/57] Add transport dialect Assisted-by: Claude Opus 4.8 --- mlir/include/CMakeLists.txt | 1 + mlir/include/Transport/CMakeLists.txt | 1 + mlir/include/Transport/IR/CMakeLists.txt | 3 + mlir/include/Transport/IR/TransportDialect.h | 24 ++++ mlir/include/Transport/IR/TransportDialect.td | 67 ++++++++++ mlir/include/Transport/IR/TransportOps.h | 26 ++++ mlir/include/Transport/IR/TransportOps.td | 117 ++++++++++++++++++ mlir/lib/CMakeLists.txt | 1 + mlir/lib/Driver/CMakeLists.txt | 1 + mlir/lib/Driver/CompilerDriver.cpp | 3 + mlir/lib/Transport/CMakeLists.txt | 1 + mlir/lib/Transport/IR/CMakeLists.txt | 13 ++ mlir/lib/Transport/IR/TransportDialect.cpp | 50 ++++++++ mlir/lib/Transport/IR/TransportOps.cpp | 24 ++++ mlir/tools/quantum-opt/CMakeLists.txt | 1 + mlir/tools/quantum-opt/quantum-opt.cpp | 3 + 16 files changed, 336 insertions(+) create mode 100644 mlir/include/Transport/CMakeLists.txt create mode 100644 mlir/include/Transport/IR/CMakeLists.txt create mode 100644 mlir/include/Transport/IR/TransportDialect.h create mode 100644 mlir/include/Transport/IR/TransportDialect.td create mode 100644 mlir/include/Transport/IR/TransportOps.h create mode 100644 mlir/include/Transport/IR/TransportOps.td create mode 100644 mlir/lib/Transport/CMakeLists.txt create mode 100644 mlir/lib/Transport/IR/CMakeLists.txt create mode 100644 mlir/lib/Transport/IR/TransportDialect.cpp create mode 100644 mlir/lib/Transport/IR/TransportOps.cpp diff --git a/mlir/include/CMakeLists.txt b/mlir/include/CMakeLists.txt index a94e4af0f9..09bb4d088e 100644 --- a/mlir/include/CMakeLists.txt +++ b/mlir/include/CMakeLists.txt @@ -11,4 +11,5 @@ add_subdirectory(QecPhysical) add_subdirectory(Quantum) add_subdirectory(QRef) add_subdirectory(RTIO) +add_subdirectory(Transport) add_subdirectory(Test) diff --git a/mlir/include/Transport/CMakeLists.txt b/mlir/include/Transport/CMakeLists.txt new file mode 100644 index 0000000000..f33061b2d8 --- /dev/null +++ b/mlir/include/Transport/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(IR) diff --git a/mlir/include/Transport/IR/CMakeLists.txt b/mlir/include/Transport/IR/CMakeLists.txt new file mode 100644 index 0000000000..d1bc88dbed --- /dev/null +++ b/mlir/include/Transport/IR/CMakeLists.txt @@ -0,0 +1,3 @@ +add_mlir_dialect(TransportOps transport) +add_mlir_doc(TransportDialect TransportDialect Transport/ -gen-dialect-doc) +add_mlir_doc(TransportOps TransportOps Transport/ -gen-op-doc) diff --git a/mlir/include/Transport/IR/TransportDialect.h b/mlir/include/Transport/IR/TransportDialect.h new file mode 100644 index 0000000000..ebfdd9ba47 --- /dev/null +++ b/mlir/include/Transport/IR/TransportDialect.h @@ -0,0 +1,24 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/Dialect.h" +#include "mlir/IR/OpDefinition.h" + +#include "Transport/IR/TransportOpsDialect.h.inc" + +#define GET_TYPEDEF_CLASSES +#include "Transport/IR/TransportOpsTypes.h.inc" diff --git a/mlir/include/Transport/IR/TransportDialect.td b/mlir/include/Transport/IR/TransportDialect.td new file mode 100644 index 0000000000..78ed32fb70 --- /dev/null +++ b/mlir/include/Transport/IR/TransportDialect.td @@ -0,0 +1,67 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TRANSPORT_DIALECT +#define TRANSPORT_DIALECT + +include "mlir/IR/OpBase.td" +include "mlir/IR/DialectBase.td" +include "mlir/IR/AttrTypeBase.td" + +//===----------------------------------------------------------------------===// +// Transport dialect definition. +//===----------------------------------------------------------------------===// + +def Transport_Dialect : Dialect { + let summary = "Runtime transport-layer ops that lower to the __catalyst__transport__* CAPI."; + let description = [{ + A thin, typed MLIR representation of the Catalyst runtime transport CAPI + (runtime/include/TransportCAPI.h). One op per controller-side CAPI entry + point; `convert-transport-to-llvm` lowers each to an `llvm.call` on the + matching `__catalyst__transport__*` symbol. The coprocessor role is an + out-of-tree backend loaded by the runtime, so only the controller side is + emitted here. + }]; + + let name = "transport"; + let cppNamespace = "::catalyst::transport"; + let useDefaultTypePrinterParser = 1; + let usePropertiesForAttributes = 1; +} + +//===----------------------------------------------------------------------===// +// Transport dialect types. +//===----------------------------------------------------------------------===// + +class Transport_Type traits = []> + : TypeDef { + let mnemonic = typeMnemonic; +} + +def Transport_SessionType : Transport_Type<"Session", "session"> { + let summary = "Opaque transport controller session handle (CatalystTransportSession*)."; +} + +def Transport_PeerType : Transport_Type<"Peer", "peer"> { + let summary = "Opaque peer-region descriptor handle (CatalystTransportPeerRef*)."; +} + +//===----------------------------------------------------------------------===// +// Transport operation base. +//===----------------------------------------------------------------------===// + +class Transport_Op traits = []> : + Op; + +#endif // TRANSPORT_DIALECT diff --git a/mlir/include/Transport/IR/TransportOps.h b/mlir/include/Transport/IR/TransportOps.h new file mode 100644 index 0000000000..0cc3624b7b --- /dev/null +++ b/mlir/include/Transport/IR/TransportOps.h @@ -0,0 +1,26 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "mlir/Bytecode/BytecodeOpInterface.h" +#include "mlir/IR/BuiltinTypes.h" +#include "mlir/IR/OpDefinition.h" +#include "mlir/IR/PatternMatch.h" +#include "mlir/Interfaces/SideEffectInterfaces.h" + +#include "Transport/IR/TransportDialect.h" + +#define GET_OP_CLASSES +#include "Transport/IR/TransportOps.h.inc" diff --git a/mlir/include/Transport/IR/TransportOps.td b/mlir/include/Transport/IR/TransportOps.td new file mode 100644 index 0000000000..ccd7b2308e --- /dev/null +++ b/mlir/include/Transport/IR/TransportOps.td @@ -0,0 +1,117 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TRANSPORT_OPS +#define TRANSPORT_OPS + +include "mlir/IR/OpBase.td" +include "mlir/IR/BuiltinAttributes.td" +include "mlir/Interfaces/SideEffectInterfaces.td" +include "Transport/IR/TransportDialect.td" + +//===----------------------------------------------------------------------===// +// Bring-up ops +//===----------------------------------------------------------------------===// + +def Transport_ControllerCreateOp : Transport_Op<"controller_create"> { + let summary = "Create a controller session from a named backend plugin .so."; + let arguments = (ins StrAttr:$backend_lib, StrAttr:$config); + let results = (outs Transport_SessionType:$session); + let assemblyFormat = "attr-dict `->` type($session)"; +} + +def Transport_ConnectOp : Transport_Op<"connect"> { + let summary = "Bring up the connection to the peer."; + let arguments = (ins Transport_SessionType:$session, StrAttr:$peer, I16Attr:$oob_port); + let results = (outs I32:$status); + let assemblyFormat = "$session attr-dict `:` functional-type($session, $status)"; +} + +def Transport_ExchangeKeysOp : Transport_Op<"exchange_keys"> { + let summary = "Exchange local and peer region handles; yields the peer descriptor."; + let arguments = (ins Transport_SessionType:$session); + let results = (outs I32:$status, Transport_PeerType:$peer); + let assemblyFormat = "$session attr-dict `:` type($session) `->` type($peer)"; +} + +def Transport_EstablishChannelOp : Transport_Op<"establish_channel"> { + let summary = "Program the data-movement channel from the local + peer regions."; + let arguments = (ins Transport_SessionType:$session, Transport_PeerType:$peer, + I32Attr:$data_path); + let results = (outs I32:$status); + let assemblyFormat = "$session `,` $peer attr-dict `:` type($session) `,` type($peer)"; +} + +def Transport_CommitWorkItemOp : Transport_Op<"commit_work_item"> { + let summary = "Build a work item (I/O sizes) in a slot before kicking rounds."; + let arguments = (ins Transport_SessionType:$session, I32Attr:$work_item_idx, + I64Attr:$in_bytes, I64Attr:$out_bytes); + let results = (outs I32:$status); + let assemblyFormat = "$session attr-dict `:` type($session)"; +} + +def Transport_StartOp : Transport_Op<"start"> { + let summary = "Start the session (non-blocking; runs until stop())."; + let arguments = (ins Transport_SessionType:$session); + let assemblyFormat = "$session attr-dict `:` type($session)"; +} + +//===----------------------------------------------------------------------===// +// Per-round ops +//===----------------------------------------------------------------------===// + +def Transport_KickOp : Transport_Op<"kick"> { + let summary = "Write the payload into the outbound slot and fire one round."; + let arguments = (ins Transport_SessionType:$session, I64:$payload, I32Attr:$work_item_idx); + let results = (outs I32:$status); + let assemblyFormat = "$session `,` $payload attr-dict `:` type($session) `,` type($payload)"; +} + +def Transport_CollectOp : Transport_Op<"collect"> { + let summary = "Wait for this round's reply and return it as a value."; + let arguments = (ins Transport_SessionType:$session, I64Attr:$bytes); + let results = (outs I64:$result); + let assemblyFormat = "$session attr-dict `:` type($session) `->` type($result)"; +} + +def Transport_LastRttNsOp : Transport_Op<"last_rtt_ns"> { + let summary = "Last round-trip time in nanoseconds."; + let arguments = (ins Transport_SessionType:$session); + let results = (outs I64:$rtt_ns); + let assemblyFormat = "$session attr-dict `:` type($session) `->` type($rtt_ns)"; +} + +//===----------------------------------------------------------------------===// +// Teardown ops +//===----------------------------------------------------------------------===// + +def Transport_StopOp : Transport_Op<"stop"> { + let summary = "Stop the session. Idempotent."; + let arguments = (ins Transport_SessionType:$session); + let assemblyFormat = "$session attr-dict `:` type($session)"; +} + +def Transport_CloseOp : Transport_Op<"close"> { + let summary = "Close the transport (releases the channel)."; + let arguments = (ins Transport_SessionType:$session); + let assemblyFormat = "$session attr-dict `:` type($session)"; +} + +def Transport_DestroyOp : Transport_Op<"destroy"> { + let summary = "Destroy the session and free it."; + let arguments = (ins Transport_SessionType:$session); + let assemblyFormat = "$session attr-dict `:` type($session)"; +} + +#endif // TRANSPORT_OPS diff --git a/mlir/lib/CMakeLists.txt b/mlir/lib/CMakeLists.txt index b4ff29b1a5..190316d1c9 100644 --- a/mlir/lib/CMakeLists.txt +++ b/mlir/lib/CMakeLists.txt @@ -13,4 +13,5 @@ add_subdirectory(QecPhysical) add_subdirectory(QRef) add_subdirectory(Quantum) add_subdirectory(RTIO) +add_subdirectory(Transport) add_subdirectory(Test) diff --git a/mlir/lib/Driver/CMakeLists.txt b/mlir/lib/Driver/CMakeLists.txt index c371e2e8e0..cd55702b17 100644 --- a/mlir/lib/Driver/CMakeLists.txt +++ b/mlir/lib/Driver/CMakeLists.txt @@ -76,6 +76,7 @@ set(LIBS ion-transforms MLIRRTIO rtio-transforms + MLIRTransport MLIRCatalystTest ${ENZYME_LIB} fmt::fmt diff --git a/mlir/lib/Driver/CompilerDriver.cpp b/mlir/lib/Driver/CompilerDriver.cpp index f3aac8efc9..8b768e590d 100644 --- a/mlir/lib/Driver/CompilerDriver.cpp +++ b/mlir/lib/Driver/CompilerDriver.cpp @@ -89,6 +89,8 @@ #include "RegisterAllPasses.h" +#include "Transport/IR/TransportDialect.h" + using namespace mlir; using namespace catalyst; using namespace catalyst::driver; @@ -181,6 +183,7 @@ void registerAllCatalystDialects(DialectRegistry ®istry) registry.insert(); registry.insert(); registry.insert(); + registry.insert(); registry.insert(); registry.insert(); registry.insert(); diff --git a/mlir/lib/Transport/CMakeLists.txt b/mlir/lib/Transport/CMakeLists.txt new file mode 100644 index 0000000000..f33061b2d8 --- /dev/null +++ b/mlir/lib/Transport/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(IR) diff --git a/mlir/lib/Transport/IR/CMakeLists.txt b/mlir/lib/Transport/IR/CMakeLists.txt new file mode 100644 index 0000000000..9554c24fe3 --- /dev/null +++ b/mlir/lib/Transport/IR/CMakeLists.txt @@ -0,0 +1,13 @@ +add_mlir_library(MLIRTransport + TransportDialect.cpp + TransportOps.cpp + + ADDITIONAL_HEADER_DIRS + ${PROJECT_SOURCE_DIR}/include/Transport + + DEPENDS + MLIRTransportOpsIncGen + + LINK_LIBS PRIVATE + MLIRLLVMDialect +) diff --git a/mlir/lib/Transport/IR/TransportDialect.cpp b/mlir/lib/Transport/IR/TransportDialect.cpp new file mode 100644 index 0000000000..96bb105661 --- /dev/null +++ b/mlir/lib/Transport/IR/TransportDialect.cpp @@ -0,0 +1,50 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Transport/IR/TransportDialect.h" + +#include "llvm/ADT/TypeSwitch.h" +#include "mlir/IR/Builders.h" +#include "mlir/IR/DialectImplementation.h" + +#include "Transport/IR/TransportOps.h" + +using namespace mlir; +using namespace catalyst::transport; + +//===----------------------------------------------------------------------===// +// Transport dialect definitions. +//===----------------------------------------------------------------------===// + +#include "Transport/IR/TransportOpsDialect.cpp.inc" + +//===----------------------------------------------------------------------===// +// Transport type definitions. +//===----------------------------------------------------------------------===// + +#define GET_TYPEDEF_CLASSES +#include "Transport/IR/TransportOpsTypes.cpp.inc" + +void TransportDialect::initialize() +{ + addTypes< +#define GET_TYPEDEF_LIST +#include "Transport/IR/TransportOpsTypes.cpp.inc" + >(); + + addOperations< +#define GET_OP_LIST +#include "Transport/IR/TransportOps.cpp.inc" + >(); +} diff --git a/mlir/lib/Transport/IR/TransportOps.cpp b/mlir/lib/Transport/IR/TransportOps.cpp new file mode 100644 index 0000000000..8d894ef69d --- /dev/null +++ b/mlir/lib/Transport/IR/TransportOps.cpp @@ -0,0 +1,24 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Transport/IR/TransportOps.h" + +#include "mlir/IR/Builders.h" +#include "mlir/IR/OpImplementation.h" + +using namespace mlir; +using namespace catalyst::transport; + +#define GET_OP_CLASSES +#include "Transport/IR/TransportOps.cpp.inc" diff --git a/mlir/tools/quantum-opt/CMakeLists.txt b/mlir/tools/quantum-opt/CMakeLists.txt index ef16ce934d..cf7a3b2053 100644 --- a/mlir/tools/quantum-opt/CMakeLists.txt +++ b/mlir/tools/quantum-opt/CMakeLists.txt @@ -33,6 +33,7 @@ set(LIBS ion-transforms MLIRRTIO rtio-transforms + MLIRTransport MLIRQecLogical MLIRQecPhysical MLIRCatalystTest diff --git a/mlir/tools/quantum-opt/quantum-opt.cpp b/mlir/tools/quantum-opt/quantum-opt.cpp index 5b9a279fd8..8e94976461 100644 --- a/mlir/tools/quantum-opt/quantum-opt.cpp +++ b/mlir/tools/quantum-opt/quantum-opt.cpp @@ -50,6 +50,8 @@ #include "RegisterAllPasses.h" +#include "Transport/IR/TransportDialect.h" + namespace test { void registerTestDialect(mlir::DialectRegistry &); } // namespace test @@ -77,6 +79,7 @@ int main(int argc, char **argv) registry.insert(); registry.insert(); registry.insert(); + registry.insert(); registry.insert(); registry.insert(); registry.insert(); From 6984afaf17a764d060aaea8e97be5732ca554801 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Tue, 21 Jul 2026 07:10:23 -0400 Subject: [PATCH 19/57] Add transport-to-llvm pass Assisted-by: Claude Opus 4.8 --- .../Transport/Transforms/CMakeLists.txt | 4 + mlir/include/Transport/Transforms/Passes.h | 30 ++ mlir/include/Transport/Transforms/Passes.td | 32 +++ mlir/lib/Transport/Transforms/CMakeLists.txt | 24 ++ .../Transport/Transforms/TransportToLLVM.cpp | 266 ++++++++++++++++++ .../Transport/ConvertTransportToLLVM.mlir | 57 ++++ 6 files changed, 413 insertions(+) create mode 100644 mlir/include/Transport/Transforms/CMakeLists.txt create mode 100644 mlir/include/Transport/Transforms/Passes.h create mode 100644 mlir/include/Transport/Transforms/Passes.td create mode 100644 mlir/lib/Transport/Transforms/CMakeLists.txt create mode 100644 mlir/lib/Transport/Transforms/TransportToLLVM.cpp create mode 100644 mlir/test/Transport/ConvertTransportToLLVM.mlir diff --git a/mlir/include/Transport/Transforms/CMakeLists.txt b/mlir/include/Transport/Transforms/CMakeLists.txt new file mode 100644 index 0000000000..fb92bac94b --- /dev/null +++ b/mlir/include/Transport/Transforms/CMakeLists.txt @@ -0,0 +1,4 @@ +set(LLVM_TARGET_DEFINITIONS Passes.td) +mlir_tablegen(Passes.h.inc -gen-pass-decls -name Transport) +add_public_tablegen_target(MLIRTransportPassIncGen) +add_mlir_doc(Passes TransportPasses ./ -gen-pass-doc) diff --git a/mlir/include/Transport/Transforms/Passes.h b/mlir/include/Transport/Transforms/Passes.h new file mode 100644 index 0000000000..78d1d1c464 --- /dev/null +++ b/mlir/include/Transport/Transforms/Passes.h @@ -0,0 +1,30 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "mlir/Pass/Pass.h" + +#include "Transport/IR/TransportDialect.h" +#include "Transport/IR/TransportOps.h" + +namespace catalyst { +namespace transport { + +#define GEN_PASS_DECL +#define GEN_PASS_REGISTRATION +#include "Transport/Transforms/Passes.h.inc" + +} // namespace transport +} // namespace catalyst diff --git a/mlir/include/Transport/Transforms/Passes.td b/mlir/include/Transport/Transforms/Passes.td new file mode 100644 index 0000000000..a33e9bd80e --- /dev/null +++ b/mlir/include/Transport/Transforms/Passes.td @@ -0,0 +1,32 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TRANSPORT_PASSES +#define TRANSPORT_PASSES + +include "mlir/Pass/PassBase.td" + +def ConvertTransportToLLVMPass : Pass<"convert-transport-to-llvm", "mlir::ModuleOp"> { + let summary = "Lower the transport dialect to LLVM dialect with runtime calls."; + let description = [{ + Lowers each `transport` op to an `llvm.call` on the matching + `__catalyst__transport__*` symbol. + }]; + + let dependentDialects = [ + "mlir::LLVM::LLVMDialect" + ]; +} + +#endif // TRANSPORT_PASSES diff --git a/mlir/lib/Transport/Transforms/CMakeLists.txt b/mlir/lib/Transport/Transforms/CMakeLists.txt new file mode 100644 index 0000000000..17ee33278d --- /dev/null +++ b/mlir/lib/Transport/Transforms/CMakeLists.txt @@ -0,0 +1,24 @@ +set(LIBRARY_NAME transport-transforms) + +file(GLOB SRC + TransportToLLVM.cpp +) + +get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS) +get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS) +set(LIBS + ${dialect_libs} + ${conversion_libs} + MLIRTransport +) + +set(DEPENDS + MLIRTransportPassIncGen +) + +add_mlir_library(${LIBRARY_NAME} STATIC ${SRC} LINK_LIBS PRIVATE ${LIBS} DEPENDS ${DEPENDS}) +target_compile_features(${LIBRARY_NAME} PUBLIC cxx_std_20) +target_include_directories(${LIBRARY_NAME} PUBLIC + . + ${PROJECT_SOURCE_DIR}/include + ${CMAKE_BINARY_DIR}/include) diff --git a/mlir/lib/Transport/Transforms/TransportToLLVM.cpp b/mlir/lib/Transport/Transforms/TransportToLLVM.cpp new file mode 100644 index 0000000000..e96f68d1f7 --- /dev/null +++ b/mlir/lib/Transport/Transforms/TransportToLLVM.cpp @@ -0,0 +1,266 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Lower the `transport` dialect to `llvm.call`s on the __catalyst__transport__* +// CAPI (runtime/include/TransportCAPI.h). Controller-side only. + +#include "llvm/ADT/Twine.h" +#include "mlir/Conversion/LLVMCommon/TypeConverter.h" +#include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" +#include "mlir/Dialect/LLVMIR/LLVMDialect.h" +#include "mlir/IR/BuiltinOps.h" +#include "mlir/Pass/Pass.h" +#include "mlir/Transforms/DialectConversion.h" + +#include "Transport/IR/TransportOps.h" +#include "Transport/Transforms/Passes.h" + +using namespace mlir; +using namespace catalyst::transport; + +namespace catalyst { +namespace transport { + +#define GEN_PASS_DEF_CONVERTTRANSPORTTOLLVMPASS +#include "Transport/Transforms/Passes.h.inc" + +namespace { + +// sizeof(CatalystTransportPeerRef) = {u32, u64, u64}; over-allocate for alignment. +constexpr int64_t kPeerRefBytes = 32; + +LLVM::LLVMPointerType ptrTy(MLIRContext *ctx) { return LLVM::LLVMPointerType::get(ctx); } + +ModuleOp moduleOf(Operation *op) { return op->getParentOfType(); } + +// Declare-or-reuse a CAPI function and emit a call to it. A null resultTy means +// the function returns void. +Value emitCall(ConversionPatternRewriter &rewriter, Location loc, ModuleOp mod, StringRef name, + ArrayRef paramTys, Type resultTy, ValueRange args) +{ + Type rty = resultTy ? resultTy : LLVM::LLVMVoidType::get(rewriter.getContext()); + auto fn = LLVM::lookupOrCreateFn(rewriter, mod, name, paramTys, rty); + assert(succeeded(fn) && "failed to declare transport CAPI function"); + auto call = LLVM::CallOp::create(rewriter, loc, *fn, args); + return call.getNumResults() ? call.getResult() : Value(); +} + +// Materialize a null-terminated global string and return a ptr to its data. +Value globalStr(ConversionPatternRewriter &rewriter, Location loc, ModuleOp mod, StringRef prefix, + StringRef value) +{ + static int counter = 0; + std::string symName = (prefix + Twine(counter++)).str(); + return LLVM::createGlobalString(loc, rewriter, symName, Twine(value).concat(Twine('\0')).str(), + LLVM::Linkage::Internal); +} + +Value constInt(ConversionPatternRewriter &rewriter, Location loc, Type ty, int64_t v) +{ + return LLVM::ConstantOp::create(rewriter, loc, ty, rewriter.getIntegerAttr(ty, v)); +} + +//===----------------------------------------------------------------------===// +// Patterns +//===----------------------------------------------------------------------===// + +struct ControllerCreateLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(ControllerCreateOp op, OpAdaptor, + ConversionPatternRewriter &rewriter) const override + { + auto *ctx = op.getContext(); + ModuleOp mod = moduleOf(op); + Value lib = globalStr(rewriter, op.getLoc(), mod, "transport_backend_", op.getBackendLib()); + Value cfg = globalStr(rewriter, op.getLoc(), mod, "transport_config_", op.getConfig()); + Value s = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__controller_create", + {ptrTy(ctx), ptrTy(ctx)}, ptrTy(ctx), {lib, cfg}); + rewriter.replaceOp(op, s); + return success(); + } +}; + +struct ConnectLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(ConnectOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override + { + auto *ctx = op.getContext(); + ModuleOp mod = moduleOf(op); + Value peer = globalStr(rewriter, op.getLoc(), mod, "transport_peer_", op.getPeer()); + Value port = constInt(rewriter, op.getLoc(), rewriter.getI16Type(), op.getOobPort()); + Value r = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__connect", + {ptrTy(ctx), ptrTy(ctx), rewriter.getI16Type()}, rewriter.getI32Type(), + {adaptor.getSession(), peer, port}); + rewriter.replaceOp(op, r); + return success(); + } +}; + +struct ExchangeKeysLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(ExchangeKeysOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override + { + auto *ctx = op.getContext(); + ModuleOp mod = moduleOf(op); + Value peerBuf = LLVM::AllocaOp::create( + rewriter, op.getLoc(), ptrTy(ctx), rewriter.getI8Type(), + constInt(rewriter, op.getLoc(), rewriter.getI64Type(), kPeerRefBytes), /*alignment=*/8); + Value r = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__exchange_keys", + {ptrTy(ctx), ptrTy(ctx)}, rewriter.getI32Type(), + {adaptor.getSession(), peerBuf}); + rewriter.replaceOp(op, {r, peerBuf}); + return success(); + } +}; + +struct EstablishChannelLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(EstablishChannelOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override + { + auto *ctx = op.getContext(); + ModuleOp mod = moduleOf(op); + Value dp = constInt(rewriter, op.getLoc(), rewriter.getI32Type(), op.getDataPath()); + Value r = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__establish_channel", + {ptrTy(ctx), rewriter.getI32Type(), ptrTy(ctx)}, rewriter.getI32Type(), + {adaptor.getSession(), dp, adaptor.getPeer()}); + rewriter.replaceOp(op, r); + return success(); + } +}; + +struct CommitWorkItemLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(CommitWorkItemOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override + { + auto *ctx = op.getContext(); + ModuleOp mod = moduleOf(op); + Value idx = constInt(rewriter, op.getLoc(), rewriter.getI32Type(), op.getWorkItemIdx()); + Value inB = constInt(rewriter, op.getLoc(), rewriter.getI64Type(), op.getInBytes()); + Value outB = constInt(rewriter, op.getLoc(), rewriter.getI64Type(), op.getOutBytes()); + Value r = emitCall( + rewriter, op.getLoc(), mod, "__catalyst__transport__commit_work_item", + {ptrTy(ctx), rewriter.getI32Type(), rewriter.getI64Type(), rewriter.getI64Type()}, + rewriter.getI32Type(), {adaptor.getSession(), idx, inB, outB}); + rewriter.replaceOp(op, r); + return success(); + } +}; + +// Void-returning single-session ops: start / stop / close / destroy. +template struct VoidSessionLowering : public OpConversionPattern { + VoidSessionLowering(const TypeConverter &tc, MLIRContext *ctx, StringRef sym) + : OpConversionPattern(tc, ctx), symbol(sym) + { + } + LogicalResult matchAndRewrite(OpT op, typename OpT::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override + { + emitCall(rewriter, op.getLoc(), op->template getParentOfType(), symbol, + {ptrTy(op.getContext())}, Type(), {adaptor.getSession()}); + rewriter.eraseOp(op); + return success(); + } + std::string symbol; +}; + +struct KickLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(KickOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override + { + auto *ctx = op.getContext(); + ModuleOp mod = moduleOf(op); + // slot = data_slot(s); store payload -> slot; kick(s, idx) + Value slot = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__data_slot", + {ptrTy(ctx)}, ptrTy(ctx), {adaptor.getSession()}); + LLVM::StoreOp::create(rewriter, op.getLoc(), adaptor.getPayload(), slot); + Value idx = constInt(rewriter, op.getLoc(), rewriter.getI32Type(), op.getWorkItemIdx()); + Value r = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__kick", + {ptrTy(ctx), rewriter.getI32Type()}, rewriter.getI32Type(), + {adaptor.getSession(), idx}); + rewriter.replaceOp(op, r); + return success(); + } +}; + +struct CollectLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(CollectOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override + { + auto *ctx = op.getContext(); + ModuleOp mod = moduleOf(op); + Value one = constInt(rewriter, op.getLoc(), rewriter.getI64Type(), 1); + Value buf = LLVM::AllocaOp::create(rewriter, op.getLoc(), ptrTy(ctx), rewriter.getI64Type(), + one, /*alignment=*/8); + Value bytes = constInt(rewriter, op.getLoc(), rewriter.getI64Type(), op.getBytes()); + emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__collect", + {ptrTy(ctx), ptrTy(ctx), rewriter.getI64Type()}, rewriter.getI32Type(), + {adaptor.getSession(), buf, bytes}); + Value loaded = LLVM::LoadOp::create(rewriter, op.getLoc(), rewriter.getI64Type(), buf); + rewriter.replaceOp(op, loaded); + return success(); + } +}; + +struct LastRttLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(LastRttNsOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override + { + Value r = + emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__last_rtt_ns", + {ptrTy(op.getContext())}, rewriter.getI64Type(), {adaptor.getSession()}); + rewriter.replaceOp(op, r); + return success(); + } +}; + +} // namespace + +struct ConvertTransportToLLVMPass + : public impl::ConvertTransportToLLVMPassBase { + using ConvertTransportToLLVMPassBase::ConvertTransportToLLVMPassBase; + + void runOnOperation() override + { + MLIRContext *ctx = &getContext(); + LLVMTypeConverter tc(ctx); + tc.addConversion([ctx](SessionType) -> Type { return LLVM::LLVMPointerType::get(ctx); }); + tc.addConversion([ctx](PeerType) -> Type { return LLVM::LLVMPointerType::get(ctx); }); + + RewritePatternSet patterns(ctx); + patterns.add(tc, ctx); + patterns.add>(tc, ctx, "__catalyst__transport__start"); + patterns.add>(tc, ctx, "__catalyst__transport__stop"); + patterns.add>(tc, ctx, "__catalyst__transport__close"); + patterns.add>(tc, ctx, "__catalyst__transport__destroy"); + + ConversionTarget target(*ctx); + target.addLegalDialect(); + target.addIllegalDialect(); + + if (failed(applyPartialConversion(getOperation(), target, std::move(patterns)))) + signalPassFailure(); + } +}; + +} // namespace transport +} // namespace catalyst diff --git a/mlir/test/Transport/ConvertTransportToLLVM.mlir b/mlir/test/Transport/ConvertTransportToLLVM.mlir new file mode 100644 index 0000000000..d9cdb1fd06 --- /dev/null +++ b/mlir/test/Transport/ConvertTransportToLLVM.mlir @@ -0,0 +1,57 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// RUN: quantum-opt %s --convert-transport-to-llvm --split-input-file | FileCheck %s + +// CHECK-DAG: llvm.func @__catalyst__transport__controller_create(!llvm.ptr, !llvm.ptr) -> !llvm.ptr +// CHECK-DAG: llvm.func @__catalyst__transport__connect(!llvm.ptr, !llvm.ptr, i16) -> i32 +// CHECK-DAG: llvm.func @__catalyst__transport__exchange_keys(!llvm.ptr, !llvm.ptr) -> i32 +// CHECK-DAG: llvm.func @__catalyst__transport__establish_channel(!llvm.ptr, i32, !llvm.ptr) -> i32 +// CHECK-DAG: llvm.func @__catalyst__transport__commit_work_item(!llvm.ptr, i32, i64, i64) -> i32 +// CHECK-DAG: llvm.func @__catalyst__transport__data_slot(!llvm.ptr) -> !llvm.ptr +// CHECK-DAG: llvm.func @__catalyst__transport__kick(!llvm.ptr, i32) -> i32 +// CHECK-DAG: llvm.func @__catalyst__transport__collect(!llvm.ptr, !llvm.ptr, i64) -> i32 +// CHECK-DAG: llvm.func @__catalyst__transport__start(!llvm.ptr) +// CHECK-DAG: llvm.func @__catalyst__transport__stop(!llvm.ptr) +// CHECK-DAG: llvm.func @__catalyst__transport__destroy(!llvm.ptr) + +// CHECK-LABEL: func.func @controller_roundtrip +func.func @controller_roundtrip() -> i64 { + // CHECK: %[[S:.*]] = llvm.call @__catalyst__transport__controller_create + %s = transport.controller_create {backend_lib = "libtransport_backend.so", config = "key=value"} -> !transport.session + // CHECK: llvm.call @__catalyst__transport__connect(%[[S]] + %c = transport.connect %s {peer = "127.0.0.1", oob_port = 18560 : i16} : (!transport.session) -> i32 + // CHECK: %[[PEER:.*]] = llvm.alloca + // CHECK: llvm.call @__catalyst__transport__exchange_keys(%[[S]], %[[PEER]]) + %cs, %peer = transport.exchange_keys %s : !transport.session -> !transport.peer + // CHECK: llvm.call @__catalyst__transport__establish_channel(%[[S]], {{.*}}, %[[PEER]]) + %e = transport.establish_channel %s, %peer {data_path = 0 : i32} : !transport.session, !transport.peer + // CHECK: llvm.call @__catalyst__transport__commit_work_item(%[[S]] + %w = transport.commit_work_item %s {work_item_idx = 0 : i32, in_bytes = 8 : i64, out_bytes = 8 : i64} : !transport.session + // CHECK: llvm.call @__catalyst__transport__start(%[[S]]) + transport.start %s : !transport.session + %payload = arith.constant 81985529216486895 : i64 + // CHECK: %[[SLOT:.*]] = llvm.call @__catalyst__transport__data_slot(%[[S]]) + // CHECK: llvm.store %{{.*}}, %[[SLOT]] + // CHECK: llvm.call @__catalyst__transport__kick(%[[S]] + %k = transport.kick %s, %payload {work_item_idx = 0 : i32} : !transport.session, i64 + // CHECK: llvm.call @__catalyst__transport__collect(%[[S]] + // CHECK: %[[RESULT:.*]] = llvm.load + %result = transport.collect %s {bytes = 8 : i64} : !transport.session -> i64 + // CHECK: llvm.call @__catalyst__transport__stop(%[[S]]) + transport.stop %s : !transport.session + // CHECK: llvm.call @__catalyst__transport__destroy(%[[S]]) + transport.destroy %s : !transport.session + return %result : i64 +} From 76305703ada3b8660a444eee9e78ca72ae2063ac Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Tue, 21 Jul 2026 09:57:42 -0400 Subject: [PATCH 20/57] Update runtime/include/Transport.hpp Co-authored-by: Mehrdad Malek <39844030+mehrdad2m@users.noreply.github.com> --- runtime/include/Transport.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index 29c4cc81f1..d90be2e428 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -135,7 +135,7 @@ class TransportSession { /** * @brief Wait for a result and write it out. * - * @param correction Output buffer to write the result into. + * @param replies Output buffer to write the result into. * @param bytes Capacity of the output buffer, in bytes. * * @return `int` From da12f304844939d1130d3dd2f84c72b56fa9c371 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Tue, 21 Jul 2026 09:58:30 -0400 Subject: [PATCH 21/57] Apply suggestion from @mehrdad2m Co-authored-by: Mehrdad Malek <39844030+mehrdad2m@users.noreply.github.com> --- runtime/include/Transport.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index d90be2e428..aecfdd68ba 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -140,7 +140,7 @@ class TransportSession { * * @return `int` */ - virtual int collect(void *correction, std::uint64_t bytes) = 0; + virtual int collect(void *replies, std::uint64_t bytes) = 0; /** * @brief Stop the engine and join. Idempotent. From 5490f086cb6607927479cbf53d4e87a73707ca5b Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Tue, 21 Jul 2026 09:58:37 -0400 Subject: [PATCH 22/57] Apply suggestion from @mehrdad2m Co-authored-by: Mehrdad Malek <39844030+mehrdad2m@users.noreply.github.com> --- runtime/include/Transport.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index aecfdd68ba..085c3b0edb 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -156,7 +156,7 @@ class TransportSession { }; /** - * @brief Controller role: writes messages out and receives corrections. + * @brief Controller role: writes messages out and receives replies. */ class ControllerSession : public TransportSession { public: From 5d639c5c7696a96790f954088d63e93025b7d329 Mon Sep 17 00:00:00 2001 From: Hong-Sheng Zheng Date: Tue, 21 Jul 2026 09:58:43 -0400 Subject: [PATCH 23/57] Apply suggestion from @mehrdad2m Co-authored-by: Mehrdad Malek <39844030+mehrdad2m@users.noreply.github.com> --- runtime/include/Transport.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index 085c3b0edb..7f8f7b07c0 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -179,7 +179,7 @@ using CoprocessorFn = std::size_t (*)(const void *in, std::size_t in_len, void * std::size_t out_cap, void *ctx); /** - * @brief Coprocessor role: receives messages, process, and returns corrections. + * @brief Coprocessor role: receives messages, process, and returns replies. */ class CoprocessorSession : public TransportSession { public: From fd1790679a015257a3f3aabebb9b52f7609408b3 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Wed, 22 Jul 2026 11:15:03 -0400 Subject: [PATCH 24/57] update --- mlir/include/Transport/IR/CMakeLists.txt | 5 + mlir/include/Transport/IR/TransportDialect.h | 1 + mlir/include/Transport/IR/TransportDialect.td | 63 +++++++++--- mlir/include/Transport/IR/TransportOps.td | 97 +++++++++++++------ mlir/lib/Transport/IR/CMakeLists.txt | 1 + mlir/lib/Transport/IR/TransportDialect.cpp | 1 + 6 files changed, 126 insertions(+), 42 deletions(-) diff --git a/mlir/include/Transport/IR/CMakeLists.txt b/mlir/include/Transport/IR/CMakeLists.txt index d1bc88dbed..d2b9081dc0 100644 --- a/mlir/include/Transport/IR/CMakeLists.txt +++ b/mlir/include/Transport/IR/CMakeLists.txt @@ -1,3 +1,8 @@ add_mlir_dialect(TransportOps transport) add_mlir_doc(TransportDialect TransportDialect Transport/ -gen-dialect-doc) add_mlir_doc(TransportOps TransportOps Transport/ -gen-op-doc) + +set(LLVM_TARGET_DEFINITIONS TransportOps.td) +mlir_tablegen(TransportEnums.h.inc -gen-enum-decls) +mlir_tablegen(TransportEnums.cpp.inc -gen-enum-defs) +add_public_tablegen_target(MLIRTransportEnumsIncGen) diff --git a/mlir/include/Transport/IR/TransportDialect.h b/mlir/include/Transport/IR/TransportDialect.h index ebfdd9ba47..57ab9b3dcb 100644 --- a/mlir/include/Transport/IR/TransportDialect.h +++ b/mlir/include/Transport/IR/TransportDialect.h @@ -19,6 +19,7 @@ #include "mlir/IR/OpDefinition.h" #include "Transport/IR/TransportOpsDialect.h.inc" +#include "Transport/IR/TransportEnums.h.inc" #define GET_TYPEDEF_CLASSES #include "Transport/IR/TransportOpsTypes.h.inc" diff --git a/mlir/include/Transport/IR/TransportDialect.td b/mlir/include/Transport/IR/TransportDialect.td index 78ed32fb70..118aae1734 100644 --- a/mlir/include/Transport/IR/TransportDialect.td +++ b/mlir/include/Transport/IR/TransportDialect.td @@ -18,20 +18,20 @@ include "mlir/IR/OpBase.td" include "mlir/IR/DialectBase.td" include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/EnumAttr.td" +include "mlir/Interfaces/SideEffectInterfaces.td" //===----------------------------------------------------------------------===// // Transport dialect definition. //===----------------------------------------------------------------------===// def Transport_Dialect : Dialect { - let summary = "Runtime transport-layer ops that lower to the __catalyst__transport__* CAPI."; + let summary = "Typed ops for setting up and driving a transport session."; let description = [{ - A thin, typed MLIR representation of the Catalyst runtime transport CAPI - (runtime/include/TransportCAPI.h). One op per controller-side CAPI entry - point; `convert-transport-to-llvm` lowers each to an `llvm.call` on the - matching `__catalyst__transport__*` symbol. The coprocessor role is an - out-of-tree backend loaded by the runtime, so only the controller side is - emitted here. + The transport dialect models a connection-oriented data-movement session + between two endpoints: creating a session, bringing up the connection, + exchanging memory handles, establishing a data path, and running rounds + of request/reply traffic until teardown. }]; let name = "transport"; @@ -41,7 +41,26 @@ def Transport_Dialect : Dialect { } //===----------------------------------------------------------------------===// -// Transport dialect types. +// Enums. +//===----------------------------------------------------------------------===// + +def Transport_Role : I32EnumAttr<"Role", "transport session role", [ + I32EnumAttrCase<"Controller", 0, "controller">, + I32EnumAttrCase<"Coprocessor", 1, "coprocessor"> + ]> { + let cppNamespace = "::catalyst::transport"; +} + +def Transport_DataPath : I32EnumAttr<"DataPath", "transport data-movement path", [ + I32EnumAttrCase<"CpuVerbs", 0, "cpu_verbs">, + I32EnumAttrCase<"GpuEngine", 1, "gpu_engine">, + I32EnumAttrCase<"Other", 2, "other"> + ]> { + let cppNamespace = "::catalyst::transport"; +} + +//===----------------------------------------------------------------------===// +// Types. //===----------------------------------------------------------------------===// class Transport_Type traits = []> @@ -49,19 +68,37 @@ class Transport_Type traits = []> let mnemonic = typeMnemonic; } +// Opaque session handle, parameterized by role. Lowers to !llvm.ptr; the role is +// compile-time only and drives op verification + the create factory selection. def Transport_SessionType : Transport_Type<"Session", "session"> { - let summary = "Opaque transport controller session handle (CatalystTransportSession*)."; + let summary = "Opaque transport session handle (CatalystTransportSession*), tagged with its role."; + let parameters = (ins EnumParameter:$role); + let assemblyFormat = "`<` $role `>`"; } -def Transport_PeerType : Transport_Type<"Peer", "peer"> { - let summary = "Opaque peer-region descriptor handle (CatalystTransportPeerRef*)."; +def Transport_TokenType : Transport_Type<"Token", "token"> { + let summary = "Handle to an in-flight async transport step, awaited with transport.barrier."; } +// Role-constrained session-type constraints for role-specific ops. +def Transport_ControllerSession : Type< + CPred<"::llvm::isa<::catalyst::transport::SessionType>($_self) && " + "::llvm::cast<::catalyst::transport::SessionType>($_self).getRole() == " + "::catalyst::transport::Role::Controller">, + "controller transport session">; + +def Transport_CoprocessorSession : Type< + CPred<"::llvm::isa<::catalyst::transport::SessionType>($_self) && " + "::llvm::cast<::catalyst::transport::SessionType>($_self).getRole() == " + "::catalyst::transport::Role::Coprocessor">, + "coprocessor transport session">; + //===----------------------------------------------------------------------===// -// Transport operation base. +// Operation base. //===----------------------------------------------------------------------===// +// All transport ops perform side-effecting I/O class Transport_Op traits = []> : - Op; + Op])>; #endif // TRANSPORT_DIALECT diff --git a/mlir/include/Transport/IR/TransportOps.td b/mlir/include/Transport/IR/TransportOps.td index ccd7b2308e..cc5406f8be 100644 --- a/mlir/include/Transport/IR/TransportOps.td +++ b/mlir/include/Transport/IR/TransportOps.td @@ -21,61 +21,104 @@ include "mlir/Interfaces/SideEffectInterfaces.td" include "Transport/IR/TransportDialect.td" //===----------------------------------------------------------------------===// -// Bring-up ops +// Session creation //===----------------------------------------------------------------------===// -def Transport_ControllerCreateOp : Transport_Op<"controller_create"> { - let summary = "Create a controller session from a named backend plugin .so."; +def Transport_CreateOp : Transport_Op<"create"> { + let summary = "Create a session from a backend plugin .so with a certain role"; + let description = [{ + Loads the backend `.so` and builds a session. The result type's role + (`!transport.session`) selects which factory the + runtime looks up and constrains the role-specific ops downstream. + }]; let arguments = (ins StrAttr:$backend_lib, StrAttr:$config); let results = (outs Transport_SessionType:$session); let assemblyFormat = "attr-dict `->` type($session)"; } +//===----------------------------------------------------------------------===// +// Bring-up ops +//===----------------------------------------------------------------------===// + def Transport_ConnectOp : Transport_Op<"connect"> { - let summary = "Bring up the connection to the peer."; + let summary = "Bring up the connection to the peer (blocking)."; let arguments = (ins Transport_SessionType:$session, StrAttr:$peer, I16Attr:$oob_port); - let results = (outs I32:$status); - let assemblyFormat = "$session attr-dict `:` functional-type($session, $status)"; + let assemblyFormat = "$session attr-dict `:` type($session)"; +} + +def Transport_ConnectAsyncOp : Transport_Op<"connect_async"> { + let summary = "connect() on a worker; await with transport.barrier."; + let arguments = (ins Transport_SessionType:$session, StrAttr:$peer, I16Attr:$oob_port); + let results = (outs Transport_TokenType:$token); + let assemblyFormat = "$session attr-dict `:` type($session) `->` type($token)"; } def Transport_ExchangeKeysOp : Transport_Op<"exchange_keys"> { - let summary = "Exchange local and peer region handles; yields the peer descriptor."; + let summary = "Exchange region handles with the peer (blocking); result kept in the session."; + let arguments = (ins Transport_SessionType:$session); + let assemblyFormat = "$session attr-dict `:` type($session)"; +} + +def Transport_ExchangeKeysAsyncOp : Transport_Op<"exchange_keys_async"> { + let summary = "exchange_keys() on a worker; await with transport.barrier."; let arguments = (ins Transport_SessionType:$session); - let results = (outs I32:$status, Transport_PeerType:$peer); - let assemblyFormat = "$session attr-dict `:` type($session) `->` type($peer)"; + let results = (outs Transport_TokenType:$token); + let assemblyFormat = "$session attr-dict `:` type($session) `->` type($token)"; +} + +def Transport_BarrierOp : Transport_Op<"barrier"> { + let summary = "Await an async step (connect_async / exchange_keys_async)."; + let arguments = (ins Transport_TokenType:$token); + let assemblyFormat = "$token attr-dict `:` type($token)"; } def Transport_EstablishChannelOp : Transport_Op<"establish_channel"> { - let summary = "Program the data-movement channel from the local + peer regions."; - let arguments = (ins Transport_SessionType:$session, Transport_PeerType:$peer, - I32Attr:$data_path); - let results = (outs I32:$status); - let assemblyFormat = "$session `,` $peer attr-dict `:` type($session) `,` type($peer)"; + let summary = "Set up the data channel used to transfer payloads each round."; + let description = [{ + Arms the data channel for the given `data_path`, using this side's + registered memory region together with the peer's region that + `exchange_keys` learned earlier (both stored in the session). After this + the channel is ready for `kick`/`collect`. + }]; + let arguments = (ins Transport_SessionType:$session, Transport_DataPath:$data_path); + let assemblyFormat = "$session $data_path attr-dict `:` type($session)"; } +//===----------------------------------------------------------------------===// +// Controller-only ops +//===----------------------------------------------------------------------===// + def Transport_CommitWorkItemOp : Transport_Op<"commit_work_item"> { let summary = "Build a work item (I/O sizes) in a slot before kicking rounds."; - let arguments = (ins Transport_SessionType:$session, I32Attr:$work_item_idx, + let arguments = (ins Transport_ControllerSession:$session, I32Attr:$work_item_idx, I64Attr:$in_bytes, I64Attr:$out_bytes); - let results = (outs I32:$status); let assemblyFormat = "$session attr-dict `:` type($session)"; } -def Transport_StartOp : Transport_Op<"start"> { - let summary = "Start the session (non-blocking; runs until stop())."; - let arguments = (ins Transport_SessionType:$session); +def Transport_KickOp : Transport_Op<"kick"> { + let summary = "Write the payload into the outbound slot and fire one round."; + let arguments = (ins Transport_ControllerSession:$session, I64:$payload, I32Attr:$work_item_idx); + let assemblyFormat = "$session `,` $payload attr-dict `:` type($session) `,` type($payload)"; +} + +//===----------------------------------------------------------------------===// +// Coprocessor-only: bind the coprocessor function (requires a coprocessor session) +//===----------------------------------------------------------------------===// + +def Transport_SetCoprocessorFnOp : Transport_Op<"set_coprocessor_fn"> { + let summary = "Bind the built-in coprocessor function (echo / on-device kernel)."; + let arguments = (ins Transport_CoprocessorSession:$session); let assemblyFormat = "$session attr-dict `:` type($session)"; } //===----------------------------------------------------------------------===// -// Per-round ops +// Run / collect / teardown //===----------------------------------------------------------------------===// -def Transport_KickOp : Transport_Op<"kick"> { - let summary = "Write the payload into the outbound slot and fire one round."; - let arguments = (ins Transport_SessionType:$session, I64:$payload, I32Attr:$work_item_idx); - let results = (outs I32:$status); - let assemblyFormat = "$session `,` $payload attr-dict `:` type($session) `,` type($payload)"; +def Transport_StartOp : Transport_Op<"start"> { + let summary = "Start the session (non-blocking; runs until stop())."; + let arguments = (ins Transport_SessionType:$session); + let assemblyFormat = "$session attr-dict `:` type($session)"; } def Transport_CollectOp : Transport_Op<"collect"> { @@ -92,10 +135,6 @@ def Transport_LastRttNsOp : Transport_Op<"last_rtt_ns"> { let assemblyFormat = "$session attr-dict `:` type($session) `->` type($rtt_ns)"; } -//===----------------------------------------------------------------------===// -// Teardown ops -//===----------------------------------------------------------------------===// - def Transport_StopOp : Transport_Op<"stop"> { let summary = "Stop the session. Idempotent."; let arguments = (ins Transport_SessionType:$session); diff --git a/mlir/lib/Transport/IR/CMakeLists.txt b/mlir/lib/Transport/IR/CMakeLists.txt index 9554c24fe3..1dbeed05e2 100644 --- a/mlir/lib/Transport/IR/CMakeLists.txt +++ b/mlir/lib/Transport/IR/CMakeLists.txt @@ -7,6 +7,7 @@ add_mlir_library(MLIRTransport DEPENDS MLIRTransportOpsIncGen + MLIRTransportEnumsIncGen LINK_LIBS PRIVATE MLIRLLVMDialect diff --git a/mlir/lib/Transport/IR/TransportDialect.cpp b/mlir/lib/Transport/IR/TransportDialect.cpp index 96bb105661..aea2c90196 100644 --- a/mlir/lib/Transport/IR/TransportDialect.cpp +++ b/mlir/lib/Transport/IR/TransportDialect.cpp @@ -28,6 +28,7 @@ using namespace catalyst::transport; //===----------------------------------------------------------------------===// #include "Transport/IR/TransportOpsDialect.cpp.inc" +#include "Transport/IR/TransportEnums.cpp.inc" //===----------------------------------------------------------------------===// // Transport type definitions. From 9cd39b52556d4721a1be0d926d0f3a4974507b14 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Wed, 22 Jul 2026 11:15:03 -0400 Subject: [PATCH 25/57] update --- mlir/include/Transport/IR/CMakeLists.txt | 5 + mlir/include/Transport/IR/TransportDialect.h | 1 + mlir/include/Transport/IR/TransportDialect.td | 63 +++++++++--- mlir/include/Transport/IR/TransportOps.td | 97 +++++++++++++------ mlir/lib/Transport/IR/CMakeLists.txt | 1 + mlir/lib/Transport/IR/TransportDialect.cpp | 1 + 6 files changed, 126 insertions(+), 42 deletions(-) diff --git a/mlir/include/Transport/IR/CMakeLists.txt b/mlir/include/Transport/IR/CMakeLists.txt index d1bc88dbed..d2b9081dc0 100644 --- a/mlir/include/Transport/IR/CMakeLists.txt +++ b/mlir/include/Transport/IR/CMakeLists.txt @@ -1,3 +1,8 @@ add_mlir_dialect(TransportOps transport) add_mlir_doc(TransportDialect TransportDialect Transport/ -gen-dialect-doc) add_mlir_doc(TransportOps TransportOps Transport/ -gen-op-doc) + +set(LLVM_TARGET_DEFINITIONS TransportOps.td) +mlir_tablegen(TransportEnums.h.inc -gen-enum-decls) +mlir_tablegen(TransportEnums.cpp.inc -gen-enum-defs) +add_public_tablegen_target(MLIRTransportEnumsIncGen) diff --git a/mlir/include/Transport/IR/TransportDialect.h b/mlir/include/Transport/IR/TransportDialect.h index ebfdd9ba47..57ab9b3dcb 100644 --- a/mlir/include/Transport/IR/TransportDialect.h +++ b/mlir/include/Transport/IR/TransportDialect.h @@ -19,6 +19,7 @@ #include "mlir/IR/OpDefinition.h" #include "Transport/IR/TransportOpsDialect.h.inc" +#include "Transport/IR/TransportEnums.h.inc" #define GET_TYPEDEF_CLASSES #include "Transport/IR/TransportOpsTypes.h.inc" diff --git a/mlir/include/Transport/IR/TransportDialect.td b/mlir/include/Transport/IR/TransportDialect.td index 78ed32fb70..118aae1734 100644 --- a/mlir/include/Transport/IR/TransportDialect.td +++ b/mlir/include/Transport/IR/TransportDialect.td @@ -18,20 +18,20 @@ include "mlir/IR/OpBase.td" include "mlir/IR/DialectBase.td" include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/EnumAttr.td" +include "mlir/Interfaces/SideEffectInterfaces.td" //===----------------------------------------------------------------------===// // Transport dialect definition. //===----------------------------------------------------------------------===// def Transport_Dialect : Dialect { - let summary = "Runtime transport-layer ops that lower to the __catalyst__transport__* CAPI."; + let summary = "Typed ops for setting up and driving a transport session."; let description = [{ - A thin, typed MLIR representation of the Catalyst runtime transport CAPI - (runtime/include/TransportCAPI.h). One op per controller-side CAPI entry - point; `convert-transport-to-llvm` lowers each to an `llvm.call` on the - matching `__catalyst__transport__*` symbol. The coprocessor role is an - out-of-tree backend loaded by the runtime, so only the controller side is - emitted here. + The transport dialect models a connection-oriented data-movement session + between two endpoints: creating a session, bringing up the connection, + exchanging memory handles, establishing a data path, and running rounds + of request/reply traffic until teardown. }]; let name = "transport"; @@ -41,7 +41,26 @@ def Transport_Dialect : Dialect { } //===----------------------------------------------------------------------===// -// Transport dialect types. +// Enums. +//===----------------------------------------------------------------------===// + +def Transport_Role : I32EnumAttr<"Role", "transport session role", [ + I32EnumAttrCase<"Controller", 0, "controller">, + I32EnumAttrCase<"Coprocessor", 1, "coprocessor"> + ]> { + let cppNamespace = "::catalyst::transport"; +} + +def Transport_DataPath : I32EnumAttr<"DataPath", "transport data-movement path", [ + I32EnumAttrCase<"CpuVerbs", 0, "cpu_verbs">, + I32EnumAttrCase<"GpuEngine", 1, "gpu_engine">, + I32EnumAttrCase<"Other", 2, "other"> + ]> { + let cppNamespace = "::catalyst::transport"; +} + +//===----------------------------------------------------------------------===// +// Types. //===----------------------------------------------------------------------===// class Transport_Type traits = []> @@ -49,19 +68,37 @@ class Transport_Type traits = []> let mnemonic = typeMnemonic; } +// Opaque session handle, parameterized by role. Lowers to !llvm.ptr; the role is +// compile-time only and drives op verification + the create factory selection. def Transport_SessionType : Transport_Type<"Session", "session"> { - let summary = "Opaque transport controller session handle (CatalystTransportSession*)."; + let summary = "Opaque transport session handle (CatalystTransportSession*), tagged with its role."; + let parameters = (ins EnumParameter:$role); + let assemblyFormat = "`<` $role `>`"; } -def Transport_PeerType : Transport_Type<"Peer", "peer"> { - let summary = "Opaque peer-region descriptor handle (CatalystTransportPeerRef*)."; +def Transport_TokenType : Transport_Type<"Token", "token"> { + let summary = "Handle to an in-flight async transport step, awaited with transport.barrier."; } +// Role-constrained session-type constraints for role-specific ops. +def Transport_ControllerSession : Type< + CPred<"::llvm::isa<::catalyst::transport::SessionType>($_self) && " + "::llvm::cast<::catalyst::transport::SessionType>($_self).getRole() == " + "::catalyst::transport::Role::Controller">, + "controller transport session">; + +def Transport_CoprocessorSession : Type< + CPred<"::llvm::isa<::catalyst::transport::SessionType>($_self) && " + "::llvm::cast<::catalyst::transport::SessionType>($_self).getRole() == " + "::catalyst::transport::Role::Coprocessor">, + "coprocessor transport session">; + //===----------------------------------------------------------------------===// -// Transport operation base. +// Operation base. //===----------------------------------------------------------------------===// +// All transport ops perform side-effecting I/O class Transport_Op traits = []> : - Op; + Op])>; #endif // TRANSPORT_DIALECT diff --git a/mlir/include/Transport/IR/TransportOps.td b/mlir/include/Transport/IR/TransportOps.td index ccd7b2308e..cc5406f8be 100644 --- a/mlir/include/Transport/IR/TransportOps.td +++ b/mlir/include/Transport/IR/TransportOps.td @@ -21,61 +21,104 @@ include "mlir/Interfaces/SideEffectInterfaces.td" include "Transport/IR/TransportDialect.td" //===----------------------------------------------------------------------===// -// Bring-up ops +// Session creation //===----------------------------------------------------------------------===// -def Transport_ControllerCreateOp : Transport_Op<"controller_create"> { - let summary = "Create a controller session from a named backend plugin .so."; +def Transport_CreateOp : Transport_Op<"create"> { + let summary = "Create a session from a backend plugin .so with a certain role"; + let description = [{ + Loads the backend `.so` and builds a session. The result type's role + (`!transport.session`) selects which factory the + runtime looks up and constrains the role-specific ops downstream. + }]; let arguments = (ins StrAttr:$backend_lib, StrAttr:$config); let results = (outs Transport_SessionType:$session); let assemblyFormat = "attr-dict `->` type($session)"; } +//===----------------------------------------------------------------------===// +// Bring-up ops +//===----------------------------------------------------------------------===// + def Transport_ConnectOp : Transport_Op<"connect"> { - let summary = "Bring up the connection to the peer."; + let summary = "Bring up the connection to the peer (blocking)."; let arguments = (ins Transport_SessionType:$session, StrAttr:$peer, I16Attr:$oob_port); - let results = (outs I32:$status); - let assemblyFormat = "$session attr-dict `:` functional-type($session, $status)"; + let assemblyFormat = "$session attr-dict `:` type($session)"; +} + +def Transport_ConnectAsyncOp : Transport_Op<"connect_async"> { + let summary = "connect() on a worker; await with transport.barrier."; + let arguments = (ins Transport_SessionType:$session, StrAttr:$peer, I16Attr:$oob_port); + let results = (outs Transport_TokenType:$token); + let assemblyFormat = "$session attr-dict `:` type($session) `->` type($token)"; } def Transport_ExchangeKeysOp : Transport_Op<"exchange_keys"> { - let summary = "Exchange local and peer region handles; yields the peer descriptor."; + let summary = "Exchange region handles with the peer (blocking); result kept in the session."; + let arguments = (ins Transport_SessionType:$session); + let assemblyFormat = "$session attr-dict `:` type($session)"; +} + +def Transport_ExchangeKeysAsyncOp : Transport_Op<"exchange_keys_async"> { + let summary = "exchange_keys() on a worker; await with transport.barrier."; let arguments = (ins Transport_SessionType:$session); - let results = (outs I32:$status, Transport_PeerType:$peer); - let assemblyFormat = "$session attr-dict `:` type($session) `->` type($peer)"; + let results = (outs Transport_TokenType:$token); + let assemblyFormat = "$session attr-dict `:` type($session) `->` type($token)"; +} + +def Transport_BarrierOp : Transport_Op<"barrier"> { + let summary = "Await an async step (connect_async / exchange_keys_async)."; + let arguments = (ins Transport_TokenType:$token); + let assemblyFormat = "$token attr-dict `:` type($token)"; } def Transport_EstablishChannelOp : Transport_Op<"establish_channel"> { - let summary = "Program the data-movement channel from the local + peer regions."; - let arguments = (ins Transport_SessionType:$session, Transport_PeerType:$peer, - I32Attr:$data_path); - let results = (outs I32:$status); - let assemblyFormat = "$session `,` $peer attr-dict `:` type($session) `,` type($peer)"; + let summary = "Set up the data channel used to transfer payloads each round."; + let description = [{ + Arms the data channel for the given `data_path`, using this side's + registered memory region together with the peer's region that + `exchange_keys` learned earlier (both stored in the session). After this + the channel is ready for `kick`/`collect`. + }]; + let arguments = (ins Transport_SessionType:$session, Transport_DataPath:$data_path); + let assemblyFormat = "$session $data_path attr-dict `:` type($session)"; } +//===----------------------------------------------------------------------===// +// Controller-only ops +//===----------------------------------------------------------------------===// + def Transport_CommitWorkItemOp : Transport_Op<"commit_work_item"> { let summary = "Build a work item (I/O sizes) in a slot before kicking rounds."; - let arguments = (ins Transport_SessionType:$session, I32Attr:$work_item_idx, + let arguments = (ins Transport_ControllerSession:$session, I32Attr:$work_item_idx, I64Attr:$in_bytes, I64Attr:$out_bytes); - let results = (outs I32:$status); let assemblyFormat = "$session attr-dict `:` type($session)"; } -def Transport_StartOp : Transport_Op<"start"> { - let summary = "Start the session (non-blocking; runs until stop())."; - let arguments = (ins Transport_SessionType:$session); +def Transport_KickOp : Transport_Op<"kick"> { + let summary = "Write the payload into the outbound slot and fire one round."; + let arguments = (ins Transport_ControllerSession:$session, I64:$payload, I32Attr:$work_item_idx); + let assemblyFormat = "$session `,` $payload attr-dict `:` type($session) `,` type($payload)"; +} + +//===----------------------------------------------------------------------===// +// Coprocessor-only: bind the coprocessor function (requires a coprocessor session) +//===----------------------------------------------------------------------===// + +def Transport_SetCoprocessorFnOp : Transport_Op<"set_coprocessor_fn"> { + let summary = "Bind the built-in coprocessor function (echo / on-device kernel)."; + let arguments = (ins Transport_CoprocessorSession:$session); let assemblyFormat = "$session attr-dict `:` type($session)"; } //===----------------------------------------------------------------------===// -// Per-round ops +// Run / collect / teardown //===----------------------------------------------------------------------===// -def Transport_KickOp : Transport_Op<"kick"> { - let summary = "Write the payload into the outbound slot and fire one round."; - let arguments = (ins Transport_SessionType:$session, I64:$payload, I32Attr:$work_item_idx); - let results = (outs I32:$status); - let assemblyFormat = "$session `,` $payload attr-dict `:` type($session) `,` type($payload)"; +def Transport_StartOp : Transport_Op<"start"> { + let summary = "Start the session (non-blocking; runs until stop())."; + let arguments = (ins Transport_SessionType:$session); + let assemblyFormat = "$session attr-dict `:` type($session)"; } def Transport_CollectOp : Transport_Op<"collect"> { @@ -92,10 +135,6 @@ def Transport_LastRttNsOp : Transport_Op<"last_rtt_ns"> { let assemblyFormat = "$session attr-dict `:` type($session) `->` type($rtt_ns)"; } -//===----------------------------------------------------------------------===// -// Teardown ops -//===----------------------------------------------------------------------===// - def Transport_StopOp : Transport_Op<"stop"> { let summary = "Stop the session. Idempotent."; let arguments = (ins Transport_SessionType:$session); diff --git a/mlir/lib/Transport/IR/CMakeLists.txt b/mlir/lib/Transport/IR/CMakeLists.txt index 9554c24fe3..1dbeed05e2 100644 --- a/mlir/lib/Transport/IR/CMakeLists.txt +++ b/mlir/lib/Transport/IR/CMakeLists.txt @@ -7,6 +7,7 @@ add_mlir_library(MLIRTransport DEPENDS MLIRTransportOpsIncGen + MLIRTransportEnumsIncGen LINK_LIBS PRIVATE MLIRLLVMDialect diff --git a/mlir/lib/Transport/IR/TransportDialect.cpp b/mlir/lib/Transport/IR/TransportDialect.cpp index 96bb105661..aea2c90196 100644 --- a/mlir/lib/Transport/IR/TransportDialect.cpp +++ b/mlir/lib/Transport/IR/TransportDialect.cpp @@ -28,6 +28,7 @@ using namespace catalyst::transport; //===----------------------------------------------------------------------===// #include "Transport/IR/TransportOpsDialect.cpp.inc" +#include "Transport/IR/TransportEnums.cpp.inc" //===----------------------------------------------------------------------===// // Transport type definitions. From 7b3ff71cd922d0d03c26703979726fe62a44f9c5 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Wed, 22 Jul 2026 11:29:19 -0400 Subject: [PATCH 26/57] update --- mlir/include/RegisterAllPasses.h | 2 + mlir/include/Transport/CMakeLists.txt | 1 + mlir/lib/Driver/CMakeLists.txt | 1 + mlir/lib/Transport/CMakeLists.txt | 1 + mlir/lib/Transport/Transforms/CMakeLists.txt | 1 + .../Transport/Transforms/TransportToLLVM.cpp | 193 +++++++++++------- mlir/tools/quantum-opt/CMakeLists.txt | 1 + 7 files changed, 122 insertions(+), 78 deletions(-) diff --git a/mlir/include/RegisterAllPasses.h b/mlir/include/RegisterAllPasses.h index 791fe4a784..e92a0850d4 100644 --- a/mlir/include/RegisterAllPasses.h +++ b/mlir/include/RegisterAllPasses.h @@ -25,6 +25,7 @@ #include "QecPhysical/Transforms/Passes.h" #include "Quantum/Transforms/Passes.h" #include "RTIO/Transforms/Passes.h" +#include "Transport/Transforms/Passes.h" #include "Test/Transforms/Passes.h" #include "hlo-extensions/Transforms/Passes.h" @@ -44,6 +45,7 @@ inline void registerAllPasses() qref::registerQRefPasses(); quantum::registerQuantumPasses(); rtio::registerRTIOPasses(); + transport::registerTransportPasses(); test::registerTestPasses(); } diff --git a/mlir/include/Transport/CMakeLists.txt b/mlir/include/Transport/CMakeLists.txt index f33061b2d8..9f57627c32 100644 --- a/mlir/include/Transport/CMakeLists.txt +++ b/mlir/include/Transport/CMakeLists.txt @@ -1 +1,2 @@ add_subdirectory(IR) +add_subdirectory(Transforms) diff --git a/mlir/lib/Driver/CMakeLists.txt b/mlir/lib/Driver/CMakeLists.txt index cd55702b17..c4f6318cba 100644 --- a/mlir/lib/Driver/CMakeLists.txt +++ b/mlir/lib/Driver/CMakeLists.txt @@ -77,6 +77,7 @@ set(LIBS MLIRRTIO rtio-transforms MLIRTransport + transport-transforms MLIRCatalystTest ${ENZYME_LIB} fmt::fmt diff --git a/mlir/lib/Transport/CMakeLists.txt b/mlir/lib/Transport/CMakeLists.txt index f33061b2d8..9f57627c32 100644 --- a/mlir/lib/Transport/CMakeLists.txt +++ b/mlir/lib/Transport/CMakeLists.txt @@ -1 +1,2 @@ add_subdirectory(IR) +add_subdirectory(Transforms) diff --git a/mlir/lib/Transport/Transforms/CMakeLists.txt b/mlir/lib/Transport/Transforms/CMakeLists.txt index 17ee33278d..efd3bd5462 100644 --- a/mlir/lib/Transport/Transforms/CMakeLists.txt +++ b/mlir/lib/Transport/Transforms/CMakeLists.txt @@ -14,6 +14,7 @@ set(LIBS set(DEPENDS MLIRTransportPassIncGen + MLIRTransportEnumsIncGen ) add_mlir_library(${LIBRARY_NAME} STATIC ${SRC} LINK_LIBS PRIVATE ${LIBS} DEPENDS ${DEPENDS}) diff --git a/mlir/lib/Transport/Transforms/TransportToLLVM.cpp b/mlir/lib/Transport/Transforms/TransportToLLVM.cpp index e96f68d1f7..4fbc1ae56c 100644 --- a/mlir/lib/Transport/Transforms/TransportToLLVM.cpp +++ b/mlir/lib/Transport/Transforms/TransportToLLVM.cpp @@ -13,15 +13,15 @@ // limitations under the License. // Lower the `transport` dialect to `llvm.call`s on the __catalyst__transport__* -// CAPI (runtime/include/TransportCAPI.h). Controller-side only. +// CAPI (runtime/include/TransportCAPI.h). -#include "llvm/ADT/Twine.h" #include "mlir/Conversion/LLVMCommon/TypeConverter.h" #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/Pass/Pass.h" #include "mlir/Transforms/DialectConversion.h" +#include "llvm/ADT/Twine.h" #include "Transport/IR/TransportOps.h" #include "Transport/Transforms/Passes.h" @@ -37,15 +37,12 @@ namespace transport { namespace { -// sizeof(CatalystTransportPeerRef) = {u32, u64, u64}; over-allocate for alignment. -constexpr int64_t kPeerRefBytes = 32; - LLVM::LLVMPointerType ptrTy(MLIRContext *ctx) { return LLVM::LLVMPointerType::get(ctx); } +IntegerType i32Ty(MLIRContext *ctx) { return IntegerType::get(ctx, 32); } +IntegerType i64Ty(MLIRContext *ctx) { return IntegerType::get(ctx, 64); } ModuleOp moduleOf(Operation *op) { return op->getParentOfType(); } -// Declare-or-reuse a CAPI function and emit a call to it. A null resultTy means -// the function returns void. Value emitCall(ConversionPatternRewriter &rewriter, Location loc, ModuleOp mod, StringRef name, ArrayRef paramTys, Type resultTy, ValueRange args) { @@ -56,7 +53,6 @@ Value emitCall(ConversionPatternRewriter &rewriter, Location loc, ModuleOp mod, return call.getNumResults() ? call.getResult() : Value(); } -// Materialize a null-terminated global string and return a ptr to its data. Value globalStr(ConversionPatternRewriter &rewriter, Location loc, ModuleOp mod, StringRef prefix, StringRef value) { @@ -75,107 +71,132 @@ Value constInt(ConversionPatternRewriter &rewriter, Location loc, Type ty, int64 // Patterns //===----------------------------------------------------------------------===// -struct ControllerCreateLowering : public OpConversionPattern { +struct CreateLowering : public OpConversionPattern { using OpConversionPattern::OpConversionPattern; - LogicalResult matchAndRewrite(ControllerCreateOp op, OpAdaptor, + LogicalResult matchAndRewrite(CreateOp op, OpAdaptor, ConversionPatternRewriter &rewriter) const override { auto *ctx = op.getContext(); ModuleOp mod = moduleOf(op); + auto sessTy = cast(op.getSession().getType()); Value lib = globalStr(rewriter, op.getLoc(), mod, "transport_backend_", op.getBackendLib()); Value cfg = globalStr(rewriter, op.getLoc(), mod, "transport_config_", op.getConfig()); - Value s = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__controller_create", - {ptrTy(ctx), ptrTy(ctx)}, ptrTy(ctx), {lib, cfg}); + Value role = + constInt(rewriter, op.getLoc(), i32Ty(ctx), static_cast(sessTy.getRole())); + Value s = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__create", + {ptrTy(ctx), ptrTy(ctx), i32Ty(ctx)}, ptrTy(ctx), {lib, cfg, role}); rewriter.replaceOp(op, s); return success(); } }; -struct ConnectLowering : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - LogicalResult matchAndRewrite(ConnectOp op, OpAdaptor adaptor, +template struct ConnectLoweringBase : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(OpT op, typename OpT::Adaptor adaptor, ConversionPatternRewriter &rewriter) const override { auto *ctx = op.getContext(); - ModuleOp mod = moduleOf(op); + ModuleOp mod = op->template getParentOfType(); Value peer = globalStr(rewriter, op.getLoc(), mod, "transport_peer_", op.getPeer()); - Value port = constInt(rewriter, op.getLoc(), rewriter.getI16Type(), op.getOobPort()); - Value r = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__connect", - {ptrTy(ctx), ptrTy(ctx), rewriter.getI16Type()}, rewriter.getI32Type(), - {adaptor.getSession(), peer, port}); - rewriter.replaceOp(op, r); + Value port = constInt(rewriter, op.getLoc(), IntegerType::get(ctx, 16), op.getOobPort()); + if (Async) { + Value r = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__connect_async", + {ptrTy(ctx), ptrTy(ctx), IntegerType::get(ctx, 16)}, i64Ty(ctx), + {adaptor.getSession(), peer, port}); + rewriter.replaceOp(op, r); + } + else { + emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__connect", + {ptrTy(ctx), ptrTy(ctx), IntegerType::get(ctx, 16)}, i32Ty(ctx), + {adaptor.getSession(), peer, port}); + rewriter.eraseOp(op); + } return success(); } }; +using ConnectLowering = ConnectLoweringBase; +using ConnectAsyncLowering = ConnectLoweringBase; -struct ExchangeKeysLowering : public OpConversionPattern { - using OpConversionPattern::OpConversionPattern; - LogicalResult matchAndRewrite(ExchangeKeysOp op, OpAdaptor adaptor, +template +struct ExchangeKeysLoweringBase : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(OpT op, typename OpT::Adaptor adaptor, ConversionPatternRewriter &rewriter) const override { auto *ctx = op.getContext(); - ModuleOp mod = moduleOf(op); - Value peerBuf = LLVM::AllocaOp::create( - rewriter, op.getLoc(), ptrTy(ctx), rewriter.getI8Type(), - constInt(rewriter, op.getLoc(), rewriter.getI64Type(), kPeerRefBytes), /*alignment=*/8); - Value r = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__exchange_keys", - {ptrTy(ctx), ptrTy(ctx)}, rewriter.getI32Type(), - {adaptor.getSession(), peerBuf}); - rewriter.replaceOp(op, {r, peerBuf}); + ModuleOp mod = op->template getParentOfType(); + if (Async) { + Value r = + emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__exchange_keys_async", + {ptrTy(ctx)}, i64Ty(ctx), {adaptor.getSession()}); + rewriter.replaceOp(op, r); + } + else { + emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__exchange_keys", + {ptrTy(ctx)}, i32Ty(ctx), {adaptor.getSession()}); + rewriter.eraseOp(op); + } return success(); } }; +using ExchangeKeysLowering = ExchangeKeysLoweringBase; +using ExchangeKeysAsyncLowering = ExchangeKeysLoweringBase; -struct EstablishChannelLowering : public OpConversionPattern { +struct BarrierLowering : public OpConversionPattern { using OpConversionPattern::OpConversionPattern; - LogicalResult matchAndRewrite(EstablishChannelOp op, OpAdaptor adaptor, + LogicalResult matchAndRewrite(BarrierOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { auto *ctx = op.getContext(); - ModuleOp mod = moduleOf(op); - Value dp = constInt(rewriter, op.getLoc(), rewriter.getI32Type(), op.getDataPath()); - Value r = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__establish_channel", - {ptrTy(ctx), rewriter.getI32Type(), ptrTy(ctx)}, rewriter.getI32Type(), - {adaptor.getSession(), dp, adaptor.getPeer()}); - rewriter.replaceOp(op, r); + emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__barrier", + {i64Ty(ctx)}, i32Ty(ctx), {adaptor.getToken()}); + rewriter.eraseOp(op); return success(); } }; -struct CommitWorkItemLowering : public OpConversionPattern { +struct EstablishChannelLowering : public OpConversionPattern { using OpConversionPattern::OpConversionPattern; - LogicalResult matchAndRewrite(CommitWorkItemOp op, OpAdaptor adaptor, + LogicalResult matchAndRewrite(EstablishChannelOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { auto *ctx = op.getContext(); - ModuleOp mod = moduleOf(op); - Value idx = constInt(rewriter, op.getLoc(), rewriter.getI32Type(), op.getWorkItemIdx()); - Value inB = constInt(rewriter, op.getLoc(), rewriter.getI64Type(), op.getInBytes()); - Value outB = constInt(rewriter, op.getLoc(), rewriter.getI64Type(), op.getOutBytes()); - Value r = emitCall( - rewriter, op.getLoc(), mod, "__catalyst__transport__commit_work_item", - {ptrTy(ctx), rewriter.getI32Type(), rewriter.getI64Type(), rewriter.getI64Type()}, - rewriter.getI32Type(), {adaptor.getSession(), idx, inB, outB}); - rewriter.replaceOp(op, r); + Value dp = + constInt(rewriter, op.getLoc(), i32Ty(ctx), static_cast(op.getDataPath())); + emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__establish_channel", + {ptrTy(ctx), i32Ty(ctx)}, i32Ty(ctx), {adaptor.getSession(), dp}); + rewriter.eraseOp(op); return success(); } }; -// Void-returning single-session ops: start / stop / close / destroy. -template struct VoidSessionLowering : public OpConversionPattern { - VoidSessionLowering(const TypeConverter &tc, MLIRContext *ctx, StringRef sym) - : OpConversionPattern(tc, ctx), symbol(sym) +struct SetCoprocessorFnLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(SetCoprocessorFnOp op, OpAdaptor adaptor, + ConversionPatternRewriter &rewriter) const override { + emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__set_coprocessor_fn", + {ptrTy(op.getContext())}, i32Ty(op.getContext()), {adaptor.getSession()}); + rewriter.eraseOp(op); + return success(); } - LogicalResult matchAndRewrite(OpT op, typename OpT::Adaptor adaptor, +}; + +struct CommitWorkItemLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(CommitWorkItemOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { - emitCall(rewriter, op.getLoc(), op->template getParentOfType(), symbol, - {ptrTy(op.getContext())}, Type(), {adaptor.getSession()}); + auto *ctx = op.getContext(); + Value idx = constInt(rewriter, op.getLoc(), i32Ty(ctx), op.getWorkItemIdx()); + Value inB = constInt(rewriter, op.getLoc(), i64Ty(ctx), op.getInBytes()); + Value outB = constInt(rewriter, op.getLoc(), i64Ty(ctx), op.getOutBytes()); + emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__commit_work_item", + {ptrTy(ctx), i32Ty(ctx), i64Ty(ctx), i64Ty(ctx)}, i32Ty(ctx), + {adaptor.getSession(), idx, inB, outB}); rewriter.eraseOp(op); return success(); } - std::string symbol; }; struct KickLowering : public OpConversionPattern { @@ -185,15 +206,13 @@ struct KickLowering : public OpConversionPattern { { auto *ctx = op.getContext(); ModuleOp mod = moduleOf(op); - // slot = data_slot(s); store payload -> slot; kick(s, idx) Value slot = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__data_slot", {ptrTy(ctx)}, ptrTy(ctx), {adaptor.getSession()}); LLVM::StoreOp::create(rewriter, op.getLoc(), adaptor.getPayload(), slot); - Value idx = constInt(rewriter, op.getLoc(), rewriter.getI32Type(), op.getWorkItemIdx()); - Value r = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__kick", - {ptrTy(ctx), rewriter.getI32Type()}, rewriter.getI32Type(), - {adaptor.getSession(), idx}); - rewriter.replaceOp(op, r); + Value idx = constInt(rewriter, op.getLoc(), i32Ty(ctx), op.getWorkItemIdx()); + emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__kick", + {ptrTy(ctx), i32Ty(ctx)}, i32Ty(ctx), {adaptor.getSession(), idx}); + rewriter.eraseOp(op); return success(); } }; @@ -205,14 +224,14 @@ struct CollectLowering : public OpConversionPattern { { auto *ctx = op.getContext(); ModuleOp mod = moduleOf(op); - Value one = constInt(rewriter, op.getLoc(), rewriter.getI64Type(), 1); - Value buf = LLVM::AllocaOp::create(rewriter, op.getLoc(), ptrTy(ctx), rewriter.getI64Type(), - one, /*alignment=*/8); - Value bytes = constInt(rewriter, op.getLoc(), rewriter.getI64Type(), op.getBytes()); + Value one = constInt(rewriter, op.getLoc(), i64Ty(ctx), 1); + Value buf = LLVM::AllocaOp::create(rewriter, op.getLoc(), ptrTy(ctx), i64Ty(ctx), one, + /*alignment=*/8); + Value bytes = constInt(rewriter, op.getLoc(), i64Ty(ctx), op.getBytes()); emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__collect", - {ptrTy(ctx), ptrTy(ctx), rewriter.getI64Type()}, rewriter.getI32Type(), + {ptrTy(ctx), ptrTy(ctx), i64Ty(ctx)}, i32Ty(ctx), {adaptor.getSession(), buf, bytes}); - Value loaded = LLVM::LoadOp::create(rewriter, op.getLoc(), rewriter.getI64Type(), buf); + Value loaded = LLVM::LoadOp::create(rewriter, op.getLoc(), i64Ty(ctx), buf); rewriter.replaceOp(op, loaded); return success(); } @@ -223,14 +242,31 @@ struct LastRttLowering : public OpConversionPattern { LogicalResult matchAndRewrite(LastRttNsOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { - Value r = - emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__last_rtt_ns", - {ptrTy(op.getContext())}, rewriter.getI64Type(), {adaptor.getSession()}); + Value r = emitCall(rewriter, op.getLoc(), moduleOf(op), + "__catalyst__transport__last_rtt_ns", {ptrTy(op.getContext())}, + i64Ty(op.getContext()), {adaptor.getSession()}); rewriter.replaceOp(op, r); return success(); } }; +// Void-returning single-session ops: start / stop / close / destroy. +template struct VoidSessionLowering : public OpConversionPattern { + VoidSessionLowering(const TypeConverter &tc, MLIRContext *ctx, StringRef sym) + : OpConversionPattern(tc, ctx), symbol(sym) + { + } + LogicalResult matchAndRewrite(OpT op, typename OpT::Adaptor adaptor, + ConversionPatternRewriter &rewriter) const override + { + emitCall(rewriter, op.getLoc(), op->template getParentOfType(), symbol, + {ptrTy(op.getContext())}, Type(), {adaptor.getSession()}); + rewriter.eraseOp(op); + return success(); + } + std::string symbol; +}; + } // namespace struct ConvertTransportToLLVMPass @@ -242,12 +278,13 @@ struct ConvertTransportToLLVMPass MLIRContext *ctx = &getContext(); LLVMTypeConverter tc(ctx); tc.addConversion([ctx](SessionType) -> Type { return LLVM::LLVMPointerType::get(ctx); }); - tc.addConversion([ctx](PeerType) -> Type { return LLVM::LLVMPointerType::get(ctx); }); + tc.addConversion([ctx](TokenType) -> Type { return IntegerType::get(ctx, 64); }); RewritePatternSet patterns(ctx); - patterns.add(tc, ctx); + patterns.add(tc, ctx); patterns.add>(tc, ctx, "__catalyst__transport__start"); patterns.add>(tc, ctx, "__catalyst__transport__stop"); patterns.add>(tc, ctx, "__catalyst__transport__close"); diff --git a/mlir/tools/quantum-opt/CMakeLists.txt b/mlir/tools/quantum-opt/CMakeLists.txt index cf7a3b2053..81847edd38 100644 --- a/mlir/tools/quantum-opt/CMakeLists.txt +++ b/mlir/tools/quantum-opt/CMakeLists.txt @@ -34,6 +34,7 @@ set(LIBS MLIRRTIO rtio-transforms MLIRTransport + transport-transforms MLIRQecLogical MLIRQecPhysical MLIRCatalystTest From bdfab4c57a348b25d7c8be21045e56a3bd2f04db Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Wed, 22 Jul 2026 11:59:34 -0400 Subject: [PATCH 27/57] format --- mlir/include/Transport/IR/TransportDialect.h | 2 +- mlir/lib/Transport/IR/TransportDialect.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mlir/include/Transport/IR/TransportDialect.h b/mlir/include/Transport/IR/TransportDialect.h index 57ab9b3dcb..675578a74c 100644 --- a/mlir/include/Transport/IR/TransportDialect.h +++ b/mlir/include/Transport/IR/TransportDialect.h @@ -18,8 +18,8 @@ #include "mlir/IR/Dialect.h" #include "mlir/IR/OpDefinition.h" -#include "Transport/IR/TransportOpsDialect.h.inc" #include "Transport/IR/TransportEnums.h.inc" +#include "Transport/IR/TransportOpsDialect.h.inc" #define GET_TYPEDEF_CLASSES #include "Transport/IR/TransportOpsTypes.h.inc" diff --git a/mlir/lib/Transport/IR/TransportDialect.cpp b/mlir/lib/Transport/IR/TransportDialect.cpp index aea2c90196..c2c41d2800 100644 --- a/mlir/lib/Transport/IR/TransportDialect.cpp +++ b/mlir/lib/Transport/IR/TransportDialect.cpp @@ -27,8 +27,8 @@ using namespace catalyst::transport; // Transport dialect definitions. //===----------------------------------------------------------------------===// -#include "Transport/IR/TransportOpsDialect.cpp.inc" #include "Transport/IR/TransportEnums.cpp.inc" +#include "Transport/IR/TransportOpsDialect.cpp.inc" //===----------------------------------------------------------------------===// // Transport type definitions. From a5150205e70ef88e53283c1d818ceb8a320b9c82 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Wed, 22 Jul 2026 12:00:44 -0400 Subject: [PATCH 28/57] format --- mlir/include/RegisterAllPasses.h | 3 ++- mlir/lib/Transport/Transforms/TransportToLLVM.cpp | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/mlir/include/RegisterAllPasses.h b/mlir/include/RegisterAllPasses.h index e92a0850d4..a5195536c5 100644 --- a/mlir/include/RegisterAllPasses.h +++ b/mlir/include/RegisterAllPasses.h @@ -25,10 +25,11 @@ #include "QecPhysical/Transforms/Passes.h" #include "Quantum/Transforms/Passes.h" #include "RTIO/Transforms/Passes.h" -#include "Transport/Transforms/Passes.h" #include "Test/Transforms/Passes.h" #include "hlo-extensions/Transforms/Passes.h" +#include "Transport/Transforms/Passes.h" + namespace catalyst { inline void registerAllPasses() diff --git a/mlir/lib/Transport/Transforms/TransportToLLVM.cpp b/mlir/lib/Transport/Transforms/TransportToLLVM.cpp index 4fbc1ae56c..0a3b7cab90 100644 --- a/mlir/lib/Transport/Transforms/TransportToLLVM.cpp +++ b/mlir/lib/Transport/Transforms/TransportToLLVM.cpp @@ -15,13 +15,13 @@ // Lower the `transport` dialect to `llvm.call`s on the __catalyst__transport__* // CAPI (runtime/include/TransportCAPI.h). +#include "llvm/ADT/Twine.h" #include "mlir/Conversion/LLVMCommon/TypeConverter.h" #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/Pass/Pass.h" #include "mlir/Transforms/DialectConversion.h" -#include "llvm/ADT/Twine.h" #include "Transport/IR/TransportOps.h" #include "Transport/Transforms/Passes.h" @@ -242,9 +242,9 @@ struct LastRttLowering : public OpConversionPattern { LogicalResult matchAndRewrite(LastRttNsOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { - Value r = emitCall(rewriter, op.getLoc(), moduleOf(op), - "__catalyst__transport__last_rtt_ns", {ptrTy(op.getContext())}, - i64Ty(op.getContext()), {adaptor.getSession()}); + Value r = + emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__last_rtt_ns", + {ptrTy(op.getContext())}, i64Ty(op.getContext()), {adaptor.getSession()}); rewriter.replaceOp(op, r); return success(); } @@ -283,8 +283,8 @@ struct ConvertTransportToLLVMPass RewritePatternSet patterns(ctx); patterns.add(tc, ctx); + SetCoprocessorFnLowering, CommitWorkItemLowering, KickLowering, + CollectLowering, LastRttLowering>(tc, ctx); patterns.add>(tc, ctx, "__catalyst__transport__start"); patterns.add>(tc, ctx, "__catalyst__transport__stop"); patterns.add>(tc, ctx, "__catalyst__transport__close"); From e4a779b1ed1253a3402fb0f193000e6a78ab92e4 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Thu, 23 Jul 2026 10:58:32 -0400 Subject: [PATCH 29/57] add include to lsp --- mlir/tools/quantum-lsp-server/quantum-lsp-server.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mlir/tools/quantum-lsp-server/quantum-lsp-server.cpp b/mlir/tools/quantum-lsp-server/quantum-lsp-server.cpp index 9cc18cbcc2..0330c721b0 100644 --- a/mlir/tools/quantum-lsp-server/quantum-lsp-server.cpp +++ b/mlir/tools/quantum-lsp-server/quantum-lsp-server.cpp @@ -30,6 +30,8 @@ #include "Quantum/IR/QuantumDialect.h" #include "RTIO/IR/RTIODialect.h" +#include "Transport/IR/TransportDialect.h" + int main(int argc, char **argv) { mlir::DialectRegistry registry; @@ -44,6 +46,7 @@ int main(int argc, char **argv) registry.insert(); registry.insert(); registry.insert(); + registry.insert(); registry.insert(); registry.insert(); From 44e0d6ace199f217a50f26c5aa18f3d2696eab0e Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Thu, 23 Jul 2026 11:33:29 -0400 Subject: [PATCH 30/57] add changelog --- doc/releases/changelog-dev.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md index 35bb08012e..1b271f2e16 100644 --- a/doc/releases/changelog-dev.md +++ b/doc/releases/changelog-dev.md @@ -15,6 +15,10 @@ * A new runtime transport layer for remote/local executors is introduced. [(#3043)](https://github.com/PennyLaneAI/catalyst/pull/3043) +* A new `Transport` MLIR dialect is added, providing typed ops for driving a transport session's + lifecycle at the IR level. + [(#3047)](https://github.com/PennyLaneAI/catalyst/pull/3047) + * A `BufferizableOpInterface` implementation is now added for `catalyst.launch_kernel` operation and it is now bufferizable. [(#3024)](https://github.com/PennyLaneAI/catalyst/pull/3024) From 0340a861a2eef0904ea794f0101b3c3b1434ef87 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Thu, 23 Jul 2026 17:53:09 -0400 Subject: [PATCH 31/57] initial copy --- .../lib/transport/common/CompletionQueue.cpp | 19 ++ .../lib/transport/common/CompletionQueue.hpp | 28 ++ runtime/lib/transport/common/Context.cpp | 41 +++ runtime/lib/transport/common/Context.hpp | 47 ++++ runtime/lib/transport/common/Decoder.hpp | 42 +++ .../lib/transport/common/DecoderPlugin.cpp | 63 +++++ .../lib/transport/common/DecoderPlugin.hpp | 65 +++++ runtime/lib/transport/common/Error.hpp | 36 +++ runtime/lib/transport/common/Handshake.hpp | 30 +++ runtime/lib/transport/common/MemoryRegion.cpp | 95 +++++++ runtime/lib/transport/common/MemoryRegion.hpp | 78 ++++++ runtime/lib/transport/common/OobSocket.cpp | 105 ++++++++ runtime/lib/transport/common/OobSocket.hpp | 59 +++++ .../lib/transport/common/ProtectionDomain.cpp | 19 ++ .../lib/transport/common/ProtectionDomain.hpp | 41 +++ runtime/lib/transport/common/QpState.hpp | 47 ++++ runtime/lib/transport/common/QueuePair.cpp | 132 ++++++++++ runtime/lib/transport/common/QueuePair.hpp | 38 +++ runtime/lib/transport/common/WireProtocol.hpp | 46 ++++ .../lib/transport/common/test/Test_Common.cpp | 44 ++++ .../transport/common/test/Test_Decoder.cpp | 22 ++ .../common/test/Test_WireProtocol.cpp | 32 +++ .../cpu_verbs/base/CpuSessionBase.cpp | 239 ++++++++++++++++++ .../cpu_verbs/base/CpuSessionBase.hpp | 76 ++++++ .../controller/CpuControllerSession.cpp | 83 ++++++ .../controller/CpuControllerSession.hpp | 79 ++++++ .../coprocessor/CpuCoprocessorSession.cpp | 48 ++++ .../coprocessor/CpuCoprocessorSession.hpp | 53 ++++ .../coprocessor/decoders/steane_plugin.cpp | 50 ++++ .../cpu_verbs/cpu_libibverbs_main.cpp | 94 +++++++ .../lib/transport/cpu_verbs/run_loopback.sh | 20 ++ .../cpu_verbs/test/Test_CpuLibibverbs.cpp | 227 +++++++++++++++++ 32 files changed, 2098 insertions(+) create mode 100644 runtime/lib/transport/common/CompletionQueue.cpp create mode 100644 runtime/lib/transport/common/CompletionQueue.hpp create mode 100644 runtime/lib/transport/common/Context.cpp create mode 100644 runtime/lib/transport/common/Context.hpp create mode 100644 runtime/lib/transport/common/Decoder.hpp create mode 100644 runtime/lib/transport/common/DecoderPlugin.cpp create mode 100644 runtime/lib/transport/common/DecoderPlugin.hpp create mode 100644 runtime/lib/transport/common/Error.hpp create mode 100644 runtime/lib/transport/common/Handshake.hpp create mode 100644 runtime/lib/transport/common/MemoryRegion.cpp create mode 100644 runtime/lib/transport/common/MemoryRegion.hpp create mode 100644 runtime/lib/transport/common/OobSocket.cpp create mode 100644 runtime/lib/transport/common/OobSocket.hpp create mode 100644 runtime/lib/transport/common/ProtectionDomain.cpp create mode 100644 runtime/lib/transport/common/ProtectionDomain.hpp create mode 100644 runtime/lib/transport/common/QpState.hpp create mode 100644 runtime/lib/transport/common/QueuePair.cpp create mode 100644 runtime/lib/transport/common/QueuePair.hpp create mode 100644 runtime/lib/transport/common/WireProtocol.hpp create mode 100644 runtime/lib/transport/common/test/Test_Common.cpp create mode 100644 runtime/lib/transport/common/test/Test_Decoder.cpp create mode 100644 runtime/lib/transport/common/test/Test_WireProtocol.cpp create mode 100644 runtime/lib/transport/cpu_verbs/base/CpuSessionBase.cpp create mode 100644 runtime/lib/transport/cpu_verbs/base/CpuSessionBase.hpp create mode 100644 runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp create mode 100644 runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp create mode 100644 runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.cpp create mode 100644 runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp create mode 100644 runtime/lib/transport/cpu_verbs/coprocessor/decoders/steane_plugin.cpp create mode 100644 runtime/lib/transport/cpu_verbs/cpu_libibverbs_main.cpp create mode 100755 runtime/lib/transport/cpu_verbs/run_loopback.sh create mode 100644 runtime/lib/transport/cpu_verbs/test/Test_CpuLibibverbs.cpp diff --git a/runtime/lib/transport/common/CompletionQueue.cpp b/runtime/lib/transport/common/CompletionQueue.cpp new file mode 100644 index 0000000000..2cdf92b044 --- /dev/null +++ b/runtime/lib/transport/common/CompletionQueue.cpp @@ -0,0 +1,19 @@ +#include "CompletionQueue.hpp" + +#include + +#include "Error.hpp" + +namespace rdma::devices::common { +CompletionQueue::CompletionQueue(std::shared_ptr ctx, int depth) : ctx_(std::move(ctx)) +{ + cq_ = ibv_create_cq(ctx_->get(), depth, nullptr, nullptr, 0); + RDMA_CHECK(cq_, "ibv_create_cq"); +} +CompletionQueue::~CompletionQueue() +{ + if (cq_) + ibv_destroy_cq(cq_); +} +ibv_cq *CompletionQueue::get() const { return cq_; } +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/CompletionQueue.hpp b/runtime/lib/transport/common/CompletionQueue.hpp new file mode 100644 index 0000000000..13babf311c --- /dev/null +++ b/runtime/lib/transport/common/CompletionQueue.hpp @@ -0,0 +1,28 @@ +#pragma once +#include + +#include + +#include "Context.hpp" + +namespace rdma::devices::common { + +/** + * @class CompletionQueue class. + * + * @brief An RAII wrapper for an RDMA Completion Queue (`ibv_cq`). + */ +class CompletionQueue { + public: + CompletionQueue() = delete; + CompletionQueue(std::shared_ptr ctx, int depth); + ~CompletionQueue(); + CompletionQueue(const CompletionQueue &) = delete; + CompletionQueue &operator=(const CompletionQueue &) = delete; + ibv_cq *get() const; + + private: + std::shared_ptr ctx_; + ibv_cq *cq_ = nullptr; +}; +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/Context.cpp b/runtime/lib/transport/common/Context.cpp new file mode 100644 index 0000000000..b47c08fdda --- /dev/null +++ b/runtime/lib/transport/common/Context.cpp @@ -0,0 +1,41 @@ +#include "Context.hpp" + +#include + +#include "Error.hpp" +namespace rdma::devices::common { +Context::Context(const std::string &dev_name) +{ + int n = 0; + ibv_device **devs = ibv_get_device_list(&n); + RDMA_CHECK(devs && n > 0, "ibv_get_device_list"); + auto it = std::find_if(devs, devs + n, + [&](ibv_device *d) { return dev_name == ibv_get_device_name(d); }); + ibv_device *dev = (it != devs + n) ? *it : nullptr; + if (!dev) { + ibv_free_device_list(devs); + RDMA_FAIL("device %s not found", dev_name.c_str()); + } + ctx_ = ibv_open_device(dev); + ibv_free_device_list(devs); + RDMA_CHECK(ctx_, "ibv_open_device(%s)", dev_name.c_str()); +} +Context::~Context() +{ + if (ctx_) + ibv_close_device(ctx_); +} +ibv_context *Context::get() const { return ctx_; } +ibv_port_attr Context::port_attr(std::uint8_t port) const +{ + ibv_port_attr attr{}; + RDMA_CHECK(ibv_query_port(ctx_, port, &attr) == 0, "ibv_query_port(%u)", port); + return attr; +} +ibv_gid Context::gid(std::uint8_t port, int idx) const +{ + ibv_gid gid{}; + RDMA_CHECK(ibv_query_gid(ctx_, port, idx, &gid) == 0, "ibv_query_gid(%u,%d)", port, idx); + return gid; +} +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/Context.hpp b/runtime/lib/transport/common/Context.hpp new file mode 100644 index 0000000000..2e9eee6517 --- /dev/null +++ b/runtime/lib/transport/common/Context.hpp @@ -0,0 +1,47 @@ +#pragma once +#include +#include + +#include + +namespace rdma::devices::common { +/** + * @class Context + * @brief RAII wrapper for an RDMA device context (`ibv_context`). + * + * Manages unique ownership of a physical NIC handle. Exposes network + * attributes, port status, and GID table properties. + */ +class Context { + public: + /** + * @brief Opens an RDMA device by its system name (e.g., "mlx5_0"). + */ + explicit Context(const std::string &dev_name); // open by name + + /** + * @brief Closes the physical RDMA device handle. + */ + ~Context(); + Context(const Context &) = delete; + Context &operator=(const Context &) = delete; + + /** + * @brief Returns the raw underlying device context pointer. + */ + ibv_context *get() const; + + /** + * @brief Accesses hardware attributes for a specific physical port. + */ + ibv_port_attr port_attr(std::uint8_t port) const; + + /** + * @brief Retrieves a Global Identifier (GID) from the device table. + */ + ibv_gid gid(std::uint8_t port, int idx) const; + + private: + ibv_context *ctx_ = nullptr; +}; +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/Decoder.hpp b/runtime/lib/transport/common/Decoder.hpp new file mode 100644 index 0000000000..aac578349b --- /dev/null +++ b/runtime/lib/transport/common/Decoder.hpp @@ -0,0 +1,42 @@ +#pragma once +#include +#include +#include +#include + +#include "DecoderPlugin.hpp" + +namespace rdma::devices::common { + +// A per-shot decode compute: reads the syndrome from `in` (in_len bytes) and +// writes the correction into `out` (out_len bytes), in place on the caller's +// buffers. Must not throw. +class Decoder { + public: + virtual ~Decoder() = default; + virtual void run(const void *in, std::size_t in_len, void *out, std::size_t out_len) = 0; +}; + +// Passthrough: out = in for min(in_len, out_len) bytes. Default / self-test. +class EchoDecoder : public Decoder { + public: + void run(const void *in, std::size_t in_len, void *out, std::size_t out_len) override + { + std::memcpy(out, in, std::min(in_len, out_len)); + } +}; + +// Adapts a dlopen'd DecoderPlugin (see DecoderPlugin.hpp) to the Decoder API. +class PluginDecoder : public Decoder { + public: + explicit PluginDecoder(std::unique_ptr plugin) : plugin_(std::move(plugin)) {} + void run(const void *in, std::size_t in_len, void *out, std::size_t out_len) override + { + plugin_->fn()(plugin_->ctx(), in, in_len, out, out_len); + } + + private: + std::unique_ptr plugin_; +}; + +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/DecoderPlugin.cpp b/runtime/lib/transport/common/DecoderPlugin.cpp new file mode 100644 index 0000000000..5f7f32e695 --- /dev/null +++ b/runtime/lib/transport/common/DecoderPlugin.cpp @@ -0,0 +1,63 @@ +#include "DecoderPlugin.hpp" + +#include +#include + +#include "Error.hpp" + +namespace rdma::devices::common { + +DecoderPlugin::DecoderPlugin(const std::string &path) +{ + handle_ = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); + RDMA_CHECK(handle_, "dlopen(%s): %s", path.c_str(), dlerror()); + fn_ = reinterpret_cast(dlsym(handle_, "decode")); + if (!fn_) { + const char *e = dlerror(); + dlclose(handle_); + handle_ = nullptr; + RDMA_FAIL("decoder %s missing 'decode' symbol: %s", path.c_str(), e ? e : "?"); + } + // Optional ctx lifecycle (a pure decoder omits both). + auto create = reinterpret_cast(dlsym(handle_, "decoder_create")); + destroy_ = reinterpret_cast(dlsym(handle_, "decoder_destroy")); + if (create) { + ctx_ = create(); + } +} + +void DecoderPlugin::reset() noexcept +{ + if (destroy_ && ctx_) { + destroy_(ctx_); + } + if (handle_) { + dlclose(handle_); + } + handle_ = nullptr; + fn_ = nullptr; + ctx_ = nullptr; + destroy_ = nullptr; +} + +DecoderPlugin::~DecoderPlugin() { reset(); } + +DecoderPlugin::DecoderPlugin(DecoderPlugin &&other) noexcept + : handle_(std::exchange(other.handle_, nullptr)), fn_(std::exchange(other.fn_, nullptr)), + ctx_(std::exchange(other.ctx_, nullptr)), destroy_(std::exchange(other.destroy_, nullptr)) +{ +} + +DecoderPlugin &DecoderPlugin::operator=(DecoderPlugin &&other) noexcept +{ + if (this != &other) { + reset(); + handle_ = std::exchange(other.handle_, nullptr); + fn_ = std::exchange(other.fn_, nullptr); + ctx_ = std::exchange(other.ctx_, nullptr); + destroy_ = std::exchange(other.destroy_, nullptr); + } + return *this; +} + +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/DecoderPlugin.hpp b/runtime/lib/transport/common/DecoderPlugin.hpp new file mode 100644 index 0000000000..ef3a79c561 --- /dev/null +++ b/runtime/lib/transport/common/DecoderPlugin.hpp @@ -0,0 +1,65 @@ +#pragma once +#include +#include + +namespace rdma::devices::common { + +/** + * @class DecoderPlugin + * @brief RAII manager for a dynamically loaded decoder shared library. + * + * Handles the automatic loading (@c dlopen), symbol resolution, optional + * context lifecycle (@c decoder_create / @c decoder_destroy), and resource + * cleanup upon destruction or movement. + */ +class DecoderPlugin { + public: + /** + * @brief Function pointer signature for the decoding operation. + */ + using Fn = void (*)(void *ctx, const void *in, std::size_t in_len, void *out, + std::size_t out_len); + + /** + * @brief Loads the shared library and resolves the decoder symbols. + * @param path The filesystem path to the shared library (.so). + * @throw Throw runtime errors if loading or symbol resolution fails. + */ + explicit DecoderPlugin(const std::string &path); // dlopen + resolve; throws on failure + + /** + * @brief Destructor. Automatically releases context and unloads the + * library. + */ + ~DecoderPlugin(); + DecoderPlugin(DecoderPlugin &&o) noexcept; + DecoderPlugin &operator=(DecoderPlugin &&o) noexcept; + DecoderPlugin(const DecoderPlugin &) = delete; + DecoderPlugin &operator=(const DecoderPlugin &) = delete; + + /** + * @brief Retrieves the resolved decoding function pointer. + * @return The decoding function pointer, or @c nullptr if uninitialized. + */ + Fn fn() const noexcept { return fn_; } + + /** + * @brief Retrieves the optional plugin context pointer. + * @return Pointer to the internal context instance, or @c nullptr if none + * exists. + */ + void *ctx() const noexcept { return ctx_; } + + private: + /** + * @brief Safely releases all held resources and resets pointers to @c + * nullptr. + */ + void reset() noexcept; + void *handle_ = nullptr; // Dynamic library handle returned by dlopen. + Fn fn_ = nullptr; // Function pointer to the 'decode' symbol. + void *ctx_ = nullptr; // Optional plugin context instance. + void (*destroy_)(void *) = nullptr; +}; + +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/Error.hpp b/runtime/lib/transport/common/Error.hpp new file mode 100644 index 0000000000..c9c3cb60bc --- /dev/null +++ b/runtime/lib/transport/common/Error.hpp @@ -0,0 +1,36 @@ +#pragma once +#include +#include +#include +#include + +namespace rdma::devices::common { +class RdmaError : public std::runtime_error { + public: + using std::runtime_error::runtime_error; +}; + +/** + * Throw RdmaError with a preformatted message. + */ +[[noreturn]] inline void rdma_throw(const char *msg) { throw RdmaError(msg); } +} // namespace rdma::devices::common + +// Unconditionally fail with "file:line: msg (errno=..)" context. +#define RDMA_FAIL(...) \ + do { \ + char rdma_msg_[256]; \ + std::snprintf(rdma_msg_, sizeof(rdma_msg_), __VA_ARGS__); \ + char rdma_full_[512]; \ + std::snprintf(rdma_full_, sizeof(rdma_full_), "%s:%d: %s (errno=%d: %s)", __FILE__, \ + __LINE__, rdma_msg_, errno, std::strerror(errno)); \ + ::rdma::devices::common::rdma_throw(rdma_full_); \ + } while (0) + +// Throw RdmaError with file:line + errno when cond is false. +#define RDMA_CHECK(cond, ...) \ + do { \ + if (!(cond)) { \ + RDMA_FAIL(__VA_ARGS__); \ + } \ + } while (0) diff --git a/runtime/lib/transport/common/Handshake.hpp b/runtime/lib/transport/common/Handshake.hpp new file mode 100644 index 0000000000..2c3ad17b69 --- /dev/null +++ b/runtime/lib/transport/common/Handshake.hpp @@ -0,0 +1,30 @@ +#pragma once +#include + +namespace rdma::devices::common { + +/** + * @struct QpInfo + * @brief Connection metadata exchanged out-of-band to connect remote Queue + * Pairs. + */ +struct QpInfo { + std::uint32_t qpn; // Unique Queue Pair Number hardware identifier. + std::uint32_t psn; // Starting Packet Sequence Number for flow validation. + std::uint8_t gid[16]; // 128-bit Global Identifier routing address. +}; + +/** + * @struct HandshakeMsg + * @brief Message exchanged once over the OOB TCP socket, after the MR exists, + * so QP identity and MR handle are swapped together. + */ +struct HandshakeMsg { + QpInfo fwd; // forward QP (controller -> coprocessor) + QpInfo bwd; // backward QP (coprocessor -> controller) + std::uint64_t mr_vaddr; // where the peer writes into us + std::uint32_t mr_rkey; + std::uint32_t mtu_enum; // ibv_mtu enum +}; + +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/MemoryRegion.cpp b/runtime/lib/transport/common/MemoryRegion.cpp new file mode 100644 index 0000000000..1e5b7a627d --- /dev/null +++ b/runtime/lib/transport/common/MemoryRegion.cpp @@ -0,0 +1,95 @@ +#include "MemoryRegion.hpp" + +#include +#include +#include +#include + +#include "Error.hpp" + +namespace rdma::devices::common { + +/** + * @brief Register caller-owned host memory (borrowed; the region does not own + * it). + */ +MemoryRegion::MemoryRegion(std::shared_ptr pd, void *addr, std::size_t length, + MemAccess access) + : MemoryRegion(std::move(pd), addr, length, access, nullptr) +{ +} + +/** + * @brief Register caller-provided memory, keeping `backing` alive for the MR's + * lifetime. + */ +MemoryRegion::MemoryRegion(std::shared_ptr pd, void *addr, std::size_t length, + MemAccess access, std::shared_ptr backing) + : pd_(std::move(pd)), backing_buffer_(std::move(backing)) +{ + mr_ = ibv_reg_mr(pd_->get(), addr, length, static_cast(access)); + RDMA_CHECK(mr_, "ibv_reg_mr"); +} + +/** + * @brief Register a dma-buf region (e.g. exported GPU memory); does not own the + * buffer. + */ +MemoryRegion::MemoryRegion(std::shared_ptr pd, std::uint64_t offset, + std::size_t length, std::uint64_t iova, int fd, MemAccess access) + : pd_(std::move(pd)) +{ + mr_ = ibv_reg_dmabuf_mr(pd_->get(), offset, length, iova, fd, static_cast(access)); + RDMA_CHECK(mr_, "ibv_reg_dmabuf_mr"); +} + +/** + * @brief Allocate + own an aligned host buffer, then register it. + */ +MemoryRegion MemoryRegion::alloc_host(std::shared_ptr pd, std::size_t length, + std::size_t alignment, MemAccess access) +{ + // aligned_alloc requires a power-of-two alignment and a size that is a + // multiple of it; guard alignment first so a bad value throws rather than + // dividing by zero. + RDMA_CHECK(std::has_single_bit(alignment), "alignment must be a power of two, got %zu", + alignment); + std::size_t rounded = ((length + alignment - 1) / alignment) * alignment; + void *buf = std::aligned_alloc(alignment, rounded); + RDMA_CHECK(buf, "aligned_alloc(%zu)", rounded); + std::memset(buf, 0, rounded); + return MemoryRegion(std::move(pd), buf, length, access, std::shared_ptr(buf, std::free)); +} + +MemoryRegion::~MemoryRegion() +{ + if (mr_) + ibv_dereg_mr(mr_); +} + +MemoryRegion::MemoryRegion(MemoryRegion &&other) noexcept + : pd_(std::move(other.pd_)), mr_(std::exchange(other.mr_, nullptr)), + backing_buffer_(std::move(other.backing_buffer_)) +{ +} + +MemoryRegion &MemoryRegion::operator=(MemoryRegion &&other) noexcept +{ + if (this != &other) { + if (mr_) { + ibv_dereg_mr(mr_); // release our current MR before taking other's + } + pd_ = std::move(other.pd_); + mr_ = std::exchange(other.mr_, nullptr); + backing_buffer_ = std::move(other.backing_buffer_); + } + return *this; +} + +ibv_mr *MemoryRegion::get() const noexcept { return mr_; } +void *MemoryRegion::addr() const noexcept { return mr_ ? mr_->addr : nullptr; } +std::size_t MemoryRegion::length() const noexcept { return mr_ ? mr_->length : 0; } +std::uint32_t MemoryRegion::lkey() const noexcept { return mr_ ? mr_->lkey : 0; } +std::uint32_t MemoryRegion::rkey() const noexcept { return mr_ ? mr_->rkey : 0; } + +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/MemoryRegion.hpp b/runtime/lib/transport/common/MemoryRegion.hpp new file mode 100644 index 0000000000..c3b2cb56ae --- /dev/null +++ b/runtime/lib/transport/common/MemoryRegion.hpp @@ -0,0 +1,78 @@ +#pragma once +#include +#include +#include + +#include + +#include "ProtectionDomain.hpp" + +namespace rdma::devices::common { + +/** + * @enum MemAccess flag. + * @brief Type-safe hardware access permissions for registered Memory Regions. + */ +enum class MemAccess : int { + LOCAL_WRITE = IBV_ACCESS_LOCAL_WRITE, + REMOTE_WRITE = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_LOCAL_WRITE, + REMOTE_READ = IBV_ACCESS_REMOTE_READ, +}; + +/** + * @brief Bitwise OR operator for combining MemAccess flags. + */ +constexpr MemAccess operator|(MemAccess a, MemAccess b) +{ + return static_cast(static_cast(a) | static_cast(b)); +} + +/** + * @class MemoryRegion + * @brief RAII wrapper for an `ibv_mr`, managing hardware registration and + * backing storage. + */ +class MemoryRegion { + public: + // register caller-provided memory; region does not own the buffer. + MemoryRegion(std::shared_ptr pd, void *addr, std::size_t length, + MemAccess access); + // register caller-provided memory and keep a shared_ptr for lifetime. + MemoryRegion(std::shared_ptr pd, void *addr, std::size_t length, + MemAccess access, std::shared_ptr backing); + // register a dma-buf (does not own the buffer). + MemoryRegion(std::shared_ptr pd, std::uint64_t offset, std::size_t length, + std::uint64_t iova, int fd, MemAccess access); + // allocate + own an aligned host buffer, then register it. + static MemoryRegion alloc_host(std::shared_ptr pd, std::size_t length, + std::size_t alignment, MemAccess access); + ~MemoryRegion(); + + MemoryRegion(const MemoryRegion &) = delete; + MemoryRegion &operator=(const MemoryRegion &) = delete; + MemoryRegion(MemoryRegion &&o) noexcept; + MemoryRegion &operator=(MemoryRegion &&o) noexcept; + + ibv_mr *get() const noexcept; + + /// @brief Returns the base virtual address of the registered region. + void *addr() const noexcept; + + /// @brief Returns the total capacity of the memory region in bytes. + std::size_t length() const noexcept; + + /// @brief Returns the local key required for posting local work requests. + std::uint32_t lkey() const noexcept; + + /// @brief Returns the remote key required by peers for one-sided RDMA + /// operations. + std::uint32_t rkey() const noexcept; + + private: + std::shared_ptr pd_; // keeps PD alive + ibv_mr *mr_ = nullptr; + // Keeps the backing storage alive until after the MR is deregistered; + // null for borrowed / dma-buf regions that own nothing. + std::shared_ptr backing_buffer_; +}; +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/OobSocket.cpp b/runtime/lib/transport/common/OobSocket.cpp new file mode 100644 index 0000000000..ccdf244123 --- /dev/null +++ b/runtime/lib/transport/common/OobSocket.cpp @@ -0,0 +1,105 @@ +#include "OobSocket.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "Error.hpp" + +namespace rdma::devices::common { + +namespace { +void set_tcp_nodelay(int fd) +{ + int one = 1; + setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)); +} +} // namespace + +FdGuard tcp_listen_accept(std::uint16_t port) +{ + FdGuard listener(socket(AF_INET, SOCK_STREAM, 0)); + RDMA_CHECK(listener.valid(), "socket"); + int one = 1; + setsockopt(listener.get(), SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one)); + sockaddr_in sa{ + .sin_family = AF_INET, + .sin_port = htons(port), + .sin_addr = {.s_addr = INADDR_ANY}, + }; + RDMA_CHECK(bind(listener.get(), reinterpret_cast(&sa), sizeof(sa)) == 0, "bind(%u)", + port); + RDMA_CHECK(listen(listener.get(), 1) == 0, "listen"); + FdGuard client(accept(listener.get(), nullptr, nullptr)); + RDMA_CHECK(client.valid(), "accept"); + set_tcp_nodelay(client.get()); + return client; // listener closed by its FdGuard on return +} + +FdGuard tcp_connect(const char *host, std::uint16_t port) +{ + // getaddrinfo resolves both numeric IPs ("127.0.0.1") and hostnames + // ("localhost", "node01"). IPv4-only. + addrinfo hints{ + .ai_family = AF_INET, + .ai_socktype = SOCK_STREAM, + }; + char port_str[6]; + std::snprintf(port_str, sizeof(port_str), "%u", port); + addrinfo *res = nullptr; + int rc = getaddrinfo(host, port_str, &hints, &res); + RDMA_CHECK(rc == 0, "getaddrinfo(%s:%s): %s", host, port_str, gai_strerror(rc)); + std::unique_ptr res_guard(res, freeaddrinfo); + + for (int attempt = 0; attempt < 200; ++attempt) { + FdGuard s(socket(res->ai_family, res->ai_socktype, res->ai_protocol)); + RDMA_CHECK(s.valid(), "socket"); + if (connect(s.get(), res->ai_addr, res->ai_addrlen) == 0) { + set_tcp_nodelay(s.get()); + return s; + } + using namespace std::chrono_literals; + std::this_thread::sleep_for(50ms); + } + RDMA_FAIL("tcp_connect(%s:%u) failed after 200 attempts", host, port); +} + +void send_exact(int fd, const void *buf, std::size_t n) +{ + std::size_t done = 0; + const char *p = static_cast(buf); + while (done < n) { + ssize_t r = ::send(fd, p + done, n - done, 0); + if (r < 0) { + if (errno == EINTR) + continue; // interrupted by signal; retry + RDMA_FAIL("send: %s", std::strerror(errno)); + } + done += static_cast(r); + } +} + +void recv_exact(int fd, void *buf, std::size_t n) +{ + std::size_t done = 0; + char *p = static_cast(buf); + while (done < n) { + ssize_t r = ::recv(fd, p + done, n - done, 0); + if (r < 0) { + if (errno == EINTR) + continue; // interrupted by signal; retry + RDMA_FAIL("recv: %s", std::strerror(errno)); + } + RDMA_CHECK(r > 0, "recv: peer closed connection (%zu/%zu bytes)", done, n); + done += static_cast(r); + } +} + +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/OobSocket.hpp b/runtime/lib/transport/common/OobSocket.hpp new file mode 100644 index 0000000000..ce31997976 --- /dev/null +++ b/runtime/lib/transport/common/OobSocket.hpp @@ -0,0 +1,59 @@ +#pragma once +#include +#include +#include +#include + +namespace rdma::devices::common { + +// RAII handle for a socket file descriptor. +class FdGuard { + public: + FdGuard() noexcept = default; + explicit FdGuard(int fd) noexcept : fd_(fd) {} + ~FdGuard() { reset(); } + + FdGuard(FdGuard &&other) noexcept : fd_(other.release()) {} + FdGuard &operator=(FdGuard &&o) noexcept + { + if (this != &o) + reset(o.release()); + return *this; + } + FdGuard(const FdGuard &) = delete; + FdGuard &operator=(const FdGuard &) = delete; + + [[nodiscard]] int get() const noexcept { return fd_; } + [[nodiscard]] bool valid() const noexcept { return fd_ >= 0; } + explicit operator bool() const noexcept { return valid(); } + + /** + * @brief Closes the current socket and takes ownership of a new one. + * @param new_fd The new socket descriptor to manage. + */ + void reset(int new_fd = -1) noexcept + { + if (fd_ >= 0) + ::close(fd_); + fd_ = new_fd; + } + + /** + * @brief Releases ownership of the socket without closing it. + * @return The raw socket file descriptor. + */ + [[nodiscard]] int release() noexcept { return std::exchange(fd_, -1); } + + private: + int fd_ = -1; +}; + +// OOB TCP handshake helpers. +FdGuard tcp_listen_accept(std::uint16_t port); +FdGuard tcp_connect(const char *host, std::uint16_t port); + +// Blocking exact-length IO over an OOB socket. +void send_exact(int fd, const void *buf, std::size_t n); +void recv_exact(int fd, void *buf, std::size_t n); + +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/ProtectionDomain.cpp b/runtime/lib/transport/common/ProtectionDomain.cpp new file mode 100644 index 0000000000..0ad4d40b61 --- /dev/null +++ b/runtime/lib/transport/common/ProtectionDomain.cpp @@ -0,0 +1,19 @@ +#include "ProtectionDomain.hpp" + +#include + +#include "Error.hpp" + +namespace rdma::devices::common { +ProtectionDomain::ProtectionDomain(std::shared_ptr ctx) : ctx_(std::move(ctx)) +{ + pd_ = ibv_alloc_pd(ctx_->get()); + RDMA_CHECK(pd_, "ibv_alloc_pd"); +} +ProtectionDomain::~ProtectionDomain() +{ + if (pd_) + ibv_dealloc_pd(pd_); +} +ibv_pd *ProtectionDomain::get() const { return pd_; } +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/ProtectionDomain.hpp b/runtime/lib/transport/common/ProtectionDomain.hpp new file mode 100644 index 0000000000..67e25b5ab2 --- /dev/null +++ b/runtime/lib/transport/common/ProtectionDomain.hpp @@ -0,0 +1,41 @@ +#pragma once +#include + +#include + +#include "Context.hpp" + +namespace rdma::devices::common { +/** + * @class ProtectionDomain + * @brief RAII wrapper for an `ibv_pd` resource, managing memory protection + * domains. + */ +class ProtectionDomain { + public: + /** + * @brief Allocates an InfiniBand protection domain tied to the lifecycle of + * the context. + * @param ctx Shared pointer to the underlying hardware context. + */ + explicit ProtectionDomain(std::shared_ptr ctx); + ProtectionDomain() = delete; + + /** + * @brief Automatically releases the underlying `ibv_pd` resource. + */ + ~ProtectionDomain(); + ProtectionDomain(const ProtectionDomain &) = delete; + ProtectionDomain &operator=(const ProtectionDomain &) = delete; + + /** + * @brief Returns a raw pointer to the underlying verbs protection domain. + * @return Raw pointer to `ibv_pd`. + */ + ibv_pd *get() const; + + private: + std::shared_ptr ctx_; // keeps the Context alive + ibv_pd *pd_ = nullptr; // Low-level verbs protection domain handle. +}; +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/QpState.hpp b/runtime/lib/transport/common/QpState.hpp new file mode 100644 index 0000000000..a5a310f047 --- /dev/null +++ b/runtime/lib/transport/common/QpState.hpp @@ -0,0 +1,47 @@ +#pragma once +#include "Error.hpp" + +namespace rdma::devices::common { + +enum class QpState { RESET, INIT, RTR, RTS, ERROR }; + +inline const char *to_string(QpState s) +{ + switch (s) { + case QpState::RESET: + return "Reset"; + case QpState::INIT: + return "Init"; + case QpState::RTR: + return "Rtr"; + case QpState::RTS: + return "Rts"; + case QpState::ERROR: + return "Error"; + } + return "?"; +} + +// Checks whether a given transition is valid. +constexpr bool is_valid_transition(QpState from, QpState to) +{ + if (to == QpState::ERROR || to == QpState::RESET) + return true; + switch (from) { + case QpState::RESET: + return to == QpState::INIT; + case QpState::INIT: + return to == QpState::RTR; + case QpState::RTR: + return to == QpState::RTS; + default: + return false; + } +} + +class BadTransition : public RdmaError { + public: + using RdmaError::RdmaError; +}; + +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/QueuePair.cpp b/runtime/lib/transport/common/QueuePair.cpp new file mode 100644 index 0000000000..fcb4eed0ab --- /dev/null +++ b/runtime/lib/transport/common/QueuePair.cpp @@ -0,0 +1,132 @@ +#include "QueuePair.hpp" + +#include +#include +#include + +#include "Error.hpp" + +namespace rdma::devices::common { + +namespace { +// RC QP tuning attributes for ibv_modify_qp (RTR/RTS). Encodings per the IB +// spec; see ibv_modify_qp(3). +constexpr std::uint8_t MIN_RNR_TIMER = 12; // RNR NAK timer code +constexpr std::uint8_t HOP_LIMIT = 64; // GRH hop limit +constexpr std::uint8_t QP_TIMEOUT = 14; // ACK timeout code +constexpr std::uint8_t RETRY_CNT = 7; // transport retry count +constexpr std::uint8_t RNR_RETRY = 7; // RNR retry count (7 = infinite) +constexpr std::uint8_t MAX_RD_ATOMIC = 1; // outstanding RDMA read/atomic ops +} // namespace + +QueuePair::QueuePair(std::shared_ptr pd, std::shared_ptr send_cq, + std::shared_ptr recv_cq, int max_send_wr, int max_inline) + : pd_(std::move(pd)), send_cq_(std::move(send_cq)), recv_cq_(std::move(recv_cq)) +{ + ibv_qp_init_attr a{ + .send_cq = send_cq_->get(), + .recv_cq = recv_cq_->get(), + .cap = + { + .max_send_wr = static_cast(max_send_wr), + .max_recv_wr = 1, + .max_send_sge = 1, + .max_recv_sge = 1, + .max_inline_data = static_cast(max_inline), + }, + .qp_type = IBV_QPT_RC, + .sq_sig_all = 0, + }; + qp_ = ibv_create_qp(pd_->get(), &a); + RDMA_CHECK(qp_, "ibv_create_qp"); +} + +QueuePair::~QueuePair() +{ + if (qp_) + ibv_destroy_qp(qp_); +} + +ibv_qp *QueuePair::get() const { return qp_; } +std::uint32_t QueuePair::qpn() const { return qp_->qp_num; } +QpState QueuePair::state() const { return state_; } + +void QueuePair::check_transition(QpState to) const +{ + if (!is_valid_transition(state_, to)) { + char m[128]; + std::snprintf(m, sizeof(m), "invalid QP transition %s -> %s", to_string(state_), + to_string(to)); + throw BadTransition(m); + } +} + +void QueuePair::modify(QpState to, ibv_qp_attr &attr, int mask, const char *what) +{ + check_transition(to); + // ibv_modify_qp returns the error code directly; errno is unreliable for + // it. + int rc = ibv_modify_qp(qp_, &attr, mask); + // ibv_modify_qp returns an errno-style code directly (errno itself is + // unreliable for it), so translate rc, not errno, to a readable message. + RDMA_CHECK(rc == 0, "%s rc=%d (%s)", what, rc, std::strerror(rc)); + state_ = to; // advance only after a successful modify +} + +void QueuePair::to_init(std::uint8_t port) +{ + ibv_qp_attr a{ + .qp_state = IBV_QPS_INIT, + .qp_access_flags = + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ, + .pkey_index = 0, + .port_num = port, + }; + modify(QpState::INIT, a, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS, + "modify_to_init"); +} + +void QueuePair::to_rtr(std::uint32_t dest_qpn, std::uint32_t dest_psn, + const std::uint8_t dest_gid[16], int sgid_idx, std::uint8_t port, + std::uint32_t mtu_enum) +{ + ibv_qp_attr a{ + .qp_state = IBV_QPS_RTR, + .path_mtu = static_cast(mtu_enum), + .rq_psn = dest_psn, + .dest_qp_num = dest_qpn, + .ah_attr = + { + .grh = {.sgid_index = static_cast(sgid_idx), .hop_limit = HOP_LIMIT}, + .is_global = 1, + .port_num = port, + }, + .max_dest_rd_atomic = MAX_RD_ATOMIC, + .min_rnr_timer = MIN_RNR_TIMER, + }; + // dgid is a 16-byte union filled from the peer's raw GID after the + // aggregate init (it can't be brace-initialized from a runtime array). + std::memcpy(&a.ah_attr.grh.dgid, dest_gid, 16); + modify(QpState::RTR, a, + IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | + IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER, + "modify_to_rtr"); +} + +void QueuePair::to_rts(std::uint32_t sq_psn) +{ + ibv_qp_attr a{ + .qp_state = IBV_QPS_RTS, + .sq_psn = sq_psn, + .max_rd_atomic = MAX_RD_ATOMIC, + .timeout = QP_TIMEOUT, + .retry_cnt = RETRY_CNT, + .rnr_retry = RNR_RETRY, + }; + modify(QpState::RTS, a, + IBV_QP_STATE | IBV_QP_TIMEOUT | IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | + IBV_QP_MAX_QP_RD_ATOMIC, + "modify_to_rts"); +} + +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/QueuePair.hpp b/runtime/lib/transport/common/QueuePair.hpp new file mode 100644 index 0000000000..c29e5e7f1a --- /dev/null +++ b/runtime/lib/transport/common/QueuePair.hpp @@ -0,0 +1,38 @@ +#pragma once +#include +#include + +#include + +#include "CompletionQueue.hpp" +#include "ProtectionDomain.hpp" +#include "QpState.hpp" + +namespace rdma::devices::common { +class QueuePair { + public: + QueuePair(std::shared_ptr pd, std::shared_ptr send_cq, + std::shared_ptr recv_cq, int max_send_wr, int max_inline = 0); + ~QueuePair(); + QueuePair(const QueuePair &) = delete; + QueuePair &operator=(const QueuePair &) = delete; + + ibv_qp *get() const; + std::uint32_t qpn() const; + QpState state() const; + + void to_init(std::uint8_t port); + void to_rtr(std::uint32_t dest_qpn, std::uint32_t dest_psn, const std::uint8_t dest_gid[16], + int sgid_idx, std::uint8_t port, std::uint32_t mtu_enum); + void to_rts(std::uint32_t sq_psn); + + private: + void check_transition(QpState to) const; + void modify(QpState to, ibv_qp_attr &attr, int mask, const char *what); + std::shared_ptr pd_; // keeps PD + (transitively) Context alive + std::shared_ptr send_cq_; // keeps the CQ alive + std::shared_ptr recv_cq_; + ibv_qp *qp_ = nullptr; + QpState state_ = QpState::RESET; +}; +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/WireProtocol.hpp b/runtime/lib/transport/common/WireProtocol.hpp new file mode 100644 index 0000000000..aac5f058b3 --- /dev/null +++ b/runtime/lib/transport/common/WireProtocol.hpp @@ -0,0 +1,46 @@ +#pragma once +#include +#include + +namespace rdma::devices::common { + +// Ring size, identical on both sides; power of two so index is a mask. +inline constexpr std::size_t K_RING_SLOTS = 256; + +// Selective-signalling stride (flow control on the pipelined send paths). +inline constexpr std::uint32_t SIGNAL_EVERY = 64; + +// Validation salt: packed into Payload.value low bits in the RTT self-test. +inline constexpr std::uint32_t SALT = 0xC0DE1515u; + +// Demo/loopback payload the controller ships each shot (stand-in for a real +// measurement outcome). Its (echo) decode is what the self-test checks. +inline constexpr std::uint64_t DEMO_SYNDROME = 0x0123456789ABCDEFull; + +// 16 B wire frame. +// Note: this application payload size is unrelated to the network MTU (the QP's +// max packet size, negotiated at RTR). The size here is far below any MTU, so +// each transfer is always a single packet. +#pragma pack(push, 1) +struct Payload { + std::uint64_t value; + std::uint32_t seq_num; + std::uint32_t pad; +}; +#pragma pack(pop) +static_assert(sizeof(Payload) == 16, "Payload must be exactly 16 B"); + +// Some controller DMA engine requires 64-B aligned. Rings are +// 64-B-strided slots; only the leading Payload (16 B) is transferred per slot. +struct alignas(64) PayloadSlot { + Payload p; + std::uint8_t pad_[48]; +}; +static_assert(sizeof(PayloadSlot) == 64, "PayloadSlot must be exactly 64 B"); +static_assert(alignof(PayloadSlot) == 64, "PayloadSlot must be 64-B aligned"); + +// Receive ring is K_RING_SLOTS PayloadSlots; the peer writes slot[cursor % +// K_RING_SLOTS]. K_RING_SLOTS must be a power of two. +inline constexpr std::size_t REGION_BYTES = K_RING_SLOTS * sizeof(PayloadSlot); + +} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/test/Test_Common.cpp b/runtime/lib/transport/common/test/Test_Common.cpp new file mode 100644 index 0000000000..bf5839c324 --- /dev/null +++ b/runtime/lib/transport/common/test/Test_Common.cpp @@ -0,0 +1,44 @@ +#include + +#include +#include + +#include "Context.hpp" +#include "QpState.hpp" + +using namespace rdma::devices::common; + +static bool have_rxe() +{ + int n = 0; + ibv_device **devs = ibv_get_device_list(&n); + bool found = false; + for (int i = 0; i < n; ++i) + if (std::string(ibv_get_device_name(devs[i])) == "rxe0") + found = true; + if (devs) + ibv_free_device_list(devs); + return found; +} + +TEST_CASE("QpState transitions gate the RC bring-up edges", "[common]") +{ + REQUIRE(is_valid_transition(QpState::RESET, QpState::INIT)); + REQUIRE(is_valid_transition(QpState::INIT, QpState::RTR)); + REQUIRE(is_valid_transition(QpState::RTR, QpState::RTS)); + REQUIRE(is_valid_transition(QpState::RTS, QpState::ERROR)); + REQUIRE(is_valid_transition(QpState::RTS, QpState::RESET)); + REQUIRE_FALSE(is_valid_transition(QpState::RESET, QpState::RTR)); + REQUIRE_FALSE(is_valid_transition(QpState::INIT, QpState::RTS)); +} + +TEST_CASE("Context opens rxe0 with an active port", "[common]") +{ + if (!have_rxe()) + SKIP("no rxe0 RDMA device"); + Context ctx("rxe0"); + REQUIRE(ctx.get() != nullptr); + ibv_port_attr pa = ctx.port_attr(1); + REQUIRE(pa.state == IBV_PORT_ACTIVE); + REQUIRE(pa.active_mtu > 0); +} diff --git a/runtime/lib/transport/common/test/Test_Decoder.cpp b/runtime/lib/transport/common/test/Test_Decoder.cpp new file mode 100644 index 0000000000..5c43b931bc --- /dev/null +++ b/runtime/lib/transport/common/test/Test_Decoder.cpp @@ -0,0 +1,22 @@ +#include + +#include + +#include "Decoder.hpp" + +using namespace rdma::devices::common; + +TEST_CASE("EchoDecoder copies min(in_len, out_len) low bytes", "[decoder]") +{ + const std::uint64_t in = 0x0123456789ABCDEFull; + std::uint64_t out = 0; + EchoDecoder d; + d.run(&in, 8, &out, 8); + REQUIRE(out == in); // full 8-byte echo + out = 0; + d.run(&in, 1, &out, 8); // 1-byte syndrome + REQUIRE(out == 0xEFull); + out = 0; + d.run(&in, 8, &out, 2); // 2-byte correction window + REQUIRE(out == 0xCDEFull); +} diff --git a/runtime/lib/transport/common/test/Test_WireProtocol.cpp b/runtime/lib/transport/common/test/Test_WireProtocol.cpp new file mode 100644 index 0000000000..e40c254fc6 --- /dev/null +++ b/runtime/lib/transport/common/test/Test_WireProtocol.cpp @@ -0,0 +1,32 @@ +#include + +#include + +#include "WireProtocol.hpp" + +using namespace rdma::devices::common; + +TEST_CASE("Payload is the 16 B wire frame") +{ + STATIC_REQUIRE(sizeof(Payload) == 16); + STATIC_REQUIRE(offsetof(Payload, value) == 0); + STATIC_REQUIRE(offsetof(Payload, seq_num) == 8); + STATIC_REQUIRE(offsetof(Payload, pad) == 12); +} + +TEST_CASE("PayloadSlot is a 64 B slot") +{ + STATIC_REQUIRE(sizeof(PayloadSlot) == 64); + STATIC_REQUIRE(alignof(PayloadSlot) == 64); + STATIC_REQUIRE(offsetof(PayloadSlot, p) == 0); +} + +TEST_CASE("Ring geometry and constants match the trampoline") +{ + STATIC_REQUIRE(K_RING_SLOTS == 256); + STATIC_REQUIRE((K_RING_SLOTS & (K_RING_SLOTS - 1)) == 0); + STATIC_REQUIRE(REGION_BYTES == K_RING_SLOTS * sizeof(PayloadSlot)); + STATIC_REQUIRE(SIGNAL_EVERY == 64); + STATIC_REQUIRE(SALT == 0xC0DE1515u); + STATIC_REQUIRE(DEMO_SYNDROME == 0x0123456789ABCDEFull); +} diff --git a/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.cpp b/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.cpp new file mode 100644 index 0000000000..c4ead091dd --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.cpp @@ -0,0 +1,239 @@ +#include "CpuSessionBase.hpp" + +#include +#include +#include +#include +#include +#include + +#include "Error.hpp" +#include "Handshake.hpp" +#include "WireProtocol.hpp" + +namespace rdma::devices::cpu_libibverbs { +using namespace catalyst::transport; +using namespace rdma::devices::common; + +namespace { +// RDMA device port; rxe0 is single-port -> 1. +constexpr std::uint8_t PORT = 1; +// Max CQEs taken per non-blocking batch reap. +constexpr int REAP_BATCH = 16; +// QP inline capacity for inline sends (>= sizeof(Payload), 16 B). +constexpr int INLINE_MAX = 256; +constexpr int SQ_DEPTH = 4096; +constexpr int CQ_DEPTH = 4096; +} // namespace + +CpuSessionBase::CpuSessionBase(std::string dev, int gid_idx) + : dev_name_(std::move(dev)), gid_idx_(gid_idx) +{ +} + +int CpuSessionBase::connect(const ConnectInfo &info) +{ + ctx_ = std::make_shared(dev_name_); + pd_ = std::make_shared(ctx_); + fwd_cq_ = std::make_shared(ctx_, CQ_DEPTH); + bwd_cq_ = std::make_shared(ctx_, CQ_DEPTH); + fwd_qp_ = std::make_shared(pd_, fwd_cq_, fwd_cq_, SQ_DEPTH, INLINE_MAX); + bwd_qp_ = std::make_shared(pd_, bwd_cq_, bwd_cq_, SQ_DEPTH, INLINE_MAX); + fwd_qp_->to_init(PORT); + bwd_qp_->to_init(PORT); + active_mtu_ = static_cast(ctx_->port_attr(PORT).active_mtu); + mygid_ = ctx_->gid(PORT, gid_idx_); + const bool listen = oob_listens(); + oob_fd_ = + listen ? tcp_listen_accept(info.oob_port) : tcp_connect(info.peer.c_str(), info.oob_port); + return 0; +} + +MemRegion CpuSessionBase::alloc_memory(std::size_t size, MemKind kind, std::uint32_t /*access*/) +{ + RDMA_CHECK(kind == MemKind::CpuRam, "cpu_libibverbs: only MemKind::CpuRam supported"); + caller_memory_regions_.push_back(MemoryRegion::alloc_host( + pd_, size, 4096, MemAccess::LOCAL_WRITE | MemAccess::REMOTE_WRITE)); + const MemoryRegion &mr = caller_memory_regions_.back(); + return MemRegion{ + .addr = mr.addr(), + .size = size, + .lkey = mr.lkey(), + .rkey = mr.rkey(), + .kind = MemKind::CpuRam, + }; +} + +PeerRef CpuSessionBase::exchange_keys(const MemRegion &local) +{ + HandshakeMsg my{ + .fwd = {.qpn = fwd_qp_->qpn(), .psn = 0}, + .bwd = {.qpn = bwd_qp_->qpn(), .psn = 0}, + .mr_vaddr = reinterpret_cast(local.addr), + .mr_rkey = local.rkey, + .mtu_enum = active_mtu_, + }; + // gid is a 16-byte array copied from the local GID after the aggregate + // init. + std::memcpy(my.fwd.gid, &mygid_, sizeof(mygid_)); + std::memcpy(my.bwd.gid, &mygid_, sizeof(mygid_)); + HandshakeMsg peer{}; + const int fd = oob_fd_.get(); + const bool listen = oob_listens(); + if (listen) { + send_exact(fd, &my, sizeof(my)); + recv_exact(fd, &peer, sizeof(peer)); + } + else { + recv_exact(fd, &peer, sizeof(peer)); + send_exact(fd, &my, sizeof(my)); + } + const std::uint32_t mtu = std::min(my.mtu_enum, peer.mtu_enum); + fwd_qp_->to_rtr(peer.fwd.qpn, peer.fwd.psn, peer.fwd.gid, gid_idx_, PORT, mtu); + bwd_qp_->to_rtr(peer.bwd.qpn, peer.bwd.psn, peer.bwd.gid, gid_idx_, PORT, mtu); + fwd_qp_->to_rts(my.fwd.psn); + bwd_qp_->to_rts(my.bwd.psn); + return PeerRef{ + .rkey = peer.mr_rkey, + .remote_addr = peer.mr_vaddr, + .size = REGION_BYTES, + }; +} + +void CpuSessionBase::establish_channel(const ChannelDesc &desc, const MemRegion &local, + const PeerRef &peer) +{ + RDMA_CHECK(local.size >= REGION_BYTES, "region too small for ring: %zu < %zu", local.size, + REGION_BYTES); + desc_ = desc; + local_ = local; + peer_ = peer; + send_buf_ = + common::MemoryRegion::alloc_host(pd_, sizeof(Payload), 64, common::MemAccess::LOCAL_WRITE); +} + +void CpuSessionBase::post_write(ibv_qp *qp, std::uint64_t cursor, bool inline_data, bool signaled) +{ + auto *send = send_payload(); // value already written by the caller + send->seq_num = static_cast(cursor + 1); + send->pad = 0; + ibv_sge sge{ + .addr = reinterpret_cast(send), + .length = sizeof(Payload), // 16 B on the wire, into slot's first 16 B + .lkey = send_buf_->lkey(), + }; + ibv_send_wr wr{ + .sg_list = &sge, + .num_sge = 1, + .opcode = IBV_WR_RDMA_WRITE, + .send_flags = static_cast((inline_data ? IBV_SEND_INLINE : 0) | + (signaled ? IBV_SEND_SIGNALED : 0)), + }; + wr.wr.rdma.remote_addr = + peer_.remote_addr + (cursor & (K_RING_SLOTS - 1)) * sizeof(PayloadSlot); + wr.wr.rdma.rkey = peer_.rkey; + ibv_send_wr *bad = nullptr; + RDMA_CHECK(ibv_post_send(qp, &wr, &bad) == 0, "ibv_post_send"); +} + +// Non-blocking batch reap of `cq`: take whatever completions are ready (up to +// REAP_BATCH) and decrement `outstanding`. With drain=true, keep polling until +// every signaled send has completed (teardown), guarded so a lost CQE can't +// hang the join. +void CpuSessionBase::reap(ibv_cq *cq, int &outstanding, bool drain) +{ + std::array wc{}; + int empty = 0; + constexpr int DRAIN_MAX_EMPTY = 1000000; + do { + int n = ibv_poll_cq(cq, static_cast(wc.size()), wc.data()); + if (n == 0) { + if (!drain) + return; + if (++empty >= DRAIN_MAX_EMPTY) + return; + continue; + } + empty = 0; + for (int k = 0; k < n; ++k) { + RDMA_CHECK(wc[k].status == IBV_WC_SUCCESS, "CQE status=%d", wc[k].status); + --outstanding; + } + } while (drain && outstanding > 0); +} + +Payload *CpuSessionBase::poll_message_arrival(std::uint64_t cursor, std::stop_token st) +{ + // Slots are reused (K_RING_SLOTS is a power of two). The ring contains + // 64 B PayloadSlots; the peer writes the 16 B Payload into each slot's + // head. + auto *ring = reinterpret_cast(local_.addr); + Payload *slot = &ring[cursor & (K_RING_SLOTS - 1)].p; + // Poll seq_num with acquire ordering: once it updates, value (written + // before it in the single RDMA_WRITE) is present, and the acquire keeps its + // read from being hoisted ahead. + std::atomic_ref seq_ref(slot->seq_num); + const auto expected = static_cast(cursor + 1); + while (seq_ref.load(std::memory_order_acquire) != expected) { + if (st.stop_requested()) + return nullptr; + std::this_thread::yield(); + } + return slot; +} + +void CpuSessionBase::start() +{ + stop(); + failed_.store(false, std::memory_order_relaxed); + error_ = nullptr; + completed_.store(0, std::memory_order_relaxed); + last_word_.store(0, std::memory_order_relaxed); + // jthread injects the stop_token. A data-path RDMA_CHECK throws RdmaError; + // it must not escape the thread function (that would std::terminate). + // Capture it into error_ and publish via failed_ (release) so collect() + // can rethrow the real exception. + auto body = [this](std::stop_token st) { + try { + run(st); + } + catch (...) { + error_ = std::current_exception(); + failed_.store(true, std::memory_order_release); + } + }; + engine_ = std::jthread(body); +} + +int CpuSessionBase::collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) +{ + while (completed_.load(std::memory_order_acquire) == 0) { + if (failed_.load(std::memory_order_acquire)) + std::rethrow_exception(error_); // surface the engine's real error + if (!engine_.joinable() || engine_.get_stop_token().stop_requested()) + break; + std::this_thread::yield(); + } + if (failed_.load(std::memory_order_acquire)) + std::rethrow_exception(error_); + // Stopped before any round completed -> no data (non-exceptional). + if (completed_.load(std::memory_order_acquire) == 0) + return -1; + if (n > 0 && outputs && outputs[0]) { + const std::uint64_t w = last_word_.load(std::memory_order_relaxed); + const std::size_t nb = + output_bytes ? std::min(output_bytes[0], sizeof(w)) : sizeof(w); + std::memcpy(outputs[0], &w, nb); + } + return 0; +} + +void CpuSessionBase::stop() +{ + if (engine_.joinable()) { + engine_.request_stop(); + engine_.join(); + } +} + +} // namespace rdma::devices::cpu_libibverbs diff --git a/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.hpp b/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.hpp new file mode 100644 index 0000000000..3053445d26 --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.hpp @@ -0,0 +1,76 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "CompletionQueue.hpp" +#include "Context.hpp" +#include "MemoryRegion.hpp" +#include "OobSocket.hpp" +#include "ProtectionDomain.hpp" +#include "QueuePair.hpp" +#include "Transport.hpp" +#include "WireProtocol.hpp" + +namespace rdma::devices::cpu_libibverbs { +using namespace catalyst::transport; + +// Shared lifecycle for coprocessor and controller roles. +class CpuSessionBase : public TransportSession { + public: + explicit CpuSessionBase(std::string dev = "rxe0", int gid_idx = 1); + ~CpuSessionBase() override { stop(); } + + int connect(const ConnectInfo &info) override; + MemRegion alloc_memory(std::size_t size, MemKind kind, std::uint32_t access) override; + PeerRef exchange_keys(const MemRegion &local) override; + void establish_channel(const ChannelDesc &desc, const MemRegion &local, + const PeerRef &peer) override; + void start() override; + int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) override; + void stop() override; + + protected: + // The role-specific engine loop (runs on engine_). + virtual void run(std::stop_token st) = 0; + // True for the coprocessor (listens/sends first on the OOB socket). + virtual bool oob_listens() const = 0; + + void post_write(ibv_qp *qp, std::uint64_t cursor, bool inline_data, bool signaled); + void reap(ibv_cq *cq, int &outstanding, bool drain); + common::Payload *poll_message_arrival(std::uint64_t cursor, std::stop_token st); + common::Payload *send_payload() + { + return reinterpret_cast(send_buf_->addr()); + } + + std::string dev_name_; + int gid_idx_; + ibv_gid mygid_{}; // local GID, cached in connect() for exchange_keys() + std::uint32_t active_mtu_ = 0; // local active_mtu enum, cached in connect() + std::shared_ptr ctx_; + std::shared_ptr pd_; + std::shared_ptr fwd_cq_, bwd_cq_; + std::shared_ptr fwd_qp_, bwd_qp_; + common::FdGuard oob_fd_; + std::vector caller_memory_regions_; + std::optional send_buf_; // local send source (one Payload) + MemRegion local_{}; + PeerRef peer_{}; + ChannelDesc desc_{}; + // failed_ (release) publishes error_; collect() acquire-loads it and + // rethrows. + std::atomic failed_{false}; + std::exception_ptr error_; + std::atomic completed_{0}; + std::atomic last_word_{0}; + std::jthread engine_; +}; + +} // namespace rdma::devices::cpu_libibverbs diff --git a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp new file mode 100644 index 0000000000..95328a53fc --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp @@ -0,0 +1,83 @@ +#include "CpuControllerSession.hpp" + +#include +#include +#include +#include + +#include "WireProtocol.hpp" + +namespace rdma::devices::cpu_libibverbs { +using namespace rdma::devices::common; + +namespace { +std::uint64_t now_ns() +{ + return static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} +} // namespace + +void CpuControllerSession::Impl::start() +{ + stop(); // drain any leftovers + reset (idempotent) + next_send_ = 0; + next_recv_ = 0; + signaled_outstanding_ = 0; + rtt_ns_ = 0; +} + +void CpuControllerSession::Impl::stop() +{ + if (fwd_cq_) + reap(fwd_cq_->get(), signaled_outstanding_, /*drain=*/true); + CpuSessionBase::stop(); // no engine thread runs for the controller; harmless join +} + +void CpuControllerSession::Impl::commit_work_item(std::uint32_t /*work_item_idx*/, + std::uint64_t in_bytes, std::uint64_t out_bytes) +{ + in_bytes_ = in_bytes; + out_bytes_ = out_bytes; +} + +void *CpuControllerSession::Impl::data_slot() +{ + // Current round's outbound slot: the caller writes up to in_bytes_ here, then kick()s. + return &send_payload()->value; +} + +int CpuControllerSession::Impl::kick(std::uint32_t /*work_item_idx*/) +{ + // The payload was written into data_slot() by the caller; fire one round. + kick_ns_ = now_ns(); + const bool sig = (next_send_ % SIGNAL_EVERY == 0); + post_write(fwd_qp_->get(), next_send_, /*inline_data=*/true, sig); + if (sig) { + ++signaled_outstanding_; + reap(fwd_cq_->get(), signaled_outstanding_, /*drain=*/false); // lazy: free the SQ + } + ++next_send_; + return 0; +} + +int CpuControllerSession::Impl::collect(void *const *outputs, const std::uint64_t *output_bytes, + std::size_t n) +{ + std::stop_token none; // blocking wait for this round's reply + Payload *r = poll_message_arrival(next_recv_, none); + if (!r) + return -1; + rtt_ns_ = now_ns() - kick_ns_; + ++next_recv_; + if (n > 0 && outputs && outputs[0]) { + const std::size_t cap = output_bytes ? output_bytes[0] : out_bytes_; + const std::size_t nb = std::min(cap, sizeof(r->value)); + std::memcpy(outputs[0], &r->value, nb); + } + return 0; +} + +} // namespace rdma::devices::cpu_libibverbs diff --git a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp new file mode 100644 index 0000000000..2844c522a3 --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp @@ -0,0 +1,79 @@ +#pragma once +#include + +#include "CpuSessionBase.hpp" + +namespace rdma::devices::cpu_libibverbs { + +// Controller role: caller-driven. The caller commits a work item +// (I/O sizes), writes the outbound payload into data_slot(), kick()s one round, +// then collect()s the reply. No internal engine thread. +class CpuControllerSession : public ControllerSession { + public: + explicit CpuControllerSession(std::string dev = "rxe0", int gid_idx = 1) + : base_(std::move(dev), gid_idx) + { + } + + int connect(const ConnectInfo &info) override { return base_.connect(info); } + MemRegion alloc_memory(std::size_t size, MemKind kind, std::uint32_t access) override + { + return base_.alloc_memory(size, kind, access); + } + PeerRef exchange_keys(const MemRegion &local) override { return base_.exchange_keys(local); } + void establish_channel(const ChannelDesc &desc, const MemRegion &local, + const PeerRef &peer) override + { + base_.establish_channel(desc, local, peer); + } + void start() override { base_.start(); } + int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) override + { + return base_.collect(outputs, output_bytes, n); + } + void stop() override { base_.stop(); } + std::uint64_t last_rtt_ns() const override { return base_.last_rtt_ns(); } + + // ControllerSession interface. + void commit_work_item(std::uint32_t work_item_idx, std::uint64_t in_bytes, + std::uint64_t out_bytes) override + { + base_.commit_work_item(work_item_idx, in_bytes, out_bytes); + } + int kick(std::uint32_t work_item_idx = 0) override { return base_.kick(work_item_idx); } + void *data_slot() override { return base_.data_slot(); } + + private: + // Caller-driven controller over the shared session primitives. run() is unused + // (no engine thread); start()/stop()/collect() are overridden for the + // synchronous kick model. + class Impl : public CpuSessionBase { + public: + using CpuSessionBase::CpuSessionBase; + ~Impl() { stop(); } + + void start() override; + void stop() override; + int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) override; + std::uint64_t last_rtt_ns() const override { return rtt_ns_; } + + void commit_work_item(std::uint32_t work_item_idx, std::uint64_t in_bytes, + std::uint64_t out_bytes); + int kick(std::uint32_t work_item_idx); + void *data_slot(); + + protected: + void run(std::stop_token) override {} // unused: controller is caller-driven + bool oob_listens() const override { return false; } + + private: + std::uint64_t in_bytes_ = sizeof(common::Payload::value); + std::uint64_t out_bytes_ = sizeof(common::Payload::value); + std::uint64_t next_send_ = 0, next_recv_ = 0; + int signaled_outstanding_ = 0; + std::uint64_t kick_ns_ = 0, rtt_ns_ = 0; + }; + Impl base_; +}; + +} // namespace rdma::devices::cpu_libibverbs diff --git a/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.cpp b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.cpp new file mode 100644 index 0000000000..7a2fc02e08 --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.cpp @@ -0,0 +1,48 @@ +#include "CpuCoprocessorSession.hpp" + +#include +#include + +#include "WireProtocol.hpp" + +namespace rdma::devices::cpu_libibverbs { +using namespace rdma::devices::common; + +void CpuCoprocessorSession::set_coprocessor_fn(CoprocessorFn fn, void *ctx) +{ + base_.coproc_fn_ = fn; + base_.coproc_ctx_ = ctx; +} + +// Coprocessor: wait for a message, run the coprocessor function into the send +// buffer in place, then send the result. A null fn is the built-in echo +// (passthrough). Replies are inline + selectively signaled; the bwd CQ is +// reaped in batches at signal points. +void CpuCoprocessorSession::Impl::run(std::stop_token st) +{ + int signaled_outstanding = 0; + for (std::uint64_t c = 0; !st.stop_requested(); c++) { + Payload *r = poll_message_arrival(c, st); // the incoming message + if (!r) { + reap(bwd_cq_->get(), signaled_outstanding, /*drain=*/true); + return; + } + last_word_.store(r->value, std::memory_order_relaxed); + completed_.fetch_add(1, std::memory_order_release); + Payload *send = send_payload(); + send->value = 0; // deterministic high bytes when the result is shorter + if (coproc_fn_) + coproc_fn_(&r->value, sizeof(r->value), &send->value, sizeof(send->value), coproc_ctx_); + else + send->value = r->value; // built-in echo + const bool sig = (c % SIGNAL_EVERY == 0); + post_write(bwd_qp_->get(), c, /*inline_data=*/true, sig); + if (sig) { + ++signaled_outstanding; + reap(bwd_cq_->get(), signaled_outstanding, /*drain=*/false); + } + } + reap(bwd_cq_->get(), signaled_outstanding, /*drain=*/true); +} + +} // namespace rdma::devices::cpu_libibverbs diff --git a/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp new file mode 100644 index 0000000000..0897153e25 --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp @@ -0,0 +1,53 @@ +#pragma once +#include +#include + +#include "CpuSessionBase.hpp" + +namespace rdma::devices::cpu_libibverbs { + +// Coprocessor role: receives messages, runs the coprocessor function, and +// returns the result. The function is bound via set_coprocessor_fn; nullptr +// selects the built-in echo (passthrough self-test). +class CpuCoprocessorSession : public CoprocessorSession { + public: + explicit CpuCoprocessorSession(std::string dev = "rxe0", int gid_idx = 1) + : base_(std::move(dev), gid_idx) + { + } + + int connect(const ConnectInfo &info) override { return base_.connect(info); } + MemRegion alloc_memory(std::size_t size, MemKind kind, std::uint32_t access) override + { + return base_.alloc_memory(size, kind, access); + } + PeerRef exchange_keys(const MemRegion &local) override { return base_.exchange_keys(local); } + void establish_channel(const ChannelDesc &desc, const MemRegion &local, + const PeerRef &peer) override + { + base_.establish_channel(desc, local, peer); + } + void start() override { base_.start(); } + int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) override + { + return base_.collect(outputs, output_bytes, n); + } + void stop() override { base_.stop(); } + + void set_coprocessor_fn(CoprocessorFn fn, void *ctx) override; + + private: + class Impl : public CpuSessionBase { + public: + using CpuSessionBase::CpuSessionBase; + ~Impl() { stop(); } + CoprocessorFn coproc_fn_ = nullptr; // nullptr -> built-in echo + void *coproc_ctx_ = nullptr; + protected: + void run(std::stop_token st) override; + bool oob_listens() const override { return true; } + }; + Impl base_; +}; + +} // namespace rdma::devices::cpu_libibverbs diff --git a/runtime/lib/transport/cpu_verbs/coprocessor/decoders/steane_plugin.cpp b/runtime/lib/transport/cpu_verbs/coprocessor/decoders/steane_plugin.cpp new file mode 100644 index 0000000000..7050733a78 --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/coprocessor/decoders/steane_plugin.cpp @@ -0,0 +1,50 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +/** + * @file steane_plugin.cpp + * Defines a decoder for the [[7,1,3]] Steane code. + */ + +#include +#include +#include + +/** + * @brief A hard-coded decoder for the [[7,1,3]] Steane code with a static + * Tanner graph mapping. + * + * @note The FTQC (Fault-Tolerant Quantum Computing) compilation + * pipeline processes and dispatches either X-check or Z-check syndromes + * independently per execution call, so each call carries a single 3-bit + * check. This behavior may be unified in future pipeline iterations. + * + * @param ctx Pointer to a common::Context instance. + * @param in Pointer to inbound syndrome measurements + * @param in_len The length of the inbound syndrome/message. + * @param out Pointer to the index of the detected error qubit. + * @param out_len The length of the outbound message. + * side. + */ +extern "C" void decode(void * /*ctx*/, const void *in, std::size_t in_len, void *out, + std::size_t out_len) +{ + std::uint64_t syndrome = 0; + std::memcpy(&syndrome, in, in_len < sizeof(syndrome) ? in_len : sizeof(syndrome)); + // One 3-bit check (X or Z) per call; the nonzero index selects the single + // corrected qubit (0 => no error). + const std::uint32_t check = syndrome & 0x7u; + const std::uint64_t correction = static_cast(check ? (1u << (check - 1)) : 0u); + std::memset(out, 0, out_len); + std::memcpy(out, &correction, out_len < sizeof(correction) ? out_len : sizeof(correction)); +} diff --git a/runtime/lib/transport/cpu_verbs/cpu_libibverbs_main.cpp b/runtime/lib/transport/cpu_verbs/cpu_libibverbs_main.cpp new file mode 100644 index 0000000000..41f9b34ea0 --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/cpu_libibverbs_main.cpp @@ -0,0 +1,94 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "CpuControllerSession.hpp" +#include "CpuCoprocessorSession.hpp" +#include "WireProtocol.hpp" + +using namespace catalyst::transport; +using namespace rdma::devices::cpu_libibverbs; +using namespace rdma::devices::common; // REGION_BYTES, DEMO_SYNDROME, Payload + +int main(int argc, char **argv) +{ + std::string role = "coprocessor", dev = "rxe0", peer = "127.0.0.1"; + int gid = 1; + std::uint16_t port = 18560; + for (int i = 1; i + 1 < argc; i += 2) { + std::string k = argv[i], v = argv[i + 1]; + if (k == "--role") + role = v; + else if (k == "--dev") + dev = v; + else if (k == "--gid") + gid = std::atoi(v.c_str()); + else if (k == "--peer") + peer = v; + else if (k == "--port") + port = static_cast(std::atoi(v.c_str())); + } + const bool is_coprocessor = (role == "coprocessor"); + + std::unique_ptr s; + CpuCoprocessorSession *coproc = nullptr; + CpuControllerSession *controller = nullptr; + if (is_coprocessor) { + auto up = std::make_unique(dev, gid); + coproc = up.get(); + s = std::move(up); + } + else { + auto up = std::make_unique(dev, gid); + controller = up.get(); + s = std::move(up); + } + + ConnectInfo ci{ + .peer = peer, + .oob_port = port, + }; + s->connect(ci); + MemRegion m = s->alloc_memory(REGION_BYTES, MemKind::CpuRam, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + PeerRef p = s->exchange_keys(m); + ChannelDesc desc{ + .data_path = DataPath::CpuVerbs, + }; + s->establish_channel(desc, m, p); + + std::uint64_t got = 0; + void *outs[1] = {&got}; + std::uint64_t obytes[1] = {sizeof(got)}; + if (coproc) { + coproc->set_coprocessor_fn(nullptr, nullptr); // built-in echo + coproc->start(); + std::this_thread::sleep_for(std::chrono::seconds(3)); // serve ~3 s + coproc->collect(outs, obytes, 1); + coproc->stop(); + } + else { + // Controller: commit a work item, write the syndrome into + // data_slot(), kick one round, then collect the correction. + controller->commit_work_item(/*work_item_idx=*/0, /*in_bytes=*/sizeof(std::uint64_t), + /*out_bytes=*/sizeof(std::uint64_t)); + controller->start(); + const std::uint64_t syndrome = DEMO_SYNDROME; + std::memcpy(controller->data_slot(), &syndrome, sizeof(syndrome)); + controller->kick(0); + controller->collect(outs, obytes, 1); + controller->stop(); + } + + // Echo coprocessor: both roles observe the demo syndrome. + const std::uint64_t expect = DEMO_SYNDROME; + const bool pass = (got == expect); + std::fprintf(stderr, "[%s] got=0x%llx expect=0x%llx -> %s\n", role.c_str(), + static_cast(got), static_cast(expect), + pass ? "PASS" : "FAIL"); + return pass ? 0 : 1; +} diff --git a/runtime/lib/transport/cpu_verbs/run_loopback.sh b/runtime/lib/transport/cpu_verbs/run_loopback.sh new file mode 100755 index 0000000000..ec7ace051f --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/run_loopback.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Two-process SoftRoCE round-trip on rxe0/lo: coprocessor (listens, built-in echo) +# + controller (connects, kicks one syndrome). Both echo -> both observe DEMO_SYNDROME. +set -u +DEV=${DEV:-rxe0}; GID=${GID:-1}; PORT=${PORT:-18560} +BIN=${BIN:-"$(cd "$(dirname "$0")" && pwd)/../build/cpu_libibverbs/cpu_libibverbs_main"} +[ -x "$BIN" ] || { echo "binary not found: $BIN (build the cpu_libibverbs_main target first)"; exit 2; } + +timeout 30 "$BIN" --role coprocessor --dev "$DEV" --gid "$GID" --port "$PORT" \ + >/tmp/cvl_coproc.log 2>&1 & +SPID=$! +sleep 1 +timeout 30 "$BIN" --role controller --dev "$DEV" --gid "$GID" --peer 127.0.0.1 --port "$PORT" \ + >/tmp/cvl_ctrl.log 2>&1 +CRC=$? +wait $SPID; SRC=$? +echo "=== coprocessor ==="; cat /tmp/cvl_coproc.log +echo "=== controller ==="; cat /tmp/cvl_ctrl.log +echo "coprocessor rc=$SRC controller rc=$CRC" +[ "$SRC" = 0 ] && [ "$CRC" = 0 ] && echo "LOOPBACK: PASS" || { echo "LOOPBACK: FAIL"; exit 1; } diff --git a/runtime/lib/transport/cpu_verbs/test/Test_CpuLibibverbs.cpp b/runtime/lib/transport/cpu_verbs/test/Test_CpuLibibverbs.cpp new file mode 100644 index 0000000000..87578d78a0 --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/test/Test_CpuLibibverbs.cpp @@ -0,0 +1,227 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include "CpuControllerSession.hpp" +#include "CpuCoprocessorSession.hpp" +#include "WireProtocol.hpp" + +using namespace catalyst::transport; +using namespace rdma::devices::cpu_libibverbs; +using namespace rdma::devices::common; // DEMO_SYNDROME, REGION_BYTES + +static bool have_rxe() +{ + int n = 0; + ibv_device **devs = ibv_get_device_list(&n); + bool found = false; + for (int i = 0; i < n; ++i) + if (std::string(ibv_get_device_name(devs[i])) == "rxe0") + found = true; + if (devs) + ibv_free_device_list(devs); + return found; +} + +// A custom coprocessor function (bitwise-invert) to exercise the set_coprocessor_fn +// path with a non-null, non-echo function. +static std::size_t invert_fn(const void *in, std::size_t in_len, void *out, std::size_t out_cap, + void * /*ctx*/) +{ + std::uint64_t v = 0; + std::memcpy(&v, in, std::min(in_len, sizeof(v))); + v = ~v; + const std::size_t n = std::min(out_cap, sizeof(v)); + std::memcpy(out, &v, n); + return n; +} + +TEST_CASE("controller and coprocessor connect: both reach INIT and open the " + "OOB channel", + "[cpu_libibverbs]") +{ + if (!have_rxe()) + SKIP("no rxe0 RDMA device"); + const std::uint16_t port = 18590; + int coproc_rc = -99; + std::thread t([&] { + CpuCoprocessorSession coproc("rxe0", 1); + ConnectInfo ci{ + .peer = "127.0.0.1", + .oob_port = port, + }; + coproc_rc = coproc.connect(ci); + }); + CpuControllerSession controller("rxe0", 1); + ConnectInfo ci{ + .peer = "127.0.0.1", + .oob_port = port, + }; + int controller_rc = controller.connect(ci); + t.join(); + REQUIRE(controller_rc == 0); + REQUIRE(coproc_rc == 0); +} + +TEST_CASE("alloc_memory registers host RAM and exchange_keys swaps regions", "[cpu_libibverbs]") +{ + if (!have_rxe()) + SKIP("no rxe0 RDMA device"); + const std::uint16_t port = 18591; + const std::size_t SIZE = REGION_BYTES; + std::uint32_t coproc_rkey = 0; + std::uint64_t coproc_peer_addr = 0; + std::uint64_t coproc_peer_size = 0; + std::thread t([&] { + CpuCoprocessorSession coproc("rxe0", 1); + ConnectInfo ci{ + .peer = "127.0.0.1", + .oob_port = port, + }; + coproc.connect(ci); + MemRegion m = coproc.alloc_memory(SIZE, MemKind::CpuRam, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + coproc_rkey = m.rkey; + PeerRef p = coproc.exchange_keys(m); + coproc_peer_addr = p.remote_addr; + coproc_peer_size = p.size; + }); + CpuControllerSession controller("rxe0", 1); + ConnectInfo ci{ + .peer = "127.0.0.1", + .oob_port = port, + }; + controller.connect(ci); + MemRegion mine = controller.alloc_memory(SIZE, MemKind::CpuRam, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + PeerRef peer = controller.exchange_keys(mine); + t.join(); + + REQUIRE(mine.addr != nullptr); + REQUIRE(mine.lkey != 0); + REQUIRE(mine.rkey != 0); + REQUIRE(peer.size == SIZE); + REQUIRE(peer.rkey == coproc_rkey); + REQUIRE(coproc_peer_addr == reinterpret_cast(mine.addr)); + REQUIRE(coproc_peer_size == SIZE); +} + +TEST_CASE("round-trip: coprocessor gets request, controller gets bounced reply", "[cpu_libibverbs]") +{ + if (!have_rxe()) + SKIP("no rxe0 RDMA device"); + const std::uint16_t port = 18593; + const std::size_t SIZE = REGION_BYTES; + std::uint64_t coproc_got = 0; + std::thread t([&] { + CpuCoprocessorSession coproc("rxe0", 1); + ConnectInfo ci{ + .peer = "127.0.0.1", + .oob_port = port, + }; + coproc.connect(ci); + MemRegion m = coproc.alloc_memory(SIZE, MemKind::CpuRam, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + PeerRef p = coproc.exchange_keys(m); + ChannelDesc desc{ + .data_path = DataPath::CpuVerbs, + }; + coproc.establish_channel(desc, m, p); + coproc.set_coprocessor_fn(nullptr, nullptr); // built-in echo + coproc.start(); + void *outs[1] = {&coproc_got}; + std::uint64_t obytes[1] = {sizeof(coproc_got)}; + coproc.collect(outs, obytes, 1); + coproc.stop(); + }); + CpuControllerSession controller("rxe0", 1); + ConnectInfo ci{ + .peer = "127.0.0.1", + .oob_port = port, + }; + controller.connect(ci); + MemRegion m = controller.alloc_memory(SIZE, MemKind::CpuRam, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + PeerRef p = controller.exchange_keys(m); + ChannelDesc desc{ + .data_path = DataPath::CpuVerbs, + }; + controller.establish_channel(desc, m, p); + controller.commit_work_item(0, sizeof(std::uint64_t), sizeof(std::uint64_t)); + controller.start(); + const std::uint64_t syndrome = DEMO_SYNDROME; + std::memcpy(controller.data_slot(), &syndrome, sizeof(syndrome)); + controller.kick(0); + std::uint64_t controller_got = 0; + void *outs[1] = {&controller_got}; + std::uint64_t obytes[1] = {sizeof(controller_got)}; + controller.collect(outs, obytes, 1); + controller.stop(); + t.join(); + + REQUIRE(coproc_got == DEMO_SYNDROME); + REQUIRE(controller_got == DEMO_SYNDROME); // echoed back unchanged +} + +TEST_CASE("round-trip with a custom coprocessor function runs on the coprocessor", + "[cpu_libibverbs]") +{ + if (!have_rxe()) + SKIP("no rxe0 RDMA device"); + const std::uint16_t port = 18595; + const std::size_t SIZE = REGION_BYTES; + std::thread t([&] { + CpuCoprocessorSession coproc("rxe0", 1); + ConnectInfo ci{ + .peer = "127.0.0.1", + .oob_port = port, + }; + coproc.connect(ci); + MemRegion m = coproc.alloc_memory(SIZE, MemKind::CpuRam, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + PeerRef p = coproc.exchange_keys(m); + ChannelDesc desc{ + .data_path = DataPath::CpuVerbs, + }; + coproc.establish_channel(desc, m, p); + coproc.set_coprocessor_fn(invert_fn, nullptr); + coproc.start(); + std::uint64_t got = 0; + void *outs[1] = {&got}; + std::uint64_t obytes[1] = {sizeof(got)}; + coproc.collect(outs, obytes, 1); + coproc.stop(); + }); + CpuControllerSession controller("rxe0", 1); + ConnectInfo ci{ + .peer = "127.0.0.1", + .oob_port = port, + }; + controller.connect(ci); + MemRegion m = controller.alloc_memory(SIZE, MemKind::CpuRam, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + PeerRef p = controller.exchange_keys(m); + ChannelDesc desc{ + .data_path = DataPath::CpuVerbs, + }; + controller.establish_channel(desc, m, p); + controller.commit_work_item(0, sizeof(std::uint64_t), sizeof(std::uint64_t)); + controller.start(); + const std::uint64_t syndrome = DEMO_SYNDROME; + std::memcpy(controller.data_slot(), &syndrome, sizeof(syndrome)); + controller.kick(0); + std::uint64_t got = 0; + void *outs[1] = {&got}; + std::uint64_t obytes[1] = {sizeof(got)}; + controller.collect(outs, obytes, 1); + controller.stop(); + t.join(); + + REQUIRE(got == ~DEMO_SYNDROME); // controller received the coprocessor's result + REQUIRE(got != DEMO_SYNDROME); // and it is not a mere echo +} From e34c06a730053da895ff8d81952c49ee8e372f0f Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Fri, 24 Jul 2026 09:27:12 -0400 Subject: [PATCH 32/57] Refine transport dialect: string data_path, get_session, typed round buffers, drop close --- mlir/include/Transport/IR/TransportDialect.td | 17 +-- mlir/include/Transport/IR/TransportOps.td | 144 +++++++++++------- 2 files changed, 92 insertions(+), 69 deletions(-) diff --git a/mlir/include/Transport/IR/TransportDialect.td b/mlir/include/Transport/IR/TransportDialect.td index 118aae1734..bf090f81dd 100644 --- a/mlir/include/Transport/IR/TransportDialect.td +++ b/mlir/include/Transport/IR/TransportDialect.td @@ -51,13 +51,6 @@ def Transport_Role : I32EnumAttr<"Role", "transport session role", [ let cppNamespace = "::catalyst::transport"; } -def Transport_DataPath : I32EnumAttr<"DataPath", "transport data-movement path", [ - I32EnumAttrCase<"CpuVerbs", 0, "cpu_verbs">, - I32EnumAttrCase<"GpuEngine", 1, "gpu_engine">, - I32EnumAttrCase<"Other", 2, "other"> - ]> { - let cppNamespace = "::catalyst::transport"; -} //===----------------------------------------------------------------------===// // Types. @@ -68,19 +61,19 @@ class Transport_Type traits = []> let mnemonic = typeMnemonic; } -// Opaque session handle, parameterized by role. Lowers to !llvm.ptr; the role is -// compile-time only and drives op verification + the create factory selection. +// Opaque session handle, parameterized by its role. The role is a compile-time tag +// that selects which role-specific ops the session may take part in. def Transport_SessionType : Transport_Type<"Session", "session"> { - let summary = "Opaque transport session handle (CatalystTransportSession*), tagged with its role."; + let summary = "An opaque transport session handle, tagged with its role."; let parameters = (ins EnumParameter:$role); let assemblyFormat = "`<` $role `>`"; } def Transport_TokenType : Transport_Type<"Token", "token"> { - let summary = "Handle to an in-flight async transport step, awaited with transport.barrier."; + let summary = "A handle to an in-flight asynchronous step, awaited with transport.barrier."; } -// Role-constrained session-type constraints for role-specific ops. +// Role-constrained session types used by the role-specific ops. def Transport_ControllerSession : Type< CPred<"::llvm::isa<::catalyst::transport::SessionType>($_self) && " "::llvm::cast<::catalyst::transport::SessionType>($_self).getRole() == " diff --git a/mlir/include/Transport/IR/TransportOps.td b/mlir/include/Transport/IR/TransportOps.td index cc5406f8be..21dda4b077 100644 --- a/mlir/include/Transport/IR/TransportOps.td +++ b/mlir/include/Transport/IR/TransportOps.td @@ -17,140 +17,170 @@ include "mlir/IR/OpBase.td" include "mlir/IR/BuiltinAttributes.td" +include "mlir/IR/CommonTypeConstraints.td" include "mlir/Interfaces/SideEffectInterfaces.td" include "Transport/IR/TransportDialect.td" +// A round's payload or reply buffer: a 1-D memref (buffer form) or tensor (value form). +def Transport_Buffer : MemRefRankOf<[I1, I8, I16, I32, I64, Index], [1]>; +def Transport_TensorBuffer : 1DTensorOf<[I1, I8, I16, I32, I64, Index]>; +def Transport_AnyBuffer : AnyTypeOf<[Transport_Buffer, Transport_TensorBuffer]>; + //===----------------------------------------------------------------------===// // Session creation //===----------------------------------------------------------------------===// def Transport_CreateOp : Transport_Op<"create"> { - let summary = "Create a session from a backend plugin .so with a certain role"; + let summary = "Create a transport session for a role."; let description = [{ - Loads the backend `.so` and builds a session. The result type's role - (`!transport.session`) selects which factory the - runtime looks up and constrains the role-specific ops downstream. + Creates a session on the backend selected by `backend_lib` and `config`; the + result type's role picks controller or coprocessor. When `key` is set, the + session is registered under it so `transport.get_session` can resolve it from + another function. }]; - let arguments = (ins StrAttr:$backend_lib, StrAttr:$config); + let arguments = (ins StrAttr:$backend_lib, StrAttr:$config, + DefaultValuedStrAttr:$key); let results = (outs Transport_SessionType:$session); - let assemblyFormat = "attr-dict `->` type($session)"; + let assemblyFormat = "attr-dict `->` qualified(type($session))"; } //===----------------------------------------------------------------------===// -// Bring-up ops +// Connection bring-up //===----------------------------------------------------------------------===// def Transport_ConnectOp : Transport_Op<"connect"> { - let summary = "Bring up the connection to the peer (blocking)."; + let summary = "Connect to the peer (blocking)."; let arguments = (ins Transport_SessionType:$session, StrAttr:$peer, I16Attr:$oob_port); - let assemblyFormat = "$session attr-dict `:` type($session)"; + let assemblyFormat = "$session attr-dict `:` qualified(type($session))"; } def Transport_ConnectAsyncOp : Transport_Op<"connect_async"> { - let summary = "connect() on a worker; await with transport.barrier."; + let summary = "Non-blocking connect; await the token with transport.barrier."; let arguments = (ins Transport_SessionType:$session, StrAttr:$peer, I16Attr:$oob_port); let results = (outs Transport_TokenType:$token); - let assemblyFormat = "$session attr-dict `:` type($session) `->` type($token)"; + let assemblyFormat = "$session attr-dict `:` qualified(type($session)) `->` type($token)"; } def Transport_ExchangeKeysOp : Transport_Op<"exchange_keys"> { - let summary = "Exchange region handles with the peer (blocking); result kept in the session."; + let summary = "Exchange memory-region handles with the peer (blocking)."; let arguments = (ins Transport_SessionType:$session); - let assemblyFormat = "$session attr-dict `:` type($session)"; + let assemblyFormat = "$session attr-dict `:` qualified(type($session))"; } def Transport_ExchangeKeysAsyncOp : Transport_Op<"exchange_keys_async"> { - let summary = "exchange_keys() on a worker; await with transport.barrier."; + let summary = "Non-blocking exchange_keys; await the token with transport.barrier."; let arguments = (ins Transport_SessionType:$session); let results = (outs Transport_TokenType:$token); - let assemblyFormat = "$session attr-dict `:` type($session) `->` type($token)"; + let assemblyFormat = "$session attr-dict `:` qualified(type($session)) `->` type($token)"; } def Transport_BarrierOp : Transport_Op<"barrier"> { - let summary = "Await an async step (connect_async / exchange_keys_async)."; + let summary = "Await an asynchronous bring-up step."; let arguments = (ins Transport_TokenType:$token); let assemblyFormat = "$token attr-dict `:` type($token)"; } def Transport_EstablishChannelOp : Transport_Op<"establish_channel"> { - let summary = "Set up the data channel used to transfer payloads each round."; + let summary = "Arm the data channel for the given data path."; let description = [{ - Arms the data channel for the given `data_path`, using this side's - registered memory region together with the peer's region that - `exchange_keys` learned earlier (both stored in the session). After this - the channel is ready for `kick`/`collect`. + Prepares the data channel selected by `data_path` for the rounds that follow. + Runs after `exchange_keys` and before `kick`/`collect`. }]; - let arguments = (ins Transport_SessionType:$session, Transport_DataPath:$data_path); - let assemblyFormat = "$session $data_path attr-dict `:` type($session)"; + let arguments = (ins Transport_SessionType:$session, StrAttr:$data_path); + let assemblyFormat = "$session $data_path attr-dict `:` qualified(type($session))"; } //===----------------------------------------------------------------------===// -// Controller-only ops +// Round setup //===----------------------------------------------------------------------===// def Transport_CommitWorkItemOp : Transport_Op<"commit_work_item"> { - let summary = "Build a work item (I/O sizes) in a slot before kicking rounds."; + let summary = "Declare a round's request/reply byte sizes (controller)."; let arguments = (ins Transport_ControllerSession:$session, I32Attr:$work_item_idx, I64Attr:$in_bytes, I64Attr:$out_bytes); - let assemblyFormat = "$session attr-dict `:` type($session)"; -} - -def Transport_KickOp : Transport_Op<"kick"> { - let summary = "Write the payload into the outbound slot and fire one round."; - let arguments = (ins Transport_ControllerSession:$session, I64:$payload, I32Attr:$work_item_idx); - let assemblyFormat = "$session `,` $payload attr-dict `:` type($session) `,` type($payload)"; + let assemblyFormat = "$session attr-dict `:` qualified(type($session))"; } -//===----------------------------------------------------------------------===// -// Coprocessor-only: bind the coprocessor function (requires a coprocessor session) -//===----------------------------------------------------------------------===// - def Transport_SetCoprocessorFnOp : Transport_Op<"set_coprocessor_fn"> { - let summary = "Bind the built-in coprocessor function (echo / on-device kernel)."; - let arguments = (ins Transport_CoprocessorSession:$session); - let assemblyFormat = "$session attr-dict `:` type($session)"; + let summary = "Bind the function run per received message (coprocessor)."; + let description = [{ + Binds the function the coprocessor applies to each received message before + replying. `symbol` is the name the coprocessor resolves and invokes. + }]; + let arguments = (ins Transport_CoprocessorSession:$session, + DefaultValuedStrAttr:$symbol); + let assemblyFormat = "$session attr-dict `:` qualified(type($session))"; } //===----------------------------------------------------------------------===// -// Run / collect / teardown +// Execution //===----------------------------------------------------------------------===// def Transport_StartOp : Transport_Op<"start"> { - let summary = "Start the session (non-blocking; runs until stop())."; + let summary = "Start the session; runs until stop (non-blocking)."; let arguments = (ins Transport_SessionType:$session); - let assemblyFormat = "$session attr-dict `:` type($session)"; + let assemblyFormat = "$session attr-dict `:` qualified(type($session))"; +} + +def Transport_KickOp : Transport_Op<"kick"> { + let summary = "Send a payload as one round's request (controller)."; + let arguments = (ins Transport_ControllerSession:$session, Transport_AnyBuffer:$payload, + I32Attr:$work_item_idx); + let assemblyFormat = "$session `,` $payload attr-dict `:` qualified(type($session)) `,` type($payload)"; } def Transport_CollectOp : Transport_Op<"collect"> { - let summary = "Wait for this round's reply and return it as a value."; - let arguments = (ins Transport_SessionType:$session, I64Attr:$bytes); - let results = (outs I64:$result); - let assemblyFormat = "$session attr-dict `:` type($session) `->` type($result)"; + let summary = "Receive the current round's reply."; + let description = [{ + Receives the current round's reply in one of two forms: a value form that + returns it as a tensor `$result`, or a destination-passing form that writes + it into the `$dest` buffer. The buffer shape gives the expected reply size. + }]; + let arguments = (ins Transport_SessionType:$session, Optional:$dest); + let results = (outs Optional:$result); + let assemblyFormat = [{ + $session (`,` $dest^)? attr-dict `:` qualified(type($session)) + (`,` type($dest)^)? (`->` type($result)^)? + }]; } def Transport_LastRttNsOp : Transport_Op<"last_rtt_ns"> { - let summary = "Last round-trip time in nanoseconds."; + let summary = "Round-trip time of the last round, in nanoseconds."; let arguments = (ins Transport_SessionType:$session); let results = (outs I64:$rtt_ns); - let assemblyFormat = "$session attr-dict `:` type($session) `->` type($rtt_ns)"; + let assemblyFormat = "$session attr-dict `:` qualified(type($session)) `->` type($rtt_ns)"; } +//===----------------------------------------------------------------------===// +// Teardown +//===----------------------------------------------------------------------===// + def Transport_StopOp : Transport_Op<"stop"> { - let summary = "Stop the session. Idempotent."; + let summary = "Stop the session (idempotent)."; let arguments = (ins Transport_SessionType:$session); - let assemblyFormat = "$session attr-dict `:` type($session)"; + let assemblyFormat = "$session attr-dict `:` qualified(type($session))"; } -def Transport_CloseOp : Transport_Op<"close"> { - let summary = "Close the transport (releases the channel)."; +def Transport_DestroyOp : Transport_Op<"destroy"> { + let summary = "Destroy the session and release it."; let arguments = (ins Transport_SessionType:$session); - let assemblyFormat = "$session attr-dict `:` type($session)"; + let assemblyFormat = "$session attr-dict `:` qualified(type($session))"; } -def Transport_DestroyOp : Transport_Op<"destroy"> { - let summary = "Destroy the session and free it."; - let arguments = (ins Transport_SessionType:$session); - let assemblyFormat = "$session attr-dict `:` type($session)"; +//===----------------------------------------------------------------------===// +// Session resolution +//===----------------------------------------------------------------------===// + +def Transport_GetSessionOp : Transport_Op<"get_session"> { + let summary = "Resolve a session by role and key."; + let description = [{ + Returns the session that `transport.create` registered under (role, `key`), + where the role is taken from the result type. Lets a session created in one + function be used in another without passing it as a value. + }]; + let arguments = (ins DefaultValuedStrAttr:$key); + let results = (outs Transport_SessionType:$session); + let assemblyFormat = "attr-dict `:` qualified(type($session))"; } #endif // TRANSPORT_OPS From ed2a082cee96c56376e8835f0f0d17638443fed8 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Fri, 24 Jul 2026 10:29:15 -0400 Subject: [PATCH 33/57] add tests --- mlir/test/Transport/RoleVerify.mlir | 43 +++++++++++++++++++ mlir/test/Transport/SmokeTest.mlir | 65 +++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 mlir/test/Transport/RoleVerify.mlir create mode 100644 mlir/test/Transport/SmokeTest.mlir diff --git a/mlir/test/Transport/RoleVerify.mlir b/mlir/test/Transport/RoleVerify.mlir new file mode 100644 index 0000000000..a1ea8f2669 --- /dev/null +++ b/mlir/test/Transport/RoleVerify.mlir @@ -0,0 +1,43 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// RUN: quantum-opt --split-input-file --verify-diagnostics %s + +// Role safety: controller-only ops reject a coprocessor session and vice versa. + +func.func @kick_requires_controller() { + %c = transport.create {backend_lib = "x", config = "c"} -> !transport.session + %buf = memref.alloc() : memref<1xi8> + // expected-error @+1 {{operand #0 must be}} + transport.kick %c, %buf {work_item_idx = 0 : i32} : !transport.session, memref<1xi8> + return +} + +// ----- + +func.func @commit_requires_controller() { + %c = transport.create {backend_lib = "x", config = "c"} -> !transport.session + // expected-error @+1 {{operand #0 must be}} + transport.commit_work_item %c {work_item_idx = 0 : i32, in_bytes = 8 : i64, out_bytes = 8 : i64} : !transport.session + return +} + +// ----- + +func.func @set_coprocessor_fn_requires_coprocessor() { + %c = transport.create {backend_lib = "x", config = "c"} -> !transport.session + // expected-error @+1 {{operand #0 must be}} + transport.set_coprocessor_fn %c {symbol = "decode"} : !transport.session + return +} diff --git a/mlir/test/Transport/SmokeTest.mlir b/mlir/test/Transport/SmokeTest.mlir new file mode 100644 index 0000000000..daf2048c19 --- /dev/null +++ b/mlir/test/Transport/SmokeTest.mlir @@ -0,0 +1,65 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// RUN: quantum-opt %s | quantum-opt | FileCheck %s + +// Smoke test for the transport dialect + +// CHECK-LABEL: func.func @transport_smoketest +func.func @transport_smoketest(%payload: memref, %reply: memref) { + // CHECK: transport.create {{.*}} -> !transport.session + %ct = transport.create {backend_lib = "libbackend.so", config = "cfg"} -> !transport.session + // CHECK: transport.create {{.*}} -> !transport.session + %co = transport.create {backend_lib = "libbackend.so", config = "cfg"} -> !transport.session + + // CHECK: transport.connect_async %{{.*}} : !transport.session -> !transport.token + %t1 = transport.connect_async %co {peer = "127.0.0.1", oob_port = 18590 : i16} : !transport.session -> !transport.token + // CHECK: transport.connect %{{.*}} : !transport.session + transport.connect %ct {peer = "127.0.0.1", oob_port = 18590 : i16} : !transport.session + // CHECK: transport.barrier %{{.*}} : !transport.token + transport.barrier %t1 : !transport.token + // CHECK: transport.exchange_keys_async %{{.*}} : !transport.session -> !transport.token + %t2 = transport.exchange_keys_async %co : !transport.session -> !transport.token + transport.barrier %t2 : !transport.token + // CHECK: transport.exchange_keys %{{.*}} : !transport.session + transport.exchange_keys %ct : !transport.session + + // CHECK: transport.establish_channel %{{.*}} "cpu_verbs" : !transport.session + transport.establish_channel %ct "cpu_verbs" : !transport.session + // CHECK: transport.establish_channel %{{.*}} "gpu_engine" : !transport.session + transport.establish_channel %co "gpu_engine" : !transport.session + + // CHECK: transport.set_coprocessor_fn %{{.*}} {symbol = "foo"} : !transport.session + transport.set_coprocessor_fn %co {symbol = "foo"} : !transport.session + // CHECK: transport.commit_work_item %{{.*}} : !transport.session + transport.commit_work_item %ct {work_item_idx = 0 : i32, in_bytes = 8 : i64, out_bytes = 8 : i64} : !transport.session + + transport.start %co : !transport.session + transport.start %ct : !transport.session + + // CHECK: transport.get_session : !transport.session + %ct2 = transport.get_session : !transport.session + + // CHECK: transport.kick %{{.*}}, %{{.*}} {work_item_idx = 0 : i32} : !transport.session, memref + transport.kick %ct2, %payload {work_item_idx = 0 : i32} : !transport.session, memref + // CHECK: transport.collect %{{.*}}, %{{.*}} : !transport.session, memref + transport.collect %ct2, %reply : !transport.session, memref + // CHECK: transport.last_rtt_ns %{{.*}} : !transport.session -> i64 + %rtt = transport.last_rtt_ns %ct2 : !transport.session -> i64 + + transport.stop %ct : !transport.session + transport.destroy %ct : !transport.session + transport.destroy %co : !transport.session + return +} From a6493b268eff1248e4d96447498fba91ca1b63e0 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Fri, 24 Jul 2026 10:33:15 -0400 Subject: [PATCH 34/57] fix --- mlir/tools/quantum-lsp-server/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/mlir/tools/quantum-lsp-server/CMakeLists.txt b/mlir/tools/quantum-lsp-server/CMakeLists.txt index 5a6bc26be6..5a121ccd27 100644 --- a/mlir/tools/quantum-lsp-server/CMakeLists.txt +++ b/mlir/tools/quantum-lsp-server/CMakeLists.txt @@ -18,6 +18,7 @@ set(LIBS MLIRRTIO MLIRQecLogical MLIRQecPhysical + MLIRTransport ) add_llvm_executable(quantum-lsp-server quantum-lsp-server.cpp) From 1c2ea7a8271a21f0f69d95e7baf22f1499c668e0 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Fri, 24 Jul 2026 10:47:27 -0400 Subject: [PATCH 35/57] update get_session --- .../Transport/Transforms/TransportToLLVM.cpp | 85 ++++++++++++++----- .../Transport/ConvertTransportToLLVM.mlir | 78 ++++++++++++----- 2 files changed, 119 insertions(+), 44 deletions(-) diff --git a/mlir/lib/Transport/Transforms/TransportToLLVM.cpp b/mlir/lib/Transport/Transforms/TransportToLLVM.cpp index 0a3b7cab90..9cc7bf609e 100644 --- a/mlir/lib/Transport/Transforms/TransportToLLVM.cpp +++ b/mlir/lib/Transport/Transforms/TransportToLLVM.cpp @@ -15,13 +15,13 @@ // Lower the `transport` dialect to `llvm.call`s on the __catalyst__transport__* // CAPI (runtime/include/TransportCAPI.h). -#include "llvm/ADT/Twine.h" #include "mlir/Conversion/LLVMCommon/TypeConverter.h" #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/Pass/Pass.h" #include "mlir/Transforms/DialectConversion.h" +#include "llvm/ADT/Twine.h" #include "Transport/IR/TransportOps.h" #include "Transport/Transforms/Passes.h" @@ -67,6 +67,22 @@ Value constInt(ConversionPatternRewriter &rewriter, Location loc, Type ty, int64 return LLVM::ConstantOp::create(rewriter, loc, ty, rewriter.getIntegerAttr(ty, v)); } + +// From a lowered 1-D memref descriptor (an LLVM struct), extract the aligned data +// pointer and the buffer's size in bytes (num elements * element byte width). +std::pair memrefPtrAndBytes(ConversionPatternRewriter &rewriter, Location loc, + Value descriptor, MemRefType memTy) +{ + Value ptr = LLVM::ExtractValueOp::create(rewriter, loc, descriptor, ArrayRef{1}); + Value nelem = LLVM::ExtractValueOp::create(rewriter, loc, descriptor, ArrayRef{3, 0}); + Type elemTy = memTy.getElementType(); + int64_t elemBytes = + isa(elemTy) ? 8 : (elemTy.getIntOrFloatBitWidth() + 7) / 8; + Value ebytes = constInt(rewriter, loc, i64Ty(rewriter.getContext()), elemBytes); + Value bytes = LLVM::MulOp::create(rewriter, loc, nelem, ebytes); + return {ptr, bytes}; +} + //===----------------------------------------------------------------------===// // Patterns //===----------------------------------------------------------------------===// @@ -81,10 +97,12 @@ struct CreateLowering : public OpConversionPattern { auto sessTy = cast(op.getSession().getType()); Value lib = globalStr(rewriter, op.getLoc(), mod, "transport_backend_", op.getBackendLib()); Value cfg = globalStr(rewriter, op.getLoc(), mod, "transport_config_", op.getConfig()); + Value key = globalStr(rewriter, op.getLoc(), mod, "transport_key_", op.getKey()); Value role = constInt(rewriter, op.getLoc(), i32Ty(ctx), static_cast(sessTy.getRole())); Value s = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__create", - {ptrTy(ctx), ptrTy(ctx), i32Ty(ctx)}, ptrTy(ctx), {lib, cfg, role}); + {ptrTy(ctx), ptrTy(ctx), i32Ty(ctx), ptrTy(ctx)}, ptrTy(ctx), + {lib, cfg, role, key}); rewriter.replaceOp(op, s); return success(); } @@ -161,10 +179,10 @@ struct EstablishChannelLowering : public OpConversionPattern ConversionPatternRewriter &rewriter) const override { auto *ctx = op.getContext(); - Value dp = - constInt(rewriter, op.getLoc(), i32Ty(ctx), static_cast(op.getDataPath())); + Value dp = globalStr(rewriter, op.getLoc(), moduleOf(op), "transport_data_path_", + op.getDataPath()); emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__establish_channel", - {ptrTy(ctx), i32Ty(ctx)}, i32Ty(ctx), {adaptor.getSession(), dp}); + {ptrTy(ctx), ptrTy(ctx)}, i32Ty(ctx), {adaptor.getSession(), dp}); rewriter.eraseOp(op); return success(); } @@ -175,8 +193,11 @@ struct SetCoprocessorFnLowering : public OpConversionPattern LogicalResult matchAndRewrite(SetCoprocessorFnOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { - emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__set_coprocessor_fn", - {ptrTy(op.getContext())}, i32Ty(op.getContext()), {adaptor.getSession()}); + auto *ctx = op.getContext(); + ModuleOp mod = moduleOf(op); + Value sym = globalStr(rewriter, op.getLoc(), mod, "transport_coproc_fn_", op.getSymbol()); + emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__set_coprocessor_fn", + {ptrTy(ctx), ptrTy(ctx)}, i32Ty(ctx), {adaptor.getSession(), sym}); rewriter.eraseOp(op); return success(); } @@ -206,9 +227,14 @@ struct KickLowering : public OpConversionPattern { { auto *ctx = op.getContext(); ModuleOp mod = moduleOf(op); + auto memTy = dyn_cast(op.getPayload().getType()); + if (!memTy) + return rewriter.notifyMatchFailure(op, "kick payload must be bufferized (memref)"); + auto [srcPtr, bytes] = + memrefPtrAndBytes(rewriter, op.getLoc(), adaptor.getPayload(), memTy); Value slot = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__data_slot", {ptrTy(ctx)}, ptrTy(ctx), {adaptor.getSession()}); - LLVM::StoreOp::create(rewriter, op.getLoc(), adaptor.getPayload(), slot); + LLVM::MemcpyOp::create(rewriter, op.getLoc(), slot, srcPtr, bytes, /*isVolatile=*/false); Value idx = constInt(rewriter, op.getLoc(), i32Ty(ctx), op.getWorkItemIdx()); emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__kick", {ptrTy(ctx), i32Ty(ctx)}, i32Ty(ctx), {adaptor.getSession(), idx}); @@ -224,15 +250,14 @@ struct CollectLowering : public OpConversionPattern { { auto *ctx = op.getContext(); ModuleOp mod = moduleOf(op); - Value one = constInt(rewriter, op.getLoc(), i64Ty(ctx), 1); - Value buf = LLVM::AllocaOp::create(rewriter, op.getLoc(), ptrTy(ctx), i64Ty(ctx), one, - /*alignment=*/8); - Value bytes = constInt(rewriter, op.getLoc(), i64Ty(ctx), op.getBytes()); + if (!op.getDest()) + return rewriter.notifyMatchFailure(op, "collect must be bufferized (dest-passing form)"); + auto memTy = cast(op.getDest().getType()); + auto [dstPtr, bytes] = memrefPtrAndBytes(rewriter, op.getLoc(), adaptor.getDest(), memTy); emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__collect", {ptrTy(ctx), ptrTy(ctx), i64Ty(ctx)}, i32Ty(ctx), - {adaptor.getSession(), buf, bytes}); - Value loaded = LLVM::LoadOp::create(rewriter, op.getLoc(), i64Ty(ctx), buf); - rewriter.replaceOp(op, loaded); + {adaptor.getSession(), dstPtr, bytes}); + rewriter.eraseOp(op); return success(); } }; @@ -242,9 +267,9 @@ struct LastRttLowering : public OpConversionPattern { LogicalResult matchAndRewrite(LastRttNsOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { - Value r = - emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__last_rtt_ns", - {ptrTy(op.getContext())}, i64Ty(op.getContext()), {adaptor.getSession()}); + Value r = emitCall(rewriter, op.getLoc(), moduleOf(op), + "__catalyst__transport__last_rtt_ns", {ptrTy(op.getContext())}, + i64Ty(op.getContext()), {adaptor.getSession()}); rewriter.replaceOp(op, r); return success(); } @@ -267,6 +292,25 @@ template struct VoidSessionLowering : public OpConversionPattern< std::string symbol; }; +// get_session: look the session up by role from the runtime registry (populated at create). +struct GetSessionLowering : public OpConversionPattern { + using OpConversionPattern::OpConversionPattern; + LogicalResult matchAndRewrite(GetSessionOp op, OpAdaptor, + ConversionPatternRewriter &rewriter) const override + { + auto *ctx = op.getContext(); + ModuleOp mod = moduleOf(op); + auto sessTy = cast(op.getSession().getType()); + Value role = + constInt(rewriter, op.getLoc(), i32Ty(ctx), static_cast(sessTy.getRole())); + Value key = globalStr(rewriter, op.getLoc(), mod, "transport_key_", op.getKey()); + Value s = emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__get_session", + {i32Ty(ctx), ptrTy(ctx)}, ptrTy(ctx), {role, key}); + rewriter.replaceOp(op, s); + return success(); + } +}; + } // namespace struct ConvertTransportToLLVMPass @@ -283,11 +327,10 @@ struct ConvertTransportToLLVMPass RewritePatternSet patterns(ctx); patterns.add(tc, ctx); + SetCoprocessorFnLowering, CommitWorkItemLowering, KickLowering, CollectLowering, + LastRttLowering, GetSessionLowering>(tc, ctx); patterns.add>(tc, ctx, "__catalyst__transport__start"); patterns.add>(tc, ctx, "__catalyst__transport__stop"); - patterns.add>(tc, ctx, "__catalyst__transport__close"); patterns.add>(tc, ctx, "__catalyst__transport__destroy"); ConversionTarget target(*ctx); diff --git a/mlir/test/Transport/ConvertTransportToLLVM.mlir b/mlir/test/Transport/ConvertTransportToLLVM.mlir index d9cdb1fd06..e2590bb88d 100644 --- a/mlir/test/Transport/ConvertTransportToLLVM.mlir +++ b/mlir/test/Transport/ConvertTransportToLLVM.mlir @@ -14,10 +14,10 @@ // RUN: quantum-opt %s --convert-transport-to-llvm --split-input-file | FileCheck %s -// CHECK-DAG: llvm.func @__catalyst__transport__controller_create(!llvm.ptr, !llvm.ptr) -> !llvm.ptr +// CHECK-DAG: llvm.func @__catalyst__transport__create(!llvm.ptr, !llvm.ptr, i32, !llvm.ptr) -> !llvm.ptr // CHECK-DAG: llvm.func @__catalyst__transport__connect(!llvm.ptr, !llvm.ptr, i16) -> i32 -// CHECK-DAG: llvm.func @__catalyst__transport__exchange_keys(!llvm.ptr, !llvm.ptr) -> i32 -// CHECK-DAG: llvm.func @__catalyst__transport__establish_channel(!llvm.ptr, i32, !llvm.ptr) -> i32 +// CHECK-DAG: llvm.func @__catalyst__transport__exchange_keys(!llvm.ptr) -> i32 +// CHECK-DAG: llvm.func @__catalyst__transport__establish_channel(!llvm.ptr, !llvm.ptr) -> i32 // CHECK-DAG: llvm.func @__catalyst__transport__commit_work_item(!llvm.ptr, i32, i64, i64) -> i32 // CHECK-DAG: llvm.func @__catalyst__transport__data_slot(!llvm.ptr) -> !llvm.ptr // CHECK-DAG: llvm.func @__catalyst__transport__kick(!llvm.ptr, i32) -> i32 @@ -26,32 +26,64 @@ // CHECK-DAG: llvm.func @__catalyst__transport__stop(!llvm.ptr) // CHECK-DAG: llvm.func @__catalyst__transport__destroy(!llvm.ptr) -// CHECK-LABEL: func.func @controller_roundtrip -func.func @controller_roundtrip() -> i64 { - // CHECK: %[[S:.*]] = llvm.call @__catalyst__transport__controller_create - %s = transport.controller_create {backend_lib = "libtransport_backend.so", config = "key=value"} -> !transport.session +// Controller: create (role in the result type) -> bring-up -> kick/collect over buffers. +// CHECK-LABEL: func.func @controller +func.func @controller(%syndrome: memref, %correction: memref) { + // CHECK: %[[S:.*]] = llvm.call @__catalyst__transport__create({{.*}}) : (!llvm.ptr, !llvm.ptr, i32, !llvm.ptr) -> !llvm.ptr + %s = transport.create {backend_lib = "libbackend.so", config = "cfg"} -> !transport.session // CHECK: llvm.call @__catalyst__transport__connect(%[[S]] - %c = transport.connect %s {peer = "127.0.0.1", oob_port = 18560 : i16} : (!transport.session) -> i32 - // CHECK: %[[PEER:.*]] = llvm.alloca - // CHECK: llvm.call @__catalyst__transport__exchange_keys(%[[S]], %[[PEER]]) - %cs, %peer = transport.exchange_keys %s : !transport.session -> !transport.peer - // CHECK: llvm.call @__catalyst__transport__establish_channel(%[[S]], {{.*}}, %[[PEER]]) - %e = transport.establish_channel %s, %peer {data_path = 0 : i32} : !transport.session, !transport.peer + transport.connect %s {peer = "127.0.0.1", oob_port = 18560 : i16} : !transport.session + // CHECK: llvm.call @__catalyst__transport__exchange_keys(%[[S]]) + transport.exchange_keys %s : !transport.session + // CHECK: llvm.call @__catalyst__transport__establish_channel(%[[S]] + transport.establish_channel %s "cpu_verbs" : !transport.session // CHECK: llvm.call @__catalyst__transport__commit_work_item(%[[S]] - %w = transport.commit_work_item %s {work_item_idx = 0 : i32, in_bytes = 8 : i64, out_bytes = 8 : i64} : !transport.session + transport.commit_work_item %s {work_item_idx = 0 : i32, in_bytes = 8 : i64, out_bytes = 8 : i64} : !transport.session // CHECK: llvm.call @__catalyst__transport__start(%[[S]]) - transport.start %s : !transport.session - %payload = arith.constant 81985529216486895 : i64 + transport.start %s : !transport.session // CHECK: %[[SLOT:.*]] = llvm.call @__catalyst__transport__data_slot(%[[S]]) - // CHECK: llvm.store %{{.*}}, %[[SLOT]] + // CHECK: "llvm.intr.memcpy"(%[[SLOT]] // CHECK: llvm.call @__catalyst__transport__kick(%[[S]] - %k = transport.kick %s, %payload {work_item_idx = 0 : i32} : !transport.session, i64 + transport.kick %s, %syndrome {work_item_idx = 0 : i32} : !transport.session, memref // CHECK: llvm.call @__catalyst__transport__collect(%[[S]] - // CHECK: %[[RESULT:.*]] = llvm.load - %result = transport.collect %s {bytes = 8 : i64} : !transport.session -> i64 + transport.collect %s, %correction : !transport.session, memref // CHECK: llvm.call @__catalyst__transport__stop(%[[S]]) - transport.stop %s : !transport.session + transport.stop %s : !transport.session // CHECK: llvm.call @__catalyst__transport__destroy(%[[S]]) - transport.destroy %s : !transport.session - return %result : i64 + transport.destroy %s : !transport.session + return +} + +// ----- + +// Coprocessor: create + bind the coprocessor function symbol + async bring-up. +// CHECK-LABEL: func.func @coprocessor +func.func @coprocessor() { + // CHECK: %[[C:.*]] = llvm.call @__catalyst__transport__create({{.*}}) : (!llvm.ptr, !llvm.ptr, i32, !llvm.ptr) -> !llvm.ptr + %c = transport.create {backend_lib = "libbackend.so", config = "cfg"} -> !transport.session + // CHECK: llvm.call @__catalyst__transport__connect_async(%[[C]] + %t = transport.connect_async %c {peer = "127.0.0.1", oob_port = 18560 : i16} : !transport.session -> !transport.token + // CHECK: llvm.call @__catalyst__transport__barrier + transport.barrier %t : !transport.token + // CHECK: llvm.call @__catalyst__transport__set_coprocessor_fn(%[[C]], {{.*}}) : (!llvm.ptr, !llvm.ptr) -> i32 + transport.set_coprocessor_fn %c {symbol = "foo"} : !transport.session + // CHECK: llvm.call @__catalyst__transport__destroy(%[[C]]) + transport.destroy %c : !transport.session + return +} + +// ----- + +// get_session resolves a session by (role from result type, key) via the runtime registry. +// CHECK-DAG: llvm.func @__catalyst__transport__get_session(i32, !llvm.ptr) -> !llvm.ptr +// CHECK-LABEL: func.func @resolve +func.func @resolve(%syndrome: memref, %correction: memref) { + // CHECK: %[[R:.*]] = llvm.mlir.constant(0 : i32) : i32 + // CHECK: %[[S:.*]] = llvm.call @__catalyst__transport__get_session(%[[R]], {{.*}}) : (i32, !llvm.ptr) -> !llvm.ptr + %s = transport.get_session {key = "cop0"} : !transport.session + // CHECK: llvm.call @__catalyst__transport__kick(%[[S]] + transport.kick %s, %syndrome {work_item_idx = 0 : i32} : !transport.session, memref + // CHECK: llvm.call @__catalyst__transport__collect(%[[S]] + transport.collect %s, %correction : !transport.session, memref + return } From 88874665e1abb2454e0c977fe8373d56807a60c2 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Fri, 24 Jul 2026 10:48:01 -0400 Subject: [PATCH 36/57] add changelog --- doc/releases/changelog-dev.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md index 1b271f2e16..d13084e3fc 100644 --- a/doc/releases/changelog-dev.md +++ b/doc/releases/changelog-dev.md @@ -19,6 +19,10 @@ lifecycle at the IR level. [(#3047)](https://github.com/PennyLaneAI/catalyst/pull/3047) +* A `convert-transport-to-llvm` pass is added, lowering the `Transport` dialect ops to the + transport runtime CAPI. + [(#3048)](https://github.com/PennyLaneAI/catalyst/pull/3048) + * A `BufferizableOpInterface` implementation is now added for `catalyst.launch_kernel` operation and it is now bufferizable. [(#3024)](https://github.com/PennyLaneAI/catalyst/pull/3024) From f6d14266c7c0bedf4fccc1abd480cc90f0bb8af6 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Fri, 24 Jul 2026 11:53:57 -0400 Subject: [PATCH 37/57] refine the CAPI --- runtime/include/Transport.hpp | 24 +- runtime/include/TransportCAPI.h | 66 +++--- runtime/lib/transport/TransportCAPI.cpp | 301 +++++++++++++++++------- 3 files changed, 252 insertions(+), 139 deletions(-) diff --git a/runtime/include/Transport.hpp b/runtime/include/Transport.hpp index 7f8f7b07c0..8cb21da9e5 100644 --- a/runtime/include/Transport.hpp +++ b/runtime/include/Transport.hpp @@ -19,15 +19,6 @@ namespace catalyst::transport { -/** - * @brief Data-plane strategy: which engine issues the transfer. - */ -enum class DataPath : std::uint8_t { - CpuVerbs, // Plain ibverbs on CPU. - GpuEngine, // Gpu-initiated comms. - Other, -}; - /** * @brief Memory kind: selects the allocation and registration path. */ @@ -70,7 +61,7 @@ struct PeerRef { * @brief Configuration for the data-movement channel a session uses. */ struct ChannelDesc { - DataPath data_path = DataPath::CpuVerbs; + std::string data_path = "cpu_verbs"; }; /** @@ -102,11 +93,10 @@ class TransportSession { * * @param size Size of the region in bytes. * @param kind Memory kind selecting the allocation and registration path. - * @param access Access flags for the registration. * * @return `MemRegion` The allocated and registered region. */ - virtual MemRegion alloc_memory(std::size_t size, MemKind kind, std::uint32_t access) = 0; + virtual MemRegion alloc_memory(std::size_t size, MemKind kind) = 0; /** * @brief Advertise a local region and receive the peer's region over the out-of-band channel. @@ -133,14 +123,16 @@ class TransportSession { virtual void start() = 0; /** - * @brief Wait for a result and write it out. + * @brief Wait for a result and scatter it into the reply buffers. * - * @param replies Output buffer to write the result into. - * @param bytes Capacity of the output buffer, in bytes. + * @param replies Array of `n` buffers to write the results into. + * @param replies_bytes Array of `n` capacities (bytes), one per reply buffer. + * @param n Number of reply buffers. * * @return `int` */ - virtual int collect(void *replies, std::uint64_t bytes) = 0; + virtual int collect(void *const *replies, const std::uint64_t *replies_bytes, + std::size_t n) = 0; /** * @brief Stop the engine and join. Idempotent. diff --git a/runtime/include/TransportCAPI.h b/runtime/include/TransportCAPI.h index c1fb6817a3..cf05791f6e 100644 --- a/runtime/include/TransportCAPI.h +++ b/runtime/include/TransportCAPI.h @@ -40,46 +40,46 @@ enum { CATALYST_TRANSPORT_ERR_STUCK = -4, // Something got stuck }; -// DataPath enum (mirrors catalyst::transport::DataPath) +// Session role (mirrors catalyst::transport::Role in the dialect). enum { - CATALYST_TRANSPORT_PATH_CPU_VERBS = 0, - CATALYST_TRANSPORT_PATH_GPU_ENGINE = 1, - CATALYST_TRANSPORT_PATH_OTHER = 2, + CATALYST_TRANSPORT_ROLE_CONTROLLER = 0, + CATALYST_TRANSPORT_ROLE_COPROCESSOR = 1, }; -// MemKind enum (mirrors catalyst::transport::MemKind) -enum { - CATALYST_TRANSPORT_MEM_CPU_RAM = 0, - CATALYST_TRANSPORT_MEM_GPU_HBM = 1, - CATALYST_TRANSPORT_MEM_DDR = 2, - CATALYST_TRANSPORT_MEM_OTHER = 3, -}; - -// Remote peer region descriptor -typedef struct { - uint32_t rkey; - uint64_t remote_addr; - uint64_t size; -} CatalystTransportPeerRef; - -// Create a controller session from a named backend plugin `.so` (dlopen'd by the runtime). -// `config` is the backend's "key=value;..." string. Returns NULL on failure. -CatalystTransportSession *__catalyst__transport__controller_create(const char *backend_lib, - const char *config); - -void __catalyst__transport__close(CatalystTransportSession *s); -int __catalyst__transport__connect(CatalystTransportSession *s, const char *peer, - uint16_t oob_port); -int __catalyst__transport__exchange_keys(CatalystTransportSession *s, - CatalystTransportPeerRef *out); -int __catalyst__transport__establish_channel(CatalystTransportSession *s, int32_t data_path, - const CatalystTransportPeerRef *peer); +// Create a session from a named backend plugin `.so` (dlopen'd by the runtime). `role` selects +// which factory symbol is looked up (controller vs coprocessor). `config` is the backend's string. +// Returns NULL on failure. +// `key` registers the session under (role, key) for later get_session; empty = not registered. +CatalystTransportSession *__catalyst__transport__create(const char *backend_lib, const char *config, + int32_t role, const char *key); + +// Resolve the live session registered at create under (`role`, `key`). +// Lets a session brought up in one function be used in another. +CatalystTransportSession *__catalyst__transport__get_session(int32_t role, const char *key); + +// Bring-up. The peer region learned in exchange_keys is kept inside the session, so +// establish_channel takes no peer handle. The *_async variants run on a worker thread and return a +// token to await with barrier. +int __catalyst__transport__connect(CatalystTransportSession *s, const char *peer, uint16_t oob_port); +int64_t __catalyst__transport__connect_async(CatalystTransportSession *s, const char *peer, + uint16_t oob_port); +int __catalyst__transport__exchange_keys(CatalystTransportSession *s); +int64_t __catalyst__transport__exchange_keys_async(CatalystTransportSession *s); +int __catalyst__transport__barrier(int64_t token); +int __catalyst__transport__establish_channel(CatalystTransportSession *s, const char *data_path); + +// Coprocessor-only: bind the function run per received message, resolved by runtime symbol name. +int __catalyst__transport__set_coprocessor_fn(CatalystTransportSession *s, const char *symbol); + +// Controller-only: work items + kick. int __catalyst__transport__commit_work_item(CatalystTransportSession *s, uint32_t work_item_idx, uint64_t in_bytes, uint64_t out_bytes); void *__catalyst__transport__data_slot(CatalystTransportSession *s); -void __catalyst__transport__start(CatalystTransportSession *s); int __catalyst__transport__kick(CatalystTransportSession *s, uint32_t work_item_idx); -int __catalyst__transport__collect(CatalystTransportSession *s, void *correction, uint64_t bytes); + +// Run / collect / teardown. +void __catalyst__transport__start(CatalystTransportSession *s); +int __catalyst__transport__collect(CatalystTransportSession *s, void *reply, uint64_t reply_bytes); uint64_t __catalyst__transport__last_rtt_ns(CatalystTransportSession *s); void __catalyst__transport__stop(CatalystTransportSession *s); void __catalyst__transport__destroy(CatalystTransportSession *s); diff --git a/runtime/lib/transport/TransportCAPI.cpp b/runtime/lib/transport/TransportCAPI.cpp index 429541508a..92d6565c03 100644 --- a/runtime/lib/transport/TransportCAPI.cpp +++ b/runtime/lib/transport/TransportCAPI.cpp @@ -14,12 +14,18 @@ #include "TransportCAPI.h" -#include -#include +#include +#include +#include #include +#include +#include #include #include +#include #include +#include +#include #include "DynamicLibraryLoader.hpp" #include "Transport.hpp" @@ -28,51 +34,132 @@ using catalyst::transport::ChannelDesc; using catalyst::transport::ConnectInfo; using catalyst::transport::ControllerSession; -using catalyst::transport::DataPath; +using catalyst::transport::CoprocessorFn; +using catalyst::transport::CoprocessorSession; +using catalyst::transport::MemKind; using catalyst::transport::MemRegion; using catalyst::transport::PeerRef; +using catalyst::transport::TransportSession; -// The opaque handle +// The opaque handle. `sess` is the base type; the concrete role is chosen by the factory at +// create. The peer region learned in exchange_keys is kept here so establish_channel needs no peer +// argument (matches the dialect, where the peer is session-held). struct CatalystTransportSession { std::unique_ptr backend; - ControllerSession *sess = nullptr; // heap-allocated by the backend factory + TransportSession *sess = nullptr; // heap-allocated by the backend factory + MemRegion reply; // local region advertised in exchange_keys + bool reply_ready = false; + PeerRef peer; // peer region learned in exchange_keys + bool peer_ready = false; }; namespace { -DataPath to_data_path(std::int32_t p) +// (role, key) -> live session registry. Populated at create, read by get_session so a session +// brought up in one function can be resolved in another. +std::unordered_map g_registry; + +std::string registry_key(std::int32_t role, const char *key) { - switch (p) { - case CATALYST_TRANSPORT_PATH_CPU_VERBS: - return DataPath::CpuVerbs; - case CATALYST_TRANSPORT_PATH_GPU_ENGINE: - return DataPath::GpuEngine; - case CATALYST_TRANSPORT_PATH_OTHER: - default: - return DataPath::Other; - } + return std::to_string(role) + "/" + (key ? key : ""); } -template int guard(Fn &&fn) +// Local reply region size, provisioned automatically at exchange_keys. +constexpr std::uint64_t kReplyBytes = 16 * 1024; + +// Run fn, logging and swallowing any exception. Returns fn()'s result, or CATALYST_TRANSPORT_ERR +// if it threw; when fn() returns void there is nothing to return. +template auto guard(Fn &&fn) -> decltype(fn()) { try { return fn(); } catch (const std::exception &e) { std::cerr << "[transport] " << e.what() << "\n"; - return CATALYST_TRANSPORT_ERR; } catch (...) { + } + if constexpr (!std::is_void_v) { return CATALYST_TRANSPORT_ERR; } } +// Provision the local reply region on first use (idempotent). +void ensure_reply(CatalystTransportSession *s) +{ + if (!s->reply_ready) { + s->reply = s->sess->alloc_memory(kReplyBytes, MemKind::CpuRam); + s->reply_ready = true; + } +} + +ControllerSession *as_controller(CatalystTransportSession *s) +{ + return s ? dynamic_cast(s->sess) : nullptr; +} + +// Bring-up bodies shared by the blocking and async (worker-thread) entry points. +int do_connect(CatalystTransportSession *s, std::string peer, std::uint16_t oob_port) +{ + ConnectInfo info; + info.peer = std::move(peer); + info.oob_port = oob_port; + return s->sess->connect(info); +} + +int do_exchange_keys(CatalystTransportSession *s) +{ + ensure_reply(s); + s->peer = s->sess->exchange_keys(s->reply); + s->peer_ready = true; + return CATALYST_TRANSPORT_OK; +} + +// Built-in fallback coprocessor function: echo the input back. +std::size_t echo_fn(const void *in, std::size_t in_len, void *out, std::size_t out_cap, void *) +{ + std::size_t n = std::min(in_len, out_cap); + if (n && in && out) { + std::memcpy(out, in, n); + } + return n; +} + +// Async task registry: connect_async / exchange_keys_async run on a worker thread and return a +// token; barrier awaits it. Tokens start at 1 so a 0 return can signal a dispatch failure. +std::mutex g_async_mtx; +std::int64_t g_next_token = 1; +std::unordered_map> g_async_tasks; + +std::int64_t dispatch_async(std::function fn) +{ + std::lock_guard lk(g_async_mtx); + std::int64_t token = g_next_token++; + g_async_tasks.emplace(token, std::async(std::launch::async, std::move(fn))); + return token; +} + +int await_token(std::int64_t token) +{ + std::future fut; + { + std::lock_guard lk(g_async_mtx); + auto it = g_async_tasks.find(token); + if (it == g_async_tasks.end()) { + return CATALYST_TRANSPORT_ERR; + } + fut = std::move(it->second); + g_async_tasks.erase(it); + } + return guard([&] { return fut.get(); }); +} + } // namespace extern "C" { -CatalystTransportSession *__catalyst__transport__controller_create(const char *backend_lib, - const char *config) +CatalystTransportSession *__catalyst__transport__create(const char *backend_lib, const char *config, + std::int32_t role, const char *key) { try { if (!backend_lib || !*backend_lib) { @@ -81,18 +168,29 @@ CatalystTransportSession *__catalyst__transport__controller_create(const char *b } auto h = std::make_unique(); h->backend = std::make_unique(backend_lib); - auto *factory = h->backend->getSymbol( - CATALYST_TRANSPORT_CONTROLLER_FACTORY_SYMBOL); - h->sess = factory(config ? config : ""); + const char *cfg = config ? config : ""; + if (role == CATALYST_TRANSPORT_ROLE_COPROCESSOR) { + auto *factory = h->backend->getSymbol( + CATALYST_TRANSPORT_COPROCESSOR_FACTORY_SYMBOL); + h->sess = factory(cfg); + } + else { + auto *factory = h->backend->getSymbol( + CATALYST_TRANSPORT_CONTROLLER_FACTORY_SYMBOL); + h->sess = factory(cfg); + } if (!h->sess) { - std::cerr << "[transport] backend factory returned null for config: " - << (config ? config : "") << "\n"; + std::cerr << "[transport] backend factory returned null for config: " << cfg << "\n"; return nullptr; } - return h.release(); + auto *raw = h.release(); + if (key && *key) { + g_registry[registry_key(role, key)] = raw; // resolved later via get_session + } + return raw; } catch (const std::exception &e) { - std::cerr << "[transport] controller_create: " << e.what() << "\n"; + std::cerr << "[transport] create: " << e.what() << "\n"; return nullptr; } catch (...) { @@ -106,44 +204,71 @@ int __catalyst__transport__connect(CatalystTransportSession *s, const char *peer if (!s || !s->sess) { return CATALYST_TRANSPORT_ERR; } - return guard([&] { - ConnectInfo info; - info.peer = peer ? peer : ""; - info.oob_port = oob_port; - return s->sess->connect(info); - }); + return guard([&] { return do_connect(s, peer ? peer : "", oob_port); }); +} + +std::int64_t __catalyst__transport__connect_async(CatalystTransportSession *s, const char *peer, + std::uint16_t oob_port) +{ + if (!s || !s->sess) { + return 0; + } + return dispatch_async( + [s, p = std::string(peer ? peer : ""), oob_port] { return do_connect(s, p, oob_port); }); +} + +int __catalyst__transport__exchange_keys(CatalystTransportSession *s) +{ + if (!s || !s->sess) { + return CATALYST_TRANSPORT_ERR; + } + return guard([&] { return do_exchange_keys(s); }); } -int __catalyst__transport__exchange_keys(CatalystTransportSession *s, CatalystTransportPeerRef *out) +std::int64_t __catalyst__transport__exchange_keys_async(CatalystTransportSession *s) +{ + if (!s || !s->sess) { + return 0; + } + return dispatch_async([s] { return do_exchange_keys(s); }); +} + +int __catalyst__transport__barrier(std::int64_t token) { return await_token(token); } + +int __catalyst__transport__establish_channel(CatalystTransportSession *s, const char *data_path) { if (!s || !s->sess) { return CATALYST_TRANSPORT_ERR; } return guard([&] { - PeerRef p = s->sess->exchange_keys(MemRegion{}); - if (out) { - out->rkey = p.rkey; - out->remote_addr = p.remote_addr; - out->size = p.size; - } + ChannelDesc desc; + desc.data_path = data_path ? data_path : ""; // opaque; the backend interprets it + s->sess->establish_channel(desc, s->reply, s->peer); return CATALYST_TRANSPORT_OK; }); } -int __catalyst__transport__establish_channel(CatalystTransportSession *s, std::int32_t data_path, - const CatalystTransportPeerRef *peer) +int __catalyst__transport__set_coprocessor_fn(CatalystTransportSession *s, const char *symbol) { - if (!s || !s->sess || !peer) { + if (!s || !s->sess) { return CATALYST_TRANSPORT_ERR; } return guard([&] { - ChannelDesc desc; - desc.data_path = to_data_path(data_path); - PeerRef p; - p.rkey = peer->rkey; - p.remote_addr = peer->remote_addr; - p.size = peer->size; - s->sess->establish_channel(desc, MemRegion{}, p); + auto *co = dynamic_cast(s->sess); + if (!co) { + std::cerr << "[transport] set_coprocessor_fn on a non-coprocessor session\n"; + return CATALYST_TRANSPORT_ERR; + } + // Empty symbol selects the built-in echo; a named-but-unresolved symbol is a hard error + CoprocessorFn fn = &echo_fn; + if (symbol && *symbol) { + fn = reinterpret_cast(dlsym(RTLD_DEFAULT, symbol)); + if (!fn) { + std::cerr << "[transport] set_coprocessor_fn: symbol not found: " << symbol << "\n"; + return CATALYST_TRANSPORT_ERR; + } + } + co->set_coprocessor_fn(fn, nullptr); return CATALYST_TRANSPORT_OK; }); } @@ -152,50 +277,51 @@ int __catalyst__transport__commit_work_item(CatalystTransportSession *s, std::uint32_t work_item_idx, std::uint64_t in_bytes, std::uint64_t out_bytes) { - if (!s || !s->sess) { + auto *c = as_controller(s); + if (!c) { return CATALYST_TRANSPORT_ERR; } return guard([&] { - s->sess->commit_work_item(work_item_idx, in_bytes, out_bytes); + if (s->reply_ready && out_bytes > s->reply.size) { + std::cerr << "[transport] commit_work_item: out_bytes (" << out_bytes + << ") exceeds the reply region (" << s->reply.size << ")\n"; + return CATALYST_TRANSPORT_ERR; + } + c->commit_work_item(work_item_idx, in_bytes, out_bytes); return CATALYST_TRANSPORT_OK; }); } void *__catalyst__transport__data_slot(CatalystTransportSession *s) { - if (!s || !s->sess) { - return nullptr; - } - + auto *c = as_controller(s); void *slot = nullptr; - try { - slot = s->sess->data_slot(); - } - catch (const std::exception &e) { - std::cerr << "[transport] data_slot: " << e.what() << "\n"; - return nullptr; - } - catch (...) { - return nullptr; + if (c) { + guard([&] { slot = c->data_slot(); }); } return slot; } int __catalyst__transport__kick(CatalystTransportSession *s, std::uint32_t work_item_idx) { - if (!s || !s->sess) { + auto *c = as_controller(s); + if (!c) { return CATALYST_TRANSPORT_ERR; } - return guard([&] { return s->sess->kick(work_item_idx); }); + return guard([&] { return c->kick(work_item_idx); }); } -int __catalyst__transport__collect(CatalystTransportSession *s, void *correction, - std::uint64_t bytes) +int __catalyst__transport__collect(CatalystTransportSession *s, void *reply, + std::uint64_t reply_bytes) { if (!s || !s->sess) { return CATALYST_TRANSPORT_ERR; } - return guard([&] { return s->sess->collect(correction, bytes); }); + return guard([&] { + void *replies[1] = {reply}; + std::uint64_t replies_bytes[1] = {reply_bytes}; + return s->sess->collect(replies, replies_bytes, 1); + }); } std::uint64_t __catalyst__transport__last_rtt_ns(CatalystTransportSession *s) @@ -208,28 +334,27 @@ std::uint64_t __catalyst__transport__last_rtt_ns(CatalystTransportSession *s) void __catalyst__transport__start(CatalystTransportSession *s) { - if (!s || !s->sess) { - return; - } - try { - s->sess->start(); - } - catch (const std::exception &e) { - std::cerr << "[transport] start: " << e.what() << "\n"; - } - catch (...) { + if (s && s->sess) { + guard([&] { s->sess->start(); }); } } void __catalyst__transport__stop(CatalystTransportSession *s) { if (s && s->sess) { - try { - s->sess->stop(); - } - catch (...) { - } + guard([&] { s->sess->stop(); }); + } +} + +CatalystTransportSession *__catalyst__transport__get_session(std::int32_t role, const char *key) +{ + auto it = g_registry.find(registry_key(role, key)); + if (it == g_registry.end()) { + std::cerr << "[transport] get_session: no session registered for role " << role << " key '" + << (key ? key : "") << "'\n"; + return nullptr; } + return it->second; } void __catalyst__transport__destroy(CatalystTransportSession *s) @@ -237,15 +362,11 @@ void __catalyst__transport__destroy(CatalystTransportSession *s) if (!s) { return; } + for (auto it = g_registry.begin(); it != g_registry.end();) + it = (it->second == s) ? g_registry.erase(it) : std::next(it); delete s->sess; // owned by the backend factory s->backend.reset(); delete s; } -void __catalyst__transport__close(CatalystTransportSession *s) -{ - __catalyst__transport__stop(s); - __catalyst__transport__destroy(s); -} - } // extern "C" From 266d3d95d4027a771ddb4722718a72914ed39464 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Fri, 24 Jul 2026 11:54:39 -0400 Subject: [PATCH 38/57] Add tests --- runtime/tests/CMakeLists.txt | 20 +++++ runtime/tests/Test_Transport.cpp | 84 +++++++++++++++++++ .../tests/stubs/stub_transport_backend.cpp | 64 ++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 runtime/tests/Test_Transport.cpp create mode 100644 runtime/tests/stubs/stub_transport_backend.cpp diff --git a/runtime/tests/CMakeLists.txt b/runtime/tests/CMakeLists.txt index bd9f9ba4d0..3d92940200 100644 --- a/runtime/tests/CMakeLists.txt +++ b/runtime/tests/CMakeLists.txt @@ -166,3 +166,23 @@ target_link_libraries(runner_tests_rsdecomp_runtime PRIVATE ) catch_discover_tests(runner_tests_rsdecomp_runtime) + +# Transport CAPI test suite +if(ENABLE_TRANSPORT) + add_library(stub_transport_backend SHARED stubs/stub_transport_backend.cpp) + target_include_directories(stub_transport_backend PRIVATE ${runtime_includes}) + set_property(TARGET stub_transport_backend PROPERTY POSITION_INDEPENDENT_CODE ON) + + add_executable(runner_tests_transport) + target_sources(runner_tests_transport PRIVATE Test_Transport.cpp) + target_link_libraries(runner_tests_transport PRIVATE + Catch2WithMain + catalyst_runtime_testing + rt_transport + ) + target_compile_definitions(runner_tests_transport PRIVATE + STUB_BACKEND_PATH="$") + add_dependencies(runner_tests_transport stub_transport_backend) + + catch_discover_tests(runner_tests_transport) +endif() diff --git a/runtime/tests/Test_Transport.cpp b/runtime/tests/Test_Transport.cpp new file mode 100644 index 0000000000..664a4dc00a --- /dev/null +++ b/runtime/tests/Test_Transport.cpp @@ -0,0 +1,84 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Unit tests for the transport CAPI session registry + +#include + +#include "catch2/catch_test_macros.hpp" + +#include "TransportCAPI.h" + +namespace { +constexpr std::int32_t kController = CATALYST_TRANSPORT_ROLE_CONTROLLER; +constexpr std::int32_t kCoprocessor = CATALYST_TRANSPORT_ROLE_COPROCESSOR; +constexpr const char *kStub = STUB_BACKEND_PATH; + +CatalystTransportSession *make(std::int32_t role, const char *key) +{ + return __catalyst__transport__create(kStub, "cfg", role, key); +} +} // namespace + +TEST_CASE("create registers a session resolvable by (role, key)", "[transport]") +{ + auto *s = make(kController, "reg_ctrl"); + REQUIRE(s != nullptr); + CHECK(__catalyst__transport__get_session(kController, "reg_ctrl") == s); + // Unknown key and mismatched role both miss. + CHECK(__catalyst__transport__get_session(kController, "reg_absent") == nullptr); + CHECK(__catalyst__transport__get_session(kCoprocessor, "reg_ctrl") == nullptr); + __catalyst__transport__destroy(s); + // destroy unregisters. + CHECK(__catalyst__transport__get_session(kController, "reg_ctrl") == nullptr); +} + +TEST_CASE("role disambiguates the same key", "[transport]") +{ + auto *ct = make(kController, "dis_key"); + auto *co = make(kCoprocessor, "dis_key"); + REQUIRE(ct != nullptr); + REQUIRE(co != nullptr); + REQUIRE(ct != co); + CHECK(__catalyst__transport__get_session(kController, "dis_key") == ct); + CHECK(__catalyst__transport__get_session(kCoprocessor, "dis_key") == co); + __catalyst__transport__destroy(ct); + __catalyst__transport__destroy(co); +} + +TEST_CASE("an empty key is not registered", "[transport]") +{ + auto *s = make(kCoprocessor, ""); + REQUIRE(s != nullptr); + CHECK(__catalyst__transport__get_session(kCoprocessor, "") == nullptr); + __catalyst__transport__destroy(s); +} + +TEST_CASE("re-create under the same key overwrites", "[transport]") +{ + auto *s1 = make(kController, "ovr_key"); + auto *s2 = make(kController, "ovr_key"); + REQUIRE(s1 != s2); + CHECK(__catalyst__transport__get_session(kController, "ovr_key") == s2); + __catalyst__transport__destroy(s1); + // s1 was already overwritten, so s2 remains resolvable until its own destroy. + CHECK(__catalyst__transport__get_session(kController, "ovr_key") == s2); + __catalyst__transport__destroy(s2); + CHECK(__catalyst__transport__get_session(kController, "ovr_key") == nullptr); +} + +TEST_CASE("get_session on an unregistered role/key returns null", "[transport]") +{ + CHECK(__catalyst__transport__get_session(kController, "never_created") == nullptr); +} diff --git a/runtime/tests/stubs/stub_transport_backend.cpp b/runtime/tests/stubs/stub_transport_backend.cpp new file mode 100644 index 0000000000..bea1840743 --- /dev/null +++ b/runtime/tests/stubs/stub_transport_backend.cpp @@ -0,0 +1,64 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A no-op transport backend for Test_Transport: implements the controller/coprocessor factory ABI +// with sessions whose methods do nothing, so the transport CAPI (create/get_session/registry, +// argument plumbing) can be unit-tested without a NIC. + +#include +#include + +#include "Transport.hpp" +#include "TransportBackend.h" + +using namespace catalyst::transport; + +namespace { + +struct StubController : ControllerSession { + std::uint64_t slot = 0; + int connect(const ConnectInfo &) override { return 0; } + MemRegion alloc_memory(std::size_t, MemKind) override { return {}; } + PeerRef exchange_keys(const MemRegion &) override { return {}; } + void establish_channel(const ChannelDesc &, const MemRegion &, const PeerRef &) override {} + void start() override {} + int collect(void *const *, const std::uint64_t *, std::size_t) override { return 0; } + void stop() override {} + void commit_work_item(std::uint32_t, std::uint64_t, std::uint64_t) override {} + int kick(std::uint32_t) override { return 0; } + void *data_slot() override { return &slot; } +}; + +struct StubCoprocessor : CoprocessorSession { + int connect(const ConnectInfo &) override { return 0; } + MemRegion alloc_memory(std::size_t, MemKind) override { return {}; } + PeerRef exchange_keys(const MemRegion &) override { return {}; } + void establish_channel(const ChannelDesc &, const MemRegion &, const PeerRef &) override {} + void start() override {} + int collect(void *const *, const std::uint64_t *, std::size_t) override { return 0; } + void stop() override {} + void set_coprocessor_fn(CoprocessorFn, void *) override {} +}; + +} // namespace + +extern "C" ControllerSession *CatalystTransportControllerFactory(const char *) +{ + return new StubController(); +} + +extern "C" CoprocessorSession *CatalystTransportCoprocessorFactory(const char *) +{ + return new StubCoprocessor(); +} From 61d3a0bbd8f19dbeb94eeb736c6e653564f3be4b Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Fri, 24 Jul 2026 11:57:28 -0400 Subject: [PATCH 39/57] format --- runtime/include/TransportCAPI.h | 5 +++-- runtime/lib/transport/TransportCAPI.cpp | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/runtime/include/TransportCAPI.h b/runtime/include/TransportCAPI.h index cf05791f6e..08a5138c78 100644 --- a/runtime/include/TransportCAPI.h +++ b/runtime/include/TransportCAPI.h @@ -53,14 +53,15 @@ enum { CatalystTransportSession *__catalyst__transport__create(const char *backend_lib, const char *config, int32_t role, const char *key); -// Resolve the live session registered at create under (`role`, `key`). +// Resolve the live session registered at create under (`role`, `key`). // Lets a session brought up in one function be used in another. CatalystTransportSession *__catalyst__transport__get_session(int32_t role, const char *key); // Bring-up. The peer region learned in exchange_keys is kept inside the session, so // establish_channel takes no peer handle. The *_async variants run on a worker thread and return a // token to await with barrier. -int __catalyst__transport__connect(CatalystTransportSession *s, const char *peer, uint16_t oob_port); +int __catalyst__transport__connect(CatalystTransportSession *s, const char *peer, + uint16_t oob_port); int64_t __catalyst__transport__connect_async(CatalystTransportSession *s, const char *peer, uint16_t oob_port); int __catalyst__transport__exchange_keys(CatalystTransportSession *s); diff --git a/runtime/lib/transport/TransportCAPI.cpp b/runtime/lib/transport/TransportCAPI.cpp index 92d6565c03..1ec26a5c32 100644 --- a/runtime/lib/transport/TransportCAPI.cpp +++ b/runtime/lib/transport/TransportCAPI.cpp @@ -49,7 +49,7 @@ struct CatalystTransportSession { TransportSession *sess = nullptr; // heap-allocated by the backend factory MemRegion reply; // local region advertised in exchange_keys bool reply_ready = false; - PeerRef peer; // peer region learned in exchange_keys + PeerRef peer; // peer region learned in exchange_keys bool peer_ready = false; }; From a14e1fda91419887956ba2b84996e33053215c13 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Fri, 24 Jul 2026 12:02:14 -0400 Subject: [PATCH 40/57] add tests --- runtime/tests/Test_Transport.cpp | 58 +++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/runtime/tests/Test_Transport.cpp b/runtime/tests/Test_Transport.cpp index 664a4dc00a..512fc3350e 100644 --- a/runtime/tests/Test_Transport.cpp +++ b/runtime/tests/Test_Transport.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Unit tests for the transport CAPI session registry +// Unit tests for the transport CAPI session registry and per-call behavior #include @@ -82,3 +82,59 @@ TEST_CASE("get_session on an unregistered role/key returns null", "[transport]") { CHECK(__catalyst__transport__get_session(kController, "never_created") == nullptr); } + +TEST_CASE("set_coprocessor_fn: an empty symbol binds the built-in echo", "[transport]") +{ + auto *s = make(kCoprocessor, ""); + REQUIRE(s != nullptr); + CHECK(__catalyst__transport__set_coprocessor_fn(s, "") == CATALYST_TRANSPORT_OK); + CHECK(__catalyst__transport__set_coprocessor_fn(s, nullptr) == CATALYST_TRANSPORT_OK); + __catalyst__transport__destroy(s); +} + +TEST_CASE("set_coprocessor_fn: an unresolved symbol is an error", "[transport]") +{ + auto *s = make(kCoprocessor, ""); + REQUIRE(s != nullptr); + CHECK(__catalyst__transport__set_coprocessor_fn(s, "catalyst_no_such_symbol_xyz") == + CATALYST_TRANSPORT_ERR); + __catalyst__transport__destroy(s); +} + +TEST_CASE("set_coprocessor_fn on a controller session is an error", "[transport]") +{ + auto *s = make(kController, ""); + REQUIRE(s != nullptr); + CHECK(__catalyst__transport__set_coprocessor_fn(s, "") == CATALYST_TRANSPORT_ERR); + __catalyst__transport__destroy(s); +} + +TEST_CASE("null session arguments are rejected without crashing", "[transport]") +{ + CHECK(__catalyst__transport__connect(nullptr, "127.0.0.1", 0) == CATALYST_TRANSPORT_ERR); + CHECK(__catalyst__transport__exchange_keys(nullptr) == CATALYST_TRANSPORT_ERR); + CHECK(__catalyst__transport__establish_channel(nullptr, "cpu_verbs") == CATALYST_TRANSPORT_ERR); + CHECK(__catalyst__transport__set_coprocessor_fn(nullptr, "") == CATALYST_TRANSPORT_ERR); + CHECK(__catalyst__transport__commit_work_item(nullptr, 0, 0, 0) == CATALYST_TRANSPORT_ERR); + CHECK(__catalyst__transport__kick(nullptr, 0) == CATALYST_TRANSPORT_ERR); + std::uint8_t buf[4] = {}; + CHECK(__catalyst__transport__collect(nullptr, buf, sizeof(buf)) == CATALYST_TRANSPORT_ERR); + CHECK(__catalyst__transport__data_slot(nullptr) == nullptr); + CHECK(__catalyst__transport__last_rtt_ns(nullptr) == 0); + // The void entry points must simply not crash on null. + __catalyst__transport__start(nullptr); + __catalyst__transport__stop(nullptr); + __catalyst__transport__destroy(nullptr); + SUCCEED(); +} + +TEST_CASE("commit_work_item rejects a reply larger than the provisioned region", "[transport]") +{ + auto *s = make(kController, ""); + REQUIRE(s != nullptr); + // exchange_keys provisions the local reply region (the stub reports a zero-size region). + REQUIRE(__catalyst__transport__exchange_keys(s) == CATALYST_TRANSPORT_OK); + CHECK(__catalyst__transport__commit_work_item(s, 0, 0, 1) == CATALYST_TRANSPORT_ERR); + CHECK(__catalyst__transport__commit_work_item(s, 0, 0, 0) == CATALYST_TRANSPORT_OK); + __catalyst__transport__destroy(s); +} From 747c0ed20ae7966576b1d410b924189b7e3bfadf Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Fri, 24 Jul 2026 12:10:02 -0400 Subject: [PATCH 41/57] format --- .../Transport/Transforms/TransportToLLVM.cpp | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/mlir/lib/Transport/Transforms/TransportToLLVM.cpp b/mlir/lib/Transport/Transforms/TransportToLLVM.cpp index 9cc7bf609e..ce5e1698a2 100644 --- a/mlir/lib/Transport/Transforms/TransportToLLVM.cpp +++ b/mlir/lib/Transport/Transforms/TransportToLLVM.cpp @@ -15,13 +15,13 @@ // Lower the `transport` dialect to `llvm.call`s on the __catalyst__transport__* // CAPI (runtime/include/TransportCAPI.h). +#include "llvm/ADT/Twine.h" #include "mlir/Conversion/LLVMCommon/TypeConverter.h" #include "mlir/Dialect/LLVMIR/FunctionCallUtils.h" #include "mlir/Dialect/LLVMIR/LLVMDialect.h" #include "mlir/IR/BuiltinOps.h" #include "mlir/Pass/Pass.h" #include "mlir/Transforms/DialectConversion.h" -#include "llvm/ADT/Twine.h" #include "Transport/IR/TransportOps.h" #include "Transport/Transforms/Passes.h" @@ -67,7 +67,6 @@ Value constInt(ConversionPatternRewriter &rewriter, Location loc, Type ty, int64 return LLVM::ConstantOp::create(rewriter, loc, ty, rewriter.getIntegerAttr(ty, v)); } - // From a lowered 1-D memref descriptor (an LLVM struct), extract the aligned data // pointer and the buffer's size in bytes (num elements * element byte width). std::pair memrefPtrAndBytes(ConversionPatternRewriter &rewriter, Location loc, @@ -76,8 +75,7 @@ std::pair memrefPtrAndBytes(ConversionPatternRewriter &rewriter, L Value ptr = LLVM::ExtractValueOp::create(rewriter, loc, descriptor, ArrayRef{1}); Value nelem = LLVM::ExtractValueOp::create(rewriter, loc, descriptor, ArrayRef{3, 0}); Type elemTy = memTy.getElementType(); - int64_t elemBytes = - isa(elemTy) ? 8 : (elemTy.getIntOrFloatBitWidth() + 7) / 8; + int64_t elemBytes = isa(elemTy) ? 8 : (elemTy.getIntOrFloatBitWidth() + 7) / 8; Value ebytes = constInt(rewriter, loc, i64Ty(rewriter.getContext()), elemBytes); Value bytes = LLVM::MulOp::create(rewriter, loc, nelem, ebytes); return {ptr, bytes}; @@ -251,7 +249,8 @@ struct CollectLowering : public OpConversionPattern { auto *ctx = op.getContext(); ModuleOp mod = moduleOf(op); if (!op.getDest()) - return rewriter.notifyMatchFailure(op, "collect must be bufferized (dest-passing form)"); + return rewriter.notifyMatchFailure(op, + "collect must be bufferized (dest-passing form)"); auto memTy = cast(op.getDest().getType()); auto [dstPtr, bytes] = memrefPtrAndBytes(rewriter, op.getLoc(), adaptor.getDest(), memTy); emitCall(rewriter, op.getLoc(), mod, "__catalyst__transport__collect", @@ -267,9 +266,9 @@ struct LastRttLowering : public OpConversionPattern { LogicalResult matchAndRewrite(LastRttNsOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override { - Value r = emitCall(rewriter, op.getLoc(), moduleOf(op), - "__catalyst__transport__last_rtt_ns", {ptrTy(op.getContext())}, - i64Ty(op.getContext()), {adaptor.getSession()}); + Value r = + emitCall(rewriter, op.getLoc(), moduleOf(op), "__catalyst__transport__last_rtt_ns", + {ptrTy(op.getContext())}, i64Ty(op.getContext()), {adaptor.getSession()}); rewriter.replaceOp(op, r); return success(); } @@ -327,8 +326,8 @@ struct ConvertTransportToLLVMPass RewritePatternSet patterns(ctx); patterns.add(tc, ctx); + SetCoprocessorFnLowering, CommitWorkItemLowering, KickLowering, + CollectLowering, LastRttLowering, GetSessionLowering>(tc, ctx); patterns.add>(tc, ctx, "__catalyst__transport__start"); patterns.add>(tc, ctx, "__catalyst__transport__stop"); patterns.add>(tc, ctx, "__catalyst__transport__destroy"); From fb3c784d95b6cd73f68c6e128f4fd7251202f3a5 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Fri, 24 Jul 2026 12:22:55 -0400 Subject: [PATCH 42/57] update namespace and test --- runtime/lib/transport/CMakeLists.txt | 18 +++ runtime/lib/transport/common/CMakeLists.txt | 20 +++ .../lib/transport/common/CompletionQueue.cpp | 18 ++- .../lib/transport/common/CompletionQueue.hpp | 18 ++- runtime/lib/transport/common/Context.cpp | 27 +++- runtime/lib/transport/common/Context.hpp | 18 ++- runtime/lib/transport/common/Decoder.hpp | 42 ------ .../lib/transport/common/DecoderPlugin.cpp | 63 --------- .../lib/transport/common/DecoderPlugin.hpp | 65 ---------- runtime/lib/transport/common/Error.hpp | 20 ++- runtime/lib/transport/common/Handshake.hpp | 20 ++- runtime/lib/transport/common/MemoryRegion.cpp | 18 ++- runtime/lib/transport/common/MemoryRegion.hpp | 18 ++- runtime/lib/transport/common/OobSocket.cpp | 18 ++- runtime/lib/transport/common/OobSocket.hpp | 18 ++- .../lib/transport/common/ProtectionDomain.cpp | 18 ++- .../lib/transport/common/ProtectionDomain.hpp | 18 ++- runtime/lib/transport/common/QpState.hpp | 18 ++- runtime/lib/transport/common/QueuePair.cpp | 20 ++- runtime/lib/transport/common/QueuePair.hpp | 18 ++- runtime/lib/transport/common/WireProtocol.hpp | 18 ++- .../transport/common/test/Test_Decoder.cpp | 22 ---- .../lib/transport/cpu_verbs/CMakeLists.txt | 35 +++++ .../transport/cpu_verbs/CpuBackendConfig.hpp | 52 ++++++++ .../cpu_verbs/base/CpuSessionBase.cpp | 29 +++-- .../cpu_verbs/base/CpuSessionBase.hpp | 20 ++- .../controller/CpuControllerFactory.cpp | 32 +++++ .../controller/CpuControllerSession.cpp | 30 +++-- .../controller/CpuControllerSession.hpp | 25 +++- .../coprocessor/CpuCoprocessorFactory.cpp | 35 +++++ .../coprocessor/CpuCoprocessorSession.cpp | 20 ++- .../coprocessor/CpuCoprocessorSession.hpp | 23 +++- .../steane_decoder_fn.cpp} | 38 +++--- .../cpu_verbs/cpu_libibverbs_main.cpp | 94 -------------- .../cpu_verbs/cpu_verbs_selftest.cpp | 121 ++++++++++++++++++ .../lib/transport/cpu_verbs/run_loopback.sh | 17 ++- runtime/tests/CMakeLists.txt | 28 ++++ .../Test_TransportCommon.cpp} | 16 ++- .../Test_TransportCpuVerbs.cpp} | 38 +++--- .../Test_TransportWireProtocol.cpp} | 16 ++- 40 files changed, 804 insertions(+), 398 deletions(-) create mode 100644 runtime/lib/transport/common/CMakeLists.txt delete mode 100644 runtime/lib/transport/common/Decoder.hpp delete mode 100644 runtime/lib/transport/common/DecoderPlugin.cpp delete mode 100644 runtime/lib/transport/common/DecoderPlugin.hpp delete mode 100644 runtime/lib/transport/common/test/Test_Decoder.cpp create mode 100644 runtime/lib/transport/cpu_verbs/CMakeLists.txt create mode 100644 runtime/lib/transport/cpu_verbs/CpuBackendConfig.hpp create mode 100644 runtime/lib/transport/cpu_verbs/controller/CpuControllerFactory.cpp create mode 100644 runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorFactory.cpp rename runtime/lib/transport/cpu_verbs/coprocessor/{decoders/steane_plugin.cpp => coprocessor_functions/steane_decoder_fn.cpp} (50%) delete mode 100644 runtime/lib/transport/cpu_verbs/cpu_libibverbs_main.cpp create mode 100644 runtime/lib/transport/cpu_verbs/cpu_verbs_selftest.cpp rename runtime/{lib/transport/common/test/Test_Common.cpp => tests/Test_TransportCommon.cpp} (66%) rename runtime/{lib/transport/cpu_verbs/test/Test_CpuLibibverbs.cpp => tests/Test_TransportCpuVerbs.cpp} (86%) rename runtime/{lib/transport/common/test/Test_WireProtocol.cpp => tests/Test_TransportWireProtocol.cpp} (58%) diff --git a/runtime/lib/transport/CMakeLists.txt b/runtime/lib/transport/CMakeLists.txt index b2cb3a38c1..e1b1a521c9 100644 --- a/runtime/lib/transport/CMakeLists.txt +++ b/runtime/lib/transport/CMakeLists.txt @@ -1,3 +1,14 @@ +############################################### +# transport backends: shared dependencies # +############################################### + +find_package(Threads REQUIRED) +find_library(IBVERBS_LIB ibverbs) +if(NOT IBVERBS_LIB) + message(FATAL_ERROR + "ENABLE_TRANSPORT requires libibverbs (e.g. libibverbs-dev / rdma-core), but it was not found.") +endif() + ############################################### # library rt_transport # ############################################### @@ -17,3 +28,10 @@ target_link_libraries(rt_transport PRIVATE ) set_property(TARGET rt_transport PROPERTY POSITION_INDEPENDENT_CODE ON) + +############################################### +# transport backends # +############################################### + +add_subdirectory(common) +add_subdirectory(cpu_verbs) diff --git a/runtime/lib/transport/common/CMakeLists.txt b/runtime/lib/transport/common/CMakeLists.txt new file mode 100644 index 0000000000..f4dfb8d961 --- /dev/null +++ b/runtime/lib/transport/common/CMakeLists.txt @@ -0,0 +1,20 @@ +############################################### +# library transport_common +# Device-agnostic RDMA verbs primitives +# shared by all transport backends +############################################### + +add_library(transport_common STATIC + CompletionQueue.cpp + Context.cpp + MemoryRegion.cpp + OobSocket.cpp + ProtectionDomain.cpp + QueuePair.cpp +) + +target_include_directories(transport_common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + +target_link_libraries(transport_common PUBLIC ${IBVERBS_LIB} Threads::Threads) + +set_property(TARGET transport_common PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/runtime/lib/transport/common/CompletionQueue.cpp b/runtime/lib/transport/common/CompletionQueue.cpp index 2cdf92b044..6cf2b09982 100644 --- a/runtime/lib/transport/common/CompletionQueue.cpp +++ b/runtime/lib/transport/common/CompletionQueue.cpp @@ -1,10 +1,24 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include "CompletionQueue.hpp" #include #include "Error.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { CompletionQueue::CompletionQueue(std::shared_ptr ctx, int depth) : ctx_(std::move(ctx)) { cq_ = ibv_create_cq(ctx_->get(), depth, nullptr, nullptr, 0); @@ -16,4 +30,4 @@ CompletionQueue::~CompletionQueue() ibv_destroy_cq(cq_); } ibv_cq *CompletionQueue::get() const { return cq_; } -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/CompletionQueue.hpp b/runtime/lib/transport/common/CompletionQueue.hpp index 13babf311c..f48daee25c 100644 --- a/runtime/lib/transport/common/CompletionQueue.hpp +++ b/runtime/lib/transport/common/CompletionQueue.hpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include @@ -5,7 +19,7 @@ #include "Context.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { /** * @class CompletionQueue class. @@ -25,4 +39,4 @@ class CompletionQueue { std::shared_ptr ctx_; ibv_cq *cq_ = nullptr; }; -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/Context.cpp b/runtime/lib/transport/common/Context.cpp index b47c08fdda..dceb8c73ae 100644 --- a/runtime/lib/transport/common/Context.cpp +++ b/runtime/lib/transport/common/Context.cpp @@ -1,9 +1,24 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include "Context.hpp" #include +#include #include "Error.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { Context::Context(const std::string &dev_name) { int n = 0; @@ -13,8 +28,14 @@ Context::Context(const std::string &dev_name) [&](ibv_device *d) { return dev_name == ibv_get_device_name(d); }); ibv_device *dev = (it != devs + n) ? *it : nullptr; if (!dev) { + std::string avail; + for (int i = 0; i < n; ++i) { + avail += (i ? ", " : ""); + avail += ibv_get_device_name(devs[i]); + } ibv_free_device_list(devs); - RDMA_FAIL("device %s not found", dev_name.c_str()); + RDMA_FAIL("device %s not found (available: %s)", dev_name.c_str(), + avail.empty() ? "" : avail.c_str()); } ctx_ = ibv_open_device(dev); ibv_free_device_list(devs); @@ -38,4 +59,4 @@ ibv_gid Context::gid(std::uint8_t port, int idx) const RDMA_CHECK(ibv_query_gid(ctx_, port, idx, &gid) == 0, "ibv_query_gid(%u,%d)", port, idx); return gid; } -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/Context.hpp b/runtime/lib/transport/common/Context.hpp index 2e9eee6517..34494e1c2b 100644 --- a/runtime/lib/transport/common/Context.hpp +++ b/runtime/lib/transport/common/Context.hpp @@ -1,10 +1,24 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include #include #include -namespace rdma::devices::common { +namespace catalyst::transport::common { /** * @class Context * @brief RAII wrapper for an RDMA device context (`ibv_context`). @@ -44,4 +58,4 @@ class Context { private: ibv_context *ctx_ = nullptr; }; -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/Decoder.hpp b/runtime/lib/transport/common/Decoder.hpp deleted file mode 100644 index aac578349b..0000000000 --- a/runtime/lib/transport/common/Decoder.hpp +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once -#include -#include -#include -#include - -#include "DecoderPlugin.hpp" - -namespace rdma::devices::common { - -// A per-shot decode compute: reads the syndrome from `in` (in_len bytes) and -// writes the correction into `out` (out_len bytes), in place on the caller's -// buffers. Must not throw. -class Decoder { - public: - virtual ~Decoder() = default; - virtual void run(const void *in, std::size_t in_len, void *out, std::size_t out_len) = 0; -}; - -// Passthrough: out = in for min(in_len, out_len) bytes. Default / self-test. -class EchoDecoder : public Decoder { - public: - void run(const void *in, std::size_t in_len, void *out, std::size_t out_len) override - { - std::memcpy(out, in, std::min(in_len, out_len)); - } -}; - -// Adapts a dlopen'd DecoderPlugin (see DecoderPlugin.hpp) to the Decoder API. -class PluginDecoder : public Decoder { - public: - explicit PluginDecoder(std::unique_ptr plugin) : plugin_(std::move(plugin)) {} - void run(const void *in, std::size_t in_len, void *out, std::size_t out_len) override - { - plugin_->fn()(plugin_->ctx(), in, in_len, out, out_len); - } - - private: - std::unique_ptr plugin_; -}; - -} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/DecoderPlugin.cpp b/runtime/lib/transport/common/DecoderPlugin.cpp deleted file mode 100644 index 5f7f32e695..0000000000 --- a/runtime/lib/transport/common/DecoderPlugin.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#include "DecoderPlugin.hpp" - -#include -#include - -#include "Error.hpp" - -namespace rdma::devices::common { - -DecoderPlugin::DecoderPlugin(const std::string &path) -{ - handle_ = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); - RDMA_CHECK(handle_, "dlopen(%s): %s", path.c_str(), dlerror()); - fn_ = reinterpret_cast(dlsym(handle_, "decode")); - if (!fn_) { - const char *e = dlerror(); - dlclose(handle_); - handle_ = nullptr; - RDMA_FAIL("decoder %s missing 'decode' symbol: %s", path.c_str(), e ? e : "?"); - } - // Optional ctx lifecycle (a pure decoder omits both). - auto create = reinterpret_cast(dlsym(handle_, "decoder_create")); - destroy_ = reinterpret_cast(dlsym(handle_, "decoder_destroy")); - if (create) { - ctx_ = create(); - } -} - -void DecoderPlugin::reset() noexcept -{ - if (destroy_ && ctx_) { - destroy_(ctx_); - } - if (handle_) { - dlclose(handle_); - } - handle_ = nullptr; - fn_ = nullptr; - ctx_ = nullptr; - destroy_ = nullptr; -} - -DecoderPlugin::~DecoderPlugin() { reset(); } - -DecoderPlugin::DecoderPlugin(DecoderPlugin &&other) noexcept - : handle_(std::exchange(other.handle_, nullptr)), fn_(std::exchange(other.fn_, nullptr)), - ctx_(std::exchange(other.ctx_, nullptr)), destroy_(std::exchange(other.destroy_, nullptr)) -{ -} - -DecoderPlugin &DecoderPlugin::operator=(DecoderPlugin &&other) noexcept -{ - if (this != &other) { - reset(); - handle_ = std::exchange(other.handle_, nullptr); - fn_ = std::exchange(other.fn_, nullptr); - ctx_ = std::exchange(other.ctx_, nullptr); - destroy_ = std::exchange(other.destroy_, nullptr); - } - return *this; -} - -} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/DecoderPlugin.hpp b/runtime/lib/transport/common/DecoderPlugin.hpp deleted file mode 100644 index ef3a79c561..0000000000 --- a/runtime/lib/transport/common/DecoderPlugin.hpp +++ /dev/null @@ -1,65 +0,0 @@ -#pragma once -#include -#include - -namespace rdma::devices::common { - -/** - * @class DecoderPlugin - * @brief RAII manager for a dynamically loaded decoder shared library. - * - * Handles the automatic loading (@c dlopen), symbol resolution, optional - * context lifecycle (@c decoder_create / @c decoder_destroy), and resource - * cleanup upon destruction or movement. - */ -class DecoderPlugin { - public: - /** - * @brief Function pointer signature for the decoding operation. - */ - using Fn = void (*)(void *ctx, const void *in, std::size_t in_len, void *out, - std::size_t out_len); - - /** - * @brief Loads the shared library and resolves the decoder symbols. - * @param path The filesystem path to the shared library (.so). - * @throw Throw runtime errors if loading or symbol resolution fails. - */ - explicit DecoderPlugin(const std::string &path); // dlopen + resolve; throws on failure - - /** - * @brief Destructor. Automatically releases context and unloads the - * library. - */ - ~DecoderPlugin(); - DecoderPlugin(DecoderPlugin &&o) noexcept; - DecoderPlugin &operator=(DecoderPlugin &&o) noexcept; - DecoderPlugin(const DecoderPlugin &) = delete; - DecoderPlugin &operator=(const DecoderPlugin &) = delete; - - /** - * @brief Retrieves the resolved decoding function pointer. - * @return The decoding function pointer, or @c nullptr if uninitialized. - */ - Fn fn() const noexcept { return fn_; } - - /** - * @brief Retrieves the optional plugin context pointer. - * @return Pointer to the internal context instance, or @c nullptr if none - * exists. - */ - void *ctx() const noexcept { return ctx_; } - - private: - /** - * @brief Safely releases all held resources and resets pointers to @c - * nullptr. - */ - void reset() noexcept; - void *handle_ = nullptr; // Dynamic library handle returned by dlopen. - Fn fn_ = nullptr; // Function pointer to the 'decode' symbol. - void *ctx_ = nullptr; // Optional plugin context instance. - void (*destroy_)(void *) = nullptr; -}; - -} // namespace rdma::devices::common diff --git a/runtime/lib/transport/common/Error.hpp b/runtime/lib/transport/common/Error.hpp index c9c3cb60bc..e64e2abdf1 100644 --- a/runtime/lib/transport/common/Error.hpp +++ b/runtime/lib/transport/common/Error.hpp @@ -1,10 +1,24 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include #include #include #include -namespace rdma::devices::common { +namespace catalyst::transport::common { class RdmaError : public std::runtime_error { public: using std::runtime_error::runtime_error; @@ -14,7 +28,7 @@ class RdmaError : public std::runtime_error { * Throw RdmaError with a preformatted message. */ [[noreturn]] inline void rdma_throw(const char *msg) { throw RdmaError(msg); } -} // namespace rdma::devices::common +} // namespace catalyst::transport::common // Unconditionally fail with "file:line: msg (errno=..)" context. #define RDMA_FAIL(...) \ @@ -24,7 +38,7 @@ class RdmaError : public std::runtime_error { char rdma_full_[512]; \ std::snprintf(rdma_full_, sizeof(rdma_full_), "%s:%d: %s (errno=%d: %s)", __FILE__, \ __LINE__, rdma_msg_, errno, std::strerror(errno)); \ - ::rdma::devices::common::rdma_throw(rdma_full_); \ + ::catalyst::transport::common::rdma_throw(rdma_full_); \ } while (0) // Throw RdmaError with file:line + errno when cond is false. diff --git a/runtime/lib/transport/common/Handshake.hpp b/runtime/lib/transport/common/Handshake.hpp index 2c3ad17b69..bb448b3483 100644 --- a/runtime/lib/transport/common/Handshake.hpp +++ b/runtime/lib/transport/common/Handshake.hpp @@ -1,7 +1,21 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include -namespace rdma::devices::common { +namespace catalyst::transport::common { /** * @struct QpInfo @@ -16,7 +30,7 @@ struct QpInfo { /** * @struct HandshakeMsg - * @brief Message exchanged once over the OOB TCP socket, after the MR exists, + * @brief Message exchanged once over the OOB TCP socket, after the MR exists, * so QP identity and MR handle are swapped together. */ struct HandshakeMsg { @@ -27,4 +41,4 @@ struct HandshakeMsg { std::uint32_t mtu_enum; // ibv_mtu enum }; -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/MemoryRegion.cpp b/runtime/lib/transport/common/MemoryRegion.cpp index 1e5b7a627d..c5c5c586ed 100644 --- a/runtime/lib/transport/common/MemoryRegion.cpp +++ b/runtime/lib/transport/common/MemoryRegion.cpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include "MemoryRegion.hpp" #include @@ -7,7 +21,7 @@ #include "Error.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { /** * @brief Register caller-owned host memory (borrowed; the region does not own @@ -92,4 +106,4 @@ std::size_t MemoryRegion::length() const noexcept { return mr_ ? mr_->length : 0 std::uint32_t MemoryRegion::lkey() const noexcept { return mr_ ? mr_->lkey : 0; } std::uint32_t MemoryRegion::rkey() const noexcept { return mr_ ? mr_->rkey : 0; } -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/MemoryRegion.hpp b/runtime/lib/transport/common/MemoryRegion.hpp index c3b2cb56ae..8f88f862da 100644 --- a/runtime/lib/transport/common/MemoryRegion.hpp +++ b/runtime/lib/transport/common/MemoryRegion.hpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include #include @@ -7,7 +21,7 @@ #include "ProtectionDomain.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { /** * @enum MemAccess flag. @@ -75,4 +89,4 @@ class MemoryRegion { // null for borrowed / dma-buf regions that own nothing. std::shared_ptr backing_buffer_; }; -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/OobSocket.cpp b/runtime/lib/transport/common/OobSocket.cpp index ccdf244123..6621b09717 100644 --- a/runtime/lib/transport/common/OobSocket.cpp +++ b/runtime/lib/transport/common/OobSocket.cpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include "OobSocket.hpp" #include @@ -13,7 +27,7 @@ #include "Error.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { namespace { void set_tcp_nodelay(int fd) @@ -102,4 +116,4 @@ void recv_exact(int fd, void *buf, std::size_t n) } } -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/OobSocket.hpp b/runtime/lib/transport/common/OobSocket.hpp index ce31997976..ca807b1c48 100644 --- a/runtime/lib/transport/common/OobSocket.hpp +++ b/runtime/lib/transport/common/OobSocket.hpp @@ -1,10 +1,24 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include #include #include #include -namespace rdma::devices::common { +namespace catalyst::transport::common { // RAII handle for a socket file descriptor. class FdGuard { @@ -56,4 +70,4 @@ FdGuard tcp_connect(const char *host, std::uint16_t port); void send_exact(int fd, const void *buf, std::size_t n); void recv_exact(int fd, void *buf, std::size_t n); -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/ProtectionDomain.cpp b/runtime/lib/transport/common/ProtectionDomain.cpp index 0ad4d40b61..89b2cc44e6 100644 --- a/runtime/lib/transport/common/ProtectionDomain.cpp +++ b/runtime/lib/transport/common/ProtectionDomain.cpp @@ -1,10 +1,24 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include "ProtectionDomain.hpp" #include #include "Error.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { ProtectionDomain::ProtectionDomain(std::shared_ptr ctx) : ctx_(std::move(ctx)) { pd_ = ibv_alloc_pd(ctx_->get()); @@ -16,4 +30,4 @@ ProtectionDomain::~ProtectionDomain() ibv_dealloc_pd(pd_); } ibv_pd *ProtectionDomain::get() const { return pd_; } -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/ProtectionDomain.hpp b/runtime/lib/transport/common/ProtectionDomain.hpp index 67e25b5ab2..c4ba7bd8c7 100644 --- a/runtime/lib/transport/common/ProtectionDomain.hpp +++ b/runtime/lib/transport/common/ProtectionDomain.hpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include @@ -5,7 +19,7 @@ #include "Context.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { /** * @class ProtectionDomain * @brief RAII wrapper for an `ibv_pd` resource, managing memory protection @@ -38,4 +52,4 @@ class ProtectionDomain { std::shared_ptr ctx_; // keeps the Context alive ibv_pd *pd_ = nullptr; // Low-level verbs protection domain handle. }; -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/QpState.hpp b/runtime/lib/transport/common/QpState.hpp index a5a310f047..0270c040ab 100644 --- a/runtime/lib/transport/common/QpState.hpp +++ b/runtime/lib/transport/common/QpState.hpp @@ -1,7 +1,21 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include "Error.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { enum class QpState { RESET, INIT, RTR, RTS, ERROR }; @@ -44,4 +58,4 @@ class BadTransition : public RdmaError { using RdmaError::RdmaError; }; -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/QueuePair.cpp b/runtime/lib/transport/common/QueuePair.cpp index fcb4eed0ab..147c118188 100644 --- a/runtime/lib/transport/common/QueuePair.cpp +++ b/runtime/lib/transport/common/QueuePair.cpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include "QueuePair.hpp" #include @@ -6,7 +20,7 @@ #include "Error.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { namespace { // RC QP tuning attributes for ibv_modify_qp (RTR/RTS). Encodings per the IB @@ -29,7 +43,7 @@ QueuePair::QueuePair(std::shared_ptr pd, std::shared_ptr(max_send_wr), - .max_recv_wr = 1, + .max_recv_wr = 4, .max_send_sge = 1, .max_recv_sge = 1, .max_inline_data = static_cast(max_inline), @@ -129,4 +143,4 @@ void QueuePair::to_rts(std::uint32_t sq_psn) "modify_to_rts"); } -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/QueuePair.hpp b/runtime/lib/transport/common/QueuePair.hpp index c29e5e7f1a..3f2698f53a 100644 --- a/runtime/lib/transport/common/QueuePair.hpp +++ b/runtime/lib/transport/common/QueuePair.hpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include #include @@ -8,7 +22,7 @@ #include "ProtectionDomain.hpp" #include "QpState.hpp" -namespace rdma::devices::common { +namespace catalyst::transport::common { class QueuePair { public: QueuePair(std::shared_ptr pd, std::shared_ptr send_cq, @@ -35,4 +49,4 @@ class QueuePair { ibv_qp *qp_ = nullptr; QpState state_ = QpState::RESET; }; -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/WireProtocol.hpp b/runtime/lib/transport/common/WireProtocol.hpp index aac5f058b3..5e68b045b6 100644 --- a/runtime/lib/transport/common/WireProtocol.hpp +++ b/runtime/lib/transport/common/WireProtocol.hpp @@ -1,8 +1,22 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include #include -namespace rdma::devices::common { +namespace catalyst::transport::common { // Ring size, identical on both sides; power of two so index is a mask. inline constexpr std::size_t K_RING_SLOTS = 256; @@ -43,4 +57,4 @@ static_assert(alignof(PayloadSlot) == 64, "PayloadSlot must be 64-B aligned"); // K_RING_SLOTS]. K_RING_SLOTS must be a power of two. inline constexpr std::size_t REGION_BYTES = K_RING_SLOTS * sizeof(PayloadSlot); -} // namespace rdma::devices::common +} // namespace catalyst::transport::common diff --git a/runtime/lib/transport/common/test/Test_Decoder.cpp b/runtime/lib/transport/common/test/Test_Decoder.cpp deleted file mode 100644 index 5c43b931bc..0000000000 --- a/runtime/lib/transport/common/test/Test_Decoder.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include - -#include - -#include "Decoder.hpp" - -using namespace rdma::devices::common; - -TEST_CASE("EchoDecoder copies min(in_len, out_len) low bytes", "[decoder]") -{ - const std::uint64_t in = 0x0123456789ABCDEFull; - std::uint64_t out = 0; - EchoDecoder d; - d.run(&in, 8, &out, 8); - REQUIRE(out == in); // full 8-byte echo - out = 0; - d.run(&in, 1, &out, 8); // 1-byte syndrome - REQUIRE(out == 0xEFull); - out = 0; - d.run(&in, 8, &out, 2); // 2-byte correction window - REQUIRE(out == 0xCDEFull); -} diff --git a/runtime/lib/transport/cpu_verbs/CMakeLists.txt b/runtime/lib/transport/cpu_verbs/CMakeLists.txt new file mode 100644 index 0000000000..56cf7e8d50 --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/CMakeLists.txt @@ -0,0 +1,35 @@ +############################################### +# CPU-verbs transport backend # +############################################### + +add_library(cpu_verbs_impl STATIC + base/CpuSessionBase.cpp + controller/CpuControllerSession.cpp + coprocessor/CpuCoprocessorSession.cpp +) +target_include_directories(cpu_verbs_impl PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} # CpuBackendConfig.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/base + ${CMAKE_CURRENT_SOURCE_DIR}/controller + ${CMAKE_CURRENT_SOURCE_DIR}/coprocessor + ${runtime_includes} # Transport.hpp, TransportBackend.h +) +target_link_libraries(cpu_verbs_impl PUBLIC transport_common) +set_property(TARGET cpu_verbs_impl PROPERTY POSITION_INDEPENDENT_CODE ON) + +# Controller library +add_library(catalyst_transport_cpu_controller SHARED controller/CpuControllerFactory.cpp) +target_link_libraries(catalyst_transport_cpu_controller PRIVATE cpu_verbs_impl) + +# Coprocessor library +add_library(catalyst_transport_cpu_coprocessor SHARED coprocessor/CpuCoprocessorFactory.cpp) +target_link_libraries(catalyst_transport_cpu_coprocessor PRIVATE cpu_verbs_impl) + +# Reference coprocessor function (Steane decode), built as a standalone shared lib +# exporting a CoprocessorFn symbol. +add_library(steane_coprocessor SHARED coprocessor/coprocessor_functions/steane_decoder_fn.cpp) +set_property(TARGET steane_coprocessor PROPERTY POSITION_INDEPENDENT_CODE ON) + +# Self-contained loopback application for testing +add_executable(cpu_verbs_selftest cpu_verbs_selftest.cpp) +target_link_libraries(cpu_verbs_selftest PRIVATE cpu_verbs_impl) diff --git a/runtime/lib/transport/cpu_verbs/CpuBackendConfig.hpp b/runtime/lib/transport/cpu_verbs/CpuBackendConfig.hpp new file mode 100644 index 0000000000..a50369a354 --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/CpuBackendConfig.hpp @@ -0,0 +1,52 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include +#include + +namespace catalyst::transport::cpu_verbs { + +// Construction parameters parsed from a backend factory `config` string +// ("key=value;..."). Connection parameters (peer/oob_port) are not here — they +// arrive later via connect(). Recognised keys: `dev`, `gid`. +struct CpuConfig { + std::string dev = "rxe0"; + int gid = 1; +}; + +inline CpuConfig parse_cpu_config(const std::string &config) +{ + CpuConfig cfg; + for (std::size_t pos = 0; pos < config.size();) { + const std::size_t sep = config.find(';', pos); + const std::size_t end = (sep == std::string::npos) ? config.size() : sep; + const std::string_view tok(config.data() + pos, end - pos); + if (const std::size_t eq = tok.find('='); eq != std::string_view::npos) { + const std::string_view key = tok.substr(0, eq); + const std::string val(tok.substr(eq + 1)); + if (key == "dev") + cfg.dev = val; + else if (key == "gid") + cfg.gid = std::atoi(val.c_str()); + } + if (sep == std::string::npos) + break; + pos = sep + 1; + } + return cfg; +} + +} // namespace catalyst::transport::cpu_verbs diff --git a/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.cpp b/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.cpp index c4ead091dd..355356f0d6 100644 --- a/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.cpp +++ b/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.cpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include "CpuSessionBase.hpp" #include @@ -11,9 +25,9 @@ #include "Handshake.hpp" #include "WireProtocol.hpp" -namespace rdma::devices::cpu_libibverbs { +namespace catalyst::transport::cpu_verbs { using namespace catalyst::transport; -using namespace rdma::devices::common; +using namespace catalyst::transport::common; namespace { // RDMA device port; rxe0 is single-port -> 1. @@ -205,7 +219,7 @@ void CpuSessionBase::start() engine_ = std::jthread(body); } -int CpuSessionBase::collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) +int CpuSessionBase::collect(void *replies, std::uint64_t bytes) { while (completed_.load(std::memory_order_acquire) == 0) { if (failed_.load(std::memory_order_acquire)) @@ -219,11 +233,10 @@ int CpuSessionBase::collect(void *const *outputs, const std::uint64_t *output_by // Stopped before any round completed -> no data (non-exceptional). if (completed_.load(std::memory_order_acquire) == 0) return -1; - if (n > 0 && outputs && outputs[0]) { + if (replies) { const std::uint64_t w = last_word_.load(std::memory_order_relaxed); - const std::size_t nb = - output_bytes ? std::min(output_bytes[0], sizeof(w)) : sizeof(w); - std::memcpy(outputs[0], &w, nb); + const std::size_t nb = std::min(bytes, sizeof(w)); + std::memcpy(replies, &w, nb); } return 0; } @@ -236,4 +249,4 @@ void CpuSessionBase::stop() } } -} // namespace rdma::devices::cpu_libibverbs +} // namespace catalyst::transport::cpu_verbs diff --git a/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.hpp b/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.hpp index 3053445d26..7b50aab953 100644 --- a/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.hpp +++ b/runtime/lib/transport/cpu_verbs/base/CpuSessionBase.hpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include #include @@ -18,7 +32,7 @@ #include "Transport.hpp" #include "WireProtocol.hpp" -namespace rdma::devices::cpu_libibverbs { +namespace catalyst::transport::cpu_verbs { using namespace catalyst::transport; // Shared lifecycle for coprocessor and controller roles. @@ -33,7 +47,7 @@ class CpuSessionBase : public TransportSession { void establish_channel(const ChannelDesc &desc, const MemRegion &local, const PeerRef &peer) override; void start() override; - int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) override; + int collect(void *replies, std::uint64_t bytes) override; void stop() override; protected: @@ -73,4 +87,4 @@ class CpuSessionBase : public TransportSession { std::jthread engine_; }; -} // namespace rdma::devices::cpu_libibverbs +} // namespace catalyst::transport::cpu_verbs diff --git a/runtime/lib/transport/cpu_verbs/controller/CpuControllerFactory.cpp b/runtime/lib/transport/cpu_verbs/controller/CpuControllerFactory.cpp new file mode 100644 index 0000000000..afa787f159 --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/controller/CpuControllerFactory.cpp @@ -0,0 +1,32 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Plugin entry point for the CPU-verbs controller backend. The runtime dlopen's +// this .so and resolves CatalystTransportControllerFactory (see TransportBackend.h). + +#include + +#include "CpuBackendConfig.hpp" +#include "CpuControllerSession.hpp" +#include "TransportBackend.h" + +namespace { +catalyst::transport::ControllerSession *make_cpu_controller(const std::string &config) +{ + const auto cfg = catalyst::transport::cpu_verbs::parse_cpu_config(config); + return new catalyst::transport::cpu_verbs::CpuControllerSession(cfg.dev, cfg.gid); +} +} // namespace + +GENERATE_TRANSPORT_CONTROLLER_FACTORY(CatalystTransportController, make_cpu_controller) diff --git a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp index 95328a53fc..d75662e8c7 100644 --- a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp +++ b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include "CpuControllerSession.hpp" #include @@ -7,8 +21,8 @@ #include "WireProtocol.hpp" -namespace rdma::devices::cpu_libibverbs { -using namespace rdma::devices::common; +namespace catalyst::transport::cpu_verbs { +using namespace catalyst::transport::common; namespace { std::uint64_t now_ns() @@ -63,8 +77,7 @@ int CpuControllerSession::Impl::kick(std::uint32_t /*work_item_idx*/) return 0; } -int CpuControllerSession::Impl::collect(void *const *outputs, const std::uint64_t *output_bytes, - std::size_t n) +int CpuControllerSession::Impl::collect(void *replies, std::uint64_t bytes) { std::stop_token none; // blocking wait for this round's reply Payload *r = poll_message_arrival(next_recv_, none); @@ -72,12 +85,11 @@ int CpuControllerSession::Impl::collect(void *const *outputs, const std::uint64_ return -1; rtt_ns_ = now_ns() - kick_ns_; ++next_recv_; - if (n > 0 && outputs && outputs[0]) { - const std::size_t cap = output_bytes ? output_bytes[0] : out_bytes_; - const std::size_t nb = std::min(cap, sizeof(r->value)); - std::memcpy(outputs[0], &r->value, nb); + if (replies) { + const std::size_t nb = std::min(bytes, sizeof(r->value)); + std::memcpy(replies, &r->value, nb); } return 0; } -} // namespace rdma::devices::cpu_libibverbs +} // namespace catalyst::transport::cpu_verbs diff --git a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp index 2844c522a3..8b5569cccd 100644 --- a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp +++ b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp @@ -1,9 +1,23 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include #include "CpuSessionBase.hpp" -namespace rdma::devices::cpu_libibverbs { +namespace catalyst::transport::cpu_verbs { // Controller role: caller-driven. The caller commits a work item // (I/O sizes), writes the outbound payload into data_slot(), kick()s one round, @@ -27,10 +41,7 @@ class CpuControllerSession : public ControllerSession { base_.establish_channel(desc, local, peer); } void start() override { base_.start(); } - int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) override - { - return base_.collect(outputs, output_bytes, n); - } + int collect(void *replies, std::uint64_t bytes) override { return base_.collect(replies, bytes); } void stop() override { base_.stop(); } std::uint64_t last_rtt_ns() const override { return base_.last_rtt_ns(); } @@ -54,7 +65,7 @@ class CpuControllerSession : public ControllerSession { void start() override; void stop() override; - int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) override; + int collect(void *replies, std::uint64_t bytes) override; std::uint64_t last_rtt_ns() const override { return rtt_ns_; } void commit_work_item(std::uint32_t work_item_idx, std::uint64_t in_bytes, @@ -76,4 +87,4 @@ class CpuControllerSession : public ControllerSession { Impl base_; }; -} // namespace rdma::devices::cpu_libibverbs +} // namespace catalyst::transport::cpu_verbs diff --git a/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorFactory.cpp b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorFactory.cpp new file mode 100644 index 0000000000..50fe36cfbc --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorFactory.cpp @@ -0,0 +1,35 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Plugin entry point for the CPU-verbs coprocessor backend. A loader dlopen's +// this .so and resolves CatalystTransportCoprocessorFactory (see TransportBackend.h). +// The coprocessor function is bound after construction via set_coprocessor_fn() +// (by the coprocessor-side harness); a freshly built session defaults to the +// built-in echo. + +#include + +#include "CpuBackendConfig.hpp" +#include "CpuCoprocessorSession.hpp" +#include "TransportBackend.h" + +namespace { +catalyst::transport::CoprocessorSession *make_cpu_coprocessor(const std::string &config) +{ + const auto cfg = catalyst::transport::cpu_verbs::parse_cpu_config(config); + return new catalyst::transport::cpu_verbs::CpuCoprocessorSession(cfg.dev, cfg.gid); +} +} // namespace + +GENERATE_TRANSPORT_COPROCESSOR_FACTORY(CatalystTransportCoprocessor, make_cpu_coprocessor) diff --git a/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.cpp b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.cpp index 7a2fc02e08..c31e97f3a5 100644 --- a/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.cpp +++ b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.cpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include "CpuCoprocessorSession.hpp" #include @@ -5,8 +19,8 @@ #include "WireProtocol.hpp" -namespace rdma::devices::cpu_libibverbs { -using namespace rdma::devices::common; +namespace catalyst::transport::cpu_verbs { +using namespace catalyst::transport::common; void CpuCoprocessorSession::set_coprocessor_fn(CoprocessorFn fn, void *ctx) { @@ -45,4 +59,4 @@ void CpuCoprocessorSession::Impl::run(std::stop_token st) reap(bwd_cq_->get(), signaled_outstanding, /*drain=*/true); } -} // namespace rdma::devices::cpu_libibverbs +} // namespace catalyst::transport::cpu_verbs diff --git a/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp index 0897153e25..60357a3642 100644 --- a/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp +++ b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp @@ -1,10 +1,24 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #pragma once #include #include #include "CpuSessionBase.hpp" -namespace rdma::devices::cpu_libibverbs { +namespace catalyst::transport::cpu_verbs { // Coprocessor role: receives messages, runs the coprocessor function, and // returns the result. The function is bound via set_coprocessor_fn; nullptr @@ -28,10 +42,7 @@ class CpuCoprocessorSession : public CoprocessorSession { base_.establish_channel(desc, local, peer); } void start() override { base_.start(); } - int collect(void *const *outputs, const std::uint64_t *output_bytes, std::size_t n) override - { - return base_.collect(outputs, output_bytes, n); - } + int collect(void *replies, std::uint64_t bytes) override { return base_.collect(replies, bytes); } void stop() override { base_.stop(); } void set_coprocessor_fn(CoprocessorFn fn, void *ctx) override; @@ -50,4 +61,4 @@ class CpuCoprocessorSession : public CoprocessorSession { Impl base_; }; -} // namespace rdma::devices::cpu_libibverbs +} // namespace catalyst::transport::cpu_verbs diff --git a/runtime/lib/transport/cpu_verbs/coprocessor/decoders/steane_plugin.cpp b/runtime/lib/transport/cpu_verbs/coprocessor/coprocessor_functions/steane_decoder_fn.cpp similarity index 50% rename from runtime/lib/transport/cpu_verbs/coprocessor/decoders/steane_plugin.cpp rename to runtime/lib/transport/cpu_verbs/coprocessor/coprocessor_functions/steane_decoder_fn.cpp index 7050733a78..383b416f10 100644 --- a/runtime/lib/transport/cpu_verbs/coprocessor/decoders/steane_plugin.cpp +++ b/runtime/lib/transport/cpu_verbs/coprocessor/coprocessor_functions/steane_decoder_fn.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. /** - * @file steane_plugin.cpp - * Defines a decoder for the [[7,1,3]] Steane code. + * @file + * A reference CoprocessorFn implementing a [[7,1,3]] Steane-code decode. */ #include @@ -21,23 +21,23 @@ #include /** - * @brief A hard-coded decoder for the [[7,1,3]] Steane code with a static - * Tanner graph mapping. + * @brief A hard-coded [[7,1,3]] Steane-code decode, exposed as a CoprocessorFn + * (see Transport.hpp) — the general "run this on the coprocessor" contract that + * supersedes the old decoder-plugin ABI. * - * @note The FTQC (Fault-Tolerant Quantum Computing) compilation - * pipeline processes and dispatches either X-check or Z-check syndromes - * independently per execution call, so each call carries a single 3-bit - * check. This behavior may be unified in future pipeline iterations. + * @note The FTQC (Fault-Tolerant Quantum Computing) compilation pipeline + * dispatches either X-check or Z-check syndromes independently per call, so each + * call carries a single 3-bit check. This may be unified in future iterations. * - * @param ctx Pointer to a common::Context instance. - * @param in Pointer to inbound syndrome measurements - * @param in_len The length of the inbound syndrome/message. - * @param out Pointer to the index of the detected error qubit. - * @param out_len The length of the outbound message. - * side. + * @param in Pointer to the inbound syndrome measurements. + * @param in_len Length of the inbound syndrome, in bytes. + * @param out Pointer to the outbound correction buffer. + * @param out_cap Capacity of the outbound buffer, in bytes. + * @param ctx Opaque context (unused). + * @return Number of bytes written to @p out. */ -extern "C" void decode(void * /*ctx*/, const void *in, std::size_t in_len, void *out, - std::size_t out_len) +extern "C" std::size_t steane_coprocessor(const void *in, std::size_t in_len, void *out, + std::size_t out_cap, void * /*ctx*/) { std::uint64_t syndrome = 0; std::memcpy(&syndrome, in, in_len < sizeof(syndrome) ? in_len : sizeof(syndrome)); @@ -45,6 +45,8 @@ extern "C" void decode(void * /*ctx*/, const void *in, std::size_t in_len, void // corrected qubit (0 => no error). const std::uint32_t check = syndrome & 0x7u; const std::uint64_t correction = static_cast(check ? (1u << (check - 1)) : 0u); - std::memset(out, 0, out_len); - std::memcpy(out, &correction, out_len < sizeof(correction) ? out_len : sizeof(correction)); + const std::size_t nb = out_cap < sizeof(correction) ? out_cap : sizeof(correction); + std::memset(out, 0, out_cap); + std::memcpy(out, &correction, nb); + return nb; } diff --git a/runtime/lib/transport/cpu_verbs/cpu_libibverbs_main.cpp b/runtime/lib/transport/cpu_verbs/cpu_libibverbs_main.cpp deleted file mode 100644 index 41f9b34ea0..0000000000 --- a/runtime/lib/transport/cpu_verbs/cpu_libibverbs_main.cpp +++ /dev/null @@ -1,94 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "CpuControllerSession.hpp" -#include "CpuCoprocessorSession.hpp" -#include "WireProtocol.hpp" - -using namespace catalyst::transport; -using namespace rdma::devices::cpu_libibverbs; -using namespace rdma::devices::common; // REGION_BYTES, DEMO_SYNDROME, Payload - -int main(int argc, char **argv) -{ - std::string role = "coprocessor", dev = "rxe0", peer = "127.0.0.1"; - int gid = 1; - std::uint16_t port = 18560; - for (int i = 1; i + 1 < argc; i += 2) { - std::string k = argv[i], v = argv[i + 1]; - if (k == "--role") - role = v; - else if (k == "--dev") - dev = v; - else if (k == "--gid") - gid = std::atoi(v.c_str()); - else if (k == "--peer") - peer = v; - else if (k == "--port") - port = static_cast(std::atoi(v.c_str())); - } - const bool is_coprocessor = (role == "coprocessor"); - - std::unique_ptr s; - CpuCoprocessorSession *coproc = nullptr; - CpuControllerSession *controller = nullptr; - if (is_coprocessor) { - auto up = std::make_unique(dev, gid); - coproc = up.get(); - s = std::move(up); - } - else { - auto up = std::make_unique(dev, gid); - controller = up.get(); - s = std::move(up); - } - - ConnectInfo ci{ - .peer = peer, - .oob_port = port, - }; - s->connect(ci); - MemRegion m = s->alloc_memory(REGION_BYTES, MemKind::CpuRam, - IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); - PeerRef p = s->exchange_keys(m); - ChannelDesc desc{ - .data_path = DataPath::CpuVerbs, - }; - s->establish_channel(desc, m, p); - - std::uint64_t got = 0; - void *outs[1] = {&got}; - std::uint64_t obytes[1] = {sizeof(got)}; - if (coproc) { - coproc->set_coprocessor_fn(nullptr, nullptr); // built-in echo - coproc->start(); - std::this_thread::sleep_for(std::chrono::seconds(3)); // serve ~3 s - coproc->collect(outs, obytes, 1); - coproc->stop(); - } - else { - // Controller: commit a work item, write the syndrome into - // data_slot(), kick one round, then collect the correction. - controller->commit_work_item(/*work_item_idx=*/0, /*in_bytes=*/sizeof(std::uint64_t), - /*out_bytes=*/sizeof(std::uint64_t)); - controller->start(); - const std::uint64_t syndrome = DEMO_SYNDROME; - std::memcpy(controller->data_slot(), &syndrome, sizeof(syndrome)); - controller->kick(0); - controller->collect(outs, obytes, 1); - controller->stop(); - } - - // Echo coprocessor: both roles observe the demo syndrome. - const std::uint64_t expect = DEMO_SYNDROME; - const bool pass = (got == expect); - std::fprintf(stderr, "[%s] got=0x%llx expect=0x%llx -> %s\n", role.c_str(), - static_cast(got), static_cast(expect), - pass ? "PASS" : "FAIL"); - return pass ? 0 : 1; -} diff --git a/runtime/lib/transport/cpu_verbs/cpu_verbs_selftest.cpp b/runtime/lib/transport/cpu_verbs/cpu_verbs_selftest.cpp new file mode 100644 index 0000000000..a219442cdf --- /dev/null +++ b/runtime/lib/transport/cpu_verbs/cpu_verbs_selftest.cpp @@ -0,0 +1,121 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "CpuControllerSession.hpp" +#include "CpuCoprocessorSession.hpp" +#include "WireProtocol.hpp" + +using namespace catalyst::transport; +using namespace catalyst::transport::cpu_verbs; +using namespace catalyst::transport::common; // REGION_BYTES, DEMO_SYNDROME, Payload + +int main(int argc, char **argv) +{ + std::string role = "coprocessor", dev = "rxe0", peer = "127.0.0.1"; + int gid = 1; + std::uint16_t port = 18560; + // Message sizes (bytes). The controller role feeds these to commit_work_item; + // both roles size the collect() buffer by --correction-bytes. + std::uint32_t syndrome_bytes = sizeof(std::uint64_t); + std::uint32_t correction_bytes = sizeof(std::uint64_t); + for (int i = 1; i + 1 < argc; i += 2) { + std::string k = argv[i], v = argv[i + 1]; + if (k == "--role") + role = v; + else if (k == "--dev") + dev = v; + else if (k == "--gid") + gid = std::atoi(v.c_str()); + else if (k == "--peer") + peer = v; + else if (k == "--port") + port = static_cast(std::atoi(v.c_str())); + else if (k == "--syndrome-bytes") + syndrome_bytes = static_cast(std::strtoul(v.c_str(), nullptr, 0)); + else if (k == "--correction-bytes") + correction_bytes = static_cast(std::strtoul(v.c_str(), nullptr, 0)); + } + const bool is_coprocessor = (role == "coprocessor"); + + std::unique_ptr s; + CpuCoprocessorSession *coproc = nullptr; + CpuControllerSession *controller = nullptr; + if (is_coprocessor) { + auto up = std::make_unique(dev, gid); + coproc = up.get(); + s = std::move(up); + } + else { + auto up = std::make_unique(dev, gid); + controller = up.get(); + s = std::move(up); + } + + ConnectInfo ci{ + .peer = peer, + .oob_port = port, + }; + s->connect(ci); + MemRegion m = s->alloc_memory(REGION_BYTES, MemKind::CpuRam, + IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + PeerRef p = s->exchange_keys(m); + ChannelDesc desc{ + .data_path = DataPath::CpuVerbs, + }; + s->establish_channel(desc, m, p); + + // Reply buffer that collect() fills with up to --correction-bytes. Sized to at + // least 8 B so the echo check below can always read a full 64-bit word: `got` + // copies the leading 8 bytes of the reply and compares them to DEMO_SYNDROME. + std::vector corr(std::max(correction_bytes, sizeof(std::uint64_t)), + 0); + if (coproc) { + coproc->set_coprocessor_fn(nullptr, nullptr); // built-in echo + coproc->start(); + std::this_thread::sleep_for(std::chrono::seconds(3)); // serve ~3 s + coproc->collect(corr.data(), correction_bytes); + coproc->stop(); + } + else { + // Controller: commit a work item sized by --syndrome-bytes/--correction-bytes, + // write the syndrome into data_slot(), kick one round, collect the correction. + controller->commit_work_item(/*work_item_idx=*/0, syndrome_bytes, correction_bytes); + controller->start(); + const std::uint64_t syndrome = DEMO_SYNDROME; + std::memcpy(controller->data_slot(), &syndrome, sizeof(syndrome)); + controller->kick(0); + controller->collect(corr.data(), correction_bytes); + controller->stop(); + } + + std::uint64_t got = 0; + std::memcpy(&got, corr.data(), sizeof(got)); + + // Built-in echo coprocessor: both roles observe the demo syndrome. + const bool pass = (got == DEMO_SYNDROME); + std::fprintf(stderr, "[%s] got=0x%llx expect=0x%llx -> %s\n", role.c_str(), + static_cast(got), + static_cast(DEMO_SYNDROME), pass ? "PASS" : "FAIL"); + return pass ? 0 : 1; +} diff --git a/runtime/lib/transport/cpu_verbs/run_loopback.sh b/runtime/lib/transport/cpu_verbs/run_loopback.sh index ec7ace051f..81cb6a452f 100755 --- a/runtime/lib/transport/cpu_verbs/run_loopback.sh +++ b/runtime/lib/transport/cpu_verbs/run_loopback.sh @@ -1,10 +1,19 @@ #!/usr/bin/env bash -# Two-process SoftRoCE round-trip on rxe0/lo: coprocessor (listens, built-in echo) -# + controller (connects, kicks one syndrome). Both echo -> both observe DEMO_SYNDROME. +# +# Manual two-process loopback test for the CPU-verbs transport. Launches +# the cpu_verbs_selftest binary twice on this host, talking to each other over a +# local RDMA device (default rxe0 / SoftRoCE): +# - coprocessor: background; listens on the OOB port and runs the built-in echo +# - controller: foreground; connects, sends one syndrome, collects the reply +# The echo bounces the syndrome back, so both roles observe DEMO_SYNDROME and exit +# 0, and the script prints "LOOPBACK: PASS". +# +# Requires a working RDMA device (e.g. rxe0 SoftRoCE). +# Env overrides: DEV (rxe0), GID (1), PORT (18560), BIN (built selftest path). set -u DEV=${DEV:-rxe0}; GID=${GID:-1}; PORT=${PORT:-18560} -BIN=${BIN:-"$(cd "$(dirname "$0")" && pwd)/../build/cpu_libibverbs/cpu_libibverbs_main"} -[ -x "$BIN" ] || { echo "binary not found: $BIN (build the cpu_libibverbs_main target first)"; exit 2; } +BIN=${BIN:-"$(cd "$(dirname "$0")" && pwd)/../../../build/lib/transport/cpu_verbs/cpu_verbs_selftest"} +[ -x "$BIN" ] || { echo "binary not found: $BIN (build the cpu_verbs_selftest target first)"; exit 2; } timeout 30 "$BIN" --role coprocessor --dev "$DEV" --gid "$GID" --port "$PORT" \ >/tmp/cvl_coproc.log 2>&1 & diff --git a/runtime/tests/CMakeLists.txt b/runtime/tests/CMakeLists.txt index bd9f9ba4d0..ab1d1242ab 100644 --- a/runtime/tests/CMakeLists.txt +++ b/runtime/tests/CMakeLists.txt @@ -166,3 +166,31 @@ target_link_libraries(runner_tests_rsdecomp_runtime PRIVATE ) catch_discover_tests(runner_tests_rsdecomp_runtime) + +# Transport test suites (RDMA-backed; individual tests SKIP without an rxe0 device) +if(ENABLE_TRANSPORT) + # Device-agnostic common primitives + add_executable(runner_tests_transport_common) + target_sources(runner_tests_transport_common PRIVATE + Test_TransportCommon.cpp + Test_TransportWireProtocol.cpp + ) + target_link_libraries(runner_tests_transport_common PRIVATE + Catch2WithMain + catalyst_runtime_testing + transport_common + ) + catch_discover_tests(runner_tests_transport_common) + + # CPU-verbs backend (both roles, in-process loopback) + add_executable(runner_tests_transport_cpu) + target_sources(runner_tests_transport_cpu PRIVATE + Test_TransportCpuVerbs.cpp + ) + target_link_libraries(runner_tests_transport_cpu PRIVATE + Catch2WithMain + catalyst_runtime_testing + cpu_verbs_impl + ) + catch_discover_tests(runner_tests_transport_cpu) +endif() diff --git a/runtime/lib/transport/common/test/Test_Common.cpp b/runtime/tests/Test_TransportCommon.cpp similarity index 66% rename from runtime/lib/transport/common/test/Test_Common.cpp rename to runtime/tests/Test_TransportCommon.cpp index bf5839c324..823de48f5b 100644 --- a/runtime/lib/transport/common/test/Test_Common.cpp +++ b/runtime/tests/Test_TransportCommon.cpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include @@ -6,7 +20,7 @@ #include "Context.hpp" #include "QpState.hpp" -using namespace rdma::devices::common; +using namespace catalyst::transport::common; static bool have_rxe() { diff --git a/runtime/lib/transport/cpu_verbs/test/Test_CpuLibibverbs.cpp b/runtime/tests/Test_TransportCpuVerbs.cpp similarity index 86% rename from runtime/lib/transport/cpu_verbs/test/Test_CpuLibibverbs.cpp rename to runtime/tests/Test_TransportCpuVerbs.cpp index 87578d78a0..dd48c4ccee 100644 --- a/runtime/lib/transport/cpu_verbs/test/Test_CpuLibibverbs.cpp +++ b/runtime/tests/Test_TransportCpuVerbs.cpp @@ -1,3 +1,17 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include #include @@ -12,8 +26,8 @@ #include "WireProtocol.hpp" using namespace catalyst::transport; -using namespace rdma::devices::cpu_libibverbs; -using namespace rdma::devices::common; // DEMO_SYNDROME, REGION_BYTES +using namespace catalyst::transport::cpu_verbs; +using namespace catalyst::transport::common; // DEMO_SYNDROME, REGION_BYTES static bool have_rxe() { @@ -134,9 +148,7 @@ TEST_CASE("round-trip: coprocessor gets request, controller gets bounced reply", coproc.establish_channel(desc, m, p); coproc.set_coprocessor_fn(nullptr, nullptr); // built-in echo coproc.start(); - void *outs[1] = {&coproc_got}; - std::uint64_t obytes[1] = {sizeof(coproc_got)}; - coproc.collect(outs, obytes, 1); + coproc.collect(&coproc_got, sizeof(coproc_got)); coproc.stop(); }); CpuControllerSession controller("rxe0", 1); @@ -158,9 +170,7 @@ TEST_CASE("round-trip: coprocessor gets request, controller gets bounced reply", std::memcpy(controller.data_slot(), &syndrome, sizeof(syndrome)); controller.kick(0); std::uint64_t controller_got = 0; - void *outs[1] = {&controller_got}; - std::uint64_t obytes[1] = {sizeof(controller_got)}; - controller.collect(outs, obytes, 1); + controller.collect(&controller_got, sizeof(controller_got)); controller.stop(); t.join(); @@ -192,9 +202,7 @@ TEST_CASE("round-trip with a custom coprocessor function runs on the coprocessor coproc.set_coprocessor_fn(invert_fn, nullptr); coproc.start(); std::uint64_t got = 0; - void *outs[1] = {&got}; - std::uint64_t obytes[1] = {sizeof(got)}; - coproc.collect(outs, obytes, 1); + coproc.collect(&got, sizeof(got)); coproc.stop(); }); CpuControllerSession controller("rxe0", 1); @@ -216,12 +224,10 @@ TEST_CASE("round-trip with a custom coprocessor function runs on the coprocessor std::memcpy(controller.data_slot(), &syndrome, sizeof(syndrome)); controller.kick(0); std::uint64_t got = 0; - void *outs[1] = {&got}; - std::uint64_t obytes[1] = {sizeof(got)}; - controller.collect(outs, obytes, 1); + controller.collect(&got, sizeof(got)); controller.stop(); t.join(); - REQUIRE(got == ~DEMO_SYNDROME); // controller received the coprocessor's result - REQUIRE(got != DEMO_SYNDROME); // and it is not a mere echo + REQUIRE(got == ~DEMO_SYNDROME); // controller received the coprocessor's result + REQUIRE(got != DEMO_SYNDROME); // and it is not a mere echo } diff --git a/runtime/lib/transport/common/test/Test_WireProtocol.cpp b/runtime/tests/Test_TransportWireProtocol.cpp similarity index 58% rename from runtime/lib/transport/common/test/Test_WireProtocol.cpp rename to runtime/tests/Test_TransportWireProtocol.cpp index e40c254fc6..a15be74735 100644 --- a/runtime/lib/transport/common/test/Test_WireProtocol.cpp +++ b/runtime/tests/Test_TransportWireProtocol.cpp @@ -1,10 +1,24 @@ +// Copyright 2026 Xanadu Quantum Technologies Inc. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include #include "WireProtocol.hpp" -using namespace rdma::devices::common; +using namespace catalyst::transport::common; TEST_CASE("Payload is the 16 B wire frame") { From d45f1582663edb0211406992fb89ebc9aa76f297 Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Fri, 24 Jul 2026 12:46:29 -0400 Subject: [PATCH 43/57] format --- runtime/lib/transport/common/CompletionQueue.hpp | 4 ++-- runtime/lib/transport/common/Error.hpp | 2 +- runtime/lib/transport/common/MemoryRegion.hpp | 4 ++-- runtime/lib/transport/common/OobSocket.cpp | 6 +++--- runtime/lib/transport/common/OobSocket.hpp | 3 ++- runtime/lib/transport/common/ProtectionDomain.hpp | 4 ++-- runtime/lib/transport/common/QueuePair.hpp | 4 ++-- .../cpu_verbs/controller/CpuControllerSession.cpp | 7 +++---- .../cpu_verbs/controller/CpuControllerSession.hpp | 3 ++- .../cpu_verbs/coprocessor/CpuCoprocessorSession.hpp | 1 + runtime/tests/Test_TransportCommon.cpp | 6 +++--- runtime/tests/Test_TransportCpuVerbs.cpp | 6 +++--- runtime/tests/Test_TransportWireProtocol.cpp | 4 ++-- 13 files changed, 28 insertions(+), 26 deletions(-) diff --git a/runtime/lib/transport/common/CompletionQueue.hpp b/runtime/lib/transport/common/CompletionQueue.hpp index f48daee25c..1788ff5e95 100644 --- a/runtime/lib/transport/common/CompletionQueue.hpp +++ b/runtime/lib/transport/common/CompletionQueue.hpp @@ -15,10 +15,10 @@ #pragma once #include -#include - #include "Context.hpp" +#include + namespace catalyst::transport::common { /** diff --git a/runtime/lib/transport/common/Error.hpp b/runtime/lib/transport/common/Error.hpp index e64e2abdf1..8fbc797095 100644 --- a/runtime/lib/transport/common/Error.hpp +++ b/runtime/lib/transport/common/Error.hpp @@ -38,7 +38,7 @@ class RdmaError : public std::runtime_error { char rdma_full_[512]; \ std::snprintf(rdma_full_, sizeof(rdma_full_), "%s:%d: %s (errno=%d: %s)", __FILE__, \ __LINE__, rdma_msg_, errno, std::strerror(errno)); \ - ::catalyst::transport::common::rdma_throw(rdma_full_); \ + ::catalyst::transport::common::rdma_throw(rdma_full_); \ } while (0) // Throw RdmaError with file:line + errno when cond is false. diff --git a/runtime/lib/transport/common/MemoryRegion.hpp b/runtime/lib/transport/common/MemoryRegion.hpp index 8f88f862da..67aaedfecb 100644 --- a/runtime/lib/transport/common/MemoryRegion.hpp +++ b/runtime/lib/transport/common/MemoryRegion.hpp @@ -17,10 +17,10 @@ #include #include -#include - #include "ProtectionDomain.hpp" +#include + namespace catalyst::transport::common { /** diff --git a/runtime/lib/transport/common/OobSocket.cpp b/runtime/lib/transport/common/OobSocket.cpp index 6621b09717..2f9f5e8ff1 100644 --- a/runtime/lib/transport/common/OobSocket.cpp +++ b/runtime/lib/transport/common/OobSocket.cpp @@ -19,14 +19,14 @@ #include #include #include -#include #include +#include "Error.hpp" + +#include #include #include -#include "Error.hpp" - namespace catalyst::transport::common { namespace { diff --git a/runtime/lib/transport/common/OobSocket.hpp b/runtime/lib/transport/common/OobSocket.hpp index ca807b1c48..dced857c60 100644 --- a/runtime/lib/transport/common/OobSocket.hpp +++ b/runtime/lib/transport/common/OobSocket.hpp @@ -15,9 +15,10 @@ #pragma once #include #include -#include #include +#include + namespace catalyst::transport::common { // RAII handle for a socket file descriptor. diff --git a/runtime/lib/transport/common/ProtectionDomain.hpp b/runtime/lib/transport/common/ProtectionDomain.hpp index c4ba7bd8c7..795c0e81fc 100644 --- a/runtime/lib/transport/common/ProtectionDomain.hpp +++ b/runtime/lib/transport/common/ProtectionDomain.hpp @@ -15,10 +15,10 @@ #pragma once #include -#include - #include "Context.hpp" +#include + namespace catalyst::transport::common { /** * @class ProtectionDomain diff --git a/runtime/lib/transport/common/QueuePair.hpp b/runtime/lib/transport/common/QueuePair.hpp index 3f2698f53a..7bbffcdf2e 100644 --- a/runtime/lib/transport/common/QueuePair.hpp +++ b/runtime/lib/transport/common/QueuePair.hpp @@ -16,12 +16,12 @@ #include #include -#include - #include "CompletionQueue.hpp" #include "ProtectionDomain.hpp" #include "QpState.hpp" +#include + namespace catalyst::transport::common { class QueuePair { public: diff --git a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp index fc3294a286..c632a3d86e 100644 --- a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp +++ b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp @@ -27,10 +27,9 @@ using namespace catalyst::transport::common; namespace { std::uint64_t now_ns() { - return static_cast( - std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch()) - .count()); + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); } } // namespace diff --git a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp index 39af6d07ce..4748d86b82 100644 --- a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp +++ b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp @@ -68,7 +68,8 @@ class CpuControllerSession : public ControllerSession { void start() override; void stop() override; - int collect(void *const *replies, const std::uint64_t *replies_bytes, std::size_t n) override; + int collect(void *const *replies, const std::uint64_t *replies_bytes, + std::size_t n) override; std::uint64_t last_rtt_ns() const override { return rtt_ns_; } void commit_work_item(std::uint32_t work_item_idx, std::uint64_t in_bytes, diff --git a/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp index 8281fa73a8..c0422825c0 100644 --- a/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp +++ b/runtime/lib/transport/cpu_verbs/coprocessor/CpuCoprocessorSession.hpp @@ -57,6 +57,7 @@ class CpuCoprocessorSession : public CoprocessorSession { ~Impl() { stop(); } CoprocessorFn coproc_fn_ = nullptr; // nullptr -> built-in echo void *coproc_ctx_ = nullptr; + protected: void run(std::stop_token st) override; bool oob_listens() const override { return true; } diff --git a/runtime/tests/Test_TransportCommon.cpp b/runtime/tests/Test_TransportCommon.cpp index 823de48f5b..8fdf54dfb7 100644 --- a/runtime/tests/Test_TransportCommon.cpp +++ b/runtime/tests/Test_TransportCommon.cpp @@ -14,12 +14,12 @@ #include -#include -#include - #include "Context.hpp" #include "QpState.hpp" +#include +#include + using namespace catalyst::transport::common; static bool have_rxe() diff --git a/runtime/tests/Test_TransportCpuVerbs.cpp b/runtime/tests/Test_TransportCpuVerbs.cpp index bb5dea0666..8cb69d0fcb 100644 --- a/runtime/tests/Test_TransportCpuVerbs.cpp +++ b/runtime/tests/Test_TransportCpuVerbs.cpp @@ -18,13 +18,13 @@ #include #include -#include -#include - #include "CpuControllerSession.hpp" #include "CpuCoprocessorSession.hpp" #include "WireProtocol.hpp" +#include +#include + using namespace catalyst::transport; using namespace catalyst::transport::cpu_verbs; using namespace catalyst::transport::common; // DEMO_SYNDROME, REGION_BYTES diff --git a/runtime/tests/Test_TransportWireProtocol.cpp b/runtime/tests/Test_TransportWireProtocol.cpp index a15be74735..bbcdd65b6d 100644 --- a/runtime/tests/Test_TransportWireProtocol.cpp +++ b/runtime/tests/Test_TransportWireProtocol.cpp @@ -14,10 +14,10 @@ #include -#include - #include "WireProtocol.hpp" +#include + using namespace catalyst::transport::common; TEST_CASE("Payload is the 16 B wire frame") From ff83df88a443418f37805409d4119c8fbb0c5e39 Mon Sep 17 00:00:00 2001 From: Joseph Lee <40768758+josephleekl@users.noreply.github.com> Date: Fri, 24 Jul 2026 13:42:26 -0400 Subject: [PATCH 44/57] Update runtime/lib/transport/common/QueuePair.cpp --- runtime/lib/transport/common/QueuePair.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/runtime/lib/transport/common/QueuePair.cpp b/runtime/lib/transport/common/QueuePair.cpp index 147c118188..d7ec088292 100644 --- a/runtime/lib/transport/common/QueuePair.cpp +++ b/runtime/lib/transport/common/QueuePair.cpp @@ -118,8 +118,6 @@ void QueuePair::to_rtr(std::uint32_t dest_qpn, std::uint32_t dest_psn, .max_dest_rd_atomic = MAX_RD_ATOMIC, .min_rnr_timer = MIN_RNR_TIMER, }; - // dgid is a 16-byte union filled from the peer's raw GID after the - // aggregate init (it can't be brace-initialized from a runtime array). std::memcpy(&a.ah_attr.grh.dgid, dest_gid, 16); modify(QpState::RTR, a, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | From 853df0b38d5bc225f519688aebcff772105ceb27 Mon Sep 17 00:00:00 2001 From: Joseph Lee <40768758+josephleekl@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:42:54 -0400 Subject: [PATCH 45/57] Update runtime/tests/CMakeLists.txt --- runtime/tests/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/tests/CMakeLists.txt b/runtime/tests/CMakeLists.txt index 77339f9248..cc3ab42951 100644 --- a/runtime/tests/CMakeLists.txt +++ b/runtime/tests/CMakeLists.txt @@ -199,7 +199,7 @@ if(ENABLE_TRANSPORT) ) catch_discover_tests(runner_tests_transport_common) - # CPU-verbs backend: both roles, in-process loopback (RDMA-backed; SKIP without rxe0). + # CPU-verbs backend: both controller and coprocessor (requires RDMA - skip without rxe0). add_executable(runner_tests_transport_cpu) target_sources(runner_tests_transport_cpu PRIVATE Test_TransportCpuVerbs.cpp From 1767ee142ebed79c352544e43082e76ddf7e3c9d Mon Sep 17 00:00:00 2001 From: Joseph Lee Date: Fri, 24 Jul 2026 15:39:18 -0400 Subject: [PATCH 46/57] add comment for CPU controller commit work item --- .../lib/transport/cpu_verbs/controller/CpuControllerSession.cpp | 2 ++ .../lib/transport/cpu_verbs/controller/CpuControllerSession.hpp | 1 + 2 files changed, 3 insertions(+) diff --git a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp index c632a3d86e..68b0b7fcfb 100644 --- a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp +++ b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.cpp @@ -49,6 +49,8 @@ void CpuControllerSession::Impl::stop() CpuSessionBase::stop(); // no engine thread runs for the controller; harmless join } +// Single work item, fixed-size frame: work_item_idx is ignored; the sizes are +// just recorded (out_bytes_ caps the reply in collect()). void CpuControllerSession::Impl::commit_work_item(std::uint32_t /*work_item_idx*/, std::uint64_t in_bytes, std::uint64_t out_bytes) { diff --git a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp index 4748d86b82..d35b527bcb 100644 --- a/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp +++ b/runtime/lib/transport/cpu_verbs/controller/CpuControllerSession.hpp @@ -82,6 +82,7 @@ class CpuControllerSession : public ControllerSession { bool oob_listens() const override { return false; } private: + // Sizes from commit_work_item are fixed in CPU controller. std::uint64_t in_bytes_ = sizeof(common::Payload::value); std::uint64_t out_bytes_ = sizeof(common::Payload::value); std::uint64_t next_send_ = 0, next_recv_ = 0; From ff8a2532da004b814dbb974906e74b8c4de40ed6 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Sat, 25 Jul 2026 19:35:04 -0400 Subject: [PATCH 47/57] format --- mlir/lib/Driver/CompilerDriver.cpp | 2 +- mlir/tools/quantum-lsp-server/quantum-lsp-server.cpp | 2 +- mlir/tools/quantum-opt/quantum-opt.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mlir/lib/Driver/CompilerDriver.cpp b/mlir/lib/Driver/CompilerDriver.cpp index 5fb9fc2347..373c8edc57 100644 --- a/mlir/lib/Driver/CompilerDriver.cpp +++ b/mlir/lib/Driver/CompilerDriver.cpp @@ -89,8 +89,8 @@ #include "RegisterAllPasses.h" -#include "Transport/IR/TransportDialect.h" #include "Executor/IR/ExecutorDialect.h" +#include "Transport/IR/TransportDialect.h" using namespace mlir; using namespace catalyst; diff --git a/mlir/tools/quantum-lsp-server/quantum-lsp-server.cpp b/mlir/tools/quantum-lsp-server/quantum-lsp-server.cpp index d9878fc4da..a6714ad20b 100644 --- a/mlir/tools/quantum-lsp-server/quantum-lsp-server.cpp +++ b/mlir/tools/quantum-lsp-server/quantum-lsp-server.cpp @@ -30,8 +30,8 @@ #include "Quantum/IR/QuantumDialect.h" #include "RTIO/IR/RTIODialect.h" -#include "Transport/IR/TransportDialect.h" #include "Executor/IR/ExecutorDialect.h" +#include "Transport/IR/TransportDialect.h" int main(int argc, char **argv) { diff --git a/mlir/tools/quantum-opt/quantum-opt.cpp b/mlir/tools/quantum-opt/quantum-opt.cpp index a8698b63db..12fad60737 100644 --- a/mlir/tools/quantum-opt/quantum-opt.cpp +++ b/mlir/tools/quantum-opt/quantum-opt.cpp @@ -50,8 +50,8 @@ #include "RegisterAllPasses.h" -#include "Transport/IR/TransportDialect.h" #include "Executor/IR/ExecutorDialect.h" +#include "Transport/IR/TransportDialect.h" namespace test { void registerTestDialect(mlir::DialectRegistry &); From 0b0d0c0d191624351302ac970a633c7cf4766f23 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Sat, 25 Jul 2026 19:37:29 -0400 Subject: [PATCH 48/57] format --- mlir/include/RegisterAllPasses.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlir/include/RegisterAllPasses.h b/mlir/include/RegisterAllPasses.h index 068a24fe3c..71879efeb9 100644 --- a/mlir/include/RegisterAllPasses.h +++ b/mlir/include/RegisterAllPasses.h @@ -28,8 +28,8 @@ #include "Test/Transforms/Passes.h" #include "hlo-extensions/Transforms/Passes.h" -#include "Transport/Transforms/Passes.h" #include "Executor/Transforms/Passes.h" +#include "Transport/Transforms/Passes.h" namespace catalyst { From a53549c3571fb3985b19a8031fe1ed2779b1bb33 Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Sat, 25 Jul 2026 19:44:33 -0400 Subject: [PATCH 49/57] apply ne w catalyst-dev changes --- mlir/lib/Transport/Transforms/CMakeLists.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mlir/lib/Transport/Transforms/CMakeLists.txt b/mlir/lib/Transport/Transforms/CMakeLists.txt index efd3bd5462..5497e20f8c 100644 --- a/mlir/lib/Transport/Transforms/CMakeLists.txt +++ b/mlir/lib/Transport/Transforms/CMakeLists.txt @@ -20,6 +20,8 @@ set(DEPENDS add_mlir_library(${LIBRARY_NAME} STATIC ${SRC} LINK_LIBS PRIVATE ${LIBS} DEPENDS ${DEPENDS}) target_compile_features(${LIBRARY_NAME} PUBLIC cxx_std_20) target_include_directories(${LIBRARY_NAME} PUBLIC - . - ${PROJECT_SOURCE_DIR}/include - ${CMAKE_BINARY_DIR}/include) + $ + $ + $ + $ +) From b01d82889e35ad99abc4793a2d16ce6cd00a200b Mon Sep 17 00:00:00 2001 From: Mehrdad Malekmohammadi Date: Sun, 26 Jul 2026 21:05:36 -0400 Subject: [PATCH 50/57] Add ENABLE_TRANSPORT --- runtime/Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/runtime/Makefile b/runtime/Makefile index 84c9ba328e..c910a0adad 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -13,6 +13,7 @@ CODE_COVERAGE ?= OFF BUILD_TYPE ?= RelWithDebInfo ENABLE_OPENQASM ?= ON ENABLE_OQD ?= OFF +ENABLE_TRANSPORT ?= OFF ENABLE_ASAN ?= OFF STRICT_WARNINGS ?= ON LLVM_DIR ?= $(MK_DIR)/../mlir/llvm-project/ @@ -53,6 +54,10 @@ ifeq ($(ENABLE_OQD), ON) TEST_TARGETS += runner_tests_oqd endif +ifeq ($(ENABLE_TRANSPORT), ON) + BUILD_TARGETS += rt_transport +endif + .PHONY: help help: @echo "Please use \`make ' where is one of" @@ -78,6 +83,7 @@ configure: -DCMAKE_CXX_COMPILER_LAUNCHER=$(COMPILER_LAUNCHER) \ -DENABLE_OPENQASM=$(ENABLE_OPENQASM) \ -DENABLE_OQD=$(ENABLE_OQD) \ + -DENABLE_TRANSPORT=$(ENABLE_TRANSPORT) \ -DENABLE_CODE_COVERAGE=$(CODE_COVERAGE) \ -DPython_EXECUTABLE=$(PYTHON) \ -DENABLE_ADDRESS_SANITIZER=$(ENABLE_ASAN) \ From a3423be4518198f7ceea8020160f10cae313c914 Mon Sep 17 00:00:00 2001 From: Shuli Shu <08cnbj@gmail.com> Date: Tue, 28 Jul 2026 13:53:52 -0400 Subject: [PATCH 51/57] Initial commit --- .github/workflows/check-transport.yaml | 137 +++++++++++++++++++++++++ Makefile | 8 +- runtime/Makefile | 22 ++++ 3 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/check-transport.yaml diff --git a/.github/workflows/check-transport.yaml b/.github/workflows/check-transport.yaml new file mode 100644 index 0000000000..dd4be19c15 --- /dev/null +++ b/.github/workflows/check-transport.yaml @@ -0,0 +1,137 @@ +name: Check Transport Backends + +# Builds the runtime with ENABLE_TRANSPORT=ON and exercises the transport +# backends (CAPI + loader against a stub backend, common RDMA primitives, and +# the cpu_verbs controller/coprocessor sessions) plus the two-process loopback +# smoke test. There is no physical RDMA NIC on GitHub-hosted runners, so we +# rely on Soft-RoCE (rdma_rxe kernel module) attached to the loopback +# interface. The test binaries key on device name "rxe0". + +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + paths: + - 'runtime/lib/transport/**' + - 'runtime/tests/Test_Transport*.cpp' + - 'runtime/tests/stubs/stub_transport_backend*' + - 'runtime/tests/CMakeLists.txt' + - 'runtime/CMakeLists.txt' + - 'runtime/lib/CMakeLists.txt' + - 'runtime/Makefile' + - 'Makefile' + - '.github/workflows/check-transport.yaml' + push: + branches: [ main ] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + determine_runner: + if: github.event.pull_request.draft == false + name: Determine runner type to use + uses: ./.github/workflows/determine-workflow-runner.yml + with: + default_runner: ubuntu-24.04 + + constants: + name: "Set build matrix" + uses: ./.github/workflows/constants.yaml + needs: [determine_runner] + with: + multiple_compilers: false + runs_on: ${{ needs.determine_runner.outputs.runner_group }} + + transport-tests: + name: Transport Backend Tests (Soft-RoCE) + needs: [constants, determine_runner] + runs-on: ${{ needs.determine_runner.outputs.runner_group }} + + steps: + - name: Checkout Catalyst repo + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ needs.constants.outputs.primary_python_version }} + + - name: Install build and RDMA dependencies + run: | + sudo apt-get update + sudo apt-get install -y \ + cmake ninja-build clang make \ + libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils \ + rdma-core iproute2 \ + "linux-modules-extra-$(uname -r)" + python3 -m pip install "nanobind<2.13" pybind11 + + - name: Configure Soft-RoCE (rxe0 on loopback) + run: | + # rdma_rxe is a mainline Linux kernel module that provides a + # software RoCE device. Attach it to `lo` so the runtime's cpu_verbs + # backend has an ibverbs device named `rxe0` to open. The Catch2 + # tests SKIP when this device is absent, and run_loopback.sh + # requires it - both scenarios happen inside make test-transport. + sudo modprobe rdma_rxe + sudo rdma link add rxe0 type rxe netdev lo + rdma link show + ibv_devices + ibv_devinfo -d rxe0 + # Fail fast if rxe0 didn't come up - otherwise Catch2 tests would + # silently SKIP and the loopback would be the only thing exercising it. + ibv_devices | awk '{print $1}' | grep -q '^rxe0$' + + - name: Get Cached LLVM Source + id: cache-llvm-source + uses: actions/cache/restore@v4 + with: + path: mlir/llvm-project + key: llvm-${{ needs.constants.outputs.llvm_version }}-default-source + enableCrossOsArchive: true + + - name: Clone LLVM Submodule + if: steps.cache-llvm-source.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: llvm/llvm-project + ref: ${{ needs.constants.outputs.llvm_version }} + path: mlir/llvm-project + + - name: Build Catalyst-Runtime with ENABLE_TRANSPORT=ON + run: | + COMPILER_LAUNCHER="" \ + C_COMPILER=$(which clang) \ + CXX_COMPILER=$(which clang++) \ + LLVM_DIR="$(pwd)/mlir/llvm-project" \ + ENABLE_ASAN=OFF \ + ENABLE_TRANSPORT=ON \ + make runtime + + - name: Run transport test suite (Catch2 + loopback) + run: | + COMPILER_LAUNCHER="" \ + C_COMPILER=$(which clang) \ + CXX_COMPILER=$(which clang++) \ + LLVM_DIR="$(pwd)/mlir/llvm-project" \ + make test-runtime-transport + + - name: Upload loopback logs on failure + if: failure() + uses: actions/upload-artifact@v4 + with: + name: transport-loopback-logs + path: | + /tmp/cvl_coproc.log + /tmp/cvl_ctrl.log + if-no-files-found: ignore + retention-days: 7 diff --git a/Makefile b/Makefile index 7f7062b5b2..5462dc4215 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,7 @@ ENZYME_BUILD_DIR ?= $(MK_DIR)/mlir/Enzyme/build COVERAGE_REPORT ?= term-missing ENABLE_OPENQASM ?= ON ENABLE_OQD ?= OFF +ENABLE_TRANSPORT ?= OFF TEST_BACKEND ?= "lightning.qubit" TEST_BRAKET ?= NONE ENABLE_ASAN ?= OFF @@ -150,12 +151,12 @@ dialect-docs: $(MAKE) -C mlir dialect-docs runtime: - $(MAKE) -C runtime runtime ENABLE_OQD=$(ENABLE_OQD) + $(MAKE) -C runtime runtime ENABLE_OQD=$(ENABLE_OQD) ENABLE_TRANSPORT=$(ENABLE_TRANSPORT) oqc: $(MAKE) -C frontend/catalyst/third_party/oqc/src oqc -.PHONY: test test-runtime test-frontend lit pytest test-demos test-oqc test-toml-spec +.PHONY: test test-runtime test-runtime-transport test-frontend lit pytest test-demos test-oqc test-toml-spec test: test-runtime test-frontend test-demos test-toml-spec: @@ -164,6 +165,9 @@ test-toml-spec: test-runtime: $(MAKE) -C runtime test +test-runtime-transport: + $(MAKE) -C runtime test-transport + test-mlir: $(MAKE) -C mlir test diff --git a/runtime/Makefile b/runtime/Makefile index c910a0adad..4370e10e07 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -58,6 +58,12 @@ ifeq ($(ENABLE_TRANSPORT), ON) BUILD_TARGETS += rt_transport endif +TRANSPORT_TEST_TARGETS := \ + runner_tests_transport \ + runner_tests_transport_common \ + runner_tests_transport_cpu \ + cpu_verbs_selftest + .PHONY: help help: @echo "Please use \`make ' where is one of" @@ -118,6 +124,22 @@ ifeq ($(ENABLE_OQD), ON) $(ASAN_COMMAND) $(RT_BUILD_DIR)/tests/runner_tests_oqd endif +.PHONY: test-transport +test-transport: ENABLE_TRANSPORT=ON +test-transport: CODE_COVERAGE=OFF +test-transport: BUILD_TYPE?=RelWithDebInfo +test-transport: configure + cmake --build $(RT_BUILD_DIR) --target $(TRANSPORT_TEST_TARGETS) -j$(NPROC) + @echo "Catalyst transport CAPI/loader test suite (stub backend)" + $(ASAN_COMMAND) $(RT_BUILD_DIR)/tests/runner_tests_transport + @echo "Catalyst transport common primitives (SKIPs without rxe0)" + $(ASAN_COMMAND) $(RT_BUILD_DIR)/tests/runner_tests_transport_common + @echo "Catalyst transport cpu_verbs backend (SKIPs without rxe0)" + $(ASAN_COMMAND) $(RT_BUILD_DIR)/tests/runner_tests_transport_cpu + @echo "Catalyst transport cpu_verbs two-process loopback (requires rxe0)" + BIN=$(RT_BUILD_DIR)/lib/transport/cpu_verbs/cpu_verbs_selftest \ + $(MK_DIR)/lib/transport/cpu_verbs/run_loopback.sh + .PHONY: coverage coverage: RT_BUILD_DIR := $(RT_BUILD_DIR)_cov coverage: CODE_COVERAGE=ON From 8995011fb719f8104a0507bc952c06406a5ce7a6 Mon Sep 17 00:00:00 2001 From: Shuli Shu <08cnbj@gmail.com> Date: Tue, 28 Jul 2026 13:55:28 -0400 Subject: [PATCH 52/57] update docstr --- .github/workflows/check-transport.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/check-transport.yaml b/.github/workflows/check-transport.yaml index dd4be19c15..51d27ed211 100644 --- a/.github/workflows/check-transport.yaml +++ b/.github/workflows/check-transport.yaml @@ -1,12 +1,5 @@ name: Check Transport Backends -# Builds the runtime with ENABLE_TRANSPORT=ON and exercises the transport -# backends (CAPI + loader against a stub backend, common RDMA primitives, and -# the cpu_verbs controller/coprocessor sessions) plus the two-process loopback -# smoke test. There is no physical RDMA NIC on GitHub-hosted runners, so we -# rely on Soft-RoCE (rdma_rxe kernel module) attached to the loopback -# interface. The test binaries key on device name "rxe0". - on: pull_request: types: From cd3b0a59bbadfdfba91598337a1299149e1415a4 Mon Sep 17 00:00:00 2001 From: Shuli Shu <08cnbj@gmail.com> Date: Tue, 28 Jul 2026 14:04:53 -0400 Subject: [PATCH 53/57] try to pin to ubuntu-22.04 --- .github/workflows/check-transport.yaml | 39 ++++++++++++++------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/check-transport.yaml b/.github/workflows/check-transport.yaml index 51d27ed211..56ec32d677 100644 --- a/.github/workflows/check-transport.yaml +++ b/.github/workflows/check-transport.yaml @@ -29,25 +29,17 @@ concurrency: cancel-in-progress: true jobs: - determine_runner: - if: github.event.pull_request.draft == false - name: Determine runner type to use - uses: ./.github/workflows/determine-workflow-runner.yml - with: - default_runner: ubuntu-24.04 - constants: name: "Set build matrix" uses: ./.github/workflows/constants.yaml - needs: [determine_runner] with: multiple_compilers: false - runs_on: ${{ needs.determine_runner.outputs.runner_group }} + runs_on: ubuntu-22.04 transport-tests: name: Transport Backend Tests (Soft-RoCE) - needs: [constants, determine_runner] - runs-on: ${{ needs.determine_runner.outputs.runner_group }} + needs: [constants] + runs-on: ubuntu-22.04 steps: - name: Checkout Catalyst repo @@ -64,17 +56,28 @@ jobs: sudo apt-get install -y \ cmake ninja-build clang make \ libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils \ - rdma-core iproute2 \ - "linux-modules-extra-$(uname -r)" + rdma-core iproute2 + # Kernel modules for the running Azure kernel; rdma_rxe lives here. + sudo apt-get install -y "linux-modules-extra-$(uname -r)" \ + || sudo apt-get install -y linux-modules-extra-azure + # Refresh modules.dep so modprobe finds anything newly-installed. + sudo depmod -a + # Verify rdma_rxe.ko exists for the running kernel before we try to + # load it - otherwise `modprobe rdma_rxe` fails with an opaque + # "Module not found" and it's not obvious the package install was + # the problem. + find /lib/modules/$(uname -r) -name 'rdma_rxe*' -print + test -n "$(find /lib/modules/$(uname -r) -name 'rdma_rxe*' -print -quit)" \ + || { echo "::error::rdma_rxe.ko missing for kernel $(uname -r); the runner image does not ship a Soft-RoCE module. Check GitHub Actions image release notes or pin an older Ubuntu."; exit 1; } python3 -m pip install "nanobind<2.13" pybind11 - name: Configure Soft-RoCE (rxe0 on loopback) run: | - # rdma_rxe is a mainline Linux kernel module that provides a - # software RoCE device. Attach it to `lo` so the runtime's cpu_verbs - # backend has an ibverbs device named `rxe0` to open. The Catch2 - # tests SKIP when this device is absent, and run_loopback.sh - # requires it - both scenarios happen inside make test-transport. + # rdma_rxe provides a software RoCE device. Attach it to `lo` so the + # runtime's cpu_verbs backend has an ibverbs device named `rxe0` to + # open. The Catch2 tests SKIP when this device is absent, and + # run_loopback.sh requires it - both scenarios happen inside + # make test-transport. sudo modprobe rdma_rxe sudo rdma link add rxe0 type rxe netdev lo rdma link show From 03648e17ce1cc82338dc0cba81251d80877f4bda Mon Sep 17 00:00:00 2001 From: Shuli Shu <08cnbj@gmail.com> Date: Tue, 28 Jul 2026 14:14:07 -0400 Subject: [PATCH 54/57] try to build roce with 24.04 --- .github/workflows/check-transport.yaml | 90 +++++++++++++++++++++----- 1 file changed, 74 insertions(+), 16 deletions(-) diff --git a/.github/workflows/check-transport.yaml b/.github/workflows/check-transport.yaml index 56ec32d677..8b23b30ddc 100644 --- a/.github/workflows/check-transport.yaml +++ b/.github/workflows/check-transport.yaml @@ -29,17 +29,25 @@ concurrency: cancel-in-progress: true jobs: + determine_runner: + if: github.event.pull_request.draft == false + name: Determine runner type to use + uses: ./.github/workflows/determine-workflow-runner.yml + with: + default_runner: ubuntu-24.04 + constants: name: "Set build matrix" uses: ./.github/workflows/constants.yaml + needs: [determine_runner] with: multiple_compilers: false - runs_on: ubuntu-22.04 + runs_on: ${{ needs.determine_runner.outputs.runner_group }} transport-tests: name: Transport Backend Tests (Soft-RoCE) - needs: [constants] - runs-on: ubuntu-22.04 + needs: [constants, determine_runner] + runs-on: ${{ needs.determine_runner.outputs.runner_group }} steps: - name: Checkout Catalyst repo @@ -50,27 +58,77 @@ jobs: with: python-version: ${{ needs.constants.outputs.primary_python_version }} - - name: Install build and RDMA dependencies + - name: Install build and RDMA userland dependencies run: | sudo apt-get update sudo apt-get install -y \ cmake ninja-build clang make \ libibverbs-dev libibverbs1 ibverbs-providers ibverbs-utils \ rdma-core iproute2 - # Kernel modules for the running Azure kernel; rdma_rxe lives here. - sudo apt-get install -y "linux-modules-extra-$(uname -r)" \ - || sudo apt-get install -y linux-modules-extra-azure - # Refresh modules.dep so modprobe finds anything newly-installed. - sudo depmod -a - # Verify rdma_rxe.ko exists for the running kernel before we try to - # load it - otherwise `modprobe rdma_rxe` fails with an opaque - # "Module not found" and it's not obvious the package install was - # the problem. - find /lib/modules/$(uname -r) -name 'rdma_rxe*' -print - test -n "$(find /lib/modules/$(uname -r) -name 'rdma_rxe*' -print -quit)" \ - || { echo "::error::rdma_rxe.ko missing for kernel $(uname -r); the runner image does not ship a Soft-RoCE module. Check GitHub Actions image release notes or pin an older Ubuntu."; exit 1; } python3 -m pip install "nanobind<2.13" pybind11 + - name: Ensure rdma_rxe.ko is available for the running kernel + run: | + # The Azure kernel used by GitHub-hosted runners frequently does not + # ship a matching linux-modules-extra- package in Ubuntu apt. + # Try that first; if the module isn't present after installing, build + # it out-of-tree from the upstream stable branch that matches the + # running kernel's major.minor version. + set -euo pipefail + + KVER=$(uname -r) + echo "Running kernel: $KVER" + + have_rxe() { + find /lib/modules/"$KVER" -name 'rdma_rxe*' -print -quit | grep -q . + } + + # --- Path 1: standard Ubuntu extras package ---------------------- + sudo apt-get install -y "linux-modules-extra-$KVER" 2>/dev/null || true + sudo depmod -a + if have_rxe; then + echo "rdma_rxe.ko provided by linux-modules-extra-$KVER" + find /lib/modules/"$KVER" -name 'rdma_rxe*' -print + exit 0 + fi + + echo "linux-modules-extra unavailable or missing rdma_rxe;" \ + "falling back to out-of-tree build." + + # --- Path 2: build rdma_rxe out-of-tree -------------------------- + sudo apt-get install -y \ + build-essential bc bison flex libssl-dev libelf-dev git \ + "linux-headers-$KVER" + + # Upstream stable branch matching the running kernel's major.minor. + KVER_MAJMIN=$(echo "$KVER" | cut -d- -f1 | cut -d. -f1-2) + SRC=/tmp/linux-src + echo "Cloning linux-${KVER_MAJMIN}.y stable branch..." + git clone --depth 1 \ + --branch "linux-${KVER_MAJMIN}.y" \ + https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git \ + "$SRC" + + # Build only drivers/infiniband/sw/rxe against the running kernel's + # build tree; the RXE driver is self-contained and rarely carries + # distro patches, so upstream source usually compiles cleanly. + make -C "/lib/modules/$KVER/build" \ + M="$SRC/drivers/infiniband/sw/rxe" modules + + KO="$SRC/drivers/infiniband/sw/rxe/rdma_rxe.ko" + [ -f "$KO" ] || { echo "::error::rdma_rxe.ko not produced by build"; exit 1; } + + sudo install -D -m 644 "$KO" \ + "/lib/modules/$KVER/kernel/drivers/infiniband/sw/rxe/rdma_rxe.ko" + sudo depmod -a + + if ! have_rxe; then + echo "::error::rdma_rxe.ko not resolvable after install" + exit 1 + fi + echo "Built and installed rdma_rxe.ko from linux-${KVER_MAJMIN}.y" + modinfo rdma_rxe | head -20 + - name: Configure Soft-RoCE (rxe0 on loopback) run: | # rdma_rxe provides a software RoCE device. Attach it to `lo` so the From 00d6020796d1101f15c93a6e3e6cae49f900881f Mon Sep 17 00:00:00 2001 From: Shuli Shu <08cnbj@gmail.com> Date: Tue, 28 Jul 2026 14:19:29 -0400 Subject: [PATCH 55/57] test fix --- .github/workflows/check-transport.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-transport.yaml b/.github/workflows/check-transport.yaml index 8b23b30ddc..18ccf717d4 100644 --- a/.github/workflows/check-transport.yaml +++ b/.github/workflows/check-transport.yaml @@ -112,11 +112,29 @@ jobs: # Build only drivers/infiniband/sw/rxe against the running kernel's # build tree; the RXE driver is self-contained and rarely carries # distro patches, so upstream source usually compiles cleanly. + # + # The RXE Makefile has `obj-$(CONFIG_RDMA_RXE) += rdma_rxe.o`. When + # kbuild reads the running kernel's .config out-of-tree and the + # Azure kernel has CONFIG_RDMA_RXE unset (which is exactly why the + # extras package doesn't ship the .ko), that resolves to nothing + # and no module is built - MODPOST silently runs against zero + # objects. Force `obj-m` so the build produces rdma_rxe.ko + # regardless of the running kernel's config. + sed -i 's|obj-\$(CONFIG_RDMA_RXE)|obj-m|' \ + "$SRC/drivers/infiniband/sw/rxe/Makefile" + echo "--- patched rxe Makefile ---" + cat "$SRC/drivers/infiniband/sw/rxe/Makefile" + echo "----------------------------" + make -C "/lib/modules/$KVER/build" \ M="$SRC/drivers/infiniband/sw/rxe" modules KO="$SRC/drivers/infiniband/sw/rxe/rdma_rxe.ko" - [ -f "$KO" ] || { echo "::error::rdma_rxe.ko not produced by build"; exit 1; } + if [ ! -f "$KO" ]; then + echo "::error::rdma_rxe.ko not produced by build" + ls -la "$SRC/drivers/infiniband/sw/rxe/" + exit 1 + fi sudo install -D -m 644 "$KO" \ "/lib/modules/$KVER/kernel/drivers/infiniband/sw/rxe/rdma_rxe.ko" From b9328793ad456e9da143741531ecb10a4f076e0f Mon Sep 17 00:00:00 2001 From: Shuli Shu <08cnbj@gmail.com> Date: Tue, 28 Jul 2026 15:30:23 -0400 Subject: [PATCH 56/57] tidy up docstr --- .github/workflows/check-transport.yaml | 28 ++++---------------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/.github/workflows/check-transport.yaml b/.github/workflows/check-transport.yaml index 18ccf717d4..9728597066 100644 --- a/.github/workflows/check-transport.yaml +++ b/.github/workflows/check-transport.yaml @@ -69,11 +69,8 @@ jobs: - name: Ensure rdma_rxe.ko is available for the running kernel run: | - # The Azure kernel used by GitHub-hosted runners frequently does not - # ship a matching linux-modules-extra- package in Ubuntu apt. - # Try that first; if the module isn't present after installing, build - # it out-of-tree from the upstream stable branch that matches the - # running kernel's major.minor version. + # Try linux-modules-extra first; if the .ko isn't there, build + # rdma_rxe out-of-tree from the matching upstream stable branch. set -euo pipefail KVER=$(uname -r) @@ -109,22 +106,9 @@ jobs: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git \ "$SRC" - # Build only drivers/infiniband/sw/rxe against the running kernel's - # build tree; the RXE driver is self-contained and rarely carries - # distro patches, so upstream source usually compiles cleanly. - # - # The RXE Makefile has `obj-$(CONFIG_RDMA_RXE) += rdma_rxe.o`. When - # kbuild reads the running kernel's .config out-of-tree and the - # Azure kernel has CONFIG_RDMA_RXE unset (which is exactly why the - # extras package doesn't ship the .ko), that resolves to nothing - # and no module is built - MODPOST silently runs against zero - # objects. Force `obj-m` so the build produces rdma_rxe.ko - # regardless of the running kernel's config. + # Force obj-m; Azure kernel has CONFIG_RDMA_RXE unset. sed -i 's|obj-\$(CONFIG_RDMA_RXE)|obj-m|' \ "$SRC/drivers/infiniband/sw/rxe/Makefile" - echo "--- patched rxe Makefile ---" - cat "$SRC/drivers/infiniband/sw/rxe/Makefile" - echo "----------------------------" make -C "/lib/modules/$KVER/build" \ M="$SRC/drivers/infiniband/sw/rxe" modules @@ -149,11 +133,7 @@ jobs: - name: Configure Soft-RoCE (rxe0 on loopback) run: | - # rdma_rxe provides a software RoCE device. Attach it to `lo` so the - # runtime's cpu_verbs backend has an ibverbs device named `rxe0` to - # open. The Catch2 tests SKIP when this device is absent, and - # run_loopback.sh requires it - both scenarios happen inside - # make test-transport. + # Attach rdma_rxe to `lo` so the tests find an ibverbs device named rxe0. sudo modprobe rdma_rxe sudo rdma link add rxe0 type rxe netdev lo rdma link show From 8e28fac61e62d8b5b5d7f8008dc97fa31a85cfce Mon Sep 17 00:00:00 2001 From: Shuli Shu <08cnbj@gmail.com> Date: Tue, 28 Jul 2026 15:33:55 -0400 Subject: [PATCH 57/57] add changelog --- doc/releases/changelog-dev.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/releases/changelog-dev.md b/doc/releases/changelog-dev.md index 713d703157..c8d9f7809e 100644 --- a/doc/releases/changelog-dev.md +++ b/doc/releases/changelog-dev.md @@ -315,6 +315,9 @@ * The `/benchmark` GitHub comment trigger can now accept additional arguments and has been renamed to `!benchmark`. [(#2947)](https://github.com/PennyLaneAI/catalyst/pull/2947) +* Added CI checks for the runtime `cpu_verbs` transport backend via Soft-RoCE. + [(#3074)](https://github.com/PennyLaneAI/catalyst/pull/3074) + * The frontend now generates MLIR in reference semantics when capture is enabled. [(#2663)](https://github.com/PennyLaneAI/catalyst/pull/2663) [(#2664)](https://github.com/PennyLaneAI/catalyst/pull/2664)