Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 105 additions & 0 deletions .github/workflows/rdma_soft_roce.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
name: RDMA (Soft-RoCE)

on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main, develop ]
workflow_call:
workflow_dispatch:

env:
ccache: ON

jobs:
rdma_soft_roce:
strategy:
matrix:
mode: [ Debug, Release ]
compiler:
- { cc: gcc, cxx: g++ }
- { cc: clang, cxx: clang++ }
runs-on: ubuntu-24.04

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Install Dependencies
run: |
sudo apt-get update
sudo apt-get install -y libibverbs-dev librdmacm-dev rdma-core ibverbs-utils iproute2 perftest \
linux-headers-$(uname -r)

- name: Setup Soft-RoCE
run: |
set -e

# Build rdma_rxe kernel module from source (Azure kernel doesn't ship it)
KERNEL_VERSION=$(uname -r | cut -d '-' -f 1)
KERNEL_NAME="linux-${KERNEL_VERSION%'.0'}"
DOWNLOAD_LINK="https://cdn.kernel.org/pub/linux/kernel/v${KERNEL_VERSION%%.*}.x/${KERNEL_NAME}.tar.xz"
ETHERNET_CARD=$(ip link | awk -F ": " '$0 !~ "lo|vir|wl|^[^0-9]"{print $2;getline}' | head -1)
echo "Kernel: $(uname -r), source: ${KERNEL_NAME}, NIC: ${ETHERNET_CARD}"

wget -q ${DOWNLOAD_LINK} -O /tmp/${KERNEL_NAME}.tar.xz
tar xf /tmp/${KERNEL_NAME}.tar.xz --directory=/tmp
RXE_PATH="/tmp/${KERNEL_NAME}/drivers/infiniband/sw/rxe"
sed 's/$(CONFIG_RDMA_RXE)/m/g' ${RXE_PATH}/Makefile > ${RXE_PATH}/Kbuild
make -C /lib/modules/$(uname -r)/build M=${RXE_PATH} modules -j

# Load modules and create Soft-RoCE device
sudo modprobe ib_core
sudo modprobe rdma_ucm
sudo insmod ${RXE_PATH}/rdma_rxe.ko
sudo rdma link add rxe0 type rxe netdev ${ETHERNET_CARD}

# Verify
echo "=== RDMA links ==="
rdma link
echo "=== RDMA devices ==="
ibv_devices
echo "=== RDMA device info ==="
ibv_devinfo

- name: Install ninja-build tool
uses: seanmiddleditch/gha-setup-ninja@master

- name: ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: ${{ github.job }}-${{ matrix.compiler.cc }}-${{ matrix.mode }}

- name: Configure
run: |
cmake -B ${{github.workspace}}/build -G Ninja \
-DCMAKE_C_COMPILER=${{matrix.compiler.cc}} \
-DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} \
-DCMAKE_BUILD_TYPE=${{matrix.mode}} \
-DYLT_ENABLE_IBV=ON \
-DUSE_CCACHE=${{env.ccache}} \
-DBUILD_CORO_HTTP=OFF \
-DBUILD_STRUCT_JSON=OFF \
-DBUILD_STRUCT_XML=OFF \
-DBUILD_STRUCT_PACK=OFF \
-DBUILD_STRUCT_PB=OFF \
-DBUILD_STRUCT_YAML=OFF \
-DBUILD_EASYLOG=OFF \
-DBUILD_METRIC=OFF \
-DBUILD_REFLECTION=OFF \
-DBUILD_UTIL=OFF

- name: Build
run: cmake --build ${{github.workspace}}/build --config ${{matrix.mode}}

- name: Test coro_io ibverbs
working-directory: ${{github.workspace}}/build
run: ctest -C ${{matrix.mode}} -j 1 -V -R "ibverbs_test"

- name: Test coro_io ibverbs pressure
working-directory: ${{github.workspace}}/build
run: ctest -C ${{matrix.mode}} -j 1 -V -R "ibverbs_pressure_test"

- name: Test coro_rpc
working-directory: ${{github.workspace}}/build
run: ctest -C ${{matrix.mode}} -j 1 -V -R "coro_rpc_test"
Loading