From 0dc524bcbe5d320e5ab63e944d49cf80243a0efd Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 7 Aug 2026 12:50:37 +0200 Subject: [PATCH 1/3] lvol: drain in-flight IO before freeing xfer teardown spdk_delete_rmt_lvol_pg/destroy_xfer_task_tmo freed rmt_lvol, reqs, pdus, and rings on xfer-task timeout with no check of outstanding_io, unlike the device-remove path (spdk_wait_for_pg_io_cleanup), which correctly defers free behind an outstanding_io==0 poller. A late S3 GET or local blob-write completion then wrote through freed memory in set_req_status_and_queued (rmt->outstanding_io--, req->status, free_ring enqueue) - a heap UAF that surfaced as a stalled S3 restore ("outstanding io N, state 13") and, when completions did arrive late, as glibc heap corruption crashing later on an unrelated allocation. - add XFER_REQ_STATUS_IN_FLIGHT, set when helper_xfer_poller claims a req - spdk_delete_rmt_lvol_pg: if rmt_lvol->outstanding_io > 0, best-effort abort in-flight reqs via spdk_bdev_abort (matches by existing req cb_arg, no new bookkeeping) and defer put_io_channel/free behind a new outstanding_io-gated poller (spdk_wait_for_xfer_pg_io_cleanup_poller), mirroring the device-remove path instead of freeing synchronously - xfer->pg[lpg->id] is now only cleared once the rmt_lvol is actually drained and freed, so destroy_xfer_task_tmo's gate is meaningful - close a race where device-remove teardown wins for the same rmt_lvol: its poller now also clears xfer_task->pg[lpg->id], otherwise that flag would hang forever instead of draining - set_req_status_and_queued: return early on the (compiled-out-under- NDEBUG) assert(false) paths instead of falling through and scribbling further on already-inconsistent state No behavior change to destroy_xfer_task_tmo itself or to timeouts/retries. --- include/spdk/blob.h | 1 + lib/lvol/lvol.c | 101 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 96 insertions(+), 6 deletions(-) diff --git a/include/spdk/blob.h b/include/spdk/blob.h index 255bfcb7e5f..24b488b4869 100644 --- a/include/spdk/blob.h +++ b/include/spdk/blob.h @@ -119,6 +119,7 @@ enum xfer_req_status { XFER_REQ_STATUS_READY, XFER_REQ_STATUS_DONE, XFER_REQ_STATUS_FAILED, + XFER_REQ_STATUS_IN_FLIGHT, }; typedef enum { diff --git a/lib/lvol/lvol.c b/lib/lvol/lvol.c index 207eee14660..fe83535a5b8 100644 --- a/lib/lvol/lvol.c +++ b/lib/lvol/lvol.c @@ -3967,6 +3967,7 @@ spdk_wait_for_pg_io_cleanup_poller(void *arg) { struct remote_lvol_info *rmt_lvol = arg; struct spdk_lvs_poll_group *lpg = rmt_lvol->group; struct spdk_transfer_dev *tdev = rmt_lvol->tdev; + struct spdk_lvs_xfer *xfer_task = rmt_lvol->xfer_task; if (rmt_lvol->outstanding_io > 0) { return SPDK_POLLER_BUSY; @@ -3979,6 +3980,15 @@ spdk_wait_for_pg_io_cleanup_poller(void *arg) { rmt_lvol->channel = NULL; tdev->pg[lpg->id]--; free(rmt_lvol); + /* + * If an xfer-task teardown (spdk_delete_rmt_lvol_pg) raced this device-remove and + * deferred to us instead of freeing directly, it left xfer_task->pg[lpg->id] set so + * destroy_xfer_task_tmo would wait. We now own completing that hand-off. Harmless + * no-op when no such race happened (flag is already false). + */ + if (xfer_task) { + xfer_task->pg[lpg->id] = false; + } return -1; } @@ -4156,14 +4166,15 @@ set_req_status_and_queued(struct spdk_lvs_xfer_req *req, enum xfer_req_status st if (rmt->outstanding_io == 0) { SPDK_ERRLOG("outstanding_io underflow\n"); assert(false); - } else { - rmt->outstanding_io--; + return; } + rmt->outstanding_io--; req->status = status; if (spdk_ring_enqueue(rmt->free_ring, (void **)&req, 1, NULL) != 1) { SPDK_ERRLOG("free_ring full while handling write submit failure\n"); assert(false); + return; } } @@ -4410,6 +4421,7 @@ helper_xfer_poller(void *arg) count++; rmt_lvol->outstanding_io++; req->rmt_lvol = rmt_lvol; + req->status = XFER_REQ_STATUS_IN_FLIGHT; if (rmt_lvol->desc == NULL || rmt_lvol->channel == NULL || !rmt_lvol->status) { set_req_status_and_queued(req, XFER_REQ_STATUS_FAILED); @@ -4645,11 +4657,73 @@ read_complete_cb(void *arg, int rc) } } +static void +xfer_abort_cpl(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) +{ + spdk_bdev_free_io(bdev_io); +} + +/* + * Best-effort: nudge any req still mid-flight for this rmt_lvol so the drain below + * resolves faster. A req that already moved past the S3-GET leg (e.g. into the local + * blobstore write) has no matching bdev_io here and spdk_bdev_abort() is a harmless + * no-op for it; it will still complete and drain on its own via outstanding_io. + */ +static void +abort_inflight_reqs_for_rmt_lvol(struct spdk_lvs_xfer *xfer, struct remote_lvol_info *rmt_lvol) +{ + if (rmt_lvol->desc == NULL || rmt_lvol->channel == NULL) { + return; + } + + for (int i = 0; i < xfer->cluster_batch; i++) { + struct spdk_lvs_xfer_req *req = &xfer->reqs[i]; + + if (req->rmt_lvol == rmt_lvol && req->status == XFER_REQ_STATUS_IN_FLIGHT) { + spdk_bdev_abort(rmt_lvol->desc, rmt_lvol->channel, req, xfer_abort_cpl, NULL); + } + } +} + +/* + * Mirrors spdk_wait_for_pg_io_cleanup_poller (device-remove path): defers + * put_io_channel/free(rmt_lvol) until outstanding_io drains to zero. Additionally clears + * xfer->pg[lpg->id], which spdk_delete_rmt_lvol_pg deferred to this poller instead of + * clearing immediately, so destroy_xfer_task_tmo cannot free reqs/pdus/rings while this + * rmt_lvol's IO is still in flight. + */ +static int +spdk_wait_for_xfer_pg_io_cleanup_poller(void *arg) +{ + struct remote_lvol_info *rmt_lvol = arg; + struct spdk_lvs_poll_group *lpg = rmt_lvol->group; + struct spdk_lvs_xfer *xfer = rmt_lvol->xfer_task; + + if (rmt_lvol->outstanding_io > 0) { + return SPDK_POLLER_BUSY; + } + + spdk_poller_unregister(&rmt_lvol->cleanup_poller); + rmt_lvol->cleanup_poller = NULL; + + if (rmt_lvol->channel) { + spdk_put_io_channel(rmt_lvol->channel); + if (rmt_lvol->tdev->pg[lpg->id] > 0) { + rmt_lvol->tdev->pg[lpg->id]--; + } + } + + free(rmt_lvol); + xfer->pg[lpg->id] = false; + return -1; +} + static void spdk_delete_rmt_lvol_pg(void *arg) { struct spdk_lvs_xfer *xfer = arg; struct spdk_lvs_poll_group *lpg = NULL; struct remote_lvol_info *rmt_lvol, *tmp; + bool deferred = false; TAILQ_FOREACH(lpg, &g_lvs_poll_groups, entry) { if (lpg->thread != spdk_get_thread()) { @@ -4668,9 +4742,11 @@ spdk_delete_rmt_lvol_pg(void *arg) { continue; } - // if the poller is already set, it means the channel is in cleaning up progress, - // skip it to avoid duplicate cleanup + // if the poller is already set, it means the channel is in cleaning up progress + // (racing device-remove teardown, spdk_wait_for_pg_io_cleanup_poller); that poller + // clears xfer_task->pg[lpg->id] itself once it completes, so leave it set here. if (rmt_lvol->cleanup_poller) { + deferred = true; continue; } SPDK_NOTICELOG("destroy rmt lvol and transfer task ---: 2.\n"); @@ -4678,17 +4754,30 @@ spdk_delete_rmt_lvol_pg(void *arg) { TAILQ_REMOVE(&lpg->rmt_lvols, rmt_lvol, entry); rmt_lvol->status = false; + if (rmt_lvol->outstanding_io > 0) { + SPDK_NOTICELOG("Waiting for %lu I/Os to finish for pg %s (xfer teardown).\n", + rmt_lvol->outstanding_io, lpg->thread_name); + abort_inflight_reqs_for_rmt_lvol(xfer, rmt_lvol); + rmt_lvol->cleanup_poller = spdk_poller_register( + spdk_wait_for_xfer_pg_io_cleanup_poller, rmt_lvol, 200000); // check every 200ms + deferred = true; + break; + } + if (rmt_lvol->channel) { spdk_put_io_channel(rmt_lvol->channel); if (rmt_lvol->tdev->pg[lpg->id] > 0) { rmt_lvol->tdev->pg[lpg->id]--; } } - + free(rmt_lvol); break; } - xfer->pg[lpg->id] = false; + + if (!deferred) { + xfer->pg[lpg->id] = false; + } } static int From c879723801dd33253b5d7690a034e7527fa32ee6 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 7 Aug 2026 12:52:30 +0200 Subject: [PATCH 2/3] Update AWS SDK ref --- docker/Dockerfile_spdk_base | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docker/Dockerfile_spdk_base b/docker/Dockerfile_spdk_base index 70023a8068d..981e48c1a5e 100644 --- a/docker/Dockerfile_spdk_base +++ b/docker/Dockerfile_spdk_base @@ -29,7 +29,16 @@ RUN dnf update -y && dnf upgrade -y && \ cd .. && rm -rf openssl-3.5.6* # Clone + build AWS SDK -RUN git clone --recurse-submodules https://github.com/simplyblock-io/aws-sdk-cpp && \ +# Fork repo and ref (branch, tag, or full SHA) are build-args so CI/callers can +# pin exactly what gets built. Empty AWS_SDK_REF keeps the previous behavior: +# clone the default branch HEAD with its recorded submodules. +ARG AWS_SDK_REPO=https://github.com/simplyblock-io/aws-sdk-cpp +ARG AWS_SDK_REF=fix-segfaults +RUN git clone --recurse-submodules "${AWS_SDK_REPO}" aws-sdk-cpp && \ + if [ -n "${AWS_SDK_REF}" ]; then \ + git -C aws-sdk-cpp checkout "${AWS_SDK_REF}" && \ + git -C aws-sdk-cpp submodule update --init --recursive; \ + fi && \ touch aws-sdk-cpp/crt/aws-crt-cpp/include/aws/crt/pthread_utils.h && \ cp aws-sdk-cpp/pthread_utils.txt aws-sdk-cpp/crt/aws-crt-cpp/include/aws/crt/pthread_utils.h && \ mkdir -p aws-sdk-cpp/sdk_build && \ From 7e224cdad3e0af722ee4234d8f72e101aa225e27 Mon Sep 17 00:00:00 2001 From: Max Schettler Date: Fri, 7 Aug 2026 12:53:26 +0200 Subject: [PATCH 3/3] Adopt workflow from master --- .../docker-base-image-builder-amd.yml | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/docker-base-image-builder-amd.yml diff --git a/.github/workflows/docker-base-image-builder-amd.yml b/.github/workflows/docker-base-image-builder-amd.yml new file mode 100644 index 00000000000..d356cf827d8 --- /dev/null +++ b/.github/workflows/docker-base-image-builder-amd.yml @@ -0,0 +1,56 @@ +name: Docker Base Image Build (AMD) + +on: + workflow_dispatch: + workflow_run: + workflows: ["Docker Base Image Build (ARM)"] + types: + - completed + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@master + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: hamdysimplyblock + password: ${{ secrets.DOCKER_PASS }} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_HAMDI }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_HAMDI }} + aws-region: us-east-1 + + - name: Login to ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Build & Push + run: | + docker build -t simplyblock/spdk-core:base_image_amd64 \ + -t public.ecr.aws/simply-block/spdk-core:base_image_amd64 -f docker/Dockerfile_spdk_base . --push + + - name: Create the manifest + run: | + DH_ARM="simplyblock/spdk-core:base_image_arm64" + DH_AMD="simplyblock/spdk-core:base_image_amd64" + ECR_ARM="public.ecr.aws/simply-block/spdk-core:base_image_arm64" + ECR_AMD="public.ecr.aws/simply-block/spdk-core:base_image_amd64" + docker manifest create simplyblock/spdk-core:base_image "$DH_ARM" "$DH_AMD" + docker manifest annotate simplyblock/spdk-core:base_image "$DH_ARM" --arch arm64 + docker manifest annotate simplyblock/spdk-core:base_image "$DH_AMD" --arch amd64 + docker manifest push simplyblock/spdk-core:base_image + docker manifest create public.ecr.aws/simply-block/spdk-core:base_image "$ECR_ARM" "$ECR_AMD" + docker manifest annotate public.ecr.aws/simply-block/spdk-core:base_image "$ECR_ARM" --arch arm64 + docker manifest annotate public.ecr.aws/simply-block/spdk-core:base_image "$ECR_AMD" --arch amd64 + docker manifest push public.ecr.aws/simply-block/spdk-core:base_image