SFAM-2777 fixes

.github/workflows/docker-base-image-builder-amd.yml

@@ -0,0 +1,56 @@
+name: Docker Base Image Build (AMD)
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: ["Docker Base Image Build (ARM)"]
+    types:
+      - completed
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: hamdysimplyblock
+        password: ${{ secrets.DOCKER_PASS }}
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_HAMDI }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_HAMDI }}
+        aws-region: us-east-1
+
+    - name: Login to ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v2
+      with:
+        registry-type: public
+
+    - name: Build & Push
+      run: |
+        docker build -t simplyblock/spdk-core:base_image_amd64 \
+        -t public.ecr.aws/simply-block/spdk-core:base_image_amd64 -f docker/Dockerfile_spdk_base . --push
+
+    - name: Create the manifest
+      run: |
+        DH_ARM="simplyblock/spdk-core:base_image_arm64"
+        DH_AMD="simplyblock/spdk-core:base_image_amd64"
+        ECR_ARM="public.ecr.aws/simply-block/spdk-core:base_image_arm64"
+        ECR_AMD="public.ecr.aws/simply-block/spdk-core:base_image_amd64"
+        docker manifest create simplyblock/spdk-core:base_image "$DH_ARM" "$DH_AMD"
+        docker manifest annotate simplyblock/spdk-core:base_image "$DH_ARM" --arch arm64
+        docker manifest annotate simplyblock/spdk-core:base_image "$DH_AMD" --arch amd64
+        docker manifest push simplyblock/spdk-core:base_image
+        docker manifest create public.ecr.aws/simply-block/spdk-core:base_image "$ECR_ARM" "$ECR_AMD"
+        docker manifest annotate public.ecr.aws/simply-block/spdk-core:base_image "$ECR_ARM" --arch arm64
+        docker manifest annotate public.ecr.aws/simply-block/spdk-core:base_image "$ECR_AMD" --arch amd64
+        docker manifest push public.ecr.aws/simply-block/spdk-core:base_image

docker/Dockerfile_spdk_base

@@ -29,7 +29,16 @@ RUN dnf update -y && dnf upgrade -y && \
     cd .. && rm -rf openssl-3.5.6*
 
 # Clone + build AWS SDK
-RUN git clone --recurse-submodules https://github.com/simplyblock-io/aws-sdk-cpp && \
+# Fork repo and ref (branch, tag, or full SHA) are build-args so CI/callers can
+# pin exactly what gets built. Empty AWS_SDK_REF keeps the previous behavior:
+# clone the default branch HEAD with its recorded submodules.
+ARG AWS_SDK_REPO=https://github.com/simplyblock-io/aws-sdk-cpp
+ARG AWS_SDK_REF=fix-segfaults
+RUN git clone --recurse-submodules "${AWS_SDK_REPO}" aws-sdk-cpp && \
+    if [ -n "${AWS_SDK_REF}" ]; then \
+        git -C aws-sdk-cpp checkout "${AWS_SDK_REF}" && \
+        git -C aws-sdk-cpp submodule update --init --recursive; \
+    fi && \
     touch aws-sdk-cpp/crt/aws-crt-cpp/include/aws/crt/pthread_utils.h && \
     cp aws-sdk-cpp/pthread_utils.txt aws-sdk-cpp/crt/aws-crt-cpp/include/aws/crt/pthread_utils.h && \
     mkdir -p aws-sdk-cpp/sdk_build && \

include/spdk/blob.h

@@ -119,6 +119,7 @@ enum xfer_req_status {
 	XFER_REQ_STATUS_READY,
 	XFER_REQ_STATUS_DONE,
 	XFER_REQ_STATUS_FAILED,
+	XFER_REQ_STATUS_IN_FLIGHT,
 };
 
 typedef enum {

lib/lvol/lvol.c

@@ -3967,6 +3967,7 @@ spdk_wait_for_pg_io_cleanup_poller(void *arg) {
 	struct remote_lvol_info *rmt_lvol = arg;
 	struct spdk_lvs_poll_group *lpg = rmt_lvol->group;
 	struct spdk_transfer_dev *tdev = rmt_lvol->tdev;
+	struct spdk_lvs_xfer *xfer_task = rmt_lvol->xfer_task;
 
 	if (rmt_lvol->outstanding_io > 0) {
 		return SPDK_POLLER_BUSY;
@@ -3979,6 +3980,15 @@ spdk_wait_for_pg_io_cleanup_poller(void *arg) {
 	rmt_lvol->channel = NULL;
 	tdev->pg[lpg->id]--;
 	free(rmt_lvol);
+	/*
+	 * If an xfer-task teardown (spdk_delete_rmt_lvol_pg) raced this device-remove and
+	 * deferred to us instead of freeing directly, it left xfer_task->pg[lpg->id] set so
+	 * destroy_xfer_task_tmo would wait. We now own completing that hand-off. Harmless
+	 * no-op when no such race happened (flag is already false).
+	 */
+	if (xfer_task) {
+		xfer_task->pg[lpg->id] = false;
+	}
 	return -1;
 }
 
@@ -4156,14 +4166,15 @@ set_req_status_and_queued(struct spdk_lvs_xfer_req *req, enum xfer_req_status st
 	if (rmt->outstanding_io == 0) {
 		SPDK_ERRLOG("outstanding_io underflow\n");
 		assert(false);
-	} else {
-		rmt->outstanding_io--;
+		return;
 	}
+	rmt->outstanding_io--;
 
 	req->status = status;
 	if (spdk_ring_enqueue(rmt->free_ring, (void **)&req, 1, NULL) != 1) {
 		SPDK_ERRLOG("free_ring full while handling write submit failure\n");
 		assert(false);
+		return;
 	}
 }
 
@@ -4410,6 +4421,7 @@ helper_xfer_poller(void *arg)
 		count++;
 		rmt_lvol->outstanding_io++;
 		req->rmt_lvol = rmt_lvol;
+		req->status = XFER_REQ_STATUS_IN_FLIGHT;
 
 		if (rmt_lvol->desc == NULL || rmt_lvol->channel == NULL || !rmt_lvol->status) {
 			set_req_status_and_queued(req, XFER_REQ_STATUS_FAILED);
@@ -4645,11 +4657,73 @@ read_complete_cb(void *arg, int rc)
 	}
 }
 
+static void
+xfer_abort_cpl(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
+{
+	spdk_bdev_free_io(bdev_io);
+}
+
+/*
+ * Best-effort: nudge any req still mid-flight for this rmt_lvol so the drain below
+ * resolves faster. A req that already moved past the S3-GET leg (e.g. into the local
+ * blobstore write) has no matching bdev_io here and spdk_bdev_abort() is a harmless
+ * no-op for it; it will still complete and drain on its own via outstanding_io.
+ */
+static void
+abort_inflight_reqs_for_rmt_lvol(struct spdk_lvs_xfer *xfer, struct remote_lvol_info *rmt_lvol)
+{
+	if (rmt_lvol->desc == NULL || rmt_lvol->channel == NULL) {
+		return;
+	}
+
+	for (int i = 0; i < xfer->cluster_batch; i++) {
+		struct spdk_lvs_xfer_req *req = &xfer->reqs[i];
+
+		if (req->rmt_lvol == rmt_lvol && req->status == XFER_REQ_STATUS_IN_FLIGHT) {
+			spdk_bdev_abort(rmt_lvol->desc, rmt_lvol->channel, req, xfer_abort_cpl, NULL);
+		}
+	}
+}
+
+/*
+ * Mirrors spdk_wait_for_pg_io_cleanup_poller (device-remove path): defers
+ * put_io_channel/free(rmt_lvol) until outstanding_io drains to zero. Additionally clears
+ * xfer->pg[lpg->id], which spdk_delete_rmt_lvol_pg deferred to this poller instead of
+ * clearing immediately, so destroy_xfer_task_tmo cannot free reqs/pdus/rings while this
+ * rmt_lvol's IO is still in flight.
+ */
+static int
+spdk_wait_for_xfer_pg_io_cleanup_poller(void *arg)
+{
+	struct remote_lvol_info *rmt_lvol = arg;
+	struct spdk_lvs_poll_group *lpg = rmt_lvol->group;
+	struct spdk_lvs_xfer *xfer = rmt_lvol->xfer_task;
+
+	if (rmt_lvol->outstanding_io > 0) {
+		return SPDK_POLLER_BUSY;
+	}
+
+	spdk_poller_unregister(&rmt_lvol->cleanup_poller);
+	rmt_lvol->cleanup_poller = NULL;
+
+	if (rmt_lvol->channel) {
+		spdk_put_io_channel(rmt_lvol->channel);
+		if (rmt_lvol->tdev->pg[lpg->id] > 0) {
+			rmt_lvol->tdev->pg[lpg->id]--;
+		}
+	}
+
+	free(rmt_lvol);
+	xfer->pg[lpg->id] = false;
+	return -1;
+}
+
 static void
 spdk_delete_rmt_lvol_pg(void *arg) {
 	struct spdk_lvs_xfer *xfer = arg;
 	struct spdk_lvs_poll_group *lpg = NULL;
 	struct remote_lvol_info *rmt_lvol, *tmp;
+	bool deferred = false;
 
 	TAILQ_FOREACH(lpg, &g_lvs_poll_groups, entry) {
 		if (lpg->thread != spdk_get_thread()) {
@@ -4668,27 +4742,42 @@ spdk_delete_rmt_lvol_pg(void *arg) {
 			continue;
 		}
 
-		// if the poller is already set, it means the channel is in cleaning up progress,
-		// skip it to avoid duplicate cleanup		
+		// if the poller is already set, it means the channel is in cleaning up progress
+		// (racing device-remove teardown, spdk_wait_for_pg_io_cleanup_poller); that poller
+		// clears xfer_task->pg[lpg->id] itself once it completes, so leave it set here.
 		if (rmt_lvol->cleanup_poller) {
+			deferred = true;
 			continue;
 		}
 		SPDK_NOTICELOG("destroy rmt lvol and transfer task ---: 2.\n");
 
 		TAILQ_REMOVE(&lpg->rmt_lvols, rmt_lvol, entry);
 		rmt_lvol->status = false;
 
+		if (rmt_lvol->outstanding_io > 0) {
+			SPDK_NOTICELOG("Waiting for %lu I/Os to finish for pg %s (xfer teardown).\n",
+					rmt_lvol->outstanding_io, lpg->thread_name);
+			abort_inflight_reqs_for_rmt_lvol(xfer, rmt_lvol);
+			rmt_lvol->cleanup_poller = spdk_poller_register(
+				spdk_wait_for_xfer_pg_io_cleanup_poller, rmt_lvol, 200000); // check every 200ms
+			deferred = true;
+			break;
+		}
+
 		if (rmt_lvol->channel) {
 			spdk_put_io_channel(rmt_lvol->channel);
 			if (rmt_lvol->tdev->pg[lpg->id] > 0) {
 				rmt_lvol->tdev->pg[lpg->id]--;
 			}
 		}
-		
+
 		free(rmt_lvol);
 		break;
 	}
-	xfer->pg[lpg->id] = false;
+
+	if (!deferred) {
+		xfer->pg[lpg->id] = false;
+	}
 }
 
 static int