Harden Bulk Delete Error Handling

src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs

@@ -271,5 +271,101 @@
 
             Assert.Contains(" Deletion.3", exception.InnerException.Message);
         }
+
+        [Fact]
+        public async Task GivenBulkHardDelete_WhenSearchServiceThrowsConnectionExceptionOnSecondPage_ThenReturnsIncompleteOperationException()
+        {
+            // Arrange
+            var resourceType = "Patient";
+            var parameters = new List<Tuple<string, string>>()
+            {
+                Tuple.Create("_lastUpdated", "2000-01-01T00:00:00Z"),
+            };
+
+            var request = new ConditionalDeleteResourceRequest(
+                resourceType,
+                parameters,
+                DeleteOperation.HardDelete,
+                maxDeleteCount: 10,
+                deleteAll: false);
+
+            var searchService = Substitute.For<ISearchService>();
+            var scopedSearchService = Substitute.For<IScoped<ISearchService>>();
+            scopedSearchService.Value.Returns(searchService);
+            _searchServiceFactory.Invoke().Returns(scopedSearchService);
+
+            // First page of results - returns 5 entries with a continuation token
+            var firstPageEntries = new List<SearchResultEntry>();
+            for (int i = 0; i < 5; i++)
+            {
+                var resource = Samples.GetDefaultPatient().ToPoco<Patient>();
+                resource.Id = $"id-{i}";
+                resource.VersionId = "1";
+
+                var resourceElement = resource.ToResourceElement();
+                var rawResource = new RawResource(resource.ToJson(), FhirResourceFormat.Json, isMetaSet: false);
+                var resourceRequest = Substitute.For<ResourceRequest>();
+                var compartmentIndices = Substitute.For<CompartmentIndices>();
+                var wrapper = new ResourceWrapper(resourceElement, rawResource, resourceRequest, false, null, compartmentIndices, new List<KeyValuePair<string, string>>(), "hash");
+                firstPageEntries.Add(new SearchResultEntry(wrapper, SearchEntryMode.Match));
+            }
+
+            var callCount = 0;
+            searchService.SearchAsync(
+                Arg.Any<string>(),
+                Arg.Any<IReadOnlyList<Tuple<string, string>>>(),
+                Arg.Any<CancellationToken>(),
+                Arg.Any<bool>(),
+                Arg.Any<ResourceVersionType>(),
+                Arg.Any<bool>(),
+                Arg.Any<bool>()).Returns(callInfo =>
+                {
+                    callCount++;
+                    if (callCount == 1)
+                    {
+                        // First call returns results with continuation token
+                        return Task.FromResult(new SearchResult(firstPageEntries, "continuation-token-1", null, Array.Empty<Tuple<string, string>>()));
+                    }
+                    else
+                    {
+                        // Second call throws a connection exception (simulating network failure)
+                        throw new InvalidOperationException("A transport-level error has occurred when receiving results from the server.");
+                    }
+                });
+
+            var fhirDataStore = Substitute.For<IFhirDataStore>();
+            fhirDataStore.HardDeleteAsync(Arg.Any<ResourceKey>(), Arg.Any<bool>(), Arg.Any<bool>(), Arg.Any<CancellationToken>())
+                .Returns(Task.CompletedTask);
+
+            var scopedDataStore = new DeletionServiceScopedDataStore(fhirDataStore);
+            _dataStoreFactory.GetScopedDataStore().Returns(scopedDataStore);
+
+            // Act
+            var exception = await Assert.ThrowsAsync<IncompleteOperationException<Dictionary<string, long>>>(async () =>
+                await _service.DeleteMultipleAsync(request, CancellationToken.None));
+
+            // Assert
+            Assert.NotNull(exception);
+            Assert.NotNull(exception.InnerException);
+            Assert.IsType<AggregateException>(exception.InnerException);
+
+            var aggregateException = (AggregateException)exception.InnerException;
+            Assert.Contains(aggregateException.InnerExceptions, ex => ex is InvalidOperationException);
+
+            // Verify that partial results contain the first page of deleted resources
+            Assert.NotNull(exception.PartialResults);
+            Assert.True(exception.PartialResults.ContainsKey("Patient"));
+            Assert.Equal(5, exception.PartialResults["Patient"]);
+
+            // Verify search service was called twice (first page succeeded, second page failed)
+            await searchService.Received(2).SearchAsync(
+                Arg.Any<string>(),
+                Arg.Any<IReadOnlyList<Tuple<string, string>>>(),
+                Arg.Any<CancellationToken>(),
+                Arg.Any<bool>(),
+                Arg.Any<ResourceVersionType>(),
+                Arg.Any<bool>(),
+                Arg.Any<bool>());
+        }
     }
 }

src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs

@@ -218,6 +218,7 @@
             }
 
             // Delete the matched results...
+            List<Exception> exceptionsOutsideTasks = new List<Exception>();
             try
             {
                 while (results.Any() || !string.IsNullOrEmpty(ct))
@@ -311,6 +312,7 @@
             catch (Exception ex)
             {
                 _logger.LogError(ex, "Error deleting");
+                exceptionsOutsideTasks.Add(ex);
                 await cancellationTokenSource.CancelAsync();
             }
 
@@ -334,9 +336,9 @@
 
             resourceTypesDeleted = AppendDeleteResults(resourceTypesDeleted, deleteTasks.Where(x => x.IsCompletedSuccessfully).Select(task => task.Result));
 
-            if (deleteTasks.Any((task) => task.IsFaulted || task.IsCanceled) || tooManyIncludeResults)
+            if (deleteTasks.Any((task) => task.IsFaulted || task.IsCanceled) || tooManyIncludeResults || exceptionsOutsideTasks.Any())
             {
-                var exceptions = new List<Exception>();
+                var exceptions = new List<Exception>(exceptionsOutsideTasks);
 
                 if (tooManyIncludeResults)
                 {
@@ -395,7 +397,7 @@
                 return new ResourceWrapperOperation(deletedWrapper, true, keepHistory, null, false, false, bundleResourceContext: request.BundleResourceContext);
             }));
 
-            var partialResults = new List<(string, string, bool)>();
+            var partialResults = new List<(string ResourceType, string ResourceId, bool IsInclude)>();
             try
             {
                 using var scopedDataStore = _dataStoreFactory.GetScopedDataStore();
@@ -439,6 +441,14 @@
                     ex.InnerException,
                     ids.GroupBy(pair => pair.ResourceType).ToDictionary(group => group.Key, group => (long)group.Count()));
             }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error soft deleting");
+                await CreateAuditLog(request.ResourceType, request.DeleteOperation, true, partialResults);
+                throw new IncompleteOperationException<Dictionary<string, long>>(
+                    ex,
+                    partialResults.GroupBy(pair => pair.ResourceType).ToDictionary(group => group.Key, group => (long)group.Count()));
+            }
 
             await CreateAuditLog(
                 request.ResourceType,