diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs index 37cfe4631e..f23d8774d1 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs @@ -271,5 +271,101 @@ public async Task GivenSearchParameterDelete_WhenConcurrencyConflictExhaustsRetr Assert.Contains(" Deletion.3", exception.InnerException.Message); } + + [Fact] + public async Task GivenBulkHardDelete_WhenSearchServiceThrowsConnectionExceptionOnSecondPage_ThenReturnsIncompleteOperationException() + { + // Arrange + var resourceType = "Patient"; + var parameters = new List>() + { + Tuple.Create("_lastUpdated", "2000-01-01T00:00:00Z"), + }; + + var request = new ConditionalDeleteResourceRequest( + resourceType, + parameters, + DeleteOperation.HardDelete, + maxDeleteCount: 10, + deleteAll: false); + + var searchService = Substitute.For(); + var scopedSearchService = Substitute.For>(); + scopedSearchService.Value.Returns(searchService); + _searchServiceFactory.Invoke().Returns(scopedSearchService); + + // First page of results - returns 5 entries with a continuation token + var firstPageEntries = new List(); + for (int i = 0; i < 5; i++) + { + var resource = Samples.GetDefaultPatient().ToPoco(); + resource.Id = $"id-{i}"; + resource.VersionId = "1"; + + var resourceElement = resource.ToResourceElement(); + var rawResource = new RawResource(resource.ToJson(), FhirResourceFormat.Json, isMetaSet: false); + var resourceRequest = Substitute.For(); + var compartmentIndices = Substitute.For(); + var wrapper = new ResourceWrapper(resourceElement, rawResource, resourceRequest, false, null, compartmentIndices, new List>(), "hash"); + firstPageEntries.Add(new SearchResultEntry(wrapper, SearchEntryMode.Match)); + } + + var callCount = 0; + searchService.SearchAsync( + Arg.Any(), + Arg.Any>>(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()).Returns(callInfo => + { + callCount++; + if (callCount == 1) + { + // First call returns results with continuation token + return Task.FromResult(new SearchResult(firstPageEntries, "continuation-token-1", null, Array.Empty>())); + } + else + { + // Second call throws a connection exception (simulating network failure) + throw new InvalidOperationException("A transport-level error has occurred when receiving results from the server."); + } + }); + + var fhirDataStore = Substitute.For(); + fhirDataStore.HardDeleteAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.CompletedTask); + + var scopedDataStore = new DeletionServiceScopedDataStore(fhirDataStore); + _dataStoreFactory.GetScopedDataStore().Returns(scopedDataStore); + + // Act + var exception = await Assert.ThrowsAsync>>(async () => + await _service.DeleteMultipleAsync(request, CancellationToken.None)); + + // Assert + Assert.NotNull(exception); + Assert.NotNull(exception.InnerException); + Assert.IsType(exception.InnerException); + + var aggregateException = (AggregateException)exception.InnerException; + Assert.Contains(aggregateException.InnerExceptions, ex => ex is InvalidOperationException); + + // Verify that partial results contain the first page of deleted resources + Assert.NotNull(exception.PartialResults); + Assert.True(exception.PartialResults.ContainsKey("Patient")); + Assert.Equal(5, exception.PartialResults["Patient"]); + + // Verify search service was called twice (first page succeeded, second page failed) + await searchService.Received(2).SearchAsync( + Arg.Any(), + Arg.Any>>(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs index 04b50be38b..1d06e50d25 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs @@ -218,6 +218,7 @@ private async Task> DeleteMultipleAsyncInternal(Condit } // Delete the matched results... + List exceptionsOutsideTasks = new List(); try { while (results.Any() || !string.IsNullOrEmpty(ct)) @@ -311,6 +312,7 @@ private async Task> DeleteMultipleAsyncInternal(Condit catch (Exception ex) { _logger.LogError(ex, "Error deleting"); + exceptionsOutsideTasks.Add(ex); await cancellationTokenSource.CancelAsync(); } @@ -334,9 +336,9 @@ private async Task> DeleteMultipleAsyncInternal(Condit resourceTypesDeleted = AppendDeleteResults(resourceTypesDeleted, deleteTasks.Where(x => x.IsCompletedSuccessfully).Select(task => task.Result)); - if (deleteTasks.Any((task) => task.IsFaulted || task.IsCanceled) || tooManyIncludeResults) + if (deleteTasks.Any((task) => task.IsFaulted || task.IsCanceled) || tooManyIncludeResults || exceptionsOutsideTasks.Any()) { - var exceptions = new List(); + var exceptions = new List(exceptionsOutsideTasks); if (tooManyIncludeResults) { @@ -395,7 +397,7 @@ await CreateAuditLog( return new ResourceWrapperOperation(deletedWrapper, true, keepHistory, null, false, false, bundleResourceContext: request.BundleResourceContext); })); - var partialResults = new List<(string, string, bool)>(); + var partialResults = new List<(string ResourceType, string ResourceId, bool IsInclude)>(); try { using var scopedDataStore = _dataStoreFactory.GetScopedDataStore(); @@ -439,6 +441,14 @@ await CreateAuditLog( ex.InnerException, ids.GroupBy(pair => pair.ResourceType).ToDictionary(group => group.Key, group => (long)group.Count())); } + catch (Exception ex) + { + _logger.LogError(ex, "Error soft deleting"); + await CreateAuditLog(request.ResourceType, request.DeleteOperation, true, partialResults); + throw new IncompleteOperationException>( + ex, + partialResults.GroupBy(pair => pair.ResourceType).ToDictionary(group => group.Key, group => (long)group.Count())); + } await CreateAuditLog( request.ResourceType,