From cb676eb14f21242774789b514c0f2798306dced5 Mon Sep 17 00:00:00 2001 From: Aftab Ahmed <58620536+aftab23822@users.noreply.github.com> Date: Tue, 16 Sep 2025 20:19:57 +0500 Subject: [PATCH 1/4] Fixed #14306 and tested with 15K, 30K and 60K records. Chunking + async/await is a reliable pattern for large data processing. --- .../Storage/Operations/FileSizeCalculator.cs | 145 ++++++++++-------- 1 file changed, 83 insertions(+), 62 deletions(-) diff --git a/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs b/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs index 3e60af00da9b..e8b96af49bc9 100644 --- a/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs +++ b/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs @@ -25,82 +25,103 @@ public FileSizeCalculator(params string[] paths) public async Task ComputeSizeAsync(CancellationToken cancellationToken = default) { - await Parallel.ForEachAsync( - _paths, - cancellationToken, - async (path, token) => await Task.Factory.StartNew(() => - { - ComputeSizeRecursively(path, token); - }, - token, - TaskCreationOptions.LongRunning, - TaskScheduler.Default)); + const int ChunkSize = 1000; + var queue = new Queue(_paths); + var batch = new List(ChunkSize); - unsafe void ComputeSizeRecursively(string path, CancellationToken token) + while (!cancellationToken.IsCancellationRequested && queue.TryDequeue(out var currentPath)) { - var queue = new Queue(); - if (!Win32Helper.HasFileAttribute(path, FileAttributes.Directory)) + if (!Win32Helper.HasFileAttribute(currentPath, FileAttributes.Directory)) { - ComputeFileSize(path); + batch.Add(currentPath); } else { - queue.Enqueue(path); - - while (queue.TryDequeue(out var directory)) + try { - WIN32_FIND_DATAW findData = default; - - fixed (char* pszFilePath = directory + "\\*.*") + foreach (var file in Directory.EnumerateFiles(currentPath)) { - var hFile = PInvoke.FindFirstFileEx( - pszFilePath, - FINDEX_INFO_LEVELS.FindExInfoBasic, - &findData, - FINDEX_SEARCH_OPS.FindExSearchNameMatch, - null, - FIND_FIRST_EX_FLAGS.FIND_FIRST_EX_LARGE_FETCH); - - if (!hFile.IsNull) + if (cancellationToken.IsCancellationRequested) + break; + batch.Add(file); + if (batch.Count >= ChunkSize) { - do - { - FILE_FLAGS_AND_ATTRIBUTES attributes = (FILE_FLAGS_AND_ATTRIBUTES)findData.dwFileAttributes; - - if (attributes.HasFlag(FILE_FLAGS_AND_ATTRIBUTES.FILE_ATTRIBUTE_REPARSE_POINT)) - // Skip symbolic links and junctions - continue; - - var itemPath = Path.Combine(directory, findData.cFileName.ToString()); - - // Skip current and parent directory entries - var fileName = findData.cFileName.ToString(); - if (fileName.Equals(".", StringComparison.OrdinalIgnoreCase) || - fileName.Equals("..", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - if (attributes.HasFlag(FILE_FLAGS_AND_ATTRIBUTES.FILE_ATTRIBUTE_DIRECTORY)) - { - queue.Enqueue(itemPath); - } - else - { - ComputeFileSize(itemPath); - } - - if (token.IsCancellationRequested) - break; - } - while (PInvoke.FindNextFile(hFile, &findData)); - - PInvoke.FindClose(hFile); + ComputeFileSizeBatch(batch); + batch.Clear(); + await Task.Yield(); } } } + catch (UnauthorizedAccessException) { } + catch (IOException) { } +#if DEBUG + catch (Exception ex) + { + System.Diagnostics.Debug.WriteLine(ex); + } +#endif + + try + { + foreach (var dir in Directory.EnumerateDirectories(currentPath)) + { + if (cancellationToken.IsCancellationRequested) + break; + queue.Enqueue(dir); + } + } + catch (UnauthorizedAccessException) { } + catch (IOException) { } +#if DEBUG + catch (Exception ex) + { + System.Diagnostics.Debug.WriteLine(ex); + } +#endif + + } + + if (batch.Count >= ChunkSize) + { + ComputeFileSizeBatch(batch); + batch.Clear(); + await Task.Yield(); } } + + if (batch.Count > 0) + { + ComputeFileSizeBatch(batch); + batch.Clear(); + } + } + + private void ComputeFileSizeBatch(IEnumerable files) + { + long batchTotal = 0; + foreach (var path in files) + { + if (_computedFiles.ContainsKey(path)) + continue; + + using var hFile = PInvoke.CreateFile( + path, + (uint)FILE_ACCESS_RIGHTS.FILE_READ_ATTRIBUTES, + FILE_SHARE_MODE.FILE_SHARE_READ, + null, + FILE_CREATION_DISPOSITION.OPEN_EXISTING, + 0, + null); + + if (!hFile.IsInvalid && PInvoke.GetFileSizeEx(hFile, out long size)) + { + if (_computedFiles.TryAdd(path, size)) + batchTotal += size; + } + } + + if (batchTotal > 0) + Interlocked.Add(ref _size, batchTotal); } private long ComputeFileSize(string path) From df8e9e8c677652f0b4efab6c5a8751f1c139e6ed Mon Sep 17 00:00:00 2001 From: Aftab Ahmed <58620536+aftab23822@users.noreply.github.com> Date: Thu, 18 Sep 2025 21:16:20 +0500 Subject: [PATCH 2/4] Referencing Feature: Improving Performance, optimizations, reliability and codebase #4180 --- src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs b/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs index e8b96af49bc9..22458d69a5d6 100644 --- a/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs +++ b/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs @@ -60,7 +60,6 @@ public async Task ComputeSizeAsync(CancellationToken cancellationToken = default System.Diagnostics.Debug.WriteLine(ex); } #endif - try { foreach (var dir in Directory.EnumerateDirectories(currentPath)) From 9b48e32433a51f4af44b1e074a73f58405875d58 Mon Sep 17 00:00:00 2001 From: Aftab Ahmed <58620536+aftab23822@users.noreply.github.com> Date: Sat, 20 Sep 2025 16:36:28 +0500 Subject: [PATCH 3/4] implemented CoPilot suggestions in the PR --- .../Storage/Operations/FileSizeCalculator.cs | 97 +++++++++---------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs b/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs index 22458d69a5d6..d7d18a47435e 100644 --- a/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs +++ b/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs @@ -29,72 +29,71 @@ public async Task ComputeSizeAsync(CancellationToken cancellationToken = default var queue = new Queue(_paths); var batch = new List(ChunkSize); - while (!cancellationToken.IsCancellationRequested && queue.TryDequeue(out var currentPath)) + while (queue.TryDequeue(out var currentPath)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!Win32Helper.HasFileAttribute(currentPath, FileAttributes.Directory)) { - if (!Win32Helper.HasFileAttribute(currentPath, FileAttributes.Directory)) - { - batch.Add(currentPath); - } - else + batch.Add(currentPath); + } + else + { + try { - try + // Use EnumerateFileSystemEntries to get both files and directories in one pass + foreach (var entry in Directory.EnumerateFileSystemEntries(currentPath)) { - foreach (var file in Directory.EnumerateFiles(currentPath)) + cancellationToken.ThrowIfCancellationRequested(); + + if (Win32Helper.HasFileAttribute(entry, FileAttributes.Directory)) { - if (cancellationToken.IsCancellationRequested) - break; - batch.Add(file); - if (batch.Count >= ChunkSize) - { - ComputeFileSizeBatch(batch); - batch.Clear(); - await Task.Yield(); - } + queue.Enqueue(entry); } - } - catch (UnauthorizedAccessException) { } - catch (IOException) { } -#if DEBUG - catch (Exception ex) - { - System.Diagnostics.Debug.WriteLine(ex); - } -#endif - try - { - foreach (var dir in Directory.EnumerateDirectories(currentPath)) + else { - if (cancellationToken.IsCancellationRequested) - break; - queue.Enqueue(dir); + batch.Add(entry); } - } - catch (UnauthorizedAccessException) { } - catch (IOException) { } -#if DEBUG - catch (Exception ex) - { - System.Diagnostics.Debug.WriteLine(ex); - } -#endif + if (batch.Count >= ChunkSize) + { + await ProcessBatchAsync(batch, ChunkSize); + } + } } - - if (batch.Count >= ChunkSize) + catch (UnauthorizedAccessException) { } + catch (IOException) { } +#if DEBUG + catch (Exception ex) { - ComputeFileSizeBatch(batch); - batch.Clear(); - await Task.Yield(); + System.Diagnostics.Debug.WriteLine(ex); } +#endif } - if (batch.Count > 0) + if (batch.Count >= ChunkSize) { - ComputeFileSizeBatch(batch); - batch.Clear(); + await ProcessBatchAsync(batch, ChunkSize); } } + // Process remaining items + if (batch.Count > 0) + { + ComputeFileSizeBatch(batch); + batch.Clear(); + } + + Completed = true; + } + + private async Task ProcessBatchAsync(List batch, int chunkSize) + { + ComputeFileSizeBatch(batch); + batch.Clear(); + await Task.Yield(); + } + private void ComputeFileSizeBatch(IEnumerable files) { long batchTotal = 0; From 486f0045215622e5baafc2ba9f31d3a2dc21f967 Mon Sep 17 00:00:00 2001 From: Aftab Ahmed <58620536+aftab23822@users.noreply.github.com> Date: Sat, 20 Sep 2025 17:36:42 +0500 Subject: [PATCH 4/4] =?UTF-8?q?Changes=20Made:=201)=20Adaptive=20chunking?= =?UTF-8?q?=20=E2=86=92=20starts=20at=20500,=20doubles=20gradually=20to=20?= =?UTF-8?q?5000=20for=20speed=20on=20large=20dirs.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2) Parallel batch processing in ProcessBatchAsync using Parallel.ForEach. 3) _computedFiles check before adding to batch → avoids double-counting. 4) CancellationToken support inside parallel loops. 5) UI responsiveness with await Task.Yield() after each batch. --- .../Storage/Operations/FileSizeCalculator.cs | 168 ++++++++++-------- 1 file changed, 98 insertions(+), 70 deletions(-) diff --git a/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs b/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs index d7d18a47435e..2b91044e5bc2 100644 --- a/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs +++ b/src/Files.App/Utils/Storage/Operations/FileSizeCalculator.cs @@ -1,4 +1,4 @@ -// Copyright (c) Files Community +// Copyright (c) Files Community // Licensed under the MIT License. using System.Collections.Concurrent; @@ -25,101 +25,129 @@ public FileSizeCalculator(params string[] paths) public async Task ComputeSizeAsync(CancellationToken cancellationToken = default) { - const int ChunkSize = 1000; + int chunkSize = 500; // start small for responsiveness + const int minChunkSize = 500; + const int maxChunkSize = 5000; + var queue = new Queue(_paths); - var batch = new List(ChunkSize); + var batch = new List(chunkSize); - while (queue.TryDequeue(out var currentPath)) - { - cancellationToken.ThrowIfCancellationRequested(); + int chunksProcessed = 0; - if (!Win32Helper.HasFileAttribute(currentPath, FileAttributes.Directory)) + while (queue.TryDequeue(out var currentPath)) { - batch.Add(currentPath); - } - else - { - try + cancellationToken.ThrowIfCancellationRequested(); + + if (!Win32Helper.HasFileAttribute(currentPath, FileAttributes.Directory)) { - // Use EnumerateFileSystemEntries to get both files and directories in one pass - foreach (var entry in Directory.EnumerateFileSystemEntries(currentPath)) + if (!_computedFiles.ContainsKey(currentPath)) + batch.Add(currentPath); + } + else + { + try { - cancellationToken.ThrowIfCancellationRequested(); - - if (Win32Helper.HasFileAttribute(entry, FileAttributes.Directory)) - { - queue.Enqueue(entry); - } - else + // Use EnumerateFileSystemEntries to get both files and directories in one pass + foreach (var entry in Directory.EnumerateFileSystemEntries(currentPath)) { - batch.Add(entry); - } - - if (batch.Count >= ChunkSize) - { - await ProcessBatchAsync(batch, ChunkSize); + cancellationToken.ThrowIfCancellationRequested(); + + if (Win32Helper.HasFileAttribute(entry, FileAttributes.Directory)) + { + queue.Enqueue(entry); + } + else + { + if (!_computedFiles.ContainsKey(entry)) + batch.Add(entry); + } + + if (batch.Count >= chunkSize) + { + await ProcessBatchAsync(batch, cancellationToken).ConfigureAwait(false); + + // ✅ Adaptive tuning + if (++chunksProcessed % 5 == 0) + { + if (chunkSize < maxChunkSize) + chunkSize = Math.Min(chunkSize * 2, maxChunkSize); + } + } } } - } - catch (UnauthorizedAccessException) { } - catch (IOException) { } + catch (UnauthorizedAccessException) { } + catch (IOException) { } #if DEBUG - catch (Exception ex) + catch (Exception ex) + { + System.Diagnostics.Debug.WriteLine(ex); + } +#endif + } + + if (batch.Count >= chunkSize) { - System.Diagnostics.Debug.WriteLine(ex); + await ProcessBatchAsync(batch, cancellationToken).ConfigureAwait(false); } -#endif } - if (batch.Count >= ChunkSize) + // Process any remaining files + if (batch.Count > 0) { - await ProcessBatchAsync(batch, ChunkSize); + await ProcessBatchAsync(batch, cancellationToken).ConfigureAwait(false); } + + Completed = true; } - // Process remaining items - if (batch.Count > 0) + private async Task ProcessBatchAsync(List batch, CancellationToken token) { - ComputeFileSizeBatch(batch); + if (batch.Count == 0) return; + + var files = batch.ToArray(); batch.Clear(); - } - Completed = true; - } + long batchTotal = await Task.Run(() => + { + long localSum = 0; - private async Task ProcessBatchAsync(List batch, int chunkSize) - { - ComputeFileSizeBatch(batch); - batch.Clear(); - await Task.Yield(); - } + Parallel.ForEach( + files, + new ParallelOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount, + CancellationToken = token + }, + file => + { + try + { + using var hFile = PInvoke.CreateFile( + file, + (uint)FILE_ACCESS_RIGHTS.FILE_READ_ATTRIBUTES, + FILE_SHARE_MODE.FILE_SHARE_READ, + null, + FILE_CREATION_DISPOSITION.OPEN_EXISTING, + 0, + null); + + if (!hFile.IsInvalid && PInvoke.GetFileSizeEx(hFile, out long size)) + { + if (_computedFiles.TryAdd(file, size)) + Interlocked.Add(ref localSum, size); + } + } + catch { /* ignore bad files */ } + }); - private void ComputeFileSizeBatch(IEnumerable files) - { - long batchTotal = 0; - foreach (var path in files) - { - if (_computedFiles.ContainsKey(path)) - continue; - - using var hFile = PInvoke.CreateFile( - path, - (uint)FILE_ACCESS_RIGHTS.FILE_READ_ATTRIBUTES, - FILE_SHARE_MODE.FILE_SHARE_READ, - null, - FILE_CREATION_DISPOSITION.OPEN_EXISTING, - 0, - null); - - if (!hFile.IsInvalid && PInvoke.GetFileSizeEx(hFile, out long size)) - { - if (_computedFiles.TryAdd(path, size)) - batchTotal += size; - } - } + return localSum; + }, token).ConfigureAwait(false); if (batchTotal > 0) Interlocked.Add(ref _size, batchTotal); + + // Yield to UI to avoid freezing + await Task.Yield(); } private long ComputeFileSize(string path)