traverse/contents: let the caller choose the enumeration flags

traverse() always read with directory_handle::flags::none, so on Windows
it cannot enumerate a directory whose entries exceed what one
NtQueryDirectoryFile() call returns: read() gives back
errc::operation_would_block and the whole traversal fails. A recursive
walk does not get to pick the directories it meets, so there was no way
to ask for best-effort enumeration.

Add an enumeration_flags parameter to traverse() and contents(),
defaulted to flags::none so existing callers are unaffected.

Author: jcelerier

include/llfio/v2.0/algorithm/contents.hpp

@@ -68,7 +68,8 @@ namespace algorithm
     {
     }
 
-    friend inline result<contents_type> contents(const path_handle &dirh, contents_visitor *visitor, size_t threads, bool force_slow_path) noexcept;
+    friend inline result<contents_type> contents(const path_handle &dirh, contents_visitor *visitor, size_t threads, bool force_slow_path,
+                                                 directory_handle::flags enumeration_flags) noexcept;
 
   protected:
     struct _state_type
@@ -221,7 +222,8 @@ namespace algorithm
   in header-only code, as it is very simple.
   */
   inline result<contents_visitor::contents_type> contents(const path_handle &dirh, contents_visitor *visitor = nullptr, size_t threads = 0,
-                                                          bool force_slow_path = false) noexcept
+                                                          bool force_slow_path = false,
+                                                          directory_handle::flags enumeration_flags = directory_handle::flags::none) noexcept
   {
     contents_visitor default_visitor;
     if(visitor == nullptr)
@@ -231,7 +233,7 @@ namespace algorithm
     contents_visitor::_state_type state(dirh);
     OUTCOME_TRY(auto &&dirhpath, dirh.current_path());
     state.rootdirpathlen.store(dirhpath.native().size() + 1, std::memory_order_relaxed);
-    OUTCOME_TRY(traverse(dirh, visitor, threads, &state, force_slow_path));
+    OUTCOME_TRY(traverse(dirh, visitor, threads, &state, force_slow_path, enumeration_flags));
     return {std::move(state.contents)};
   }
 

include/llfio/v2.0/algorithm/traverse.hpp

@@ -181,8 +181,17 @@ namespace algorithm
   - Fast path, 1 thread, traversed 131,915 directories and 8,254,162 entries in 2.73 seconds (+12%).
 
   - Fast path, 16 threads, traversed 131,915 directories and 8,254,162 entries in 0.525 seconds (+46%).
+
+  `enumeration_flags` is passed to each `directory_handle::read()` performed. On Windows,
+  `NtQueryDirectoryFile()` returns at most about 64KB of entries per call however large a
+  buffer it is given (issue #137), so a directory larger than that cannot be read as an
+  atomic snapshot and `read()` fails with `errc::operation_would_block`. Pass
+  `directory_handle::flags::permit_racy_reads` if you would rather have the entries than
+  the failure: an enumeration which fits into a single call is still an atomic snapshot,
+  one which does not reports `is_snapshot()` false.
   */
-  LLFIO_HEADERS_ONLY_FUNC_SPEC result<size_t> traverse(const path_handle &dirh, traverse_visitor *visitor, size_t threads = 0, void *data = nullptr, bool force_slow_path = false) noexcept;
+  LLFIO_HEADERS_ONLY_FUNC_SPEC result<size_t> traverse(const path_handle &dirh, traverse_visitor *visitor, size_t threads = 0, void *data = nullptr, bool force_slow_path = false,
+                                                       directory_handle::flags enumeration_flags = directory_handle::flags::none) noexcept;
 
 }  // namespace algorithm
 

include/llfio/v2.0/detail/impl/traverse.ipp

@@ -46,7 +46,8 @@ LLFIO_V2_NAMESPACE_BEGIN
 namespace algorithm
 {
   LLFIO_HEADERS_ONLY_FUNC_SPEC result<size_t> traverse(const path_handle &_topdirh, traverse_visitor *visitor,
-                                                       size_t threads, void *data, bool force_slow_path) noexcept
+                                                       size_t threads, void *data, bool force_slow_path,
+                                                       directory_handle::flags enumeration_flags) noexcept
   {
     return visitor->finished(
     data,
@@ -91,6 +92,7 @@ namespace algorithm
           std::mutex lock;
           size_t max_sso_path_size;
           traverse_visitor *visitor{nullptr};
+          directory_handle::flags enumeration_flags{directory_handle::flags::none};
 #if 0
         struct workitem
         {
@@ -211,12 +213,13 @@ namespace algorithm
           size_t dirs_processed{0}, known_dirs_remaining{0}, depth_processed{0}, threads_sleeping{0},
           threads_running{0};
 
-          explicit state_t(size_t _max_sso_path_size, traverse_visitor *_visitor)
+          explicit state_t(size_t _max_sso_path_size, traverse_visitor *_visitor, directory_handle::flags _enumeration_flags)
               : max_sso_path_size(_max_sso_path_size)
               , visitor(_visitor)
+              , enumeration_flags(_enumeration_flags)
           {
           }
-        } state(use_slow_path ? size_t(-1) : LLFIO_ALGORITHM_TRAVERSE_MAX_SSO_PATH_SIZE, visitor);
+        } state(use_slow_path ? size_t(-1) : LLFIO_ALGORITHM_TRAVERSE_MAX_SSO_PATH_SIZE, visitor, enumeration_flags);
         struct worker
         {
           state_t *state{nullptr};
@@ -289,7 +292,7 @@ namespace algorithm
                 for(;;)
                 {
                   buffers = directory_handle::buffers_type{span<directory_handle::buffer_type>(entries.data(), entries.size()), std::move(buffers)};
-                  OUTCOME_TRY(buffers, mydirh->read({std::move(buffers), {}, directory_handle::filter::none}));
+                  OUTCOME_TRY(buffers, mydirh->read({std::move(buffers), state->enumeration_flags, {}, directory_handle::filter::none}));
                   if(buffers.done())
                   {
                     break;