From d9fd514d626ff33078fc4328a03996db98b3d965 Mon Sep 17 00:00:00 2001 From: Sean Arms <67096+lesserwhirls@users.noreply.github.com> Date: Tue, 31 Mar 2026 15:47:26 -0600 Subject: [PATCH] Refactor inventory management to use managed-file abstractions, part 1 * Transition thredds.inventory and thredds.filesystem abstractions from java.io/java.nio path APIs (with the exception of their OS implementations) * Add MFileDirectoryStream to support managed-file directory iteration (to replace over-dependence on java.nio.file.DirectoryStream * Begin migrating GRIB collection, partition, and index code to managed-file abstractions (just enough to compile and pass tests) --- .../test/java/thredds/inventory/TestDcm.java | 35 +----- .../thredds/inventory/TestMCollection.java | 9 +- .../java/thredds/filesystem/ControllerOS.java | 76 +++++------- .../thredds/filesystem/ControllerOS7.java | 41 +++++-- .../thredds/inventory/CollectionAbstract.java | 16 +-- .../thredds/inventory/CollectionGeneral.java | 36 +++--- .../thredds/inventory/CollectionGlob.java | 45 +++---- .../thredds/inventory/CollectionList.java | 17 ++- .../inventory/CollectionPathMatcher.java | 40 +++---- .../inventory/CollectionSingleFile.java | 14 +-- .../CollectionSpecParserAbstract.java | 22 ++-- .../java/thredds/inventory/MController.java | 32 +++-- .../java/thredds/inventory/MControllers.java | 40 ++++++- .../inventory/MFileCollectionManager.java | 28 ++--- .../inventory/MFileDirectoryStream.java | 38 ++++++ .../java/thredds/inventory/MFileFilter.java | 7 +- .../thredds/inventory/filter/RegExpMatch.java | 31 +++++ .../inventory/filter/StreamFilter.java | 3 + .../main/java/thredds/inventory/package.html | 8 +- .../inventory/partition/DirectoryBuilder.java | 112 +++++++++--------- .../partition/DirectoryCollection.java | 92 +++++++------- .../DirectoryCollectionFromIndex.java | 7 +- .../partition/DirectoryPartition.java | 17 ++- .../inventory/partition/FilePartition.java | 6 +- .../inventory/partition/IndexReader.java | 10 +- .../thredds/filesystem/s3/ControllerS3.java | 18 +-- .../filesystem/s3/TestControllerS3.java | 65 ++++++---- .../filesystem/zarr/ControllerZip.java | 16 +-- .../io/zarr/RandomAccessDirectory.java | 15 ++- .../nc2/grib/collection/GribCdmIndex.java | 109 ++++++++++------- .../collection/GribCollectionBuilder.java | 7 +- .../grib/collection/GribPartitionBuilder.java | 8 +- .../java/ucar/nc2/ui/grib/CdmIndexPanel.java | 7 +- .../java/ucar/nc2/ui/op/CdmIndexOpPanel.java | 4 +- .../nc2/ui/op/DirectoryPartitionViewer.java | 10 +- 35 files changed, 572 insertions(+), 469 deletions(-) create mode 100644 cdm/core/src/main/java/thredds/inventory/MFileDirectoryStream.java create mode 100644 cdm/core/src/main/java/thredds/inventory/filter/RegExpMatch.java diff --git a/cdm-test/src/test/java/thredds/inventory/TestDcm.java b/cdm-test/src/test/java/thredds/inventory/TestDcm.java index 3c07700075..8d70fceb27 100644 --- a/cdm-test/src/test/java/thredds/inventory/TestDcm.java +++ b/cdm-test/src/test/java/thredds/inventory/TestDcm.java @@ -1,33 +1,6 @@ /* - * Copyright (c) 1998 - 2011. University Corporation for Atmospheric Research/Unidata - * Portions of this software were developed by the Unidata Program at the - * University Corporation for Atmospheric Research. - * - * Access and use of this software shall impose the following obligations - * and understandings on the user. The user is granted the right, without - * any fee or cost, to use, copy, modify, alter, enhance and distribute - * this software, and any derivative works thereof, and its supporting - * documentation for any purpose whatsoever, provided that this entire - * notice appears in all copies of the software, derivative works and - * supporting documentation. Further, UCAR requests that the user credit - * UCAR/Unidata in any publications that result from the use of this - * software or in any product that includes this software. The names UCAR - * and/or Unidata, however, may not be used in any advertising or publicity - * to endorse or promote any products or commercial entity unless specific - * written permission is obtained from UCAR/Unidata. The user also - * understands that UCAR/Unidata is not obligated to provide the user with - * any support, consulting, training or assistance of any kind with regard - * to the use, operation and performance of this software nor to provide - * the user with any updates, revisions, new versions or "bug fixes." - * - * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, - * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING - * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, - * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION - * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE. + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. */ package thredds.inventory; @@ -41,7 +14,7 @@ import org.slf4j.LoggerFactory; import thredds.featurecollection.FeatureCollectionConfig; import thredds.featurecollection.FeatureCollectionType; -import thredds.inventory.filter.StreamFilter; +import thredds.inventory.filter.RegExpMatch; import thredds.inventory.partition.DirectoryCollection; import ucar.nc2.time.CalendarDate; import ucar.unidata.util.test.category.NeedsCdmUnitTest; @@ -146,7 +119,7 @@ public void testOlderThanInDirectoryCollection() throws IOException { // String topCollectionName, String topDirS, String olderThan, org.slf4j.Logger logger DirectoryCollection dcm = new DirectoryCollection("topCollectionName", specp.getRootDir(), true, config.olderThan, logger); - dcm.setStreamFilter(new StreamFilter(Pattern.compile(".*grib2"), true)); + dcm.setStreamFilter(new RegExpMatch(Pattern.compile(".*grib2"), true)); List fileList = dcm.getFilenames(); for (String name : fileList) diff --git a/cdm-test/src/test/java/thredds/inventory/TestMCollection.java b/cdm-test/src/test/java/thredds/inventory/TestMCollection.java index fe54e6b4e6..a9543958ce 100644 --- a/cdm-test/src/test/java/thredds/inventory/TestMCollection.java +++ b/cdm-test/src/test/java/thredds/inventory/TestMCollection.java @@ -1,7 +1,8 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package thredds.inventory; import org.junit.Test; @@ -10,7 +11,7 @@ import org.slf4j.LoggerFactory; import thredds.featurecollection.FeatureCollectionConfig; import thredds.featurecollection.FeatureCollectionType; -import thredds.inventory.filter.StreamFilter; +import thredds.inventory.filter.RegExpMatch; import thredds.inventory.partition.DirectoryCollection; import thredds.inventory.partition.TimePartition; import ucar.unidata.util.test.category.NeedsCdmUnitTest; @@ -38,10 +39,10 @@ public void testStreamFilterInDirPartition() throws IOException { Path rootPath = Paths.get(specp.getRootDir()); try (DirectoryCollection dcm = - new DirectoryCollection(config.collectionName, rootPath, true, config.olderThan, logger)) { + new DirectoryCollection(config.collectionName, rootPath.toString(), true, config.olderThan, logger)) { dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); if (specp.getFilter() != null) - dcm.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName())); + dcm.setStreamFilter(new RegExpMatch(specp.getFilter(), specp.getFilterOnName())); int count = 0; for (MFile mfile : dcm.getFilesSorted()) { diff --git a/cdm/core/src/main/java/thredds/filesystem/ControllerOS.java b/cdm/core/src/main/java/thredds/filesystem/ControllerOS.java index 22e6b438e4..1aaa024377 100644 --- a/cdm/core/src/main/java/thredds/filesystem/ControllerOS.java +++ b/cdm/core/src/main/java/thredds/filesystem/ControllerOS.java @@ -1,33 +1,6 @@ /* - * Copyright (c) 1998 - 2011. University Corporation for Atmospheric Research/Unidata - * Portions of this software were developed by the Unidata Program at the - * University Corporation for Atmospheric Research. - * - * Access and use of this software shall impose the following obligations - * and understandings on the user. The user is granted the right, without - * any fee or cost, to use, copy, modify, alter, enhance and distribute - * this software, and any derivative works thereof, and its supporting - * documentation for any purpose whatsoever, provided that this entire - * notice appears in all copies of the software, derivative works and - * supporting documentation. Further, UCAR requests that the user credit - * UCAR/Unidata in any publications that result from the use of this - * software or in any product that includes this software. The names UCAR - * and/or Unidata, however, may not be used in any advertising or publicity - * to endorse or promote any products or commercial entity unless specific - * written permission is obtained from UCAR/Unidata. The user also - * understands that UCAR/Unidata is not obligated to provide the user with - * any support, consulting, training or assistance of any kind with regard - * to the use, operation and performance of this software nor to provide - * the user with any updates, revisions, new versions or "bug fixes." - * - * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, - * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING - * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, - * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION - * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE. + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. */ package thredds.filesystem; @@ -35,10 +8,12 @@ import thredds.inventory.CollectionConfig; import thredds.inventory.MController; import thredds.inventory.MFile; +import thredds.inventory.MFileDirectoryStream; import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; import java.io.File; +import java.nio.file.DirectoryStream; import java.util.*; /** @@ -57,7 +32,7 @@ public class ControllerOS implements MController { @Nullable @Override - public Iterator getInventoryAll(CollectionConfig mc, boolean recheck) { + public DirectoryStream getInventoryAll(CollectionConfig mc, boolean recheck) { String path = mc.getDirectoryName(); if (path.startsWith("file:")) { path = path.substring(5); @@ -68,12 +43,12 @@ public Iterator getInventoryAll(CollectionConfig mc, boolean recheck) { return null; if (!cd.isDirectory()) return null; - return new FilteredIterator(mc, new MFileIteratorAll(cd), false); + return new MFileDirectoryStream(new FilteredIterator(mc, new MFileIteratorAll(cd), false)); } @Nullable @Override - public Iterator getInventoryTop(CollectionConfig mc, boolean recheck) { + public DirectoryStream getInventoryTop(CollectionConfig mc, boolean recheck) { String path = mc.getDirectoryName(); if (path.startsWith("file:")) { path = path.substring(5); @@ -84,11 +59,12 @@ public Iterator getInventoryTop(CollectionConfig mc, boolean recheck) { return null; if (!cd.isDirectory()) return null; - return new FilteredIterator(mc, new MFileIterator(cd), false); // removes subdirs + return new MFileDirectoryStream(new FilteredIterator(mc, new MFileIterator(cd), false)); // removes subdirs } @Nullable - public Iterator getSubdirs(CollectionConfig mc, boolean recheck) { + @Override + public DirectoryStream getSubdirs(CollectionConfig mc, boolean recheck) { String path = mc.getDirectoryName(); if (path.startsWith("file:")) { path = path.substring(5); @@ -99,7 +75,7 @@ public Iterator getSubdirs(CollectionConfig mc, boolean recheck) { return null; if (!cd.isDirectory()) return null; - return new FilteredIterator(mc, new MFileIterator(cd), true); // return only subdirs + return new MFileDirectoryStream(new FilteredIterator(mc, new MFileIterator(cd), true)); // return only subdirs } @@ -123,14 +99,22 @@ private static class FilteredIterator implements Iterator { } public boolean hasNext() { - next = nextFilteredFile(); /// 7 + if (next != null) { + return true; + } + next = nextFilteredFile(); return (next != null); } public MFile next() { - if (next == null) - throw new NoSuchElementException(); - return next; + if (next == null) { + if (!hasNext()) { + throw new NoSuchElementException(); + } + } + MFile result = next; + next = null; + return result; } public void remove() { @@ -140,16 +124,14 @@ public void remove() { private MFile nextFilteredFile() { if (orgIter == null) return null; - if (!orgIter.hasNext()) - return null; - MFile pdata = orgIter.next(); - while ((pdata.isDirectory() != wantDirs) || !mc.accept(pdata)) { // skip directories, and filter - if (!orgIter.hasNext()) - return null; /// 6 - pdata = orgIter.next(); + while (orgIter.hasNext()) { + MFile pdata = orgIter.next(); + if (pdata.isDirectory() == wantDirs && mc.accept(pdata)) { + return pdata; + } } - return pdata; + return null; } } diff --git a/cdm/core/src/main/java/thredds/filesystem/ControllerOS7.java b/cdm/core/src/main/java/thredds/filesystem/ControllerOS7.java index 7e7874fb20..82543e04d1 100644 --- a/cdm/core/src/main/java/thredds/filesystem/ControllerOS7.java +++ b/cdm/core/src/main/java/thredds/filesystem/ControllerOS7.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -8,11 +8,13 @@ import thredds.inventory.CollectionConfig; import thredds.inventory.MController; import thredds.inventory.MFile; +import thredds.inventory.MFileDirectoryStream; import javax.annotation.concurrent.ThreadSafe; import java.io.IOException; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.Iterator; +import java.util.NoSuchElementException; /** * Use Java 7 NIO for scanning the file system @@ -27,12 +29,12 @@ public class ControllerOS7 implements MController { //////////////////////////////////////// @Override - public Iterator getInventoryAll(CollectionConfig mc, boolean recheck) { + public DirectoryStream getInventoryAll(CollectionConfig mc, boolean recheck) { return null; } @Override - public Iterator getInventoryTop(CollectionConfig mc, boolean recheck) throws IOException { + public DirectoryStream getInventoryTop(CollectionConfig mc, boolean recheck) { String path = mc.getDirectoryName(); if (path.startsWith("file:")) { path = path.substring(5); @@ -41,10 +43,17 @@ public Iterator getInventoryTop(CollectionConfig mc, boolean recheck) thr Path cd = Paths.get(path); if (!Files.exists(cd)) return null; - return new MFileIterator(cd, new CollectionFilter(mc)); // removes subdirs + MFileDirectoryStream mfileDirStream = null; + try { + mfileDirStream = new MFileDirectoryStream(new MFileIterator(cd, new CollectionFilter(mc))); // removes subdirs + } catch (IOException ioe) { + logger.warn(ioe.getMessage(), ioe); + } + return mfileDirStream; } - public Iterator getSubdirs(CollectionConfig mc, boolean recheck) { + @Override + public DirectoryStream getSubdirs(CollectionConfig mc, boolean recheck) { return null; } @@ -68,32 +77,38 @@ public boolean accept(Path entry) { } // returns everything in the current directory - private static class MFileIterator implements Iterator { - Iterator dirStream; + private static class MFileIterator implements Iterator, AutoCloseable { + private final DirectoryStream dirStream; + private final Iterator pathIterator; MFileIterator(Path dir, DirectoryStream.Filter filter) throws IOException { if (filter != null) - dirStream = Files.newDirectoryStream(dir, filter).iterator(); + dirStream = Files.newDirectoryStream(dir, filter); else - dirStream = Files.newDirectoryStream(dir).iterator(); + dirStream = Files.newDirectoryStream(dir); + pathIterator = dirStream.iterator(); } public boolean hasNext() { - return dirStream.hasNext(); + return pathIterator.hasNext(); } public MFile next() { try { - return new MFileOS7(dirStream.next()); + return new MFileOS7(pathIterator.next()); } catch (IOException e) { - e.printStackTrace(); // LOOK we should pass this exception up - throw new RuntimeException(e); + throw new NoSuchElementException(e.getMessage()); } } public void remove() { throw new UnsupportedOperationException(); } + + @Override + public void close() throws IOException { + dirStream.close(); + } } ////////////////////////////////////////////////////////////////// diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionAbstract.java b/cdm/core/src/main/java/thredds/inventory/CollectionAbstract.java index 98cb13b69e..a21fca8bfe 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionAbstract.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionAbstract.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2017 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE.txt for license information. */ @@ -14,8 +14,6 @@ import ucar.unidata.util.StringUtil2; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Path; import java.util.*; /** @@ -109,7 +107,7 @@ public static String cleanName(String name) { protected DateExtractor dateExtractor; protected CalendarDate startCollection; protected long lastModified; - protected DirectoryStream.Filter sfilter; + protected MFileFilter sfilter; protected CollectionAbstract(String collectionName, org.slf4j.Logger logger) { this.collectionName = cleanName(collectionName); @@ -126,7 +124,7 @@ public String getIndexFilename(String suffix) { return getRoot() + "/" + collectionName + suffix; } - public void setStreamFilter(DirectoryStream.Filter filter) { + public void setStreamFilter(MFileFilter filter) { this.sfilter = filter; } @@ -243,14 +241,6 @@ CalendarDate extractRunDateWithError(MFile mfile) { } - ///////////////////////////////////////////////////////////////////////// - - public class MyStreamFilter implements DirectoryStream.Filter { - public boolean accept(Path entry) throws IOException { - return sfilter == null || sfilter.accept(entry); - } - } - protected List makeFileListSorted() throws IOException { List list = new ArrayList<>(100); try (CloseableIterator iter = getFileIterator()) { diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionGeneral.java b/cdm/core/src/main/java/thredds/inventory/CollectionGeneral.java index 4283ab9f2d..04839ef18f 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionGeneral.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionGeneral.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -7,15 +7,9 @@ import org.slf4j.Logger; import thredds.featurecollection.FeatureCollectionConfig; -import thredds.filesystem.MFileOS7; import ucar.nc2.util.CloseableIterator; import java.io.IOException; import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileTime; import java.util.*; /** @@ -27,13 +21,12 @@ */ public class CollectionGeneral extends CollectionAbstract { private final long olderThanMillis; - private final Path rootPath; public CollectionGeneral(FeatureCollectionConfig config, CollectionSpecParser specp, Logger logger) { super(config.collectionName, logger); this.root = specp.getRootDir(); - this.rootPath = Paths.get(this.root); this.olderThanMillis = parseOlderThanString(config.olderThan); + this.sfilter = specp.getMFileFilter(); } @Override @@ -46,18 +39,20 @@ public Iterable getFilesSorted() throws IOException { @Override public CloseableIterator getFileIterator() throws IOException { - return new MyFileIterator(rootPath); + return new MyFileIterator(root); } // returns everything defined by specp, checking olderThanMillis private class MyFileIterator implements CloseableIterator { - DirectoryStream dirStream; - Iterator dirStreamIterator; + DirectoryStream dirStream; + Iterator dirStreamIterator; MFile nextMFile; long now; - MyFileIterator(Path dir) throws IOException { - dirStream = Files.newDirectoryStream(dir, new MyStreamFilter()); + MyFileIterator(String dir) throws IOException { + MController controller = MControllers.create(dir); + CollectionConfig config = new CollectionConfig(collectionName, dir, false, (MFileFilter) sfilter, null); + dirStream = controller.getInventoryTop(config, true); dirStreamIterator = dirStream.iterator(); now = System.currentTimeMillis(); } @@ -71,19 +66,18 @@ public boolean hasNext() { } try { - Path nextPath = dirStreamIterator.next(); - BasicFileAttributes attr = Files.readAttributes(nextPath, BasicFileAttributes.class); - if (attr.isDirectory()) + MFile nextFile = dirStreamIterator.next(); + if (nextFile.isDirectory()) continue; // LOOK fix this - FileTime last = attr.lastModifiedTime(); - long millisSinceModified = now - last.toMillis(); + long last = nextFile.getLastModified(); + long millisSinceModified = now - last; if (millisSinceModified < olderThanMillis) continue; - nextMFile = new MFileOS7(nextPath, attr); + nextMFile = nextFile; return true; - } catch (IOException e) { + } catch (Exception e) { throw new RuntimeException(e); } } diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionGlob.java b/cdm/core/src/main/java/thredds/inventory/CollectionGlob.java index 9766da8e31..ab21e94edd 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionGlob.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionGlob.java @@ -1,16 +1,15 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package thredds.inventory; import org.slf4j.Logger; -import thredds.filesystem.MFileOS7; +import thredds.inventory.filter.WildcardMatchOnPath; import ucar.nc2.util.CloseableIterator; import java.io.IOException; -import java.nio.file.*; -import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.DirectoryStream; import java.util.*; /** @@ -47,14 +46,14 @@ * @since 5/19/14 */ public class CollectionGlob extends CollectionAbstract { - PathMatcher matcher; + MFileFilter matcher; boolean debug; int depth; public CollectionGlob(String collectionName, String glob, Logger logger) { super(collectionName, logger); - matcher = FileSystems.getDefault().getPathMatcher("glob:" + glob); + matcher = new WildcardMatchOnPath(glob); // lets suppose the first "*" indicates the top dir int pos = glob.indexOf("*"); @@ -92,25 +91,16 @@ public CloseableIterator getFileIterator() throws IOException { return new MyFileIterator(this.root); } - // from http://blog.eyallupu.com/2011/11/java-7-working-with-directories.html - public static DirectoryStream newDirectoryStream(Path dir, String glob) throws IOException { - FileSystem fs = dir.getFileSystem(); - PathMatcher matcher = fs.getPathMatcher("glob:" + glob); - DirectoryStream.Filter filter = entry -> matcher.matches(entry.getFileName()); - return fs.provider().newDirectoryStream(dir, filter); - } - private class MyFileIterator implements CloseableIterator { - DirectoryStream dirStream; - Iterator dirStreamIterator; + DirectoryStream dirStream; + Iterator dirStreamIterator; MFile nextMFile; int count, total; - Stack subdirs = new Stack<>(); + Stack subdirs = new Stack<>(); int currDepth; MyFileIterator(String topDir) throws IOException { - Path topPath = Paths.get(topDir); - dirStream = Files.newDirectoryStream(topPath); + dirStream = MControllers.newDirectoryStream(topDir); dirStreamIterator = dirStream.iterator(); } @@ -125,31 +115,30 @@ public boolean hasNext() { return false; } currDepth++; // LOOK wrong - Path nextSubdir = subdirs.pop(); - dirStream = Files.newDirectoryStream(nextSubdir); + MFile nextSubdir = subdirs.pop(); + dirStream = MControllers.newDirectoryStream(nextSubdir.getPath()); dirStreamIterator = dirStream.iterator(); } total++; - Path nextPath = dirStreamIterator.next(); - BasicFileAttributes attr = Files.readAttributes(nextPath, BasicFileAttributes.class); - if (attr.isDirectory()) { + MFile nextFile = dirStreamIterator.next(); + if (nextFile.isDirectory()) { if (currDepth < depth) - subdirs.push(nextPath); + subdirs.push(nextFile); continue; } - if (!matcher.matches(nextPath)) { + // lOOK we need Path for PathMatcher. This is still local-only if it's using FileSystems.getDefault() + if (!matcher.accept(nextFile)) { continue; } - nextMFile = new MFileOS7(nextPath, attr); + nextMFile = nextFile; return true; } catch (IOException e) { throw new RuntimeException(e); } - // if (filter == null || filter.accept(nextMFile)) return true; } } diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionList.java b/cdm/core/src/main/java/thredds/inventory/CollectionList.java index ae84dc2e11..a895dacc12 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionList.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionList.java @@ -1,17 +1,15 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package thredds.inventory; import org.slf4j.Logger; -import thredds.filesystem.MFileOS; import ucar.nc2.util.CloseableIterator; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; /** * MCollection that is initialized by specific list of MFiles. @@ -35,10 +33,9 @@ public CollectionList(String collectionName, String list, Logger logger) { String filename = s.trim(); if (filename.isEmpty()) continue; - Path p = Paths.get(filename); - if (Files.exists(p)) { - MFileOS mfile = new MFileOS(filename); - mfiles.add(new MFileOS(filename)); + MFile mfile = MFiles.createIfExists(filename); + if (mfile != null) { + mfiles.add(mfile); lastModified = Math.max(lastModified, mfile.getLastModified()); } } diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionPathMatcher.java b/cdm/core/src/main/java/thredds/inventory/CollectionPathMatcher.java index 24624fd90d..62e5890cb7 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionPathMatcher.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionPathMatcher.java @@ -1,17 +1,15 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package thredds.inventory; import org.slf4j.Logger; import thredds.featurecollection.FeatureCollectionConfig; -import thredds.filesystem.MFileOS7; import ucar.nc2.util.CloseableIterator; import java.io.IOException; -import java.nio.file.*; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileTime; +import java.nio.file.DirectoryStream; import java.util.*; /** @@ -25,8 +23,7 @@ public class CollectionPathMatcher extends CollectionAbstract { protected final FeatureCollectionConfig config; private final boolean wantSubdirs; private final long olderThanMillis; - private final Path rootPath; - private final PathMatcher matcher; + private final MFileFilter matcher; public CollectionPathMatcher(FeatureCollectionConfig config, CollectionSpecParserAbstract specp, Logger logger) { super(config.collectionName, logger); @@ -38,9 +35,8 @@ public CollectionPathMatcher(FeatureCollectionConfig config, CollectionSpecParse setDateExtractor(extract); putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); - matcher = specp.getPathMatcher(); // LOOK still need to decide what you are matching on name, path, etc + matcher = specp.getMFileFilter(); // LOOK still need to decide what you are matching on name, path, etc - this.rootPath = Paths.get(this.root); this.olderThanMillis = parseOlderThanString(config.olderThan); } @@ -64,7 +60,7 @@ private class AllFilesIterator implements CloseableIterator { OneDirIterator current; AllFilesIterator() throws IOException { - current = new OneDirIterator(rootPath, subdirs); + current = new OneDirIterator(root, subdirs); } public boolean hasNext() { @@ -99,15 +95,14 @@ public void close() throws IOException { private class OneDirIterator implements CloseableIterator { Queue subdirs; - DirectoryStream dirStream; - Iterator dirStreamIterator; + DirectoryStream dirStream; + Iterator dirStreamIterator; MFile nextMFile; long now; - OneDirIterator(Path dir, Queue subdirs) throws IOException { + OneDirIterator(String dir, Queue subdirs) throws IOException { this.subdirs = subdirs; - dirStream = Files.newDirectoryStream(dir); // , new MyStreamFilter()); LOOK dont use the - // DirectoryStream.Filter + dirStream = MControllers.newDirectoryStream(dir); dirStreamIterator = dirStream.iterator(); now = System.currentTimeMillis(); } @@ -121,24 +116,23 @@ public boolean hasNext() { } try { - Path nextPath = dirStreamIterator.next(); - BasicFileAttributes attr = Files.readAttributes(nextPath, BasicFileAttributes.class); + MFile nextFile = dirStreamIterator.next(); - if (wantSubdirs && attr.isDirectory()) { // dont filter subdirectories - subdirs.add(new OneDirIterator(nextPath, subdirs)); + if (wantSubdirs && nextFile.isDirectory()) { // dont filter subdirectories + subdirs.add(new OneDirIterator(nextFile.getPath(), subdirs)); continue; } - if (!matcher.matches(nextPath)) // otherwise apply the filter specified by the specp + if (!matcher.accept(nextFile)) // otherwise apply the filter specified by the specp continue; if (olderThanMillis > 0) { - FileTime last = attr.lastModifiedTime(); - long millisSinceModified = now - last.toMillis(); + long last = nextFile.getLastModified(); + long millisSinceModified = now - last; if (millisSinceModified < olderThanMillis) continue; } - nextMFile = new MFileOS7(nextPath, attr); + nextMFile = nextFile; return true; } catch (IOException e) { diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionSingleFile.java b/cdm/core/src/main/java/thredds/inventory/CollectionSingleFile.java index ba7244a0e5..faa81b9d6f 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionSingleFile.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionSingleFile.java @@ -1,13 +1,10 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package thredds.inventory; -import java.nio.file.Path; -import java.nio.file.Paths; - /** * A CollectionManager consisting of a single file * @@ -19,11 +16,12 @@ public class CollectionSingleFile extends CollectionList { public CollectionSingleFile(MFile file, org.slf4j.Logger logger) { super(file.getName(), logger); mfiles.add(file); - Path p = Paths.get(file.getPath()); - if (p.getParent() != null) - this.root = p.getParent().toString(); - else + try { + MFile p = file.getParent(); + this.root = p != null ? p.getPath() : System.getProperty("user.dir"); + } catch (java.io.IOException e) { this.root = System.getProperty("user.dir"); + } this.lastModified = file.getLastModified(); } diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionSpecParserAbstract.java b/cdm/core/src/main/java/thredds/inventory/CollectionSpecParserAbstract.java index 1e13bec454..df857a1862 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionSpecParserAbstract.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionSpecParserAbstract.java @@ -1,16 +1,16 @@ /* - * Copyright (c) 1998-2022 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package thredds.inventory; import com.google.re2j.Matcher; import com.google.re2j.Pattern; import ucar.unidata.util.StringUtil2; import javax.annotation.concurrent.ThreadSafe; -import java.nio.file.FileSystems; -import java.nio.file.Path; -import java.nio.file.PathMatcher; +import thredds.inventory.filter.RegExpMatch; +import thredds.inventory.filter.WildcardMatchOnPath; import java.util.Formatter; /** @@ -137,18 +137,20 @@ protected static String getDateFormatMark(String filterAndDateMark) { } } - public PathMatcher getPathMatcher() { - if (spec.startsWith("regex:") || spec.startsWith("glob:")) { // experimental - return FileSystems.getDefault().getPathMatcher(spec); + public MFileFilter getMFileFilter() { + if (spec.startsWith("regex:")) { + return new RegExpMatch(Pattern.compile(spec.substring("regex:".length())), false); + } else if (spec.startsWith("glob:")) { + return new WildcardMatchOnPath(spec.substring("glob:".length())); } else { return new BySpecp(); } } - private class BySpecp implements java.nio.file.PathMatcher { + private class BySpecp implements MFileFilter { @Override - public boolean matches(Path path) { - Matcher matcher = filter.matcher(path.getFileName().toString()); + public boolean accept(MFile path) { + Matcher matcher = filter.matcher(path.getName()); return matcher.matches(); } } diff --git a/cdm/core/src/main/java/thredds/inventory/MController.java b/cdm/core/src/main/java/thredds/inventory/MController.java index 84b38325f7..1e42dd0f23 100644 --- a/cdm/core/src/main/java/thredds/inventory/MController.java +++ b/cdm/core/src/main/java/thredds/inventory/MController.java @@ -1,13 +1,13 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package thredds.inventory; import javax.annotation.Nullable; -import java.io.IOException; -import java.util.Iterator; +import java.io.Closeable; +import java.nio.file.DirectoryStream; /** * Inventory Management Controller @@ -15,38 +15,50 @@ * @author caron * @since Jun 25, 2009 */ -public interface MController { +public interface MController extends Closeable { /** * Returns all leaves in collection, recursing into subdirectories. * * @param mc defines the collection to scan * @param recheck if false, may use cached results. otherwise must sync with File OS - * @return iterator over Mfiles, or null if collection does not exist + * @return DirectoryStream over Mfiles, or null if collection does not exist */ @Nullable - Iterator getInventoryAll(CollectionConfig mc, boolean recheck); + DirectoryStream getInventoryAll(CollectionConfig mc, boolean recheck); /** * Returns all leaves in top collection, not recursing into subdirectories. * * @param mc defines the collection to scan * @param recheck if false, may use cached results. otherwise must sync with File OS - * @return iterator over Mfiles, or null if collection does not exist + * @return DirectoryStream over Mfiles, or null if collection does not exist */ @Nullable - Iterator getInventoryTop(CollectionConfig mc, boolean recheck) throws IOException; + DirectoryStream getInventoryTop(CollectionConfig mc, boolean recheck); /** * Returns all subdirectories in top collection. * * @param mc defines the collection to scan * @param recheck if false, may use cached results. otherwise must sync with File OS - * @return iterator over Mfiles, or null if collection does not exist + * @return DirectoryStream over Mfiles, or null if collection does not exist */ @Nullable - Iterator getSubdirs(CollectionConfig mc, boolean recheck); + DirectoryStream getSubdirs(CollectionConfig mc, boolean recheck); + /** + * Get an MFile for a specific location. + * + * @param location the location + * @return MFile or null + */ + @Nullable + default MFile getMFile(String location) { + return MFiles.create(location); + } + + @Override void close(); } diff --git a/cdm/core/src/main/java/thredds/inventory/MControllers.java b/cdm/core/src/main/java/thredds/inventory/MControllers.java index 76787cf799..6a703b4599 100644 --- a/cdm/core/src/main/java/thredds/inventory/MControllers.java +++ b/cdm/core/src/main/java/thredds/inventory/MControllers.java @@ -1,10 +1,12 @@ /* - * Copyright (c) 2020 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 2020-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package thredds.inventory; +import java.io.IOException; +import java.nio.file.DirectoryStream; import java.util.ServiceLoader; import thredds.filesystem.ControllerOS; @@ -31,4 +33,40 @@ public static MController create(String location) { return mControllerProvider != null ? mControllerProvider.create() : new ControllerOS(); } + + /** + * Create a {@link DirectoryStream} of {@link MFile}s for a given location. + * + * @param location location to scan + * @return {@link DirectoryStream} of {@link MFile}s + * @throws IOException if an I/O error occurs + */ + public static DirectoryStream newDirectoryStream(String location) throws IOException { + MController controller = create(location); + CollectionConfig config = new CollectionConfig(location, location, false, null, null); + DirectoryStream stream = controller.getInventoryTop(config, true); + controller.close(); + if (stream == null) { + throw new IOException("Could not create DirectoryStream for " + location); + } + return stream; + } + + /** + * Create a {@link DirectoryStream} of {@link MFile} subdirectories for a given location. + * + * @param location location to scan + * @return {@link DirectoryStream} of {@link MFile} subdirectories + * @throws IOException if an I/O error occurs + */ + public static DirectoryStream newSubdirStream(String location) throws IOException { + MController controller = create(location); + CollectionConfig config = new CollectionConfig(location, location, false, null, null); + DirectoryStream stream = controller.getSubdirs(config, true); + controller.close(); + if (stream == null) { + throw new IOException("Could not create SubdirStream for " + location); + } + return stream; + } } diff --git a/cdm/core/src/main/java/thredds/inventory/MFileCollectionManager.java b/cdm/core/src/main/java/thredds/inventory/MFileCollectionManager.java index 7b6a0705ba..512bdfdc18 100644 --- a/cdm/core/src/main/java/thredds/inventory/MFileCollectionManager.java +++ b/cdm/core/src/main/java/thredds/inventory/MFileCollectionManager.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2020 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE.txt for license information. */ @@ -546,21 +546,19 @@ protected void reallyScan(java.util.Map map) throws IOException { long start = System.currentTimeMillis(); // lOOK: are there any circumstances where we dont need to recheck against OS, ie always use cached values? - Iterator iter = - (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true); /// NCDC - /// wants - /// subdir - /// /global/nomads/nexus/gfsanl/**/gfsanl_3_.*\.grb$ - if (iter == null) { - logger.error(collectionName + ": Invalid collection= " + mc); - continue; - } + try (java.nio.file.DirectoryStream stream = + (mc.wantSubdirs()) ? controller.getInventoryAll(mc, true) : controller.getInventoryTop(mc, true)) { - while (iter.hasNext()) { - MFile mfile = iter.next(); - mfile.setAuxInfo(mc.getAuxInfo()); - map.put(mfile.getPath(), mfile); - count++; + if (stream == null) { + logger.error(collectionName + ": Invalid collection= " + mc); + continue; + } + + for (MFile mfile : stream) { + mfile.setAuxInfo(mc.getAuxInfo()); + map.put(mfile.getPath(), mfile); + count++; + } } if (logger.isDebugEnabled()) { diff --git a/cdm/core/src/main/java/thredds/inventory/MFileDirectoryStream.java b/cdm/core/src/main/java/thredds/inventory/MFileDirectoryStream.java new file mode 100644 index 0000000000..bc5dde446d --- /dev/null +++ b/cdm/core/src/main/java/thredds/inventory/MFileDirectoryStream.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package thredds.inventory; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.util.Iterator; + +/** + * A DirectoryStream of MFile objects. + */ +public class MFileDirectoryStream implements DirectoryStream { + private final Iterator iterator; + + public MFileDirectoryStream(Iterator iterator) { + this.iterator = iterator; + } + + @Override + public Iterator iterator() { + return iterator; + } + + @Override + public void close() throws IOException { + // If the iterator itself is closeable, close it. + if (iterator instanceof AutoCloseable) { + try { + ((AutoCloseable) iterator).close(); + } catch (Exception e) { + throw new IOException(e); + } + } + } +} diff --git a/cdm/core/src/main/java/thredds/inventory/MFileFilter.java b/cdm/core/src/main/java/thredds/inventory/MFileFilter.java index 63b6f16e9e..2c503c42f9 100644 --- a/cdm/core/src/main/java/thredds/inventory/MFileFilter.java +++ b/cdm/core/src/main/java/thredds/inventory/MFileFilter.java @@ -1,22 +1,25 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package thredds.inventory; +import java.nio.file.DirectoryStream; + /** * Filter on MFiles * * @author caron * @since Jun 26, 2009 */ -public interface MFileFilter { +public interface MFileFilter extends DirectoryStream.Filter { /** * Tests if a specified MFile should be included in a file collection. * * @param mfile the MFile * @return true if the mfile should be included in the file collection; false otherwise. */ + @Override boolean accept(MFile mfile); } diff --git a/cdm/core/src/main/java/thredds/inventory/filter/RegExpMatch.java b/cdm/core/src/main/java/thredds/inventory/filter/RegExpMatch.java new file mode 100644 index 0000000000..6c358d76fe --- /dev/null +++ b/cdm/core/src/main/java/thredds/inventory/filter/RegExpMatch.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + +package thredds.inventory.filter; + +import com.google.re2j.Matcher; +import com.google.re2j.Pattern; +import thredds.inventory.MFileFilter; +import thredds.inventory.MFile; + +/** + * A java.nio.file.DirectoryStream.Filter using a regexp on the last entry of the MFile path + */ +public class RegExpMatch implements MFileFilter { + private final Pattern pattern; + private final boolean nameOnly; + + public RegExpMatch(Pattern pattern, boolean nameOnly) { + this.pattern = pattern; + this.nameOnly = nameOnly; + } + + @Override + public boolean accept(MFile mfile) { + String matchOn = nameOnly ? mfile.getName() : mfile.getPath().replace('\\', '/'); + Matcher matcher = this.pattern.matcher(matchOn); + return matcher.matches(); + } +} diff --git a/cdm/core/src/main/java/thredds/inventory/filter/StreamFilter.java b/cdm/core/src/main/java/thredds/inventory/filter/StreamFilter.java index 8277e620e6..e498662dba 100644 --- a/cdm/core/src/main/java/thredds/inventory/filter/StreamFilter.java +++ b/cdm/core/src/main/java/thredds/inventory/filter/StreamFilter.java @@ -2,6 +2,7 @@ * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package thredds.inventory.filter; import com.google.re2j.Matcher; @@ -15,7 +16,9 @@ * * @author John * @since 1/28/14 + * @deprecated use {@link RegExpMatch} which works on {@link thredds.inventory.MFile} */ +@Deprecated public class StreamFilter implements DirectoryStream.Filter { private Pattern pattern; private boolean nameOnly; diff --git a/cdm/core/src/main/java/thredds/inventory/package.html b/cdm/core/src/main/java/thredds/inventory/package.html index 84d7c6b951..97e7bc8096 100644 --- a/cdm/core/src/main/java/thredds/inventory/package.html +++ b/cdm/core/src/main/java/thredds/inventory/package.html @@ -1,5 +1,5 @@ @@ -11,9 +11,9 @@
   NOT PUBLIC API - DO NOT USE: Abstractions for tracking dataset inventory using "managed files".
-  MFile - Abstraction of java.io.File
-  MCollection - An abstract description of a collection of MFile.
-  MController - Something that knows how to obtain the MFiles using a MCollection.
+  MFile - Filesystem-agnostic abstraction of portions of java.io.File and java.nio.file.Path
+  MCollection - An abstracttion of a collections of MFiles.
+  MController - An inventory management controller that knows how to discover and obtain the MFiles for an MCollection.
   Collection - Manages a collection of MFile objects.
   CollectionManager - Manages a dynamic collection of MFile objects. Allows storing key/value pairs on MFiles
   CollectionUpdater - singleton class for background updating of Collections.
diff --git a/cdm/core/src/main/java/thredds/inventory/partition/DirectoryBuilder.java b/cdm/core/src/main/java/thredds/inventory/partition/DirectoryBuilder.java
index b5fe5e3a97..3e8ad497da 100644
--- a/cdm/core/src/main/java/thredds/inventory/partition/DirectoryBuilder.java
+++ b/cdm/core/src/main/java/thredds/inventory/partition/DirectoryBuilder.java
@@ -1,29 +1,26 @@
 /*
- * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package thredds.inventory.partition;
 
 import thredds.featurecollection.FeatureCollectionConfig;
 import thredds.inventory.CollectionUpdateType;
 import thredds.inventory.MCollection;
+import thredds.inventory.MControllers;
 import thredds.inventory.MFile;
+import thredds.inventory.MFiles;
 import ucar.nc2.util.Indent;
 import java.io.IOException;
 import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.nio.file.attribute.FileTime;
 import java.util.ArrayList;
 import java.util.Formatter;
-import java.util.Iterator;
 import java.util.List;
 
 /**
  * A Builder of DirectoryPartitions and DirectoryCollections.
- * Each DirectoryBuilder is associated with one directory, and one ncx index.
+ * Each DirectoryBuilder is associated with one directory and one ncx index.
  * This may contain collections of files (MFiles in a DirectoryCollection), or subdirectories (MCollections in a
  * DirectoryPartition).
  *
@@ -33,9 +30,9 @@
 public class DirectoryBuilder {
 
   // returns a DirectoryPartition or DirectoryCollection
-  public static MCollection factory(FeatureCollectionConfig config, Path topDir, boolean isTop, IndexReader indexReader,
-      String suffix, org.slf4j.Logger logger) throws IOException {
-    DirectoryBuilder builder = new DirectoryBuilder(config.collectionName, topDir.toString(), suffix);
+  public static MCollection factory(FeatureCollectionConfig config, String topDir, boolean isTop,
+      IndexReader indexReader, String suffix, org.slf4j.Logger logger) throws IOException {
+    DirectoryBuilder builder = new DirectoryBuilder(config.collectionName, topDir, suffix);
 
     DirectoryPartition dpart = new DirectoryPartition(config, topDir, isTop, indexReader, suffix, logger);
     if (!builder.isLeaf(indexReader)) { // its a partition
@@ -63,10 +60,10 @@ private enum PartitionStatus {
   private final String suffix;
   private final String topCollectionName; // collection name
   private final String partitionName; // partition name
-  private final Path dir; // the directory
-  private final FileTime dirLastModified; // directory last modified
-  private Path index; // TimePartition index file (ncx2 with magic = TimePartition)
-  private FileTime indexLastModified; // index last modified
+  private final String dir; // the directory
+  private final long dirLastModified; // directory last modified
+  private String index; // TimePartition index file (ncx2 with magic = TimePartition)
+  private long indexLastModified; // index last modified
   private long indexSize; // index size
 
   private boolean childrenConstructed;
@@ -74,28 +71,24 @@ private enum PartitionStatus {
   private PartitionStatus partitionStatus = PartitionStatus.unknown;
 
   public DirectoryBuilder(String topCollectionName, String dirFilename, String suffix) throws IOException {
-    this(topCollectionName, Paths.get(dirFilename), null, suffix);
+    this(topCollectionName, MControllers.create(dirFilename).getMFile(dirFilename), suffix);
   }
 
   /**
    * Create a DirectoryBuilder for the named directory
    * 
    * @param topCollectionName from config, name of the collection
-   * @param dir covers this directory
-   * @param attr file attributes, may be null
+   * @param mdir covers this directory
    */
-  public DirectoryBuilder(String topCollectionName, Path dir, BasicFileAttributes attr, String suffix)
-      throws IOException {
+  public DirectoryBuilder(String topCollectionName, MFile mdir, String suffix) throws IOException {
     this.topCollectionName = topCollectionName;
-    this.dir = dir;
+    this.dir = mdir.getPath();
     this.partitionName = DirectoryCollection.makeCollectionName(topCollectionName, dir);
     this.suffix = suffix;
 
-    if (attr == null)
-      attr = Files.readAttributes(this.dir, BasicFileAttributes.class);
-    if (!attr.isDirectory())
+    if (!mdir.isDirectory())
       throw new IllegalArgumentException("DirectoryPartitionBuilder needs a directory");
-    dirLastModified = attr.lastModifiedTime();
+    dirLastModified = mdir.getLastModified();
 
     // see if we can find the index
     findIndex();
@@ -109,12 +102,12 @@ public DirectoryBuilder(String topCollectionName, Path dir, BasicFileAttributes
    * @return true if found
    */
   public boolean findIndex() throws IOException {
-    Path indexPath = Paths.get(dir.toString(), partitionName + suffix);
-    if (Files.exists(indexPath)) {
+    String indexPath = dir + "/" + partitionName + suffix;
+    MFile indexFile = MFiles.createIfExists(indexPath);
+    if (indexFile != null) {
       this.index = indexPath;
-      BasicFileAttributes attr = Files.readAttributes(indexPath, BasicFileAttributes.class);
-      this.indexLastModified = attr.lastModifiedTime();
-      this.indexSize = attr.size();
+      this.indexLastModified = indexFile.getLastModified();
+      this.indexSize = indexFile.getLength();
       return true;
     }
     return false;
@@ -130,12 +123,11 @@ private boolean isLeaf(IndexReader indexReader) throws IOException {
     if (partitionStatus == PartitionStatus.unknown) {
 
       int countDir = 0, countFile = 0, count = 0;
-      try (DirectoryStream dirStream = Files.newDirectoryStream(dir)) {
-        Iterator iterator = dirStream.iterator();
-        while (iterator.hasNext() && count++ < 100) {
-          Path p = iterator.next();
-          BasicFileAttributes attr = Files.readAttributes(p, BasicFileAttributes.class);
-          if (attr.isDirectory())
+      try (DirectoryStream dirStream = MControllers.newDirectoryStream(dir)) {
+        for (MFile mfile : dirStream) {
+          if (count++ >= 100)
+            break;
+          if (mfile.isDirectory())
             countDir++;
           else
             countFile++;
@@ -188,11 +180,9 @@ public List constructChildrenFromIndex(IndexReader indexReader
     return children;
   }
 
-  // add a child partition from the index file (callback from constructChildren)
-  // we dont know at this point if its another partition or a gribCollection
   private class AddChild implements IndexReader.AddChildCallback {
     public void addChild(String dirName, String indexFilename, long lastModified) throws IOException {
-      Path indexPath = Paths.get(indexFilename);
+      String indexPath = dirName + "/" + indexFilename;
       DirectoryBuilder child = new DirectoryBuilder(topCollectionName, indexPath, lastModified, suffix);
       children.add(child);
     }
@@ -206,10 +196,13 @@ private class AddChildSub implements IndexReader.AddChildCallback {
     }
 
     public void addChild(String dirName, String indexFilename, long lastModified) throws IOException {
-      Path indexPath = Paths.get(dirName, indexFilename);
+      String indexPath = dirName + "/" + indexFilename;
       if (substituteParentDir) {
-        Path parent = index.getParent();
-        indexPath = parent.resolve(indexFilename);
+        int pos = index.lastIndexOf('/');
+        if (pos < 0)
+          pos = index.lastIndexOf('\\');
+        String parent = (pos >= 0) ? index.substring(0, pos) : ".";
+        indexPath = parent + "/" + indexFilename;
       }
       DirectoryBuilder child = new DirectoryBuilder(topCollectionName, indexPath, lastModified, suffix);
       children.add(child);
@@ -217,21 +210,25 @@ public void addChild(String dirName, String indexFilename, long lastModified) th
   }
 
   // coming in from the index reader
-  private DirectoryBuilder(String topCollectionName, Path indexFile, long indexLastModified, String suffix)
+  private DirectoryBuilder(String topCollectionName, String indexFile, long lastModified, String suffix)
       throws IOException {
     this.topCollectionName = topCollectionName;
-    if (Files.exists(indexFile)) {
+    MFile mIndexFile = MFiles.createIfExists(indexFile);
+    if (mIndexFile != null) {
       this.index = indexFile;
-      this.indexLastModified = FileTime.fromMillis(indexLastModified);
+      this.indexLastModified = mIndexFile.getLastModified();
     }
 
-    this.dir = indexFile.getParent();
+    int pos = indexFile.lastIndexOf('/');
+    if (pos < 0)
+      pos = indexFile.lastIndexOf('\\');
+    this.dir = (pos >= 0) ? indexFile.substring(0, pos) : ".";
     this.partitionName = DirectoryCollection.makeCollectionName(topCollectionName, dir);
 
-    BasicFileAttributes attr = Files.readAttributes(this.dir, BasicFileAttributes.class);
-    if (!attr.isDirectory())
+    MFile mdir = MFiles.create(this.dir);
+    if (!mdir.isDirectory())
       throw new IllegalArgumentException("DirectoryPartition needs a directory");
-    dirLastModified = attr.lastModifiedTime();
+    dirLastModified = mdir.getLastModified();
 
     this.suffix = suffix;
   }
@@ -244,14 +241,11 @@ private void scanForChildren() {
       System.out.printf("DirectoryBuilder.scanForChildren on %s ", dir);
 
     int count = 0;
-    try (DirectoryStream ds = Files.newDirectoryStream(dir)) {
-      for (Path p : ds) {
-        BasicFileAttributes attr = Files.readAttributes(p, BasicFileAttributes.class);
-        if (attr.isDirectory()) {
-          children.add(new DirectoryBuilder(topCollectionName, p, attr, suffix));
-          if (debug && (++count % 10 == 0))
-            System.out.printf("%d ", count);
-        }
+    try (DirectoryStream ds = MControllers.newSubdirStream(dir)) {
+      for (MFile mfile : ds) {
+        children.add(new DirectoryBuilder(topCollectionName, mfile, suffix));
+        if (debug && (++count % 10 == 0))
+          System.out.printf("%d ", count);
       }
     } catch (IOException e) {
       e.printStackTrace();
@@ -280,7 +274,7 @@ public List readFilesFromIndex(IndexReader indexReader) throws IOExceptio
    * 
    * @return directory
    */
-  public Path getDir() {
+  public String getDir() {
     return dir;
   }
 
@@ -289,7 +283,7 @@ public Path getDir() {
    * 
    * @return ncx2 file path
    */
-  public Path getIndex() {
+  public String getIndex() {
     return index;
   }
 
diff --git a/cdm/core/src/main/java/thredds/inventory/partition/DirectoryCollection.java b/cdm/core/src/main/java/thredds/inventory/partition/DirectoryCollection.java
index 1dd807cce4..e0e6886605 100644
--- a/cdm/core/src/main/java/thredds/inventory/partition/DirectoryCollection.java
+++ b/cdm/core/src/main/java/thredds/inventory/partition/DirectoryCollection.java
@@ -1,21 +1,19 @@
 /*
- * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
 package thredds.inventory.partition;
 
-import thredds.filesystem.MFileOS7;
 import thredds.inventory.CollectionAbstract;
+import thredds.inventory.CollectionConfig;
+import thredds.inventory.MController;
+import thredds.inventory.MControllers;
 import thredds.inventory.MFile;
+import thredds.inventory.MFileFilter;
 import ucar.nc2.util.CloseableIterator;
 import java.io.IOException;
 import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.nio.file.attribute.FileTime;
 import java.util.*;
 
 /**
@@ -35,10 +33,11 @@ public class DirectoryCollection extends CollectionAbstract {
    * @param dir directory for this
    * @return standard collection name, to name the index file
    */
-  public static String makeCollectionName(String topCollectionName, Path dir) {
-    int last = dir.getNameCount() - 1;
-    Path lastDir = dir.getName(last);
-    String lastDirName = lastDir.toString();
+  public static String makeCollectionName(String topCollectionName, String dir) {
+    int pos = dir.lastIndexOf('/');
+    if (pos < 0)
+      pos = dir.lastIndexOf('\\');
+    String lastDirName = (pos >= 0) ? dir.substring(pos + 1) : dir;
     return topCollectionName + "-" + lastDirName;
   }
 
@@ -49,28 +48,23 @@ public static String makeCollectionName(String topCollectionName, Path dir) {
    * @param dir directory for this
    * @return standard collection name, to name the index file
    */
-  public static Path makeCollectionIndexPath(String topCollectionName, Path dir, String suffix) {
+  public static String makeCollectionIndexPath(String topCollectionName, String dir, String suffix) {
     String collectionName = makeCollectionName(topCollectionName, dir);
-    return Paths.get(dir.toString(), collectionName + suffix);
+    return dir + "/" + collectionName + suffix;
   }
 
   ///////////////////////////////////////////////////////////////////////////////////
 
   final String topCollection;
-  final Path collectionDir; // directory for this collection
+  final String collectionDir; // directory for this collection
   final long olderThanMillis;
   final boolean isTop;
 
   public DirectoryCollection(String topCollectionName, String topDirS, boolean isTop, String olderThan,
       org.slf4j.Logger logger) {
-    this(topCollectionName, Paths.get(topDirS), isTop, olderThan, logger);
-  }
-
-  public DirectoryCollection(String topCollectionName, Path collectionDir, boolean isTop, String olderThan,
-      org.slf4j.Logger logger) {
     super(null, logger);
     this.topCollection = cleanName(topCollectionName);
-    this.collectionDir = collectionDir;
+    this.collectionDir = topDirS;
     this.collectionName = isTop ? this.topCollection : makeCollectionName(topCollection, collectionDir);
     this.isTop = isTop;
 
@@ -81,15 +75,14 @@ public DirectoryCollection(String topCollectionName, Path collectionDir, boolean
 
   @Override
   public String getRoot() {
-    return collectionDir.toString();
+    return collectionDir;
   }
 
   @Override
   public String getIndexFilename(String suffix) {
     if (isTop)
       return super.getIndexFilename(suffix);
-    Path indexPath = DirectoryCollection.makeCollectionIndexPath(topCollection, collectionDir, suffix);
-    return indexPath.toString();
+    return DirectoryCollection.makeCollectionIndexPath(topCollection, collectionDir, suffix);
   }
 
   @Override
@@ -111,22 +104,20 @@ public void close() {
   // returns everything in the current directory, subject to sfilter
   private class MyFileIterator implements CloseableIterator {
     int debugNum;
-    DirectoryStream dirStream;
-    Iterator dirStreamIterator;
+    DirectoryStream dirStream;
+    Iterator dirStreamIterator;
     MFile nextMFile;
     int count;
 
-    MyFileIterator(Path dir) throws IOException {
+    MyFileIterator(String dir) {
       if (debug) {
         debugNum = debugCount++;
         System.out.printf(" MyFileIterator %s (%d)", dir, debugNum);
       }
-      try {
-        dirStream = Files.newDirectoryStream(dir, new MyStreamFilter());
+      try (MController controller = MControllers.create(dir)) {
+        CollectionConfig config = new CollectionConfig(collectionName, dir, false, (MFileFilter) sfilter, null);
+        dirStream = controller.getInventoryTop(config, true);
         dirStreamIterator = dirStream.iterator();
-      } catch (IOException ioe) {
-        logger.error("Files.newDirectoryStream failed to open directory " + dir.getFileName(), ioe);
-        throw ioe;
       }
     }
 
@@ -141,18 +132,17 @@ public boolean hasNext() {
 
         long now = System.currentTimeMillis();
         try {
-          Path nextPath = dirStreamIterator.next();
-          BasicFileAttributes attr = Files.readAttributes(nextPath, BasicFileAttributes.class);
-          if (attr.isDirectory())
+          MFile nextFile = dirStreamIterator.next();
+          if (nextFile.isDirectory())
             continue;
-          FileTime last = attr.lastModifiedTime();
-          long millisSinceModified = now - last.toMillis();
+          long last = nextFile.getLastModified();
+          long millisSinceModified = now - last;
           if (millisSinceModified < olderThanMillis)
             continue;
-          nextMFile = new MFileOS7(nextPath, attr);
+          nextMFile = nextFile;
           return true;
 
-        } catch (IOException e) {
+        } catch (Exception e) {
           throw new RuntimeException(e);
         }
       }
@@ -181,24 +171,22 @@ public void close() throws IOException {
   private static final boolean debug = false;
   private static int debugCount;
 
-  // this idiom keeps the iterator from escaping, so that we can use try-with-resource, and ensure DirectoryStream
+  // this idiom keeps the iterator from escaping so that we can use try-with-resource, and ensure DirectoryStream
   // closes. like++
-  public void iterateOverMFileCollection(Visitor visit) throws IOException {
+  public void iterateOverMFileCollection(Visitor visit) {
     if (debug)
       System.out.printf(" iterateOverMFileCollection %s ", collectionDir);
     int count = 0;
-    try (DirectoryStream ds = Files.newDirectoryStream(collectionDir, new MyStreamFilter())) {
-      for (Path p : ds) {
-        try {
-          BasicFileAttributes attr = Files.readAttributes(p, BasicFileAttributes.class);
-          if (!attr.isDirectory())
-            visit.consume(new MFileOS7(p));
-          if (debug)
-            System.out.printf("%d ", count++);
-        } catch (IOException ioe) {
-          // catch error and skip file
-          logger.error("Failed to read attributes from file found in Files.newDirectoryStream ", ioe);
-        }
+    MController controller = MControllers.create(collectionDir);
+    CollectionConfig config = new CollectionConfig(collectionName, collectionDir, false, (MFileFilter) sfilter, null);
+    DirectoryStream ds = controller.getInventoryTop(config, true);
+    controller.close();
+    if (ds != null) {
+      for (MFile mfile : ds) {
+        if (!mfile.isDirectory())
+          visit.consume(mfile);
+        if (debug)
+          System.out.printf("%d ", count++);
       }
     }
     if (debug)
diff --git a/cdm/core/src/main/java/thredds/inventory/partition/DirectoryCollectionFromIndex.java b/cdm/core/src/main/java/thredds/inventory/partition/DirectoryCollectionFromIndex.java
index ac1d8f5695..214d0b0486 100644
--- a/cdm/core/src/main/java/thredds/inventory/partition/DirectoryCollectionFromIndex.java
+++ b/cdm/core/src/main/java/thredds/inventory/partition/DirectoryCollectionFromIndex.java
@@ -1,7 +1,8 @@
 /*
- * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package thredds.inventory.partition;
 
 import thredds.inventory.CollectionAbstract;
@@ -26,7 +27,7 @@ class DirectoryCollectionFromIndex extends CollectionAbstract {
       org.slf4j.Logger logger) {
     super(builder.getPartitionName(), logger);
     setDateExtractor(dateExtractor);
-    setRoot(builder.getDir().toString());
+    setRoot(builder.getDir());
     this.builder = builder;
     this.indexReader = indexReader;
   }
@@ -38,7 +39,7 @@ public CloseableIterator getFileIterator() throws IOException {
 
   @Override
   public String getRoot() {
-    return builder.getDir().toString();
+    return builder.getDir();
   }
 
   @Override
diff --git a/cdm/core/src/main/java/thredds/inventory/partition/DirectoryPartition.java b/cdm/core/src/main/java/thredds/inventory/partition/DirectoryPartition.java
index 94cda9780f..89f460079f 100644
--- a/cdm/core/src/main/java/thredds/inventory/partition/DirectoryPartition.java
+++ b/cdm/core/src/main/java/thredds/inventory/partition/DirectoryPartition.java
@@ -1,14 +1,14 @@
 /*
- * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package thredds.inventory.partition;
 
 import thredds.featurecollection.FeatureCollectionConfig;
 import thredds.inventory.*;
 import ucar.nc2.util.CloseableIterator;
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.*;
 
 /**
@@ -22,14 +22,14 @@
 public class DirectoryPartition extends CollectionAbstract implements PartitionManager {
 
   private final FeatureCollectionConfig config;
-  private final Path collectionDir; // directory for this collection
+  private final String collectionDir; // directory for this collection
   private final String topCollection; // config collection name,
   private final boolean isTop; // is this the top of the tree ?
   private final IndexReader indexReader;
   private final String suffix;
 
-  public DirectoryPartition(FeatureCollectionConfig config, Path collectionDir, boolean isTop, IndexReader indexReader,
-      String suffix, org.slf4j.Logger logger) {
+  public DirectoryPartition(FeatureCollectionConfig config, String collectionDir, boolean isTop,
+      IndexReader indexReader, String suffix, org.slf4j.Logger logger) {
     super(null, logger);
     this.config = config;
     this.collectionDir = collectionDir;
@@ -46,8 +46,7 @@ public DirectoryPartition(FeatureCollectionConfig config, Path collectionDir, bo
   public String getIndexFilename(String suffix) {
     if (isTop)
       return super.getIndexFilename(suffix);
-    Path indexPath = DirectoryCollection.makeCollectionIndexPath(topCollection, collectionDir, suffix);
-    return indexPath.toString();
+    return DirectoryCollection.makeCollectionIndexPath(topCollection, collectionDir, suffix);
   }
 
   @Override
@@ -55,7 +54,7 @@ public Iterable makePartitions(CollectionUpdateType forceCollection
     if (forceCollection == null)
       forceCollection = CollectionUpdateType.test;
 
-    DirectoryBuilder builder = new DirectoryBuilder(topCollection, collectionDir, null, suffix);
+    DirectoryBuilder builder = new DirectoryBuilder(topCollection, collectionDir, suffix);
     builder.constructChildren(indexReader, forceCollection);
 
     List result = new ArrayList<>();
@@ -94,7 +93,7 @@ MCollection makeChildCollection(DirectoryBuilder dpb) throws IOException {
 
   @Override
   public String getRoot() {
-    return collectionDir.toString();
+    return collectionDir;
   }
 
   // empty mfile list
diff --git a/cdm/core/src/main/java/thredds/inventory/partition/FilePartition.java b/cdm/core/src/main/java/thredds/inventory/partition/FilePartition.java
index 52c1c2248b..5e21f4d616 100644
--- a/cdm/core/src/main/java/thredds/inventory/partition/FilePartition.java
+++ b/cdm/core/src/main/java/thredds/inventory/partition/FilePartition.java
@@ -1,13 +1,13 @@
 /*
- * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package thredds.inventory.partition;
 
 import thredds.inventory.*;
 import ucar.nc2.util.CloseableIterator;
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -21,7 +21,7 @@
  */
 public class FilePartition extends DirectoryCollection implements PartitionManager {
 
-  public FilePartition(String topCollectionName, Path topDir, boolean isTop, String olderThan,
+  public FilePartition(String topCollectionName, String topDir, boolean isTop, String olderThan,
       org.slf4j.Logger logger) {
     super(topCollectionName, topDir, isTop, olderThan, logger);
   }
diff --git a/cdm/core/src/main/java/thredds/inventory/partition/IndexReader.java b/cdm/core/src/main/java/thredds/inventory/partition/IndexReader.java
index bdd0d6d2a7..cf289e66bc 100644
--- a/cdm/core/src/main/java/thredds/inventory/partition/IndexReader.java
+++ b/cdm/core/src/main/java/thredds/inventory/partition/IndexReader.java
@@ -1,12 +1,12 @@
 /*
- * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package thredds.inventory.partition;
 
 import thredds.inventory.MFile;
 import java.io.IOException;
-import java.nio.file.Path;
 import java.util.List;
 
 /**
@@ -25,7 +25,7 @@ public interface IndexReader {
    * @return true if indexFile is a partition collection
    * @throws IOException on bad things
    */
-  boolean readChildren(Path indexFile, AddChildCallback callback) throws IOException;
+  boolean readChildren(String indexFile, AddChildCallback callback) throws IOException;
 
   interface AddChildCallback {
     /**
@@ -46,7 +46,7 @@ interface AddChildCallback {
    * @return true if its a partition type index
    * @throws IOException on bad
    */
-  boolean isPartition(Path indexFile) throws IOException;
+  boolean isPartition(String indexFile) throws IOException;
 
   /**
    * Read the MFiles from a GribCollection index file
@@ -55,7 +55,7 @@ interface AddChildCallback {
    * @param result put results in this list
    * @return true if indexFile is a GribCollection collection, and read ok
    */
-  boolean readMFiles(Path indexFile, List result) throws IOException;
+  boolean readMFiles(String indexFile, List result) throws IOException;
 
 
 }
diff --git a/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java b/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java
index b123a6ab34..0af11b58a5 100644
--- a/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java
+++ b/cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2020 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE.txt for license information.
  */
 
@@ -7,6 +7,7 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.nio.file.DirectoryStream;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -26,6 +27,7 @@
 import thredds.inventory.MController;
 import thredds.inventory.MControllerProvider;
 import thredds.inventory.MFile;
+import thredds.inventory.MFileDirectoryStream;
 import thredds.inventory.s3.MFileS3;
 import ucar.unidata.io.s3.CdmS3Client;
 import ucar.unidata.io.s3.CdmS3Uri;
@@ -71,7 +73,7 @@ private void initClient() throws IOException {
   }
 
   @Override
-  public Iterator getInventoryAll(CollectionConfig mc, boolean recheck) {
+  public DirectoryStream getInventoryAll(CollectionConfig mc, boolean recheck) {
     init(mc);
     String prefix = null;
     if (initialUri.getKey().isPresent()) {
@@ -79,21 +81,23 @@ public Iterator getInventoryAll(CollectionConfig mc, boolean recheck) {
     }
     // to get all inventory, we need to make the listObject call in MFileS3Iterator without a delimiter.
     // but, we want the resulting MFile object to retain CdmS3Uri objects that continue to have a delimiter.
-    return new FilteredIterator(mc, new MFileS3Iterator(client, initialUri, prefix, limit, true), true, true);
+    return new MFileDirectoryStream(
+        new FilteredIterator(mc, new MFileS3Iterator(client, initialUri, prefix, limit, true), true, true));
   }
 
   @Override
-  public Iterator getInventoryTop(CollectionConfig mc, boolean recheck) {
+  public DirectoryStream getInventoryTop(CollectionConfig mc, boolean recheck) {
     init(mc);
     String prefix = null;
     if (initialUri.getKey().isPresent()) {
       prefix = initialUri.getKey().get();
     }
-    return new FilteredIterator(mc, new MFileS3Iterator(client, initialUri, prefix, limit, false), false);
+    return new MFileDirectoryStream(
+        new FilteredIterator(mc, new MFileS3Iterator(client, initialUri, prefix, limit, false), false));
   }
 
   @Override
-  public Iterator getSubdirs(CollectionConfig mc, boolean recheck) {
+  public DirectoryStream getSubdirs(CollectionConfig mc, boolean recheck) {
     init(mc);
     String prefix = null;
     if (initialUri.getKey().isPresent()) {
@@ -121,7 +125,7 @@ public Iterator getSubdirs(CollectionConfig mc, boolean recheck) {
         logger.error("Error creating MFile for {} bucket {}", commonPrefix, initialUri.getBucket(), e);
       }
     }
-    return new FilteredIterator(mc, mFiles.iterator(), true);
+    return new MFileDirectoryStream(new FilteredIterator(mc, mFiles.iterator(), true));
   }
 
   @Override
diff --git a/cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java b/cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java
index bb60995074..e9683fe4b0 100644
--- a/cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java
+++ b/cdm/s3/src/test/java/thredds/filesystem/s3/TestControllerS3.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 2020-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -7,7 +7,9 @@
 
 import static com.google.common.truth.Truth.assertThat;
 
+import java.io.IOException;
 import java.net.URISyntaxException;
+import java.nio.file.DirectoryStream;
 import java.util.Iterator;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
@@ -66,18 +68,22 @@ public static void setup() {
   }
 
   @Test
-  public void shouldReturnSameValueFromHasNext() throws URISyntaxException {
+  public void shouldReturnSameValueFromHasNext() throws URISyntaxException, IOException {
     final CdmS3Uri uri = new CdmS3Uri("cdms3:thredds-test-data");
     final MFileFilter filter = new WildcardMatchOnName("testData.nc");
     final CollectionConfig collectionConfig = new CollectionConfig(uri.getBucket(), uri.toString(), true, filter, null);
-    final ControllerS3 controller = new ControllerS3();
-    final Iterator iterator = controller.getInventoryTop(collectionConfig, false);
-
-    assertThat(iterator.hasNext()).isTrue();
-    assertThat(iterator.hasNext()).isTrue();
-    iterator.next();
-    assertThat(iterator.hasNext()).isFalse();
-    assertThat(iterator.hasNext()).isFalse();
+    try (ControllerS3 controller = new ControllerS3()) {
+      try (DirectoryStream stream = controller.getInventoryTop(collectionConfig, false)) {
+        assertThat(stream).isNotNull();
+        final Iterator iterator = stream.iterator();
+
+        assertThat(iterator.hasNext()).isTrue();
+        assertThat(iterator.hasNext()).isTrue();
+        iterator.next();
+        assertThat(iterator.hasNext()).isFalse();
+        assertThat(iterator.hasNext()).isFalse();
+      }
+    }
   }
 
   //////////////////////
@@ -364,10 +370,15 @@ private int topInventoryCount(CdmS3Uri uri) {
   }
 
   private int topInventoryCount(CollectionConfig collectionConfig) {
-    ControllerS3 controller = new ControllerS3();
-    controller.limit = true;
-    Iterator it = controller.getInventoryTop(collectionConfig, false);
-    return countObjects(it);
+    try (ControllerS3 controller = new ControllerS3()) {
+      controller.limit = true;
+      try (DirectoryStream stream = controller.getInventoryTop(collectionConfig, false)) {
+        assertThat(stream).isNotNull();
+        return countObjects(stream.iterator());
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
   }
 
   private void checkInventoryAllCount(CdmS3Uri uri, int expectedCount) {
@@ -376,10 +387,15 @@ private void checkInventoryAllCount(CdmS3Uri uri, int expectedCount) {
   }
 
   private void checkInventoryAllCount(CollectionConfig collectionConfig, int expectedCount) {
-    ControllerS3 controller = new ControllerS3();
-    controller.limit = true;
-    Iterator it = controller.getInventoryAll(collectionConfig, false);
-    assertThat(countObjects(it)).isEqualTo(expectedCount);
+    try (ControllerS3 controller = new ControllerS3()) {
+      controller.limit = true;
+      try (DirectoryStream stream = controller.getInventoryAll(collectionConfig, false)) {
+        assertThat(stream).isNotNull();
+        assertThat(countObjects(stream.iterator())).isEqualTo(expectedCount);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
   }
 
   private void checkSubdirsCount(CdmS3Uri uri, int expectedCount) {
@@ -388,10 +404,15 @@ private void checkSubdirsCount(CdmS3Uri uri, int expectedCount) {
   }
 
   private void checkSubdirsCount(CollectionConfig collectionConfig, int expectedCount) {
-    ControllerS3 controller = new ControllerS3();
-    controller.limit = true;
-    Iterator it = controller.getSubdirs(collectionConfig, false);
-    assertThat(countObjects(it)).isEqualTo(expectedCount);
+    try (ControllerS3 controller = new ControllerS3()) {
+      controller.limit = true;
+      try (DirectoryStream stream = controller.getSubdirs(collectionConfig, false)) {
+        assertThat(stream).isNotNull();
+        assertThat(countObjects(stream.iterator())).isEqualTo(expectedCount);
+      } catch (IOException e) {
+        throw new RuntimeException(e);
+      }
+    }
   }
 
   private int countObjects(Iterator it) {
diff --git a/cdm/zarr/src/main/java/thredds/filesystem/zarr/ControllerZip.java b/cdm/zarr/src/main/java/thredds/filesystem/zarr/ControllerZip.java
index cf37250274..365f87345c 100644
--- a/cdm/zarr/src/main/java/thredds/filesystem/zarr/ControllerZip.java
+++ b/cdm/zarr/src/main/java/thredds/filesystem/zarr/ControllerZip.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 2021-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -10,10 +10,12 @@
 import thredds.inventory.MController;
 import thredds.inventory.MControllerProvider;
 import thredds.inventory.MFile;
+import thredds.inventory.MFileDirectoryStream;
 import thredds.inventory.zarr.MFileZip;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.DirectoryStream;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
@@ -30,7 +32,7 @@ public class ControllerZip extends ControllerOS implements MController {
   private static final String prefix = "file:";
 
   @Override
-  public Iterator getInventoryAll(CollectionConfig mc, boolean recheck) {
+  public DirectoryStream getInventoryAll(CollectionConfig mc, boolean recheck) {
     String path = mc.getDirectoryName();
     if (path.startsWith(prefix)) {
       path = path.substring(prefix.length());
@@ -38,7 +40,7 @@ public Iterator getInventoryAll(CollectionConfig mc, boolean recheck) {
 
     try {
       MFileZip mfile = new MFileZip(path);
-      return new MFileIteratorLeaves(mfile);
+      return new MFileDirectoryStream(new MFileIteratorLeaves(mfile));
     } catch (IOException ioe) {
       logger.warn(ioe.getMessage(), ioe);
       return null;
@@ -46,7 +48,7 @@ public Iterator getInventoryAll(CollectionConfig mc, boolean recheck) {
   }
 
   @Override
-  public Iterator getInventoryTop(CollectionConfig mc, boolean recheck) {
+  public DirectoryStream getInventoryTop(CollectionConfig mc, boolean recheck) {
     String path = mc.getDirectoryName();
     if (path.startsWith(prefix)) {
       path = path.substring(prefix.length());
@@ -54,7 +56,7 @@ public Iterator getInventoryTop(CollectionConfig mc, boolean recheck) {
 
     try {
       MFileZip mfile = new MFileZip(path);
-      return new FilteredIterator(mfile, false);
+      return new MFileDirectoryStream(new FilteredIterator(mfile, false));
     } catch (IOException ioe) {
       logger.warn(ioe.getMessage(), ioe);
       return null;
@@ -62,7 +64,7 @@ public Iterator getInventoryTop(CollectionConfig mc, boolean recheck) {
   }
 
   @Override
-  public Iterator getSubdirs(CollectionConfig mc, boolean recheck) {
+  public DirectoryStream getSubdirs(CollectionConfig mc, boolean recheck) {
     String path = mc.getDirectoryName();
     if (path.startsWith(prefix)) {
       path = path.substring(prefix.length());
@@ -70,7 +72,7 @@ public Iterator getSubdirs(CollectionConfig mc, boolean recheck) {
 
     try {
       MFileZip mfile = new MFileZip(path);
-      return new FilteredIterator(mfile, true);
+      return new MFileDirectoryStream(new FilteredIterator(mfile, true));
     } catch (IOException ioe) {
       logger.warn(ioe.getMessage(), ioe);
       return null;
diff --git a/cdm/zarr/src/main/java/ucar/unidata/io/zarr/RandomAccessDirectory.java b/cdm/zarr/src/main/java/ucar/unidata/io/zarr/RandomAccessDirectory.java
index f1a62cbf75..d14077f3a1 100644
--- a/cdm/zarr/src/main/java/ucar/unidata/io/zarr/RandomAccessDirectory.java
+++ b/cdm/zarr/src/main/java/ucar/unidata/io/zarr/RandomAccessDirectory.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 2021-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -14,6 +14,7 @@
 
 import java.io.*;
 import java.nio.channels.WritableByteChannel;
+import java.nio.file.DirectoryStream;
 import java.util.*;
 
 /**
@@ -50,9 +51,15 @@ public RandomAccessDirectory(String location, int bufferSize) throws IOException
 
     // build children list
     this.children = new ArrayList<>();
-    MController controller = MControllers.create(location);
-    CollectionConfig cc = new CollectionConfig("children", location, false, null, null);
-    List files = sortIterator(controller.getInventoryAll(cc, false)); // standardize order
+    List files = null;
+    try (MController controller = MControllers.create(location)) {
+      CollectionConfig cc = new CollectionConfig("children", location, false, null, null);
+      try (DirectoryStream ds = controller.getInventoryAll(cc, false)) {
+        if (ds != null) {
+          files = sortIterator(ds.iterator()); // standardize order
+        }
+      }
+    }
     if (files == null) {
       return;
     }
diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java b/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java
index 8776409455..e0a460c674 100644
--- a/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java
+++ b/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2018 John Caron and University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 2013-2026 John Caron and University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -15,7 +15,7 @@
 import thredds.featurecollection.FeatureCollectionType;
 import thredds.filesystem.MFileOS;
 import thredds.inventory.*;
-import thredds.inventory.filter.StreamFilter;
+import thredds.inventory.filter.RegExpMatch;
 import thredds.inventory.partition.*;
 import ucar.nc2.dataset.DatasetUrl;
 import ucar.nc2.grib.GribIndexCache;
@@ -301,6 +301,7 @@ public static boolean updateGribCollection(FeatureCollectionConfig config, Colle
     if (logger == null)
       logger = classLogger;
 
+    logger.debug("GribCdmIndex.updateGribCollection {} {}", config.collectionName, updateType);
     long start = System.currentTimeMillis();
 
     Formatter errlog = new Formatter();
@@ -330,7 +331,7 @@ public static boolean updateGribCollection(FeatureCollectionConfig config, Colle
       if (specp.wantSubdirs()) { // its a partition
 
         try (DirectoryPartition dpart =
-            new DirectoryPartition(config, rootPath, true, new GribCdmIndex(logger), NCX_SUFFIX, logger)) {
+            new DirectoryPartition(config, rootPath.toString(), true, new GribCdmIndex(logger), NCX_SUFFIX, logger)) {
           dpart.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
           changed = updateDirectoryCollectionRecurse(isGrib1, dpart, config, updateType, logger);
         }
@@ -349,10 +350,10 @@ public static boolean updateGribCollection(FeatureCollectionConfig config, Colle
   public static boolean updateGribCollection(boolean isGrib1, MCollection dcm, CollectionUpdateType updateType,
       FeatureCollectionConfig.PartitionType ptype, Logger logger, Formatter errlog) throws IOException {
 
-    logger.debug("GribCdmIndex.updateGribCollection {} {}", dcm.getCollectionName(), updateType);
-    if (!isUpdateNeeded(dcm.getIndexFilename(NCX_SUFFIX), updateType,
-        (isGrib1 ? GribCollectionType.GRIB1 : GribCollectionType.GRIB2), logger))
-      return false;
+    boolean updateNeeded = isUpdateNeeded(dcm.getIndexFilename(NCX_SUFFIX), updateType,
+        (isGrib1 ? GribCollectionType.GRIB1 : GribCollectionType.GRIB2), logger);
+    logger.debug("GribCdmIndex.updateGribCollection (mcoll) {} {} updateNeeded={}", dcm.getCollectionName(), updateType,
+        updateNeeded);
 
     boolean changed;
     if (isGrib1) { // existing case handles correctly - make separate index for each runtime (OR) partition == runtime
@@ -362,12 +363,17 @@ public static boolean updateGribCollection(boolean isGrib1, MCollection dcm, Col
       Grib2CollectionBuilder builder = new Grib2CollectionBuilder(dcm.getCollectionName(), dcm, logger);
       changed = builder.updateNeeded(updateType) && builder.createIndex(ptype, errlog);
     }
-    return changed;
+    return changed || updateNeeded;
   }
 
   // return true if changed, exception on failure
   private static boolean updatePartition(boolean isGrib1, PartitionManager dcm, CollectionUpdateType updateType,
       Logger logger, Formatter errlog) throws IOException {
+    boolean updateNeeded = isUpdateNeeded(dcm.getIndexFilename(NCX_SUFFIX), updateType,
+        (isGrib1 ? GribCollectionType.Partition1 : GribCollectionType.Partition2), logger);
+    logger.debug("GribCdmIndex.updatePartition {} {} updateNeeded={}", dcm.getCollectionName(), updateType,
+        updateNeeded);
+
     boolean changed;
     if (isGrib1) {
       Grib1PartitionBuilder builder =
@@ -379,7 +385,7 @@ private static boolean updatePartition(boolean isGrib1, PartitionManager dcm, Co
           new Grib2PartitionBuilder(dcm.getCollectionName(), new File(dcm.getRoot()), dcm, logger);
       changed = builder.updateNeeded(updateType) && builder.createPartitionedIndex(updateType, errlog);
     }
-    return changed;
+    return changed || updateNeeded;
   }
 
 
@@ -387,17 +393,19 @@ private static boolean updateTimePartition(boolean isGrib1, TimePartition tp, Co
       Logger logger) throws IOException {
 
     logger.debug("GribCdmIndex.updateTimePartition {} {}", tp.getRoot(), updateType);
-    if (!isUpdateNeeded(tp.getIndexFilename(NCX_SUFFIX), updateType,
-        (isGrib1 ? GribCollectionType.Partition1 : GribCollectionType.Partition2), logger))
-      return false;
+    boolean updateNeeded = isUpdateNeeded(tp.getIndexFilename(NCX_SUFFIX), updateType,
+        (isGrib1 ? GribCollectionType.Partition1 : GribCollectionType.Partition2), logger);
 
     long start = System.currentTimeMillis();
     Formatter errlog = new Formatter();
+    AtomicBoolean anyChange = new AtomicBoolean(false);
 
     for (MCollection part : tp.makePartitions(updateType)) {
       try {
-        updateGribCollection(isGrib1, part, updateType, FeatureCollectionConfig.PartitionType.timePeriod, logger,
-            errlog);
+        boolean changed = updateGribCollection(isGrib1, part, updateType,
+            FeatureCollectionConfig.PartitionType.timePeriod, logger, errlog);
+        if (changed)
+          anyChange.set(true);
 
       } catch (Throwable t) {
         logger.warn("Error making partition " + part.getRoot(), t);
@@ -405,10 +413,13 @@ private static boolean updateTimePartition(boolean isGrib1, TimePartition tp, Co
       }
     } // loop over component grib collections
 
+    if (!updateNeeded && !anyChange.get())
+      return false;
 
     try {
-      boolean changed = updatePartition(isGrib1, tp, updateType, logger, errlog);
+      boolean recreated = updatePartition(isGrib1, tp, updateType, logger, errlog);
 
+      boolean changed = recreated || anyChange.get();
       long took = System.currentTimeMillis() - start;
       errlog.format(" INFO updateTimePartition %s took %d msecs%n", tp.getRoot(), took);
       logger.debug("GribCdmIndex.updateTimePartition complete ({}) on {} errlog={}", changed, tp.getRoot(), errlog);
@@ -429,7 +440,8 @@ private static boolean isUpdateNeeded(String idxFilenameOrg, CollectionUpdateTyp
     if (updateType == CollectionUpdateType.never)
       return false;
 
-    // see if index already exists
+    if (updateType == CollectionUpdateType.always)
+      return true;
     File collectionIndexFile = GribIndexCache.getExistingFileOrCache(idxFilenameOrg);
     if (collectionIndexFile != null) { // it exists
 
@@ -458,25 +470,30 @@ private static boolean isUpdateNeeded(String idxFilenameOrg, CollectionUpdateTyp
   private static boolean updateDirectoryCollectionRecurse(boolean isGrib1, DirectoryPartition dpart,
       FeatureCollectionConfig config, CollectionUpdateType updateType, Logger logger) throws IOException {
 
-    logger.debug("GribCdmIndex.updateDirectoryCollectionRecurse {} {}", dpart.getRoot(), updateType);
-    if (!isUpdateNeeded(dpart.getIndexFilename(NCX_SUFFIX), updateType,
-        (isGrib1 ? GribCollectionType.Partition1 : GribCollectionType.Partition2), logger))
-      return false;
+    boolean updateNeeded = isUpdateNeeded(dpart.getIndexFilename(NCX_SUFFIX), updateType,
+        (isGrib1 ? GribCollectionType.Partition1 : GribCollectionType.Partition2), logger);
+    logger.debug("GribCdmIndex.updateDirectoryCollectionRecurse {} {} updateNeeded={}", dpart.getRoot(), updateType,
+        updateNeeded);
 
     long start = System.currentTimeMillis();
+    AtomicBoolean anyChange = new AtomicBoolean(false);
 
     // check the children partitions first
     if (updateType != CollectionUpdateType.testIndexOnly) { // skip children on testIndexOnly
       for (MCollection part : dpart.makePartitions(updateType)) {
         part.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
         try {
+          boolean changed;
           if (part instanceof DirectoryPartition) { // LOOK if child partition fails, the parent partition doesnt know
                                                     // that - suckage
-            updateDirectoryCollectionRecurse(isGrib1, (DirectoryPartition) part, config, updateType, logger);
+            changed = updateDirectoryCollectionRecurse(isGrib1, (DirectoryPartition) part, config, updateType, logger);
           } else {
             Path partPath = Paths.get(part.getRoot());
-            updateLeafCollection(isGrib1, config, updateType, false, logger, partPath); // LOOK why not using part ??
+            changed = updateLeafCollection(isGrib1, config, updateType, false, logger, partPath); // LOOK why not using
+                                                                                                  // part ??
           }
+          if (changed)
+            anyChange.set(true);
         } catch (IllegalStateException t) {
           logger.warn("Error making partition {} '{}'", part.getRoot(), t.getMessage());
           dpart.removePartition(part); // keep on truckin; can happen if directory is empty
@@ -488,24 +505,28 @@ private static boolean updateDirectoryCollectionRecurse(boolean isGrib1, Directo
       } // loop over partitions
     }
 
+    if (!updateNeeded && !anyChange.get())
+      return false;
+
     try {
       // update the partition
       Formatter errlog = new Formatter();
-      boolean changed = updatePartition(isGrib1, dpart, updateType, logger, errlog);
+      boolean recreated = updatePartition(isGrib1, dpart, updateType, logger, errlog);
 
+      boolean changed = recreated || anyChange.get();
       long took = System.currentTimeMillis() - start;
       errlog.format(" INFO updateDirectoryCollectionRecurse %s took %d msecs%n", dpart.getRoot(), took);
       logger.debug("GribCdmIndex.updateDirectoryCollectionRecurse complete ({}) on {} errlog={}", changed,
           dpart.getRoot(), errlog);
-      return changed;
+      return changed || updateNeeded;
 
     } catch (IllegalStateException t) {
       logger.warn("Error making partition {} '{}'", dpart.getRoot(), t.getMessage());
-      return false;
+      return updateNeeded;
 
     } catch (Throwable t) {
       logger.error("Error making partition " + dpart.getRoot(), t);
-      return false;
+      return updateNeeded;
     }
   }
 
@@ -519,6 +540,7 @@ private static boolean updateDirectoryCollectionRecurse(boolean isGrib1, Directo
   private static boolean updateLeafCollection(boolean isGrib1, FeatureCollectionConfig config,
       CollectionUpdateType updateType, boolean isTop, Logger logger, Path dirPath) throws IOException {
 
+    logger.debug("GribCdmIndex.updateLeafCollection {} {} ptype={}", dirPath, updateType, config.ptype);
     if (config.ptype == FeatureCollectionConfig.PartitionType.file) {
       return updateFilePartition(isGrib1, config, updateType, isTop, logger, dirPath);
 
@@ -527,10 +549,10 @@ private static boolean updateLeafCollection(boolean isGrib1, FeatureCollectionCo
       CollectionSpecParserAbstract specp = config.getCollectionSpecParserAbstract(errlog);
 
       try (DirectoryCollection dcm =
-          new DirectoryCollection(config.collectionName, dirPath, isTop, config.olderThan, logger)) {
+          new DirectoryCollection(config.collectionName, dirPath.toString(), isTop, config.olderThan, logger)) {
         dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
         if (specp.getFilter() != null)
-          dcm.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName()));
+          dcm.setStreamFilter(new RegExpMatch(specp.getFilter(), specp.getFilterOnName()));
 
         boolean changed = updateGribCollection(isGrib1, dcm, updateType,
             FeatureCollectionConfig.PartitionType.directory, logger, errlog);
@@ -553,19 +575,20 @@ private static boolean updateLeafCollection(boolean isGrib1, FeatureCollectionCo
    */
   private static boolean updateFilePartition(boolean isGrib1, FeatureCollectionConfig config,
       CollectionUpdateType updateType, boolean isTop, Logger logger, Path dirPath) throws IOException {
+    logger.debug("GribCdmIndex.updateFilePartition {} {}", dirPath, updateType);
     long start = System.currentTimeMillis();
     Formatter errlog = new Formatter();
     CollectionSpecParserAbstract specp = config.getCollectionSpecParserAbstract(errlog);
 
-    try (FilePartition partition = new FilePartition(config.collectionName, dirPath, isTop, config.olderThan, logger)) {
+    try (FilePartition partition =
+        new FilePartition(config.collectionName, dirPath.toString(), isTop, config.olderThan, logger)) {
       partition.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
       if (specp.getFilter() != null)
-        partition.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName()));
+        partition.setStreamFilter(new RegExpMatch(specp.getFilter(), specp.getFilterOnName()));
 
       logger.debug("GribCdmIndex.updateFilePartition {} {}", partition.getCollectionName(), updateType);
-      if (!isUpdateNeeded(partition.getIndexFilename(NCX_SUFFIX), updateType,
-          (isGrib1 ? GribCollectionType.Partition1 : GribCollectionType.Partition2), logger))
-        return false;
+      boolean updateNeeded = isUpdateNeeded(partition.getIndexFilename(NCX_SUFFIX), updateType,
+          (isGrib1 ? GribCollectionType.Partition1 : GribCollectionType.Partition2), logger);
 
       AtomicBoolean anyChange = new AtomicBoolean(false); // just need a mutable boolean we can declare final
 
@@ -592,16 +615,20 @@ private static boolean updateFilePartition(boolean isGrib1, FeatureCollectionCon
         });
       }
 
+      if (!updateNeeded && !anyChange.get())
+        return false;
+
       // LOOK what if theres only one file?
 
       try {
         // redo partition index if needed, will detect if children have changed
         boolean recreated = updatePartition(isGrib1, partition, updateType, logger, errlog);
 
+        boolean changed = recreated || anyChange.get();
         long took = System.currentTimeMillis() - start;
-        if (recreated)
+        if (changed)
           logger.info("RewriteFilePartition {} took {} msecs", partition.getCollectionName(), took);
-        return recreated;
+        return changed;
 
       } catch (IllegalStateException t) {
         logger.warn("Error making partition {} '{}'", partition.getRoot(), t.getMessage());
@@ -885,9 +912,9 @@ public GribCdmIndex(Logger logger) {
 
   /// IndexReader interface
   @Override
-  public boolean readChildren(Path indexFile, AddChildCallback callback) throws IOException {
+  public boolean readChildren(String indexFile, AddChildCallback callback) throws IOException {
     logger.debug("GribCdmIndex.readChildren {}", indexFile);
-    try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile.toString())) {
+    try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile)) {
       GribCollectionType type = getType(raf);
       if (type == GribCollectionType.Partition1 || type == GribCollectionType.Partition2) {
         if (openIndex(raf, logger)) {
@@ -905,18 +932,18 @@ public boolean readChildren(Path indexFile, AddChildCallback callback) throws IO
   }
 
   @Override
-  public boolean isPartition(Path indexFile) throws IOException {
+  public boolean isPartition(String indexFile) throws IOException {
     logger.debug("GribCdmIndex.isPartition {}", indexFile);
-    try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile.toString())) {
+    try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile)) {
       GribCollectionType type = getType(raf);
       return (type == GribCollectionType.Partition1) || (type == GribCollectionType.Partition2);
     }
   }
 
   @Override
-  public boolean readMFiles(Path indexFile, List result) throws IOException {
+  public boolean readMFiles(String indexFile, List result) throws IOException {
     logger.debug("GribCdmIndex.readMFiles {}", indexFile);
-    try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile.toString())) {
+    try (RandomAccessFile raf = RandomAccessFile.acquire(indexFile)) {
       // GribCollectionType type = getType(raf);
       // if (type == GribCollectionType.GRIB1 || type == GribCollectionType.GRIB2) {
       if (openIndex(raf, logger)) {
diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionBuilder.java b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionBuilder.java
index f0cd86b0ae..6f3c8dade4 100644
--- a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionBuilder.java
+++ b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionBuilder.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -64,7 +64,8 @@ boolean updateNeeded(CollectionUpdateType ff) throws IOException {
     if (ff == CollectionUpdateType.always)
       return true;
 
-    File collectionIndexFile = GribIndexCache.getExistingFileOrCache(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX));
+    String indexFilename = dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX);
+    File collectionIndexFile = GribIndexCache.getExistingFileOrCache(indexFilename);
     if (collectionIndexFile == null)
       return true;
 
@@ -93,7 +94,7 @@ private boolean needsUpdate(CollectionUpdateType ff, File collectionIndexFile) t
     // now see if any files were deleted, by reading the index and comparing to the files there
     GribCdmIndex reader = new GribCdmIndex(logger);
     List oldFiles = new ArrayList<>();
-    reader.readMFiles(collectionIndexFile.toPath(), oldFiles);
+    reader.readMFiles(collectionIndexFile.getPath(), oldFiles);
     Set oldFileSet = new HashSet<>();
     for (MFile oldFile : oldFiles) {
       if (!newFileSet.contains(oldFile.getPath()))
diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribPartitionBuilder.java b/grib/src/main/java/ucar/nc2/grib/collection/GribPartitionBuilder.java
index 13aed76691..f3bc6e7f4e 100644
--- a/grib/src/main/java/ucar/nc2/grib/collection/GribPartitionBuilder.java
+++ b/grib/src/main/java/ucar/nc2/grib/collection/GribPartitionBuilder.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -61,8 +61,8 @@ boolean updateNeeded(CollectionUpdateType ff) throws IOException {
     if (ff == CollectionUpdateType.always)
       return true;
 
-    File collectionIndexFile =
-        GribIndexCache.getExistingFileOrCache(partitionManager.getIndexFilename(GribCdmIndex.NCX_SUFFIX));
+    String indexFilename = partitionManager.getIndexFilename(GribCdmIndex.NCX_SUFFIX);
+    File collectionIndexFile = GribIndexCache.getExistingFileOrCache(indexFilename);
     if (collectionIndexFile == null)
       return true;
 
@@ -94,7 +94,7 @@ private boolean needsUpdate(CollectionUpdateType ff, File collectionIndexFile) t
     // now see if any files were deleted
     GribCdmIndex reader = new GribCdmIndex(logger);
     List oldFiles = new ArrayList<>();
-    reader.readMFiles(collectionIndexFile.toPath(), oldFiles);
+    reader.readMFiles(collectionIndexFile.getPath(), oldFiles);
     Set oldFileSet = new HashSet<>();
     for (MFile oldFile : oldFiles) {
       if (!newFileSet.contains(oldFile.getPath()))
diff --git a/uicdm/src/main/java/ucar/nc2/ui/grib/CdmIndexPanel.java b/uicdm/src/main/java/ucar/nc2/ui/grib/CdmIndexPanel.java
index 8638950adc..8f1d68ef8b 100644
--- a/uicdm/src/main/java/ucar/nc2/ui/grib/CdmIndexPanel.java
+++ b/uicdm/src/main/java/ucar/nc2/ui/grib/CdmIndexPanel.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -37,6 +37,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.*;
 import java.util.List;
 
@@ -774,11 +775,11 @@ else if (s2 == null)
   Collection gcFiles;
   FeatureCollectionConfig config = new FeatureCollectionConfig();
 
-  public void setIndexFile(Path indexFile, FeatureCollectionConfig config) throws IOException {
+  public void setIndexFile(String indexFile, FeatureCollectionConfig config) throws IOException {
     if (gc != null)
       gc.close();
 
-    this.indexFile = indexFile;
+    this.indexFile = Paths.get(indexFile);
     this.config = config;
     gc = GribCdmIndex.openCdmIndex(indexFile.toString(), config, false, logger);
     if (gc == null)
diff --git a/uicdm/src/main/java/ucar/nc2/ui/op/CdmIndexOpPanel.java b/uicdm/src/main/java/ucar/nc2/ui/op/CdmIndexOpPanel.java
index 8b650ff7d7..fe211d5acc 100644
--- a/uicdm/src/main/java/ucar/nc2/ui/op/CdmIndexOpPanel.java
+++ b/uicdm/src/main/java/ucar/nc2/ui/op/CdmIndexOpPanel.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2019 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -46,7 +46,7 @@ public boolean process(Object o) {
     boolean err = false;
 
     try {
-      indexPanel.setIndexFile(Paths.get(command), new FeatureCollectionConfig());
+      indexPanel.setIndexFile(command, new FeatureCollectionConfig());
     } catch (FileNotFoundException ioe) {
       JOptionPane.showMessageDialog(null, "GribCdmIndexPanel cannot open " + command + "\n" + ioe.getMessage());
       err = true;
diff --git a/uicdm/src/main/java/ucar/nc2/ui/op/DirectoryPartitionViewer.java b/uicdm/src/main/java/ucar/nc2/ui/op/DirectoryPartitionViewer.java
index 11995643e2..d653d86a84 100644
--- a/uicdm/src/main/java/ucar/nc2/ui/op/DirectoryPartitionViewer.java
+++ b/uicdm/src/main/java/ucar/nc2/ui/op/DirectoryPartitionViewer.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2019 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -370,7 +370,7 @@ public void run() {
         Formatter out = new Formatter();
         GribCdmIndex indexReader = new GribCdmIndex(logger);
         try (DirectoryPartition dpart =
-            new DirectoryPartition(config, node.dir, true, indexReader, GribCdmIndex.NCX_SUFFIX, logger)) {
+            new DirectoryPartition(config, node.dir.toString(), true, indexReader, GribCdmIndex.NCX_SUFFIX, logger)) {
           dpart.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
 
           try (PartitionCollectionMutable tp = (PartitionCollectionMutable) GribCdmIndex
@@ -420,7 +420,7 @@ public void run() {
   private void cmdShowIndex(NodeInfo node) {
     try {
       // this opens the index file and constructs a GribCollection
-      Path index = node.part.getIndex();
+      String index = node.part.getIndex();
       if (index == null) {
         node.part.findIndex();
       }
@@ -463,7 +463,7 @@ private class NodeInfo {
       this.dir = dir;
 
       try {
-        part = new DirectoryBuilder(collectionName, dir, null, GribCdmIndex.NCX_SUFFIX);
+        part = new DirectoryBuilder(collectionName, dir.toString(), GribCdmIndex.NCX_SUFFIX);
         hasIndex = part.getIndex() != null;
 
       } catch (IOException e) {
@@ -473,7 +473,7 @@ private class NodeInfo {
 
     NodeInfo(DirectoryBuilder part) {
       this.part = part;
-      this.dir = part.getDir();
+      this.dir = Paths.get(part.getDir());
       this.hasIndex = part.getIndex() != null;
     }