Refactor inventory management to use managed-file abstractions, part 1

cdm-test/src/test/java/thredds/inventory/TestDcm.java

@@ -1,33 +1,6 @@
 /*
- * Copyright (c) 1998 - 2011. University Corporation for Atmospheric Research/Unidata
- * Portions of this software were developed by the Unidata Program at the
- * University Corporation for Atmospheric Research.
- *
- * Access and use of this software shall impose the following obligations
- * and understandings on the user. The user is granted the right, without
- * any fee or cost, to use, copy, modify, alter, enhance and distribute
- * this software, and any derivative works thereof, and its supporting
- * documentation for any purpose whatsoever, provided that this entire
- * notice appears in all copies of the software, derivative works and
- * supporting documentation. Further, UCAR requests that the user credit
- * UCAR/Unidata in any publications that result from the use of this
- * software or in any product that includes this software. The names UCAR
- * and/or Unidata, however, may not be used in any advertising or publicity
- * to endorse or promote any products or commercial entity unless specific
- * written permission is obtained from UCAR/Unidata. The user also
- * understands that UCAR/Unidata is not obligated to provide the user with
- * any support, consulting, training or assistance of any kind with regard
- * to the use, operation and performance of this software nor to provide
- * the user with any updates, revisions, new versions or "bug fixes."
- *
- * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
- * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
- * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
- * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
- * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
+ * See LICENSE for license information.
  */
 
 package thredds.inventory;
@@ -41,7 +14,7 @@
 import org.slf4j.LoggerFactory;
 import thredds.featurecollection.FeatureCollectionConfig;
 import thredds.featurecollection.FeatureCollectionType;
-import thredds.inventory.filter.StreamFilter;
+import thredds.inventory.filter.RegExpMatch;
 import thredds.inventory.partition.DirectoryCollection;
 import ucar.nc2.time.CalendarDate;
 import ucar.unidata.util.test.category.NeedsCdmUnitTest;
@@ -146,7 +119,7 @@ public void testOlderThanInDirectoryCollection() throws IOException {
     // String topCollectionName, String topDirS, String olderThan, org.slf4j.Logger logger
     DirectoryCollection dcm =
         new DirectoryCollection("topCollectionName", specp.getRootDir(), true, config.olderThan, logger);
-    dcm.setStreamFilter(new StreamFilter(Pattern.compile(".*grib2"), true));
+    dcm.setStreamFilter(new RegExpMatch(Pattern.compile(".*grib2"), true));
 
     List<String> fileList = dcm.getFilenames();
     for (String name : fileList)

cdm-test/src/test/java/thredds/inventory/TestMCollection.java

@@ -1,7 +1,8 @@
 /*
- * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package thredds.inventory;
 
 import org.junit.Test;
@@ -10,7 +11,7 @@
 import org.slf4j.LoggerFactory;
 import thredds.featurecollection.FeatureCollectionConfig;
 import thredds.featurecollection.FeatureCollectionType;
-import thredds.inventory.filter.StreamFilter;
+import thredds.inventory.filter.RegExpMatch;
 import thredds.inventory.partition.DirectoryCollection;
 import thredds.inventory.partition.TimePartition;
 import ucar.unidata.util.test.category.NeedsCdmUnitTest;
@@ -38,10 +39,10 @@ public void testStreamFilterInDirPartition() throws IOException {
     Path rootPath = Paths.get(specp.getRootDir());
 
     try (DirectoryCollection dcm =
-        new DirectoryCollection(config.collectionName, rootPath, true, config.olderThan, logger)) {
+        new DirectoryCollection(config.collectionName, rootPath.toString(), true, config.olderThan, logger)) {
       dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
       if (specp.getFilter() != null)
-        dcm.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName()));
+        dcm.setStreamFilter(new RegExpMatch(specp.getFilter(), specp.getFilterOnName()));
 
       int count = 0;
       for (MFile mfile : dcm.getFilesSorted()) {

cdm/core/src/main/java/thredds/filesystem/ControllerOS.java

@@ -1,44 +1,19 @@
 /*
- * Copyright (c) 1998 - 2011. University Corporation for Atmospheric Research/Unidata
- * Portions of this software were developed by the Unidata Program at the
- * University Corporation for Atmospheric Research.
- *
- * Access and use of this software shall impose the following obligations
- * and understandings on the user. The user is granted the right, without
- * any fee or cost, to use, copy, modify, alter, enhance and distribute
- * this software, and any derivative works thereof, and its supporting
- * documentation for any purpose whatsoever, provided that this entire
- * notice appears in all copies of the software, derivative works and
- * supporting documentation. Further, UCAR requests that the user credit
- * UCAR/Unidata in any publications that result from the use of this
- * software or in any product that includes this software. The names UCAR
- * and/or Unidata, however, may not be used in any advertising or publicity
- * to endorse or promote any products or commercial entity unless specific
- * written permission is obtained from UCAR/Unidata. The user also
- * understands that UCAR/Unidata is not obligated to provide the user with
- * any support, consulting, training or assistance of any kind with regard
- * to the use, operation and performance of this software nor to provide
- * the user with any updates, revisions, new versions or "bug fixes."
- *
- * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL,
- * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
- * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
- * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
- * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE.
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
+ * See LICENSE for license information.
  */
 
 package thredds.filesystem;
 
 import thredds.inventory.CollectionConfig;
 import thredds.inventory.MController;
 import thredds.inventory.MFile;
+import thredds.inventory.MFileDirectoryStream;
 
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.ThreadSafe;
 import java.io.File;
+import java.nio.file.DirectoryStream;
 import java.util.*;
 
 /**
@@ -57,7 +32,7 @@ public class ControllerOS implements MController {
 
   @Nullable
   @Override
-  public Iterator<MFile> getInventoryAll(CollectionConfig mc, boolean recheck) {
+  public DirectoryStream<MFile> getInventoryAll(CollectionConfig mc, boolean recheck) {
     String path = mc.getDirectoryName();
     if (path.startsWith("file:")) {
       path = path.substring(5);
@@ -68,12 +43,12 @@ public Iterator<MFile> getInventoryAll(CollectionConfig mc, boolean recheck) {
       return null;
     if (!cd.isDirectory())
       return null;
-    return new FilteredIterator(mc, new MFileIteratorAll(cd), false);
+    return new MFileDirectoryStream(new FilteredIterator(mc, new MFileIteratorAll(cd), false));
   }
 
   @Nullable
   @Override
-  public Iterator<MFile> getInventoryTop(CollectionConfig mc, boolean recheck) {
+  public DirectoryStream<MFile> getInventoryTop(CollectionConfig mc, boolean recheck) {
     String path = mc.getDirectoryName();
     if (path.startsWith("file:")) {
       path = path.substring(5);
@@ -84,11 +59,12 @@ public Iterator<MFile> getInventoryTop(CollectionConfig mc, boolean recheck) {
       return null;
     if (!cd.isDirectory())
       return null;
-    return new FilteredIterator(mc, new MFileIterator(cd), false); // removes subdirs
+    return new MFileDirectoryStream(new FilteredIterator(mc, new MFileIterator(cd), false)); // removes subdirs
   }
 
   @Nullable
-  public Iterator<MFile> getSubdirs(CollectionConfig mc, boolean recheck) {
+  @Override
+  public DirectoryStream<MFile> getSubdirs(CollectionConfig mc, boolean recheck) {
     String path = mc.getDirectoryName();
     if (path.startsWith("file:")) {
       path = path.substring(5);
@@ -99,7 +75,7 @@ public Iterator<MFile> getSubdirs(CollectionConfig mc, boolean recheck) {
       return null;
     if (!cd.isDirectory())
       return null;
-    return new FilteredIterator(mc, new MFileIterator(cd), true); // return only subdirs
+    return new MFileDirectoryStream(new FilteredIterator(mc, new MFileIterator(cd), true)); // return only subdirs
   }
 
 
@@ -123,14 +99,22 @@ private static class FilteredIterator implements Iterator<MFile> {
     }
 
     public boolean hasNext() {
-      next = nextFilteredFile(); /// 7
+      if (next != null) {
+        return true;
+      }
+      next = nextFilteredFile();
       return (next != null);
     }
 
     public MFile next() {
-      if (next == null)
-        throw new NoSuchElementException();
-      return next;
+      if (next == null) {
+        if (!hasNext()) {
+          throw new NoSuchElementException();
+        }
+      }
+      MFile result = next;
+      next = null;
+      return result;
     }
 
     public void remove() {
@@ -140,16 +124,14 @@ public void remove() {
     private MFile nextFilteredFile() {
       if (orgIter == null)
         return null;
-      if (!orgIter.hasNext())
-        return null;
 
-      MFile pdata = orgIter.next();
-      while ((pdata.isDirectory() != wantDirs) || !mc.accept(pdata)) { // skip directories, and filter
-        if (!orgIter.hasNext())
-          return null; /// 6
-        pdata = orgIter.next();
+      while (orgIter.hasNext()) {
+        MFile pdata = orgIter.next();
+        if (pdata.isDirectory() == wantDirs && mc.accept(pdata)) {
+          return pdata;
+        }
       }
-      return pdata;
+      return null;
     }
   }
 

cdm/core/src/main/java/thredds/filesystem/ControllerOS7.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -8,11 +8,13 @@
 import thredds.inventory.CollectionConfig;
 import thredds.inventory.MController;
 import thredds.inventory.MFile;
+import thredds.inventory.MFileDirectoryStream;
 import javax.annotation.concurrent.ThreadSafe;
 import java.io.IOException;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.Iterator;
+import java.util.NoSuchElementException;
 
 /**
  * Use Java 7 NIO for scanning the file system
@@ -27,12 +29,12 @@ public class ControllerOS7 implements MController {
   ////////////////////////////////////////
 
   @Override
-  public Iterator<MFile> getInventoryAll(CollectionConfig mc, boolean recheck) {
+  public DirectoryStream<MFile> getInventoryAll(CollectionConfig mc, boolean recheck) {
     return null;
   }
 
   @Override
-  public Iterator<MFile> getInventoryTop(CollectionConfig mc, boolean recheck) throws IOException {
+  public DirectoryStream<MFile> getInventoryTop(CollectionConfig mc, boolean recheck) {
     String path = mc.getDirectoryName();
     if (path.startsWith("file:")) {
       path = path.substring(5);
@@ -41,10 +43,17 @@ public Iterator<MFile> getInventoryTop(CollectionConfig mc, boolean recheck) thr
     Path cd = Paths.get(path);
     if (!Files.exists(cd))
       return null;
-    return new MFileIterator(cd, new CollectionFilter(mc)); // removes subdirs
+    MFileDirectoryStream mfileDirStream = null;
+    try {
+      mfileDirStream = new MFileDirectoryStream(new MFileIterator(cd, new CollectionFilter(mc))); // removes subdirs
+    } catch (IOException ioe) {
+      logger.warn(ioe.getMessage(), ioe);
+    }
+    return mfileDirStream;
   }
 
-  public Iterator<MFile> getSubdirs(CollectionConfig mc, boolean recheck) {
+  @Override
+  public DirectoryStream<MFile> getSubdirs(CollectionConfig mc, boolean recheck) {
     return null;
   }
 
@@ -68,32 +77,38 @@ public boolean accept(Path entry) {
   }
 
   // returns everything in the current directory
-  private static class MFileIterator implements Iterator<MFile> {
-    Iterator<Path> dirStream;
+  private static class MFileIterator implements Iterator<MFile>, AutoCloseable {
+    private final DirectoryStream<Path> dirStream;
+    private final Iterator<Path> pathIterator;
 
     MFileIterator(Path dir, DirectoryStream.Filter<Path> filter) throws IOException {
       if (filter != null)
-        dirStream = Files.newDirectoryStream(dir, filter).iterator();
+        dirStream = Files.newDirectoryStream(dir, filter);
       else
-        dirStream = Files.newDirectoryStream(dir).iterator();
+        dirStream = Files.newDirectoryStream(dir);
+      pathIterator = dirStream.iterator();
     }
 
     public boolean hasNext() {
-      return dirStream.hasNext();
+      return pathIterator.hasNext();
     }
 
     public MFile next() {
       try {
-        return new MFileOS7(dirStream.next());
+        return new MFileOS7(pathIterator.next());
       } catch (IOException e) {
-        e.printStackTrace(); // LOOK we should pass this exception up
-        throw new RuntimeException(e);
+        throw new NoSuchElementException(e.getMessage());
       }
     }
 
     public void remove() {
       throw new UnsupportedOperationException();
     }
+
+    @Override
+    public void close() throws IOException {
+      dirStream.close();
+    }
   }
 
   //////////////////////////////////////////////////////////////////

cdm/core/src/main/java/thredds/inventory/CollectionAbstract.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2017 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE.txt for license information.
  */
 
@@ -14,8 +14,6 @@
 import ucar.unidata.util.StringUtil2;
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Path;
 import java.util.*;
 
 /**
@@ -109,7 +107,7 @@ public static String cleanName(String name) {
   protected DateExtractor dateExtractor;
   protected CalendarDate startCollection;
   protected long lastModified;
-  protected DirectoryStream.Filter<Path> sfilter;
+  protected MFileFilter sfilter;
 
   protected CollectionAbstract(String collectionName, org.slf4j.Logger logger) {
     this.collectionName = cleanName(collectionName);
@@ -126,7 +124,7 @@ public String getIndexFilename(String suffix) {
     return getRoot() + "/" + collectionName + suffix;
   }
 
-  public void setStreamFilter(DirectoryStream.Filter<Path> filter) {
+  public void setStreamFilter(MFileFilter filter) {
     this.sfilter = filter;
   }
 
@@ -243,14 +241,6 @@ CalendarDate extractRunDateWithError(MFile mfile) {
   }
 
 
-  /////////////////////////////////////////////////////////////////////////
-
-  public class MyStreamFilter implements DirectoryStream.Filter<Path> {
-    public boolean accept(Path entry) throws IOException {
-      return sfilter == null || sfilter.accept(entry);
-    }
-  }
-
   protected List<MFile> makeFileListSorted() throws IOException {
     List<MFile> list = new ArrayList<>(100);
     try (CloseableIterator<MFile> iter = getFileIterator()) {