Refactor GRIB collection handling to use MFile abstractions

cdm-test/src/test/java/ucar/nc2/grib/TestGribIndexLocationS3.java

@@ -1,3 +1,8 @@
+/*
+ * Copyright (c) 2023-2026 University Corporation for Atmospheric Research/Unidata
+ * See LICENSE for license information.
+ */
+
 package ucar.nc2.grib;
 
 import static com.google.common.truth.Truth.assertThat;
@@ -51,7 +56,7 @@ public static List<Object[]> getTestParameters() {
 
   public TestGribIndexLocationS3(String filename, String fragment) {
     this.filename = filename + fragment;
-    this.indexFilename = filename + GribIndex.GBX9_IDX;
+    this.indexFilename = GribIndex.makeIndexFileName(filename);
     this.isGrib1 = filename.endsWith(".grib1");
   }
 
@@ -86,7 +91,7 @@ public void shouldCreateIndexInDefaultCache() throws IOException {
     assertThat(index).isNotNull();
     assertThat(index.getNRecords()).isNotEqualTo(0);
 
-    assertThat(diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).exists()).isTrue();
+    assertThat(diskCache.getCacheFile(GribIndex.makeIndexFileName(s3File.getPath())).exists()).isTrue();
   }
 
   @Test
@@ -98,13 +103,13 @@ public void shouldReuseCachedIndex() throws IOException {
     final GribIndex index =
         GribIndex.readOrCreateIndexFromSingleFile(isGrib1, s3File, CollectionUpdateType.always, logger);
     assertThat(index).isNotNull();
-    assertThat(diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).exists()).isTrue();
-    final long cacheLastModified = diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX).lastModified();
+    assertThat(diskCache.getCacheFile(GribIndex.makeIndexFileName(s3File.getPath())).exists()).isTrue();
+    final long cacheLastModified = diskCache.getCacheFile(GribIndex.makeIndexFileName(s3File.getPath())).lastModified();
 
     final GribIndex rereadIndex =
         GribIndex.readOrCreateIndexFromSingleFile(isGrib1, s3File, CollectionUpdateType.never, logger);
     assertThat(rereadIndex).isNotNull();
-    final File rereadCachedIndex = diskCache.getCacheFile(s3File.getPath() + GribIndex.GBX9_IDX);
+    final File rereadCachedIndex = diskCache.getCacheFile(GribIndex.makeIndexFileName(s3File.getPath()));
     assertThat(rereadCachedIndex.lastModified()).isEqualTo(cacheLastModified);
   }
 }

cdm/core/src/main/java/thredds/filesystem/ControllerOS.java

@@ -78,6 +78,20 @@ public DirectoryStream<MFile> getSubdirs(CollectionConfig mc, boolean recheck) {
     return new MFileDirectoryStream(new FilteredIterator(mc, new MFileIterator(cd), true)); // return only subdirs
   }
 
+  @Nullable
+  @Override
+  public DirectoryStream<MFile> getFullInventoryAtLocation(String location) {
+    if (location.startsWith("file:")) {
+      location = location.substring(5);
+    }
+
+    File cd = new File(location);
+    if (!cd.exists())
+      return null;
+    if (!cd.isDirectory())
+      return null;
+    return new MFileDirectoryStream(new MFileIterator(cd));
+  }
 
   public void close() {} // NOOP
 

cdm/core/src/main/java/thredds/filesystem/ControllerOS7.java

@@ -5,6 +5,7 @@
 
 package thredds.filesystem;
 
+import javax.annotation.Nullable;
 import thredds.inventory.CollectionConfig;
 import thredds.inventory.MController;
 import thredds.inventory.MFile;
@@ -57,6 +58,27 @@ public DirectoryStream<MFile> getSubdirs(CollectionConfig mc, boolean recheck) {
     return null;
   }
 
+  @Nullable
+  @Override
+  public DirectoryStream<MFile> getFullInventoryAtLocation(String location) {
+    if (location.startsWith("file:")) {
+      location = location.substring(5);
+    }
+
+    Path cd = Paths.get(location);
+    if (!Files.exists(cd))
+      return null;
+    if (!Files.isDirectory(cd))
+      return null;
+
+    MFileDirectoryStream stream = null;
+    try {
+      stream = new MFileDirectoryStream(new MFileIterator(cd, null));
+    } catch (IOException ioe) {
+      logger.warn(ioe.getMessage(), ioe);
+    }
+    return stream;
+  }
 
   public void close() {} // NOOP
 

cdm/core/src/main/java/thredds/inventory/CollectionAbstract.java

@@ -121,7 +121,11 @@ public String getCollectionName() {
 
   @Override
   public String getIndexFilename(String suffix) {
-    return getRoot() + "/" + collectionName + suffix;
+    MFile rootMFile = MFiles.create(getRoot());
+    MFile mfile = rootMFile.getChild(collectionName + suffix);
+    if (mfile == null)
+      throw new IllegalStateException("Cannot determine location of collection index file");
+    return mfile.getPath();
   }
 
   public void setStreamFilter(MFileFilter filter) {

cdm/core/src/main/java/thredds/inventory/CollectionList.java

@@ -42,7 +42,6 @@ public CollectionList(String collectionName, String list, Logger logger) {
 
     Collections.sort(mfiles);
     this.lastModified = lastModified;
-    this.root = System.getProperty("user.dir");
   }
 
   public CollectionList(String collectionName, String root, List<MFile> mfiles, Logger logger) {

cdm/core/src/main/java/thredds/inventory/CollectionManagerCatalog.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
 
@@ -18,6 +18,7 @@
 import java.util.Date;
 import java.util.Formatter;
 import java.util.List;
+import ucar.nc2.util.DiskCache2;
 
 /**
  * CollectionManager of datasets from a catalog.
@@ -45,7 +46,7 @@ public CollectionManagerCatalog(String collectionName, String collectionSpec, St
     }
 
     this.catalogUrl = collectionSpec;
-    this.root = System.getProperty("user.dir");
+    this.root = DiskCache2.getDefault().getRootDirectory();
   }
 
   @Override

cdm/core/src/main/java/thredds/inventory/CollectionSingleFile.java

@@ -5,6 +5,8 @@
 
 package thredds.inventory;
 
+import ucar.nc2.util.DiskCache2;
+
 /**
  * A CollectionManager consisting of a single file
  *
@@ -18,9 +20,9 @@ public CollectionSingleFile(MFile file, org.slf4j.Logger logger) {
     mfiles.add(file);
     try {
       MFile p = file.getParent();
-      this.root = p != null ? p.getPath() : System.getProperty("user.dir");
+      this.root = p != null ? p.getPath() : DiskCache2.getDefault().getRootDirectory();
     } catch (java.io.IOException e) {
-      this.root = System.getProperty("user.dir");
+      this.root = DiskCache2.getDefault().getRootDirectory();
     }
 
     this.lastModified = file.getLastModified();

cdm/core/src/main/java/thredds/inventory/CollectionSpecParser.java

@@ -1,11 +1,13 @@
 /*
- * Copyright (c) 1998-2022 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package thredds.inventory;
 
 import java.io.File;
 import java.util.Formatter;
+import ucar.nc2.util.DiskCache2;
 
 /**
  * Parses the collection specification string for local files.
@@ -39,7 +41,7 @@
 public class CollectionSpecParser extends CollectionSpecParserAbstract {
   private final static String DELIMITER = "/";
   private final static String FRAGMENT = "";
-  private final static String DEFAULT_DIR = System.getProperty("user.dir");
+  private final static String DEFAULT_DIR = DiskCache2.getDefault().getRootDirectory();
 
   /**
    * Single spec : "/topdir/** /#dateFormatMark#regExp"

cdm/core/src/main/java/thredds/inventory/MController.java

@@ -48,15 +48,13 @@ public interface MController extends Closeable {
   DirectoryStream<MFile> getSubdirs(CollectionConfig mc, boolean recheck);
 
   /**
-   * Get an MFile for a specific location.
-   * 
-   * @param location the location
-   * @return MFile or null
+   * Returns all subdirectories and leaves in a given location, not recursing into subdirectories.
+   *
+   * @param location defines the location to scan
+   * @return DirectoryStream over Mfiles, or null if the location does not exist
    */
   @Nullable
-  default MFile getMFile(String location) {
-    return MFiles.create(location);
-  }
+  DirectoryStream<MFile> getFullInventoryAtLocation(String location);
 
   @Override
   void close();

cdm/core/src/main/java/thredds/inventory/MControllers.java

@@ -43,8 +43,7 @@ public static MController create(String location) {
    */
   public static DirectoryStream<MFile> newDirectoryStream(String location) throws IOException {
     MController controller = create(location);
-    CollectionConfig config = new CollectionConfig(location, location, false, null, null);
-    DirectoryStream<MFile> stream = controller.getInventoryTop(config, true);
+    DirectoryStream<MFile> stream = controller.getFullInventoryAtLocation(location);
     controller.close();
     if (stream == null) {
       throw new IOException("Could not create DirectoryStream for " + location);

cdm/core/src/main/java/thredds/inventory/partition/DirectoryBuilder.java

@@ -71,7 +71,7 @@ private enum PartitionStatus {
   private PartitionStatus partitionStatus = PartitionStatus.unknown;
 
   public DirectoryBuilder(String topCollectionName, String dirFilename, String suffix) throws IOException {
-    this(topCollectionName, MControllers.create(dirFilename).getMFile(dirFilename), suffix);
+    this(topCollectionName, MFiles.create(dirFilename), suffix);
   }
 
   /**

cdm/core/src/test/java/thredds/inventory/TestCollectionSpecParser.java

@@ -1,3 +1,8 @@
+/*
+ * Copyright (c) 2019-2026 University Corporation for Atmospheric Research/Unidata
+ * See LICENSE for license information.
+ */
+
 package thredds.inventory;
 
 import static com.google.common.truth.Truth.assertThat;
@@ -12,6 +17,7 @@
 import org.junit.runners.Parameterized;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import ucar.nc2.util.DiskCache2;
 
 @RunWith(Parameterized.class)
 public class TestCollectionSpecParser {
@@ -67,11 +73,11 @@ public static List<Object[]> getTestParameters() throws URISyntaxException {
         {"/data/ldm/pub/decoded/netcdf/surface/metar/T*.T", "/data/ldm/pub/decoded/netcdf/surface/metar", false, "T*.T",
             null},
 
-        {"", System.getProperty("user.dir"), false, null, null},
+        {"", DiskCache2.getDefault().getRootDirectory(), false, null, null},
 
-        {".*grib1", System.getProperty("user.dir"), false, ".*grib1", null},
+        {".*grib1", DiskCache2.getDefault().getRootDirectory(), false, ".*grib1", null},
 
-        {".*\\.grib1", System.getProperty("user.dir"), false, ".*\\.grib1", null},
+        {".*\\.grib1", DiskCache2.getDefault().getRootDirectory(), false, ".*\\.grib1", null},
 
         {"dir/**/subdir/.*grib1", "dir", true, "subdir/.*grib1", null},});
   }

cdm/core/src/test/java/thredds/inventory/TestMControllers.java

@@ -0,0 +1,52 @@
+package thredds.inventory;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import org.junit.Test;
+import ucar.unidata.util.test.TestDir;
+
+public class TestMControllers {
+
+  @Test
+  public void testDirectoryStream() throws IOException {
+    String testDir = TestDir.localTestDataDir + "directory_stream";
+
+    ArrayList<String> dsMfiles = new ArrayList<>();
+    DirectoryStream<MFile> ds = MControllers.newDirectoryStream(testDir);
+    for (MFile item : ds) {
+      dsMfiles.add(Paths.get(item.getPath()).toAbsolutePath().toString());
+    }
+    ds.close();
+
+    ArrayList<String> dsFiles = new ArrayList<>();
+    DirectoryStream<Path> ds2 =
+        Files.newDirectoryStream(Paths.get(TestDir.localTestDataDir + "directory_stream").toAbsolutePath());
+    for (Path item : ds2) {
+      dsFiles.add(item.toString());
+    }
+    ds2.close();
+
+    dsMfiles.sort(null);
+    dsFiles.sort(null);
+    assertThat(dsMfiles).containsExactlyElementsIn(dsFiles);
+  }
+
+  @Test
+  public void testSubdirStream() throws IOException {
+    String testDir = TestDir.localTestDataDir + "directory_stream";
+    ArrayList<String> dsMfiles = new ArrayList<>();
+    DirectoryStream<MFile> ds = MControllers.newSubdirStream(testDir);
+    for (MFile item : ds) {
+      dsMfiles.add(Paths.get(item.getPath()).toAbsolutePath().toString());
+    }
+    ds.close();
+
+    assertThat(dsMfiles).hasSize(1);
+  }
+}

cdm/s3/src/main/java/thredds/filesystem/s3/ControllerS3.java

@@ -51,11 +51,17 @@ public ControllerS3() {}
 
   private void init(CollectionConfig mc) {
     if (mc != null) {
+      init(mc.getDirectoryName());
+    }
+  }
+
+  private void init(String location) {
+    if (location != null) {
       try {
-        initUri(mc.getDirectoryName());
+        initUri(location);
         initClient();
       } catch (IOException e) {
-        logger.error("Error initializing ControllerS3 for {}.", mc.getDirectoryName(), e);
+        logger.error("Error initializing ControllerS3 for {}.", location, e);
       }
     }
   }
@@ -128,6 +134,16 @@ public DirectoryStream<MFile> getSubdirs(CollectionConfig mc, boolean recheck) {
     return new MFileDirectoryStream(new FilteredIterator(mc, mFiles.iterator(), true));
   }
 
+  @Override
+  public DirectoryStream<MFile> getFullInventoryAtLocation(String location) {
+    init(location);
+    String prefix = null;
+    if (initialUri.getKey().isPresent()) {
+      prefix = initialUri.getKey().get();
+    }
+    return new MFileDirectoryStream(new MFileS3Iterator(client, initialUri, prefix, limit, false));
+  }
+
   @Override
   public void close() {} // NO-OP
 

cdm/s3/src/main/java/thredds/inventory/s3/MFileS3.java

@@ -1,7 +1,8 @@
 /*
- * Copyright (c) 2020 University Corporation for Atmospheric Research/Unidata
+ * Copyright (c) 2020-2026 University Corporation for Atmospheric Research/Unidata
  * See LICENSE for license information.
  */
+
 package thredds.inventory.s3;
 
 import java.io.IOException;
@@ -218,9 +219,20 @@ public String getName() {
   @Nullable
   public MFile getParent() throws IOException {
     // In general, objects to do not have parents. However, if a delimiter is set, we have a pseudo path, and then
-    // the object can have a parent.
+    // the object can have a parent. If a delimiter is not set, but the key is not null, then the parent is
+    // the top of the bucket. If a delimiter is not set, and the key is null, then we are at the top of the bucket
+    // already, and then there is no parent.
     MFile parentMfile = null;
-    if (delimiter != null) {
+    if (delimiter == null) {
+      String key = getKey();
+      if (key != null && !key.isEmpty()) {
+        String uriString = getPath();
+        int queryIdx = uriString.indexOf("?");
+        if (queryIdx > 0) {
+          parentMfile = new MFileS3(uriString.substring(0, queryIdx));
+        }
+      }
+    } else {
       // get the full path
       String currentUri = getPath();
       String frag = "";