From 40156e6a045a03f81ea2ebbf0aea88447fc78807 Mon Sep 17 00:00:00 2001 From: Sean Arms <67096+lesserwhirls@users.noreply.github.com> Date: Fri, 3 Apr 2026 09:50:12 -0600 Subject: [PATCH] Refactor inventory management to use managed-file abstractions, part 2 * Transition GRIB collection abstractions from java.io/java.nio path APIs with the exception of writing GRIB Index files, which remain local-only for now. --- .../nc2/grib/TestGribCdmIndexUpdating.java | 6 +- .../grib/TestGribCollectionCoordinates.java | 7 +- .../nc2/grib/TestGribCollectionTimeUnits.java | 5 +- .../main/java/ucar/nc2/grib/GribIndex.java | 13 ++-- .../java/ucar/nc2/grib/GribIndexCache.java | 14 ++-- .../ucar/nc2/grib/collection/GcMFile.java | 33 ++++----- .../collection/Grib1CollectionBuilder.java | 5 +- .../collection/Grib1CollectionWriter.java | 20 ++++-- .../collection/Grib1PartitionBuilder.java | 6 +- .../collection/Grib2CollectionBuilder.java | 5 +- .../collection/Grib2CollectionWriter.java | 24 ++++--- .../collection/Grib2PartitionBuilder.java | 6 +- .../nc2/grib/collection/GribCdmIndex.java | 67 +++++++++++-------- .../collection/GribCollectionBuilder.java | 12 ++-- .../collection/GribCollectionImmutable.java | 43 ++++++------ .../collection/GribCollectionMutable.java | 44 ++++++------ .../grib/collection/GribPartitionBuilder.java | 33 ++++----- .../PartitionCollectionImmutable.java | 29 ++------ .../PartitionCollectionMutable.java | 16 +++-- .../java/ucar/nc2/grib/grib1/Grib1Index.java | 24 +++++-- .../java/ucar/nc2/grib/grib2/Grib2Index.java | 25 +++++-- .../ucar/nc2/grib/collection/TestGcMFile.java | 20 ++++-- .../grib/coord/TestDiscontiguousInterval.java | 7 +- .../src/main/java/ucar/nc2/ui/MFileTable.java | 9 +++ .../java/ucar/nc2/ui/grib/CdmIndexPanel.java | 3 +- .../nc2/ui/op/DirectoryPartitionViewer.java | 4 +- 26 files changed, 268 insertions(+), 212 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/grib/TestGribCdmIndexUpdating.java b/cdm-test/src/test/java/ucar/nc2/grib/TestGribCdmIndexUpdating.java index 243263e3f4..14398ef6cd 100644 --- a/cdm-test/src/test/java/ucar/nc2/grib/TestGribCdmIndexUpdating.java +++ b/cdm-test/src/test/java/ucar/nc2/grib/TestGribCdmIndexUpdating.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2021 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -77,7 +77,7 @@ public static List getTestParameters() { public static void after(String dataDir, String newModel, FeatureCollectionConfig config, String varName, int orgLen, int remLen) { // cleanup index files created during test for a given parameter input - cleanUpIndexFiles(GribCdmIndex.getTopIndexFileFromConfig(config).getParent()); + cleanUpIndexFiles(GribCdmIndex.getTopIndexFileFromConfig(config).getPath()); } /////////////////////////////////////// @@ -98,7 +98,7 @@ public TestGribCdmIndexUpdating(String dataDir, String newModel, FeatureCollecti this.orgLen = orgLen; this.remLen = remLen; GribCdmIndex.updateGribCollection(config, CollectionUpdateType.always, logger); - this.indexFile = GribCdmIndex.getTopIndexFileFromConfig(config).getAbsolutePath(); + this.indexFile = GribCdmIndex.getTopIndexFileFromConfig(config).getPath(); } @Test diff --git a/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionCoordinates.java b/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionCoordinates.java index 5efe7a39c4..9313527982 100644 --- a/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionCoordinates.java +++ b/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionCoordinates.java @@ -1,7 +1,8 @@ /* - * Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package ucar.nc2.grib; import static com.google.common.truth.Truth.assertThat; @@ -68,7 +69,7 @@ public void testExtraCoordinates() throws IOException { config.gribConfig.setExcludeZero(true); // no longer the default boolean changed = GribCdmIndex.updateGribCollection(config, CollectionUpdateType.always, logger); - String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getAbsolutePath(); + String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getPath(); logger.debug("changed = {}", changed); @@ -126,7 +127,7 @@ public void testBestReftimeMonotonic() throws IOException { boolean changed = GribCdmIndex.updateGribCollection(config, updateMode, logger); logger.debug("changed = {}", changed); - String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getAbsolutePath(); + String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getPath(); boolean ok = true; try (NetcdfDataset ds = NetcdfDatasets.openDataset(topLevelIndex)) { diff --git a/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java b/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java index 97dd97f8ac..2c1bdf479e 100644 --- a/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java +++ b/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java @@ -1,7 +1,8 @@ /* - * Copyright (c) 1998-2022 University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ + package ucar.nc2.grib; import static com.google.common.truth.Truth.assertThat; @@ -80,7 +81,7 @@ private void checkVariableNameAndTimeAxis(FeatureCollectionConfig config, String String units, double[] values) throws IOException { final boolean changed = GribCdmIndex.updateGribCollection(config, CollectionUpdateType.always, logger); assertThat(changed).isTrue(); - final String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getAbsolutePath(); + final String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getPath(); try (NetcdfDataset netcdfDataset = NetcdfDatasets.openDataset(topLevelIndex)) { final Variable variable = diff --git a/grib/src/main/java/ucar/nc2/grib/GribIndex.java b/grib/src/main/java/ucar/nc2/grib/GribIndex.java index daa86c8bd4..4724e7b309 100644 --- a/grib/src/main/java/ucar/nc2/grib/GribIndex.java +++ b/grib/src/main/java/ucar/nc2/grib/GribIndex.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -12,7 +12,6 @@ import ucar.nc2.grib.grib1.Grib1Index; import ucar.nc2.grib.grib2.Grib2Index; import ucar.unidata.io.RandomAccessFile; -import java.io.File; import java.io.IOException; /** @@ -33,11 +32,11 @@ public boolean hasChangedSince(MFile file, long when) { String idxPath = file.getPath(); if (!idxPath.endsWith(GBX9_IDX)) idxPath += GBX9_IDX; - File idxFile = GribIndexCache.getExistingFileOrCache(idxPath); + MFile idxFile = GribIndexCache.getExistingFileOrCache(idxPath); if (idxFile == null) return true; - long idxLastModified = idxFile.lastModified(); + long idxLastModified = idxFile.getLastModified(); if (idxLastModified < file.getLastModified()) return true; return 0 < when && when < idxLastModified; @@ -47,13 +46,13 @@ public boolean hasntChangedSince(MFile file, long when) { String idxPath = file.getPath(); if (!idxPath.endsWith(GBX9_IDX)) idxPath += GBX9_IDX; - File idxFile = GribIndexCache.getExistingFileOrCache(idxPath); + MFile idxFile = GribIndexCache.getExistingFileOrCache(idxPath); if (idxFile == null) return true; - if (idxFile.lastModified() < file.getLastModified()) + if (idxFile.getLastModified() < file.getLastModified()) return true; - return 0 < when && idxFile.lastModified() < when; + return 0 < when && idxFile.getLastModified() < when; } }; ///////////////////////////////////////////////////////////////////////// diff --git a/grib/src/main/java/ucar/nc2/grib/GribIndexCache.java b/grib/src/main/java/ucar/nc2/grib/GribIndexCache.java index 2abfa72cbe..58b61aa168 100644 --- a/grib/src/main/java/ucar/nc2/grib/GribIndexCache.java +++ b/grib/src/main/java/ucar/nc2/grib/GribIndexCache.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -7,6 +7,8 @@ import ucar.nc2.grib.collection.Grib; import ucar.nc2.util.DiskCache2; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; import java.io.File; /** @@ -35,11 +37,11 @@ public static synchronized DiskCache2 getDiskCache2() { * @param fileLocation full path of original index filename * @return File, possibly in cache, may or may not exist */ - public static File getFileOrCache(String fileLocation) { - File result = getExistingFileOrCache(fileLocation); + public static MFile getFileOrCache(String fileLocation) { + MFile result = getExistingFileOrCache(fileLocation); if (result != null) return result; - return getDiskCache2().getFile(fileLocation); + return MFiles.create(getDiskCache2().getFile(fileLocation).getPath()); } /** @@ -48,7 +50,7 @@ public static File getFileOrCache(String fileLocation) { * @param fileLocation full path of original index filename * @return existing file if you can find it, else null */ - public static File getExistingFileOrCache(String fileLocation) { + public static MFile getExistingFileOrCache(String fileLocation) { File result = getDiskCache2().getExistingFileOrCache(fileLocation); if (result == null && Grib.debugGbxIndexOnly && fileLocation.endsWith(".gbx9.ncx4")) { // might create only from // gbx9 for debugging @@ -56,6 +58,6 @@ public static File getExistingFileOrCache(String fileLocation) { String maybeIndexAlreadyExists = fileLocation.substring(0, length - 10) + ".ncx4"; result = getDiskCache2().getExistingFileOrCache(maybeIndexAlreadyExists); } - return result; + return result == null ? null : MFiles.create(result.getPath()); } } diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GcMFile.java b/grib/src/main/java/ucar/nc2/grib/collection/GcMFile.java index 1976210268..733f5d8846 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GcMFile.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GcMFile.java @@ -1,21 +1,18 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package ucar.nc2.grib.collection; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.OutputStream; import javax.annotation.Nullable; -import thredds.filesystem.MFileOS; import thredds.inventory.MFile; import ucar.nc2.util.IO; import ucar.unidata.io.RandomAccessFile; import ucar.unidata.util.StringUtil2; -import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -28,7 +25,7 @@ */ public class GcMFile implements thredds.inventory.MFile { - static List makeFiles(File directory, List files, Set allFileSet) { + static List makeFiles(MFile directory, List files, Set allFileSet) { List result = new ArrayList<>(files.size()); String dirPath = StringUtil2.replace(directory.getPath(), '\\', "/"); @@ -47,12 +44,12 @@ static List makeFiles(File directory, List files, Set a } //////////////////////////////////////////////////////////////////////////////////////// - public final File directory; + public final MFile directory; public final String name; public final long lastModified, length; public final int index; - GcMFile(File directory, String name, long lastModified, long length, int index) { + GcMFile(MFile directory, String name, long lastModified, long length, int index) { this.directory = directory; this.name = name; this.lastModified = lastModified; @@ -77,8 +74,8 @@ public boolean isDirectory() { @Override public String getPath() { - String path = new File(directory, name).getPath(); - return StringUtil2.replace(path, '\\', "/"); + MFile file = directory.getChild(name); + return file == null ? null : file.getPath(); } @Override @@ -87,8 +84,8 @@ public String getName() { } @Override - public MFile getParent() { - return new MFileOS(directory); + public MFile getParent() throws IOException { + return directory; } @Override @@ -105,24 +102,28 @@ public Object getAuxInfo() { @Override public void setAuxInfo(Object info) {} - public File getDirectory() { + public MFile getDirectory() { return directory; } @Override public String toString() { - return "GcMFile{" + "directory=" + directory + ", name='" + name + '\'' + ", lastModified=" + lastModified + return "GcMFile{" + "directory=" + directory.getPath() + ", name='" + name + '\'' + ", lastModified=" + lastModified + ", length=" + length + ", index=" + index + '}'; } @Override public boolean exists() { - return new File(directory, name).exists(); + MFile file = directory.getChild(name); + return file != null && file.exists(); } @Override - public FileInputStream getInputStream() throws FileNotFoundException { - return new FileInputStream(new File(directory, name)); + public java.io.InputStream getInputStream() throws FileNotFoundException { + MFile file = directory.getChild(name); + if (file == null) + throw new FileNotFoundException(name); + return file.getInputStream(); } @Override diff --git a/grib/src/main/java/ucar/nc2/grib/collection/Grib1CollectionBuilder.java b/grib/src/main/java/ucar/nc2/grib/collection/Grib1CollectionBuilder.java index b03cd4fac2..b7a2df4c54 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/Grib1CollectionBuilder.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/Grib1CollectionBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -28,7 +28,6 @@ import ucar.nc2.time.CalendarDateRange; import ucar.nc2.time.CalendarPeriod; import ucar.nc2.util.CloseableIterator; -import java.io.File; import java.io.IOException; import java.util.*; @@ -191,7 +190,7 @@ protected boolean writeIndex(String name, String indexFilepath, CoordinateRuntim List groups2 = new ArrayList<>(); for (Object g : groups) groups2.add((Grib1CollectionWriter.Group) g); // why copy ? - File indexFileInCache = GribIndexCache.getFileOrCache(indexFilepath); + MFile indexFileInCache = GribIndexCache.getFileOrCache(indexFilepath); return writer.writeIndex(name, indexFileInCache, masterRuntime, groups2, files, type, dateRange); } diff --git a/grib/src/main/java/ucar/nc2/grib/collection/Grib1CollectionWriter.java b/grib/src/main/java/ucar/nc2/grib/collection/Grib1CollectionWriter.java index b4fa3f1d7f..9b8639700b 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/Grib1CollectionWriter.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/Grib1CollectionWriter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -9,6 +9,7 @@ import java.nio.charset.StandardCharsets; import thredds.inventory.MCollection; import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.nc2.grib.coord.Coordinate; import ucar.nc2.grib.coord.CoordinateEns; import ucar.nc2.grib.coord.CoordinateRuntime; @@ -90,15 +91,18 @@ public Set getCoordinateRuntimes() { */ // indexFile is in the cache - boolean writeIndex(String name, File idxFile, CoordinateRuntime masterRuntime, List groups, List files, + boolean writeIndex(String name, MFile idxFile, CoordinateRuntime masterRuntime, List groups, List files, GribCollectionImmutable.Type type, CalendarDateRange dateRange) throws IOException { Grib1Record first = null; // take global metadata from here boolean deleteOnClose = false; if (idxFile.exists()) { RandomAccessFile.eject(idxFile.getPath()); - if (!idxFile.delete()) { - logger.warn(" gc1 cant delete index file {}", idxFile.getPath()); + if (idxFile instanceof thredds.filesystem.MFileOS) { + File f = new File(idxFile.getPath()); + if (!f.delete()) { + logger.warn(" gc1 cant delete index file {}", idxFile.getPath()); + } } } logger.debug(" createIndex for {}", idxFile.getPath()); @@ -186,7 +190,7 @@ boolean writeIndex(String name, File idxFile, CoordinateRuntime masterRuntime, L indexBuilder.setVersion(currentVersion); // directory and mfile list - File directory = new File(dcm.getRoot()); + MFile directory = MFiles.create(dcm.getRoot()); List gcmfiles = GcMFile.makeFiles(directory, files, allFileSet); for (GcMFile gcmfile : gcmfiles) { GribCollectionProto.MFile.Builder b = GribCollectionProto.MFile.newBuilder(); @@ -230,8 +234,10 @@ boolean writeIndex(String name, File idxFile, CoordinateRuntime masterRuntime, L } finally { // remove it on failure - if (deleteOnClose && !idxFile.delete()) { - logger.error(" gc1 cant deleteOnClose index file {}", idxFile.getPath()); + if (deleteOnClose && idxFile instanceof thredds.filesystem.MFileOS) { + File f = new File(idxFile.getPath()); + if (!f.delete()) + logger.error(" gc1 cant deleteOnClose index file {}", idxFile.getPath()); } } } diff --git a/grib/src/main/java/ucar/nc2/grib/collection/Grib1PartitionBuilder.java b/grib/src/main/java/ucar/nc2/grib/collection/Grib1PartitionBuilder.java index 1a2fa1a5ce..bb6f78da92 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/Grib1PartitionBuilder.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/Grib1PartitionBuilder.java @@ -1,13 +1,13 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package ucar.nc2.grib.collection; import thredds.featurecollection.FeatureCollectionConfig; +import thredds.inventory.MFile; import thredds.inventory.partition.PartitionManager; -import java.io.File; /** * Builds Grib1 PartitionCollections (version 2) @@ -18,7 +18,7 @@ class Grib1PartitionBuilder extends GribPartitionBuilder { public static final String MAGIC_START = "Grib1Partition2Index"; // was Grib1Partition0Index - Grib1PartitionBuilder(String name, File directory, PartitionManager tpc, org.slf4j.Logger logger) { + Grib1PartitionBuilder(String name, MFile directory, PartitionManager tpc, org.slf4j.Logger logger) { super(name, tpc, logger); FeatureCollectionConfig config = null; diff --git a/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionBuilder.java b/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionBuilder.java index 79dfb95389..a5007a72ff 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionBuilder.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionBuilder.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -28,7 +28,6 @@ import ucar.nc2.time.CalendarDateRange; import ucar.nc2.time.CalendarPeriod; import ucar.nc2.util.CloseableIterator; -import java.io.File; import java.io.IOException; import java.util.*; @@ -216,7 +215,7 @@ protected boolean writeIndex(String name, String indexFilepath, CoordinateRuntim for (Object g : groups) groups2.add((Grib2CollectionWriter.Group) g); // copy to change GribCollectionBuilder.Group -> // Grib2CollectionWriter.Group - File indexFileInCache = GribIndexCache.getFileOrCache(indexFilepath); + MFile indexFileInCache = GribIndexCache.getFileOrCache(indexFilepath); return writer.writeIndex(name, indexFileInCache, masterRuntime, groups2, files, type, dateRange); } diff --git a/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionWriter.java b/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionWriter.java index a294f9cb83..eb44d35790 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionWriter.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionWriter.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -7,7 +7,9 @@ import com.google.protobuf.ByteString; import java.nio.charset.StandardCharsets; -import thredds.inventory.*; +import thredds.inventory.MCollection; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.nc2.grib.coord.Coordinate; import ucar.nc2.grib.coord.CoordinateEns; import ucar.nc2.grib.coord.CoordinateRuntime; @@ -90,15 +92,18 @@ public List getCoordinates() { * GribCollectionIndex (sizeIndex bytes) */ - boolean writeIndex(String name, File idxFile, CoordinateRuntime masterRuntime, List groups, List files, + boolean writeIndex(String name, MFile idxFile, CoordinateRuntime masterRuntime, List groups, List files, GribCollectionImmutable.Type type, CalendarDateRange dateRange) throws IOException { Grib2Record first = null; // take global metadata from here boolean deleteOnClose = false; if (idxFile.exists()) { RandomAccessFile.eject(idxFile.getPath()); - if (!idxFile.delete()) { - logger.error("gc2 cant delete index file {}", idxFile.getPath()); + if (idxFile instanceof thredds.filesystem.MFileOS) { + File f = new File(idxFile.getPath()); + if (!f.delete()) { + logger.error("gc2 cant delete index file {}", idxFile.getPath()); + } } } logger.debug(" createIndex for {}", idxFile.getPath()); @@ -180,7 +185,7 @@ boolean writeIndex(String name, File idxFile, CoordinateRuntime masterRuntime, L indexBuilder.setVersion(currentVersion); // directory and mfile list - File directory = new File(dcm.getRoot()); + MFile directory = MFiles.create(dcm.getRoot()); List gcmfiles = GcMFile.makeFiles(directory, files, allFileSet); for (GcMFile gcmfile : gcmfiles) { GribCollectionProto.MFile.Builder b = GribCollectionProto.MFile.newBuilder(); @@ -225,8 +230,11 @@ boolean writeIndex(String name, File idxFile, CoordinateRuntime masterRuntime, L } finally { // remove it on failure - if (deleteOnClose && !idxFile.delete()) - logger.error(" gc2 cant deleteOnClose index file {}", idxFile.getPath()); + if (deleteOnClose && idxFile instanceof thredds.filesystem.MFileOS) { + File f = new File(idxFile.getPath()); + if (!f.delete()) + logger.error(" gc2 cant deleteOnClose index file {}", idxFile.getPath()); + } } return true; diff --git a/grib/src/main/java/ucar/nc2/grib/collection/Grib2PartitionBuilder.java b/grib/src/main/java/ucar/nc2/grib/collection/Grib2PartitionBuilder.java index b5b0aac6b0..39d7923ee3 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/Grib2PartitionBuilder.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/Grib2PartitionBuilder.java @@ -1,13 +1,13 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package ucar.nc2.grib.collection; import thredds.featurecollection.FeatureCollectionConfig; +import thredds.inventory.MFile; import thredds.inventory.partition.PartitionManager; -import java.io.File; /** * Builds Grib2 PartitionCollections (version 2) @@ -18,7 +18,7 @@ class Grib2PartitionBuilder extends GribPartitionBuilder { public static final String MAGIC_START = "Grib2Partition2Index"; // was Grib2Partition0Index - Grib2PartitionBuilder(String name, File directory, PartitionManager tpc, org.slf4j.Logger logger) { + Grib2PartitionBuilder(String name, MFile directory, PartitionManager tpc, org.slf4j.Logger logger) { super(name, tpc, logger); FeatureCollectionConfig config = null; diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java b/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java index e0a460c674..9413c790c9 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribCdmIndex.java @@ -31,8 +31,6 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.Arrays; import java.util.Comparator; import java.util.Formatter; @@ -99,8 +97,8 @@ public static void shutdown() { ///////////////////////////////////////////////////////////////////////////////////////////////// - public static File getTopIndexFileFromConfig(FeatureCollectionConfig config) { - File indexFile = makeTopIndexFileFromConfig(config); + public static MFile getTopIndexFileFromConfig(FeatureCollectionConfig config) { + MFile indexFile = makeTopIndexFileFromConfig(config); return GribIndexCache.getExistingFileOrCache(indexFile.getPath()); } @@ -110,19 +108,19 @@ public static File getTopIndexFileFromConfig(FeatureCollectionConfig config) { * @param config use this FeatureCollectionConfig * @return index File */ - private static File makeTopIndexFileFromConfig(FeatureCollectionConfig config) { + private static MFile makeTopIndexFileFromConfig(FeatureCollectionConfig config) { Formatter errlog = new Formatter(); CollectionSpecParserAbstract specp = config.getCollectionSpecParserAbstract(errlog); String name = StringUtil2.replace(config.collectionName, '\\', "/"); // String cname = DirectoryCollection.makeCollectionName(name, Paths.get(specp.getRootDir())); - return makeIndexFile(name, new File(specp.getRootDir())); + return makeIndexFile(name, MFiles.create(specp.getRootDir())); } - static File makeIndexFile(String collectionName, File directory) { + static MFile makeIndexFile(String collectionName, MFile directory) { String nameNoBlanks = StringUtil2.replace(collectionName, ' ', "_"); - return new File(directory, nameNoBlanks + NCX_SUFFIX); + return directory.getChild(nameNoBlanks + NCX_SUFFIX); } private static String makeNameFromIndexFilename(String idxPathname) { @@ -177,7 +175,8 @@ public static GribCollectionType getType(RandomAccessFile raf) throws IOExceptio @Nullable public static GribCollectionImmutable openCdmIndex(String indexFilename, FeatureCollectionConfig config, boolean useCache, Logger logger) throws IOException { - File indexFileInCache = useCache ? GribIndexCache.getExistingFileOrCache(indexFilename) : new File(indexFilename); + MFile indexFileInCache = + useCache ? GribIndexCache.getExistingFileOrCache(indexFilename) : MFiles.create(indexFilename); if (indexFileInCache == null) return null; String indexFilenameInCache = indexFileInCache.getPath(); @@ -209,8 +208,12 @@ public static GribCollectionImmutable openCdmIndex(String indexFilename, Feature } catch (Throwable t) { logger.warn("GribCdmIndex.openCdmIndex failed on " + indexFilenameInCache, t); RandomAccessFile.eject(indexFilenameInCache); - if (!indexFileInCache.delete()) - logger.warn("failed to delete {}", indexFileInCache.getPath()); + // We don't try to delete non-local files here if it failed to open + if (indexFileInCache instanceof MFileOS) { + File f = new File(indexFileInCache.getPath()); + if (!f.delete()) + logger.warn("failed to delete {}", indexFileInCache.getPath()); + } } return result; @@ -221,7 +224,8 @@ public static GribCollectionImmutable openCdmIndex(String indexFilename, Feature @Nullable public static GribCollectionMutable openMutableGCFromIndex(String indexFilename, FeatureCollectionConfig config, boolean dataOnly, boolean useCache, Logger logger) { - File indexFileInCache = useCache ? GribIndexCache.getExistingFileOrCache(indexFilename) : new File(indexFilename); + MFile indexFileInCache = + useCache ? GribIndexCache.getExistingFileOrCache(indexFilename) : MFiles.create(indexFilename); if (indexFileInCache == null) { return null; } @@ -260,8 +264,11 @@ public static GribCollectionMutable openMutableGCFromIndex(String indexFilename, if (result == null) { RandomAccessFile.eject(indexFilenameInCache); - if (!indexFileInCache.delete()) - logger.warn("failed to delete {}", indexFileInCache.getPath()); + if (indexFileInCache instanceof MFileOS) { + File f = new File(indexFileInCache.getPath()); + if (!f.delete()) + logger.warn("failed to delete {}", indexFileInCache.getPath()); + } } return result; @@ -306,7 +313,7 @@ public static boolean updateGribCollection(FeatureCollectionConfig config, Colle Formatter errlog = new Formatter(); CollectionSpecParserAbstract specp = config.getCollectionSpecParserAbstract(errlog); - Path rootPath = Paths.get(specp.getRootDir()); + String rootPath = specp.getRootDir(); boolean isGrib1 = config.type == FeatureCollectionType.GRIB1; boolean changed; @@ -377,12 +384,12 @@ private static boolean updatePartition(boolean isGrib1, PartitionManager dcm, Co boolean changed; if (isGrib1) { Grib1PartitionBuilder builder = - new Grib1PartitionBuilder(dcm.getCollectionName(), new File(dcm.getRoot()), dcm, logger); + new Grib1PartitionBuilder(dcm.getCollectionName(), MFiles.create(dcm.getRoot()), dcm, logger); changed = builder.updateNeeded(updateType) && builder.createPartitionedIndex(updateType, errlog); } else { Grib2PartitionBuilder builder = - new Grib2PartitionBuilder(dcm.getCollectionName(), new File(dcm.getRoot()), dcm, logger); + new Grib2PartitionBuilder(dcm.getCollectionName(), MFiles.create(dcm.getRoot()), dcm, logger); changed = builder.updateNeeded(updateType) && builder.createPartitionedIndex(updateType, errlog); } return changed || updateNeeded; @@ -442,7 +449,7 @@ private static boolean isUpdateNeeded(String idxFilenameOrg, CollectionUpdateTyp if (updateType == CollectionUpdateType.always) return true; - File collectionIndexFile = GribIndexCache.getExistingFileOrCache(idxFilenameOrg); + MFile collectionIndexFile = GribIndexCache.getExistingFileOrCache(idxFilenameOrg); if (collectionIndexFile != null) { // it exists boolean bad; @@ -459,8 +466,11 @@ private static boolean isUpdateNeeded(String idxFilenameOrg, CollectionUpdateTyp if (bad) { // delete the file and remove from cache if its in there RandomAccessFile.eject(collectionIndexFile.getPath()); - if (!collectionIndexFile.delete()) - logger.warn("failed to delete {}", collectionIndexFile.getPath()); + if (collectionIndexFile instanceof MFileOS) { + File f = new File(collectionIndexFile.getPath()); + if (!f.delete()) + logger.warn("failed to delete {}", collectionIndexFile.getPath()); + } } } @@ -488,7 +498,7 @@ private static boolean updateDirectoryCollectionRecurse(boolean isGrib1, Directo // that - suckage changed = updateDirectoryCollectionRecurse(isGrib1, (DirectoryPartition) part, config, updateType, logger); } else { - Path partPath = Paths.get(part.getRoot()); + String partPath = part.getRoot(); changed = updateLeafCollection(isGrib1, config, updateType, false, logger, partPath); // LOOK why not using // part ?? } @@ -538,7 +548,7 @@ private static boolean updateDirectoryCollectionRecurse(boolean isGrib1, Directo * @return true if collection was rewritten, exception on failure */ private static boolean updateLeafCollection(boolean isGrib1, FeatureCollectionConfig config, - CollectionUpdateType updateType, boolean isTop, Logger logger, Path dirPath) throws IOException { + CollectionUpdateType updateType, boolean isTop, Logger logger, String dirPath) throws IOException { logger.debug("GribCdmIndex.updateLeafCollection {} {} ptype={}", dirPath, updateType, config.ptype); if (config.ptype == FeatureCollectionConfig.PartitionType.file) { @@ -549,7 +559,7 @@ private static boolean updateLeafCollection(boolean isGrib1, FeatureCollectionCo CollectionSpecParserAbstract specp = config.getCollectionSpecParserAbstract(errlog); try (DirectoryCollection dcm = - new DirectoryCollection(config.collectionName, dirPath.toString(), isTop, config.olderThan, logger)) { + new DirectoryCollection(config.collectionName, dirPath, isTop, config.olderThan, logger)) { dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); if (specp.getFilter() != null) dcm.setStreamFilter(new RegExpMatch(specp.getFilter(), specp.getFilterOnName())); @@ -574,14 +584,13 @@ private static boolean updateLeafCollection(boolean isGrib1, FeatureCollectionCo * @return true if partition was rewritten, exception on failure */ private static boolean updateFilePartition(boolean isGrib1, FeatureCollectionConfig config, - CollectionUpdateType updateType, boolean isTop, Logger logger, Path dirPath) throws IOException { + CollectionUpdateType updateType, boolean isTop, Logger logger, String dirPath) throws IOException { logger.debug("GribCdmIndex.updateFilePartition {} {}", dirPath, updateType); long start = System.currentTimeMillis(); Formatter errlog = new Formatter(); CollectionSpecParserAbstract specp = config.getCollectionSpecParserAbstract(errlog); - try (FilePartition partition = - new FilePartition(config.collectionName, dirPath.toString(), isTop, config.olderThan, logger)) { + try (FilePartition partition = new FilePartition(config.collectionName, dirPath, isTop, config.olderThan, logger)) { partition.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config); if (specp.getFilter() != null) partition.setStreamFilter(new RegExpMatch(specp.getFilter(), specp.getFilterOnName())); @@ -645,7 +654,7 @@ private static boolean updateFilePartition(boolean isGrib1, FeatureCollectionCon // DirectoryPartitionViewer - public static boolean makeIndex(FeatureCollectionConfig config, Formatter errlog, Path topPath) throws IOException { + public static boolean makeIndex(FeatureCollectionConfig config, Formatter errlog, String topPath) throws IOException { return false; /* * GribCdmIndex indexReader = new GribCdmIndex(); @@ -767,7 +776,7 @@ public static GribCollectionImmutable openGribCollection(FeatureCollectionConfig // update if needed boolean changed = updateGribCollection(config, updateType, logger); - File idxFile = makeTopIndexFileFromConfig(config); + MFile idxFile = makeTopIndexFileFromConfig(config); // If call to updateGribCollection shows a change happened, then collection changed. // If updateType is never (tds is in charge of updating, not TDM or some other external application), @@ -947,7 +956,7 @@ public boolean readMFiles(String indexFile, List result) throws IOExcepti // GribCollectionType type = getType(raf); // if (type == GribCollectionType.GRIB1 || type == GribCollectionType.GRIB2) { if (openIndex(raf, logger)) { - File protoDir = new File(gribCollectionIndex.getTopDir()); + MFile protoDir = MFiles.create(gribCollectionIndex.getTopDir()); int n = gribCollectionIndex.getMfilesCount(); for (int i = 0; i < n; i++) { GribCollectionProto.MFile mfilep = gribCollectionIndex.getMfiles(i); diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionBuilder.java b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionBuilder.java index 6f3c8dade4..b0f7cb5d32 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionBuilder.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionBuilder.java @@ -11,6 +11,7 @@ import thredds.inventory.CollectionUpdateType; import thredds.inventory.MCollection; import thredds.inventory.MFile; +import thredds.inventory.MFiles; import thredds.inventory.partition.PartitionManager; import thredds.inventory.partition.PartitionManagerFromIndexList; import ucar.nc2.grib.coord.Coordinate; @@ -23,7 +24,6 @@ import ucar.nc2.time.CalendarDateRange; import ucar.nc2.util.CloseableIterator; import ucar.unidata.util.StringUtil2; -import java.io.File; import java.io.IOException; import java.util.*; @@ -41,7 +41,7 @@ abstract class GribCollectionBuilder { protected GribCollectionImmutable.Type type; protected final String name; // collection name - protected final File directory; // top directory + protected final MFile directory; // top directory protected abstract List makeGroups(List allFiles, boolean singleRuntime, Formatter errlog) throws IOException; @@ -55,7 +55,7 @@ protected abstract boolean writeIndex(String name, String indexFilepath, Coordin this.isGrib1 = isGrib1; this.name = StringUtil2.replace(name, ' ', "_"); - this.directory = new File(dcm.getRoot()); + this.directory = MFiles.create(dcm.getRoot()); } boolean updateNeeded(CollectionUpdateType ff) throws IOException { @@ -65,7 +65,7 @@ boolean updateNeeded(CollectionUpdateType ff) throws IOException { return true; String indexFilename = dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX); - File collectionIndexFile = GribIndexCache.getExistingFileOrCache(indexFilename); + MFile collectionIndexFile = GribIndexCache.getExistingFileOrCache(indexFilename); if (collectionIndexFile == null) return true; @@ -75,8 +75,8 @@ boolean updateNeeded(CollectionUpdateType ff) throws IOException { return needsUpdate(ff, collectionIndexFile); } - private boolean needsUpdate(CollectionUpdateType ff, File collectionIndexFile) throws IOException { - long collectionLastModified = collectionIndexFile.lastModified(); + private boolean needsUpdate(CollectionUpdateType ff, MFile collectionIndexFile) throws IOException { + long collectionLastModified = collectionIndexFile.getLastModified(); Set newFileSet = new HashSet<>(); CollectionManager.ChangeChecker cc = GribIndex.getChangeChecker(); diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionImmutable.java b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionImmutable.java index 53dc60f9b5..67f8f43269 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionImmutable.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionImmutable.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory; import thredds.featurecollection.FeatureCollectionConfig; import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.nc2.Attribute; import ucar.nc2.AttributeContainer; import ucar.nc2.AttributeContainerMutable; @@ -44,13 +45,8 @@ import ucar.unidata.io.RandomAccessFile; import javax.annotation.concurrent.Immutable; import java.io.Closeable; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; import java.util.*; /** @@ -95,7 +91,7 @@ public boolean isTwoD() { //////////////////////////////////////////////////////////////// protected final String name; // collection name; index filename must be directory/name.ncx2 - protected final File directory; + protected final MFile directory; protected final FeatureCollectionConfig config; public final boolean isGrib1; protected final Info info; @@ -136,8 +132,8 @@ public boolean isTwoD() { indexFilename = gc.indexFilename; } else { - File indexFile = GribCdmIndex.makeIndexFile(name, directory); - File indexFileInCache = GribIndexCache.getExistingFileOrCache(indexFile.getPath()); + MFile indexFile = GribCdmIndex.makeIndexFile(name, directory); + MFile indexFileInCache = GribIndexCache.getExistingFileOrCache(indexFile.getPath()); if (indexFileInCache == null) throw new IllegalStateException(indexFile.getPath() + " does not exist, nor in cache"); indexFilename = indexFileInCache.getPath(); @@ -169,7 +165,7 @@ public String getName() { return name; } - public File getDirectory() { + public MFile getDirectory() { return directory; } @@ -913,8 +909,8 @@ public String getLocation() { @Override public long getLastModified() { - File indexFile = new File(indexFilename); - return indexFile.lastModified(); + MFile indexFile = MFiles.create(indexFilename); + return indexFile.getLastModified(); } /** @deprecated do not use */ @@ -1014,13 +1010,9 @@ private void showIndexFile(Formatter f) { if (indexFilename == null) return; f.format("indexFile=%s%n", indexFilename); - try { - Path indexFile = Paths.get(indexFilename); - BasicFileAttributes attr = Files.readAttributes(indexFile, BasicFileAttributes.class); - f.format(" size=%d lastModifiedTime=%s lastAccessTime=%s creationTime=%s%n", attr.size(), - attr.lastModifiedTime(), attr.lastAccessTime(), attr.creationTime()); - } catch (IOException e) { - e.printStackTrace(); + MFile mfile = MFiles.create(indexFilename); + if (mfile.exists()) { + f.format(" size=%d lastModifiedTime=%s%n", mfile.getLength(), CalendarDate.of(mfile.getLastModified())); } f.format("%n"); } @@ -1064,8 +1056,8 @@ public String toString() { //////////////////////////////////////// public long getIndexFileSize() { - File indexFile = new File(indexFilename); - return indexFile.length(); + MFile indexFile = MFiles.create(indexFilename); + return indexFile.getLength(); } public MFile getFile(int fileno) { @@ -1092,15 +1084,18 @@ RandomAccessFile getDataRaf(int fileno) throws IOException { // absolute location MFile mfile = fileMap.get(fileno); String filename = mfile.getPath(); - File dataFile = new File(filename); + MFile dataFile = MFiles.create(filename); // if data file does not exist, check relative location - eg may be /upc/share instead of Q: if (!dataFile.exists()) { + MFile relativeFile; if (fileMap.size() == 1) { - dataFile = new File(directory, name); // single file case + relativeFile = directory.getChild(name); // single file case } else { - dataFile = new File(directory, dataFile.getName()); // must be in same directory as the ncx file + relativeFile = directory.getChild(dataFile.getName()); // must be in same directory as the ncx file } + if (relativeFile != null) + dataFile = relativeFile; } // data file not here diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionMutable.java b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionMutable.java index 9c4b2f9e63..fb542a319f 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionMutable.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribCollectionMutable.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -10,6 +10,7 @@ import javax.annotation.Nullable; import thredds.featurecollection.FeatureCollectionConfig; import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.nc2.grib.GribIndexCache; import ucar.nc2.grib.GribTables; import ucar.nc2.grib.coord.Coordinate; @@ -33,9 +34,7 @@ import ucar.unidata.util.StringUtil2; import javax.annotation.concurrent.Immutable; import java.io.Closeable; -import java.io.File; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.io.IOException; import java.util.*; /** @@ -51,7 +50,7 @@ public class GribCollectionMutable implements Closeable { ////////////////////////////////////////////////////////// - static MFile makeIndexMFile(String collectionName, File directory) { + static MFile makeIndexMFile(String collectionName, MFile directory) { String nameNoBlanks = StringUtil2.replace(collectionName, ' ', "_"); return new GcMFile(directory, nameNoBlanks + GribCdmIndex.NCX_SUFFIX, -1, -1, -1); // LOOK dont know lastMod, size. // can it be added later? @@ -69,7 +68,7 @@ static String makeName(String collectionName, CalendarDate runtime) { protected final String name; // collection name; index filename must be directory/name.ncx2 protected final FeatureCollectionConfig config; protected final boolean isGrib1; - protected File directory; + protected MFile directory; protected String orgDirectory; // set by the builder @@ -98,7 +97,7 @@ void setCalendarDateRange(long startMsecs, long endMsecs) { private static int countGC; - protected GribCollectionMutable(String name, File directory, FeatureCollectionConfig config, boolean isGrib1) { + protected GribCollectionMutable(String name, MFile directory, FeatureCollectionConfig config, boolean isGrib1) { countGC++; this.name = name; this.directory = directory; @@ -125,7 +124,7 @@ public String getName() { return name; } - public File getDirectory() { + public MFile getDirectory() { return directory; } @@ -158,12 +157,15 @@ public List getFilenames() { } @Nullable - File getIndexParentFile() { + MFile getIndexParentFile() { if (indexRaf == null) return null; - Path index = Paths.get(indexRaf.getLocation()); - Path parent = index.getParent(); - return parent.toFile(); + MFile index = MFiles.create(indexRaf.getLocation()); + try { + return index.getParent(); + } catch (IOException e) { + return null; + } } public String getFilename(int fileno) { @@ -210,19 +212,23 @@ void setIndexRaf(RandomAccessFile indexRaf) { * @return index filename; may not exist; may be in disk cache */ private String getIndexFilepathInCache() { - File indexFile = GribCdmIndex.makeIndexFile(name, directory); + MFile indexFile = GribCdmIndex.makeIndexFile(name, directory); return GribIndexCache.getFileOrCache(indexFile.getPath()).getPath(); } // set from GribCollectionBuilderFromIndex.readFromIndex() - File setOrgDirectory(String orgDirectory) { + MFile setOrgDirectory(String orgDirectory) { this.orgDirectory = orgDirectory; - directory = new File(orgDirectory); + directory = MFiles.create(orgDirectory); if (!directory.exists()) { - File indexFile = new File(indexFilename); - File parent = indexFile.getParentFile(); - if (parent.exists()) - directory = parent; + MFile indexFile = MFiles.create(indexFilename); + try { + MFile parent = indexFile.getParent(); + if (parent != null && parent.exists()) + directory = parent; + } catch (IOException e) { + // ignore + } } return directory; } diff --git a/grib/src/main/java/ucar/nc2/grib/collection/GribPartitionBuilder.java b/grib/src/main/java/ucar/nc2/grib/collection/GribPartitionBuilder.java index f3bc6e7f4e..c98f164cb8 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/GribPartitionBuilder.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/GribPartitionBuilder.java @@ -12,6 +12,7 @@ import thredds.inventory.CollectionUpdateType; import thredds.inventory.MCollection; import thredds.inventory.MFile; +import thredds.inventory.MFiles; import thredds.inventory.partition.PartitionManager; import ucar.nc2.grib.GribIndexCache; import ucar.nc2.grib.coord.CalendarDateFactory; @@ -33,7 +34,6 @@ import ucar.unidata.util.StringUtil2; import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.*; /** @@ -62,7 +62,7 @@ boolean updateNeeded(CollectionUpdateType ff) throws IOException { return true; String indexFilename = partitionManager.getIndexFilename(GribCdmIndex.NCX_SUFFIX); - File collectionIndexFile = GribIndexCache.getExistingFileOrCache(indexFilename); + MFile collectionIndexFile = GribIndexCache.getExistingFileOrCache(indexFilename); if (collectionIndexFile == null) return true; @@ -74,16 +74,16 @@ boolean updateNeeded(CollectionUpdateType ff) throws IOException { } // LOOK need an option to only scan latest last partition or something - private boolean needsUpdate(CollectionUpdateType ff, File collectionIndexFile) throws IOException { - long collectionLastModified = collectionIndexFile.lastModified(); + private boolean needsUpdate(CollectionUpdateType ff, MFile collectionIndexFile) throws IOException { + long collectionLastModified = collectionIndexFile.getLastModified(); Set newFileSet = new HashSet<>(); for (MCollection dcm : partitionManager.makePartitions(CollectionUpdateType.test)) { String partitionIndexFilename = StringUtil2.replace(dcm.getIndexFilename(GribCdmIndex.NCX_SUFFIX), '\\', "/"); - File partitionIndexFile = GribIndexCache.getExistingFileOrCache(partitionIndexFilename); + MFile partitionIndexFile = GribIndexCache.getExistingFileOrCache(partitionIndexFilename); if (partitionIndexFile == null) // make sure each partition has an index return true; - if (collectionLastModified < partitionIndexFile.lastModified()) // and the partition index is earlier than the - // collection index + if (collectionLastModified < partitionIndexFile.getLastModified()) // and the partition index is earlier than the + // collection index return true; newFileSet.add(partitionIndexFilename); } @@ -493,11 +493,14 @@ private void makeDatasetBest(GribCollectionMutable.Dataset ds2D, boolean isCompl * GribCollectionIndex (sizeIndex bytes) */ protected boolean writeIndex(PartitionCollectionMutable pc, Formatter f) throws IOException { - File idxFile = GribIndexCache.getFileOrCache(partitionManager.getIndexFilename(GribCdmIndex.NCX_SUFFIX)); + MFile idxFile = GribIndexCache.getFileOrCache(partitionManager.getIndexFilename(GribCdmIndex.NCX_SUFFIX)); if (idxFile.exists()) { RandomAccessFile.eject(idxFile.getPath()); - if (!idxFile.delete()) - logger.error("gc2tp cant delete " + idxFile.getPath()); + if (idxFile instanceof thredds.filesystem.MFileOS) { + File f2 = new File(idxFile.getPath()); + if (!f2.delete()) + logger.error("gc2tp cant delete " + idxFile.getPath()); + } } writer = new GribCollectionWriter(null, null); @@ -542,8 +545,7 @@ protected boolean writeIndex(PartitionCollectionMutable pc, Formatter f) throws GribCollectionProto.GribCollection.Builder indexBuilder = GribCollectionProto.GribCollection.newBuilder(); indexBuilder.setName(pc.getName()); - Path topDir = pc.directory.toPath(); - String pathS = StringUtil2.replace(topDir.toString(), '\\', "/"); + String pathS = pc.getDirectory().getPath(); indexBuilder.setTopDir(pathS); // mfiles are the partition indexes @@ -597,10 +599,9 @@ protected boolean writeIndex(PartitionCollectionMutable pc, Formatter f) throws } private String makeRelativeFilename(PartitionCollectionMutable pc, PartitionCollectionMutable.Partition part) { - Path topDir = pc.directory.toPath(); - Path partPath = new File(part.getDirectory(), part.getFilename()).toPath(); - Path pathRelative = topDir.relativize(partPath); - return StringUtil2.replace(pathRelative.toString(), '\\', "/"); + MFile topDir = pc.getDirectory(); + MFile partFile = MFiles.create(part.getDirectory()).getChild(part.getFilename()); + return topDir.relativize(partFile); } /* diff --git a/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionImmutable.java b/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionImmutable.java index 91905bcfb3..67cb2ec1a1 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionImmutable.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionImmutable.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -11,6 +11,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import thredds.featurecollection.FeatureCollectionConfig; +import thredds.inventory.MFile; import ucar.nc2.grib.coord.Coordinate; import ucar.nc2.grib.coord.CoordinateRuntime; import ucar.nc2.grib.coord.CoordinateTime2D; @@ -28,7 +29,6 @@ import ucar.nc2.util.cache.SmartArrayInt; import ucar.unidata.io.RandomAccessFile; import javax.annotation.concurrent.Immutable; -import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; @@ -225,29 +225,12 @@ public org.slf4j.Logger getLogger() { } public String getIndexFilenameInCache() throws FileNotFoundException { - File file = new File(directory, filename); - File existingFile = GribIndexCache.getExistingFileOrCache(file.getPath()); + MFile file = directory.getChild(filename); + MFile existingFile = (file == null) ? null : GribIndexCache.getExistingFileOrCache(file.getPath()); if (existingFile == null) { - throw new FileNotFoundException("No index filename for partition= " + this + " looking for " + file.getName()); - } - - /* - * if (existingFile == null) { - * if (Grib.debugIndexOnly) { // we are running in debug mode where we only have the indices, not the data files - * // tricky: substitute the current root - * File orgParentDir = new File(directory); - * File currentFile = new File(PartitionCollectionImmutable.this.indexFilename); - * File currentParent = currentFile.getParentFile(); - * File currentParentWithDir = new File(currentParent, orgParentDir.getName()); - * File nestedIndex = isPartitionOfPartitions ? new File(currentParentWithDir, filename) : new File(currentParent, - * filename); // JMJ - * path = nestedIndex.getPath(); - * - * } else { - * throw new FileNotFoundException("No index filename for partition= " + this.toString()); - * } - */ + throw new FileNotFoundException("No index filename for partition= " + this + " looking for " + filename); + } return existingFile.getPath(); } diff --git a/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionMutable.java b/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionMutable.java index ac9e85af12..fd934d23c4 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionMutable.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionMutable.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ @@ -10,6 +10,8 @@ import thredds.featurecollection.FeatureCollectionConfig; import thredds.inventory.DateExtractor; import thredds.inventory.MCollection; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.nc2.dataset.DatasetUrl; import ucar.nc2.grib.GribIndexCache; import ucar.nc2.time.CalendarDate; @@ -198,15 +200,15 @@ private DateExtractor getDateExtractor() { @Nullable String getIndexFilenameInCache() { - File file = new File(directory, filename); - File existingFile = GribIndexCache.getExistingFileOrCache(file.getPath()); + MFile file = MFiles.create(directory).getChild(filename); + MFile existingFile = (file == null) ? null : GribIndexCache.getExistingFileOrCache(file.getPath()); if (existingFile == null) { // try relative to index file - File parent = getIndexParentFile(); + MFile parent = getIndexParentFile(); if (parent == null) return null; - existingFile = new File(parent, filename); - if (!existingFile.exists()) + existingFile = parent.getChild(filename); + if (existingFile == null || !existingFile.exists()) return null; } return existingFile.getPath(); @@ -304,7 +306,7 @@ public Partition(MCollection dcm) { protected boolean isPartitionOfPartitions; int[] run2part; // masterRuntime.length; which partition to use for masterRuntime i - protected PartitionCollectionMutable(String name, File directory, FeatureCollectionConfig config, boolean isGrib1, + protected PartitionCollectionMutable(String name, MFile directory, FeatureCollectionConfig config, boolean isGrib1, org.slf4j.Logger logger) { super(name, directory, config, isGrib1); this.logger = logger; diff --git a/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java b/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java index 37adc7bcbf..8614a7ae17 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java +++ b/grib/src/main/java/ucar/nc2/grib/grib1/Grib1Index.java @@ -1,20 +1,23 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package ucar.nc2.grib.grib1; import com.google.protobuf.ByteString; +import java.io.InputStream; import java.nio.charset.StandardCharsets; +import thredds.filesystem.MFileOS; +import thredds.filesystem.MFileOS7; import thredds.inventory.CollectionUpdateType; +import thredds.inventory.MFile; import ucar.nc2.NetcdfFiles; import ucar.nc2.grib.GribIndex; import ucar.nc2.grib.GribIndexCache; import ucar.nc2.stream.NcStream; import ucar.unidata.io.RandomAccessFile; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.util.*; @@ -88,15 +91,15 @@ public boolean readIndex(String filename, long gribLastModified, CollectionUpdat String idxPath = filename; if (!idxPath.endsWith(GBX9_IDX)) idxPath += GBX9_IDX; - File idxFile = GribIndexCache.getExistingFileOrCache(idxPath); + MFile idxFile = GribIndexCache.getExistingFileOrCache(idxPath); if (idxFile == null) return false; - long idxModified = idxFile.lastModified(); + long idxModified = idxFile.getLastModified(); if ((force != CollectionUpdateType.nocheck) && (idxModified < gribLastModified)) return false; // force new index if file was updated - try (FileInputStream fin = new FileInputStream(idxFile)) { + try (InputStream fin = idxFile.getInputStream()) { //// check header is ok if (!NcStream.readAndTest(fin, MAGIC_START.getBytes(StandardCharsets.UTF_8))) { logger.info("Bad magic number of grib index, on file = {}", idxFile); @@ -169,8 +172,15 @@ public boolean makeIndex(String filename, RandomAccessFile dataRaf) throws IOExc String idxPath = filename; if (!idxPath.endsWith(GBX9_IDX)) idxPath += GBX9_IDX; - File idxFile = GribIndexCache.getFileOrCache(idxPath); - File idxFileTmp = GribIndexCache.getFileOrCache(idxPath + ".tmp"); + MFile idxMFile = GribIndexCache.getFileOrCache(idxPath); + MFile idxMFileTmp = GribIndexCache.getFileOrCache(idxPath + ".tmp"); + + if (!(idxMFile instanceof MFileOS || idxMFile instanceof MFileOS7)) { + throw new IllegalArgumentException("Only local file systems are supported for index creation"); + } + + File idxFile = new File(idxMFile.getPath()); + File idxFileTmp = new File(idxMFileTmp.getPath()); RandomAccessFile raf = null; try (FileOutputStream fout = new FileOutputStream(idxFileTmp)) { diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java index a7e006be3c..f2488f343f 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/Grib2Index.java @@ -1,20 +1,23 @@ /* - * Copyright (c) 1998-2018 John Caron and University Corporation for Atmospheric Research/Unidata + * Copyright (c) 1998-2026 John Caron and University Corporation for Atmospheric Research/Unidata * See LICENSE for license information. */ package ucar.nc2.grib.grib2; import com.google.protobuf.ByteString; +import java.io.InputStream; import java.nio.charset.StandardCharsets; +import thredds.filesystem.MFileOS; +import thredds.filesystem.MFileOS7; import thredds.inventory.CollectionUpdateType; +import thredds.inventory.MFile; import ucar.nc2.NetcdfFiles; import ucar.nc2.grib.GribIndex; import ucar.nc2.grib.GribIndexCache; import ucar.nc2.stream.NcStream; import ucar.unidata.io.RandomAccessFile; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.util.*; @@ -92,15 +95,15 @@ public boolean readIndex(String filename, long gribLastModified, CollectionUpdat String idxPath = filename; if (!idxPath.endsWith(GBX9_IDX)) idxPath += GBX9_IDX; - File idxFile = GribIndexCache.getExistingFileOrCache(idxPath); + MFile idxFile = GribIndexCache.getExistingFileOrCache(idxPath); if (idxFile == null) return false; - long idxModified = idxFile.lastModified(); + long idxModified = idxFile.getLastModified(); if ((force != CollectionUpdateType.nocheck) && (idxModified < gribLastModified)) return false; // force new index if file was updated - try (FileInputStream fin = new FileInputStream(idxFile)) { + try (InputStream fin = idxFile.getInputStream()) { //// check header is ok if (!NcStream.readAndTest(fin, MAGIC_START.getBytes(StandardCharsets.UTF_8))) { logger.info("Bad magic number of grib index on file= {}", idxFile); @@ -199,12 +202,20 @@ private Grib2SectionGridDefinition readGds(Grib2IndexProto.GribGdsSection proto) //////////////////////////////////////////////////////////////////////////////// // LOOK what about extending an index ?? + // Only support creation of index files locally public boolean makeIndex(String filename, RandomAccessFile dataRaf) throws IOException { String idxPath = filename; if (!idxPath.endsWith(GBX9_IDX)) idxPath += GBX9_IDX; - File idxFile = GribIndexCache.getFileOrCache(idxPath); - File idxFileTmp = GribIndexCache.getFileOrCache(idxPath + ".tmp"); + MFile idxMFile = GribIndexCache.getFileOrCache(idxPath); + MFile idxMFileTmp = GribIndexCache.getFileOrCache(idxPath + ".tmp"); + + if (!(idxMFile instanceof MFileOS || idxMFile instanceof MFileOS7)) { + throw new IllegalArgumentException("Only local file systems are supported for index creation"); + } + + File idxFile = new File(idxMFile.getPath()); + File idxFileTmp = new File(idxMFileTmp.getPath()); boolean ok = false; RandomAccessFile raf = null; diff --git a/grib/src/test/java/ucar/nc2/grib/collection/TestGcMFile.java b/grib/src/test/java/ucar/nc2/grib/collection/TestGcMFile.java index 7e07a13f08..a7cdeefb2a 100644 --- a/grib/src/test/java/ucar/nc2/grib/collection/TestGcMFile.java +++ b/grib/src/test/java/ucar/nc2/grib/collection/TestGcMFile.java @@ -1,3 +1,8 @@ +/* + * Copyright (c) 2022-2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + package ucar.nc2.grib.collection; import static com.google.common.truth.Truth.assertThat; @@ -16,6 +21,7 @@ import org.junit.rules.TemporaryFolder; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; +import thredds.inventory.MFiles; @RunWith(Enclosed.class) public class TestGcMFile { @@ -37,7 +43,8 @@ public static List getTestParameters() { @Test public void shouldWriteFileToStream() throws IOException { final File file = createTemporaryFile(expectedSize); - final GcMFile mFile = new GcMFile(tempFolder.getRoot(), file.getName(), file.lastModified(), file.length(), 0); + final GcMFile mFile = new GcMFile(MFiles.create(tempFolder.getRoot().getPath()), file.getName(), + file.lastModified(), file.length(), 0); final long length = mFile.getLength(); assertThat(length).isEqualTo(expectedSize); @@ -51,7 +58,8 @@ public void shouldWriteFileToStream() throws IOException { @Test public void shouldWritePartialFileToStream() throws IOException { final File file = createTemporaryFile(expectedSize); - GcMFile mFile = new GcMFile(tempFolder.getRoot(), file.getName(), file.lastModified(), file.length(), 0); + GcMFile mFile = new GcMFile(MFiles.create(tempFolder.getRoot().getPath()), file.getName(), file.lastModified(), + file.length(), 0); final long length = mFile.getLength(); assertThat(length).isEqualTo(expectedSize); @@ -80,20 +88,22 @@ public static class TestGcMFileNonParameterized { @Test public void shouldReturnTrueForExistingFile() throws IOException { final File file = createTemporaryFile(0); - final GcMFile mFile = new GcMFile(tempFolder.getRoot(), file.getName(), file.lastModified(), file.length(), 0); + final GcMFile mFile = new GcMFile(MFiles.create(tempFolder.getRoot().getPath()), file.getName(), + file.lastModified(), file.length(), 0); assertThat(mFile.exists()).isEqualTo(true); } @Test public void shouldReturnFalseForNonExistingFile() { - final GcMFile mFile = new GcMFile(tempFolder.getRoot(), "notARealFile", 0, 0, 0); + final GcMFile mFile = new GcMFile(MFiles.create(tempFolder.getRoot().getPath()), "notARealFile", 0, 0, 0); assertThat(mFile.exists()).isEqualTo(false); } @Test public void shouldGetInputStream() throws IOException { final File file = createTemporaryFile(1); - final GcMFile mFile = new GcMFile(tempFolder.getRoot(), file.getName(), file.lastModified(), file.length(), 0); + final GcMFile mFile = new GcMFile(MFiles.create(tempFolder.getRoot().getPath()), file.getName(), + file.lastModified(), file.length(), 0); try (final InputStream inputStream = mFile.getInputStream()) { assertThat(inputStream.read()).isNotEqualTo(-1); } diff --git a/grib/src/test/java/ucar/nc2/grib/coord/TestDiscontiguousInterval.java b/grib/src/test/java/ucar/nc2/grib/coord/TestDiscontiguousInterval.java index b7ba843598..f11c5fd479 100644 --- a/grib/src/test/java/ucar/nc2/grib/coord/TestDiscontiguousInterval.java +++ b/grib/src/test/java/ucar/nc2/grib/coord/TestDiscontiguousInterval.java @@ -1,3 +1,8 @@ +/* + * Copyright (c) 2022-2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ + package ucar.nc2.grib.coord; import static com.google.common.truth.Truth.assertThat; @@ -118,7 +123,7 @@ public void testFeatureCollectionBest_isStrictlyMonotonicallyIncreasing() throws FeatureCollectionType.GRIB1, spec, null, null, null, "file", null); final boolean changed = GribCdmIndex.updateGribCollection(config, CollectionUpdateType.always, null); assertThat(changed).isTrue(); - final String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getAbsolutePath(); + final String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getPath(); final String varName = "Total_precipitation_surface_Mixed_intervals_Accumulation"; checkTimeCoord2D_fromIntervals_isStrictlyMonotonicallyIncreasing(topLevelIndex, varName, "Best/"); diff --git a/uicdm/src/main/java/ucar/nc2/ui/MFileTable.java b/uicdm/src/main/java/ucar/nc2/ui/MFileTable.java index eeddd16512..54fc6a9a54 100644 --- a/uicdm/src/main/java/ucar/nc2/ui/MFileTable.java +++ b/uicdm/src/main/java/ucar/nc2/ui/MFileTable.java @@ -5,6 +5,8 @@ package ucar.nc2.ui; +import thredds.filesystem.MFileOS; +import thredds.filesystem.MFileOS7; import thredds.inventory.MFile; import ucar.nc2.time.CalendarDateFormatter; import ucar.ui.widget.BAMutil; @@ -99,6 +101,13 @@ public void save() { prefs.putBeanObject("FileWindowBounds", fileWindow.getBounds()); } + public void setFiles(MFile mdir, Collection files) { + if (!(mdir instanceof MFileOS || mdir instanceof MFileOS7)) { + throw new IllegalArgumentException("Only local file systems are supported for index creation"); + } + setFiles(new File(mdir.getPath()), files); + } + public void setFiles(File dir, Collection files) { if (files == null) return; diff --git a/uicdm/src/main/java/ucar/nc2/ui/grib/CdmIndexPanel.java b/uicdm/src/main/java/ucar/nc2/ui/grib/CdmIndexPanel.java index 8f1d68ef8b..7b39d7b1c2 100644 --- a/uicdm/src/main/java/ucar/nc2/ui/grib/CdmIndexPanel.java +++ b/uicdm/src/main/java/ucar/nc2/ui/grib/CdmIndexPanel.java @@ -34,7 +34,6 @@ import javax.swing.*; import java.awt.*; import java.awt.event.ActionEvent; -import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -352,7 +351,7 @@ public void showInfo(Formatter f) { } private void showFileTable(GribCollectionImmutable gc, GribCollectionImmutable.GroupGC group) { - File dir = gc.getDirectory(); + MFile dir = gc.getDirectory(); Collection files = (group == null) ? gc.getFiles() : group.getFiles(); fileTable.setFiles(dir, files); } diff --git a/uicdm/src/main/java/ucar/nc2/ui/op/DirectoryPartitionViewer.java b/uicdm/src/main/java/ucar/nc2/ui/op/DirectoryPartitionViewer.java index d653d86a84..962058021c 100644 --- a/uicdm/src/main/java/ucar/nc2/ui/op/DirectoryPartitionViewer.java +++ b/uicdm/src/main/java/ucar/nc2/ui/op/DirectoryPartitionViewer.java @@ -438,7 +438,7 @@ private void cmdMakeIndex(NodeInfo node) { Formatter out = new Formatter(); out.format("makeTimePartitionIndex %s%n%n", node); try { - boolean ok = GribCdmIndex.makeIndex(config, out, node.dir); + boolean ok = GribCdmIndex.makeIndex(config, out, node.dir.toAbsolutePath().toString()); out.format("makeTimePartitionIndex success %s%n%n", ok); infoTA.setText(out.toString()); infoTA.gotoTop(); @@ -1015,7 +1015,7 @@ private void setGroup(GribCollectionMutable.GroupGC group) { private void showFiles(GribCollectionMutable gc, GribCollectionMutable.GroupGC group) { Collection files = (group == null) ? gc.getFiles() : group.getFiles(); - File dir = (gc == null) ? null : gc.getDirectory(); + MFile dir = (gc == null) ? null : gc.getDirectory(); fileTable.setFiles(dir, files); }