feat: Add logic to clean up logs by last modified date for TimeBasedArchiveRemover

Signed-off-by: Tianci Shen <[email protected]>

Author: Tianci Shen

logback-core/src/main/java/ch/qos/logback/core/rolling/SizeAndTimeBasedRollingPolicy.java

@@ -14,12 +14,15 @@
 package ch.qos.logback.core.rolling;
 
 import ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP.Usage;
+import ch.qos.logback.core.util.Duration;
 import ch.qos.logback.core.util.FileSize;
 
 public class SizeAndTimeBasedRollingPolicy<E> extends TimeBasedRollingPolicy<E> {
 
     FileSize maxFileSize;
 
+    Duration checkIncrement = null;
+
     @Override
     public void start() {
         SizeAndTimeBasedFNATP<E> sizeAndTimeBasedFNATP = new SizeAndTimeBasedFNATP<E>(Usage.EMBEDDED);
@@ -30,6 +33,10 @@ public void start() {
             addInfo("Archive files will be limited to [" + maxFileSize + "] each.");
         }
 
+        if (checkIncrement != null) {
+            sizeAndTimeBasedFNATP.setCheckIncrement(checkIncrement);
+        }
+
         sizeAndTimeBasedFNATP.setMaxFileSize(maxFileSize);
         timeBasedFileNamingAndTriggeringPolicy = sizeAndTimeBasedFNATP;
 
@@ -47,6 +54,10 @@ public void setMaxFileSize(FileSize aMaxFileSize) {
         this.maxFileSize = aMaxFileSize;
     }
 
+    public void setCheckIncrement(Duration checkIncrement) {
+        this.checkIncrement = checkIncrement;
+    }
+
     @Override
     public String toString() {
         return "c.q.l.core.rolling.SizeAndTimeBasedRollingPolicy@" + this.hashCode();

logback-core/src/main/java/ch/qos/logback/core/rolling/TimeBasedRollingPolicy.java

@@ -61,6 +61,7 @@ public class TimeBasedRollingPolicy<E> extends RollingPolicyBase implements Trig
     TimeBasedFileNamingAndTriggeringPolicy<E> timeBasedFileNamingAndTriggeringPolicy;
 
     boolean cleanHistoryOnStart = false;
+    boolean cleanLogsByLastModifiedDate = false;
 
     public void start() {
         // set the LR for our utility object
@@ -109,6 +110,7 @@ public void start() {
             archiveRemover = timeBasedFileNamingAndTriggeringPolicy.getArchiveRemover();
             archiveRemover.setMaxHistory(maxHistory);
             archiveRemover.setTotalSizeCap(totalSizeCap.getSize());
+            archiveRemover.setCleanLogsByLastModifiedDate(cleanLogsByLastModifiedDate);
             if (cleanHistoryOnStart) {
                 addInfo("Cleaning on start up");
                 Instant now = Instant.ofEpochMilli(timeBasedFileNamingAndTriggeringPolicy.getCurrentTime());
@@ -271,6 +273,17 @@ public void setCleanHistoryOnStart(boolean cleanHistoryOnStart) {
         this.cleanHistoryOnStart = cleanHistoryOnStart;
     }
 
+
+    /**
+     * Should archive removal use a file's last modified date to determine deletion?
+     * Default is false.
+     *
+     * @param cleanLogsByLastModifiedDate
+     */
+    public void setCleanLogsByLastModifiedDate(boolean cleanLogsByLastModifiedDate){
+        this.cleanLogsByLastModifiedDate = cleanLogsByLastModifiedDate;
+    }
+
     @Override
     public String toString() {
         return "c.q.l.core.rolling.TimeBasedRollingPolicy@" + this.hashCode();

logback-core/src/main/java/ch/qos/logback/core/rolling/helper/ArchiveRemover.java

@@ -31,5 +31,7 @@ public interface ArchiveRemover extends ContextAware {
 
     void setTotalSizeCap(long totalSizeCap);
 
+    void setCleanLogsByLastModifiedDate(boolean cleanLogsByLastModifiedDate);
+
     Future<?> cleanAsynchronously(Instant now);
 }

logback-core/src/main/java/ch/qos/logback/core/rolling/helper/SizeAndTimeBasedArchiveRemover.java

@@ -29,6 +29,10 @@ public SizeAndTimeBasedArchiveRemover(FileNamePattern fileNamePattern, RollingCa
         super(fileNamePattern, rc);
     }
 
+    File getParentDir(Instant cleanupCutoff) {
+        return getParentDir(new File(fileNamePattern.convertMultipleArguments(cleanupCutoff, 0)));
+    }
+
     protected File[] getFilesInPeriod(Instant instantOfPeriodToClean) {
         File archive0 = new File(fileNamePattern.convertMultipleArguments(instantOfPeriodToClean, 0));
         File parentDir = getParentDir(archive0);

logback-core/src/main/java/ch/qos/logback/core/rolling/helper/TimeBasedArchiveRemover.java

@@ -36,6 +36,7 @@ public class TimeBasedArchiveRemover extends ContextAwareBase implements Archive
     final RollingCalendar rc;
     private int maxHistory = CoreConstants.UNBOUNDED_HISTORY;
     private long totalSizeCap = CoreConstants.UNBOUNDED_TOTAL_SIZE_CAP;
+    private boolean cleanLogsByLastModifiedDate = false;
     final boolean parentClean;
     long lastHeartBeat = UNINITIALIZED;
 
@@ -70,10 +71,54 @@ public void clean(Instant now) {
             addInfo("Multiple periods, i.e. " + periodsElapsed
                     + " periods, seem to have elapsed. This is expected at application start.");
         }
-        for (int i = 0; i < periodsElapsed; i++) {
-            int offset = getPeriodOffsetForDeletionTarget() - i;
-            Instant instantOfPeriodToClean = rc.getEndOfNextNthPeriod(now, offset);
-            cleanPeriod(instantOfPeriodToClean);
+
+        if (cleanLogsByLastModifiedDate) {
+            // Delete old logs based on date the file was last modified
+            cleanLogsByDateModified(now);
+        } else {
+            // Delete old logs based on expected file name
+            for (int i = 0; i < periodsElapsed; i++) {
+                int offset = getPeriodOffsetForDeletionTarget() - i;
+                Instant instantOfPeriodToClean = rc.getEndOfNextNthPeriod(now, offset);
+                cleanPeriod(instantOfPeriodToClean);
+            }
+        }
+    }
+
+    /**
+     * Iterates through log files and deletes files outside the rollover window
+     * Expects the file name to occur before the date specifier
+     * Does not work well with file patterns that have auxiliary date specifiers
+     *
+     * @param now
+     */
+    private void cleanLogsByDateModified(Instant now) {
+        File filePattern = new File(fileNamePattern.getPattern());
+        String fileNameBeforeDateSpecifier = filePattern.getName().split("\\%d\\{.+\\}")[0];
+        Instant cleanupCutoff = rc.getEndOfNextNthPeriod(now, getPeriodOffsetForDeletionTarget());
+        
+        File parentDir;
+        parentDir = getParentDir(cleanupCutoff);
+        if (parentDir == null) {
+            addError("Cannot get parent directory");
+            return;
+        }
+        
+        File[] matchedFiles;
+        matchedFiles = parentDir.listFiles((dir, name) -> name.contains(fileNameBeforeDateSpecifier));
+        if (matchedFiles == null) {
+            addError("Failed to find relevant log files");
+            return;
+        }
+        
+        for (File file : matchedFiles) {
+            Instant lastModifiedDate = Instant.ofEpochMilli(file.lastModified());
+            if (cleanupCutoff.isAfter(lastModifiedDate)) {
+                checkAndDeleteFile(file);
+            }
+        }
+        if (parentClean && matchedFiles.length > 0) {
+            removeFolderIfEmpty(parentDir);
         }
     }
 
@@ -148,6 +193,10 @@ protected void descendingSort(File[] matchingFileArray, Instant instant) {
         // nothing to do in super class
     }
 
+    File getParentDir(Instant cleanupCutoff) {
+        return getParentDir(new File(fileNamePattern.convert(cleanupCutoff)));
+    }
+
     File getParentDir(File file) {
         File absolute = file.getAbsoluteFile();
         File parentDir = absolute.getParentFile();
@@ -241,6 +290,10 @@ public void setTotalSizeCap(long totalSizeCap) {
         this.totalSizeCap = totalSizeCap;
     }
 
+    public void setCleanLogsByLastModifiedDate(boolean cleanLogsByLastModifiedDate) {
+        this.cleanLogsByLastModifiedDate = cleanLogsByLastModifiedDate;
+    }
+
     public String toString() {
         return "c.q.l.core.rolling.helper.TimeBasedArchiveRemover";
     }

logback-core/src/test/java/ch/qos/logback/core/rolling/TimeBasedRollingWithArchiveRemoval_Test.java

@@ -16,9 +16,11 @@
 import static ch.qos.logback.core.CoreConstants.DAILY_DATE_PATTERN;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 
 import java.io.File;
 import java.io.FileFilter;
+import java.io.IOException;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
@@ -338,10 +340,10 @@ public void dailyChronologSizeBasedRolloverWithSecondPhase() {
         checkDirPatternCompliance(maxHistory + 1);
     }
 
-    void logTwiceAndStop(long currentTime, String fileNamePattern, int maxHistory, long durationInMillis) {
+    void logTwiceAndStop(long currentTime, String fileNamePattern, int maxHistory, long durationInMillis, boolean cleanLogsByLastModifiedDate) {
         ConfigParameters params = new ConfigParameters(currentTime).fileNamePattern(fileNamePattern)
                 .maxHistory(maxHistory);
-        buildRollingFileAppender(params, DO_CLEAN_HISTORY_ON_START);
+        buildRollingFileAppender(params, DO_CLEAN_HISTORY_ON_START, cleanLogsByLastModifiedDate);
         rfa.doAppend("Hello ----------------------------------------------------------" + new Date(currentTime));
         currentTime += durationInMillis / 2;
         add(tbrp.compressionFuture);
@@ -359,7 +361,7 @@ public void cleanHistoryOnStartWithHourPattern() {
         String fileNamePattern = randomOutputDir + "clean-%d{" + DAILY_HOUR_PATTERN + "}.txt";
         int maxHistory = 3;
         for (int i = 0; i <= 5; i++) {
-            logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR);
+            logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR, DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE);
             simulatedTime += MILLIS_IN_HOUR;
         }
         checkFileCount(expectedCountWithoutFolders(maxHistory));
@@ -379,7 +381,7 @@ public void cleanHistoryOnStartWithHourPatternWithCollisions() {
         String fileNamePattern = randomOutputDir + "clean-%d{HH}.txt";
         int maxHistory = 3;
         for (int i = 0; i <= 5; i++) {
-            logTwiceAndStop(now, fileNamePattern, maxHistory, MILLIS_IN_DAY);
+            logTwiceAndStop(now, fileNamePattern, maxHistory, MILLIS_IN_DAY, DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE);
             now = now + MILLIS_IN_HOUR;
         }
         checkFileCount(expectedCountWithoutFolders(maxHistory));
@@ -391,7 +393,7 @@ public void cleanHistoryOnStartWithDayPattern() {
         String fileNamePattern = randomOutputDir + "clean-%d{" + DAILY_DATE_PATTERN + "}.txt";
         int maxHistory = 3;
         for (int i = 0; i <= 5; i++) {
-            logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_DAY);
+            logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_DAY, DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE);
             simulatedTime += MILLIS_IN_DAY;
         }
         checkFileCount(expectedCountWithoutFolders(maxHistory));
@@ -403,12 +405,32 @@ public void cleanHistoryOnStartWithHourDayPattern() {
         String fileNamePattern = randomOutputDir + "clean-%d{yyyy-MM-dd-HH}.txt";
         int maxHistory = 3;
         for (int i = 0; i <= 5; i++) {
-            logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR);
+            logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR, DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE);
             simulatedTime += MILLIS_IN_HOUR;
         }
         checkFileCount(expectedCountWithoutFolders(maxHistory));
     }
 
+    @Test
+    public void cleanLogsByLastModifiedDateWithHourDayPattern() throws IOException {
+        long simulatedTime = WED_2016_03_23_T_230705_CET;
+        String fileNamePattern = randomOutputDir + "clean-%d{yyyy-MM-dd-HH}.txt";
+        int maxHistory = 3;
+        logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR, DO_CLEAN_LOGS_BY_LAST_MODIFIED_DATE);
+        simulatedTime += MILLIS_IN_HOUR;
+        File fileToDelete = new File(randomOutputDir + "clean-0000-00-00-00.txt");
+        if (!fileToDelete.exists()) {
+            assertTrue(fileToDelete.createNewFile());
+        }
+        assertTrue(fileToDelete.setLastModified(0));
+        for (int i = 0; i <= 2; i++) {
+            logTwiceAndStop(simulatedTime, fileNamePattern, maxHistory, MILLIS_IN_HOUR, DO_CLEAN_LOGS_BY_LAST_MODIFIED_DATE);
+            simulatedTime += MILLIS_IN_HOUR;
+        }
+        checkFileCount(expectedCountWithoutFolders(maxHistory));
+        assertFalse(fileToDelete.exists());
+    }
+
     int expectedCountWithoutFolders(int maxHistory) {
         return maxHistory + 1;
     }
@@ -422,7 +444,7 @@ int expectedCountWithFolders(int maxHistory, boolean withExtraFolder) {
         return result;
     }
 
-    void buildRollingFileAppender(ConfigParameters cp, boolean cleanHistoryOnStart) {
+    void buildRollingFileAppender(ConfigParameters cp, boolean cleanHistoryOnStart, boolean cleanLogsByLastModifiedDate) {
         rfa.setContext(context);
         rfa.setEncoder(encoder);
         tbrp.setContext(context);
@@ -431,6 +453,7 @@ void buildRollingFileAppender(ConfigParameters cp, boolean cleanHistoryOnStart)
         tbrp.setTotalSizeCap(new FileSize(cp.sizeCap));
         tbrp.setParent(rfa);
         tbrp.setCleanHistoryOnStart(cleanHistoryOnStart);
+        tbrp.setCleanLogsByLastModifiedDate(cleanLogsByLastModifiedDate);
         tbrp.timeBasedFileNamingAndTriggeringPolicy = tbfnatp;
         tbrp.timeBasedFileNamingAndTriggeringPolicy.setCurrentTime(cp.simulatedTime);
         tbrp.start();
@@ -440,10 +463,13 @@ void buildRollingFileAppender(ConfigParameters cp, boolean cleanHistoryOnStart)
 
     boolean DO_CLEAN_HISTORY_ON_START = true;
     boolean DO_NOT_CLEAN_HISTORY_ON_START = false;
+    boolean DO_CLEAN_LOGS_BY_LAST_MODIFIED_DATE = true;
+    boolean DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE = false;
+
 
     long logOverMultiplePeriods(ConfigParameters cp) {
 
-        buildRollingFileAppender(cp, DO_NOT_CLEAN_HISTORY_ON_START);
+        buildRollingFileAppender(cp, DO_NOT_CLEAN_HISTORY_ON_START, DO_NOT_CLEAN_LOGS_BY_LAST_MODIFIED_DATE);
 
         int runLength = cp.simulatedNumberOfPeriods * ticksPerPeriod;
         int startInactivityIndex = cp.startInactivity * ticksPerPeriod;