diff --git a/changes.md b/changes.md new file mode 100644 index 00000000000..3d5b820d2ad --- /dev/null +++ b/changes.md @@ -0,0 +1,23 @@ +# Changes for Negative Cache TTL on Implicit Directories + +## Summary + +Added integration test coverage to verify that the negative stat cache correctly respects the negative TTL configuration when handling implicit directories in gcsfuse. + +## Details + +1. **New Integration Test File:** + - Created `tools/integration_tests/negative_stat_cache/implicit_dir_finite_negative_stat_cache_test.go` + - Added `implicitDirFiniteNegativeStatCacheTest` suite testing the `--implicit-dirs` behavior alongside finite negative stat cache configuration. + - The test asserts that checking an implicit directory when it doesn't exist successfully populates the negative stat cache. + - It simulates an implicit directory creation by adding an object behind the scenes and ensures it remains unseen (due to negative stat cache) until the 5-second TTL expires. + +2. **Configuration Updates:** + - Appended a new `ConfigItem` within `tools/integration_tests/negative_stat_cache/setup_test.go` for the negative stat cache tests. + - The configuration runs the newly added test suite (`TestImplicitDirFiniteNegativeStatCacheTest`) with the flags `{"--metadata-cache-negative-ttl-secs=5", "--implicit-dirs"}`. + - Set the compatibility configuration parameter `hns` to `false` because Hierarchical Namespace (HNS) buckets do not conceptually support implicit directories. + +## Validation + +- Verified that all unit tests correctly pass using `go test ./...` +- Specifically executed and successfully passed the added integration test with `go test ./tools/integration_tests/negative_stat_cache/... -run TestImplicitDirFiniteNegativeStatCacheTest`. \ No newline at end of file diff --git a/tools/integration_tests/negative_stat_cache/implicit_dir_finite_negative_stat_cache_test.go b/tools/integration_tests/negative_stat_cache/implicit_dir_finite_negative_stat_cache_test.go new file mode 100644 index 00000000000..e488b50bbcc --- /dev/null +++ b/tools/integration_tests/negative_stat_cache/implicit_dir_finite_negative_stat_cache_test.go @@ -0,0 +1,110 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package negative_stat_cache + +import ( + "log" + "os" + "path" + "testing" + "time" + + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/client" + "github.com/googlecloudplatform/gcsfuse/v3/tools/integration_tests/util/setup" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type implicitDirFiniteNegativeStatCacheTest struct { + flags []string + testDir string + suite.Suite +} + +func (s *implicitDirFiniteNegativeStatCacheTest) SetupSuite() { + setup.MountGCSFuseWithGivenMountWithConfigFunc(testEnv.cfg, s.flags, testEnv.mountFunc) + setup.SetMntDir(testEnv.mountDir) +} + +func (s *implicitDirFiniteNegativeStatCacheTest) TearDownSuite() { + setup.UnmountGCSFuseWithConfig(testEnv.cfg) +} + +func (s *implicitDirFiniteNegativeStatCacheTest) SetupTest() { + s.testDir = testDirName + setup.GenerateRandomString(5) + testEnv.testDirPath = setup.SetupTestDirectory(s.testDir) +} + +func (s *implicitDirFiniteNegativeStatCacheTest) TearDownTest() { + setup.SaveGCSFuseLogFileInCaseOfFailure(s.T()) +} + +//////////////////////////////////////////////////////////////////////// +// Test scenarios +//////////////////////////////////////////////////////////////////////// + +func (s *implicitDirFiniteNegativeStatCacheTest) TestImplicitDirFiniteNegativeStatCache() { + targetDir := path.Join(testEnv.testDirPath, "implicit_dir") + + // Error should be returned as the implicit directory doesn't exist + _, err := os.Stat(targetDir) + + assert.NotNil(s.T(), err) + // Assert the underlying error is File Not Exist + assert.ErrorContains(s.T(), err, "no such file or directory") + + // Adding an object inside the path to create an implicit directory + client.CreateObjectInGCSTestDir(testEnv.ctx, testEnv.storageClient, s.testDir, path.Join("implicit_dir", "file1.txt"), "some-content", s.T()) + + // Error should be returned again, as call will not be served from GCS due to finite gcsfuse stat cache + _, err = os.Stat(targetDir) + + assert.NotNil(s.T(), err) + // Assert the underlying error is File Not Exist + assert.ErrorContains(s.T(), err, "no such file or directory") + + //Wait for Cache to expire + time.Sleep(5 * time.Second) + + // Directory should be returned, as call will be served from GCS and gcsfuse should not return from cache + fileInfo, err := os.Stat(targetDir) + + //Assert Directory is found and it is a directory + assert.NoError(s.T(), err) + assert.True(s.T(), fileInfo.IsDir()) +} + +//////////////////////////////////////////////////////////////////////// +// Test Function (Runs once before all tests) +//////////////////////////////////////////////////////////////////////// + +func TestImplicitDirFiniteNegativeStatCacheTest(t *testing.T) { + ts := &implicitDirFiniteNegativeStatCacheTest{} + + // Run tests for mounted directory if the flag is set. + if testEnv.cfg.GKEMountedDirectory != "" && testEnv.cfg.TestBucket != "" { + suite.Run(t, ts) + return + } + + // Define flag set to run the tests. + flagsSet := setup.BuildFlagSets(*testEnv.cfg, testEnv.bucketType, t.Name()) + + // Run tests. + for _, ts.flags = range flagsSet { + log.Printf("Running tests with flags: %s", ts.flags) + suite.Run(t, ts) + } +} diff --git a/tools/integration_tests/negative_stat_cache/setup_test.go b/tools/integration_tests/negative_stat_cache/setup_test.go index 5774000c816..cffe40a12fe 100644 --- a/tools/integration_tests/negative_stat_cache/setup_test.go +++ b/tools/integration_tests/negative_stat_cache/setup_test.go @@ -73,7 +73,7 @@ func TestMain(m *testing.M) { cfg.NegativeStatCache[0].GKEMountedDirectory = setup.MountedDirectory() cfg.NegativeStatCache[0].LogFile = setup.LogFile() // Initialize the slice to hold specific test configurations - cfg.NegativeStatCache[0].Configs = make([]test_suite.ConfigItem, 3) + cfg.NegativeStatCache[0].Configs = make([]test_suite.ConfigItem, 4) cfg.NegativeStatCache[0].Configs[0].Flags = []string{"--metadata-cache-negative-ttl-secs=0"} cfg.NegativeStatCache[0].Configs[0].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} cfg.NegativeStatCache[0].Configs[0].Run = "TestDisabledNegativeStatCacheTest" @@ -83,6 +83,9 @@ func TestMain(m *testing.M) { cfg.NegativeStatCache[0].Configs[2].Flags = []string{"--metadata-cache-negative-ttl-secs=-1"} cfg.NegativeStatCache[0].Configs[2].Compatible = map[string]bool{"flat": true, "hns": true, "zonal": true} cfg.NegativeStatCache[0].Configs[2].Run = "TestInfiniteNegativeStatCacheTest" + cfg.NegativeStatCache[0].Configs[3].Flags = []string{"--metadata-cache-negative-ttl-secs=5", "--implicit-dirs"} + cfg.NegativeStatCache[0].Configs[3].Compatible = map[string]bool{"flat": true, "hns": false, "zonal": true} + cfg.NegativeStatCache[0].Configs[3].Run = "TestImplicitDirFiniteNegativeStatCacheTest" } testEnv.ctx = context.Background()