diff --git a/filecontents_test.go b/filecontents_test.go new file mode 100644 index 0000000..8b9b723 --- /dev/null +++ b/filecontents_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause + +package gomodfs + +import ( + "bytes" + "context" + "net/http" + "sync/atomic" + "testing" + + "github.com/tailscale/gomodfs/store" + "github.com/tailscale/gomodfs/store/gitstore" +) + +// getFileCountingStore is a store.Store that counts GetFile calls. +type getFileCountingStore struct { + store.Store + getFiles atomic.Int64 +} + +func (s *getFileCountingStore) GetFile(ctx context.Context, h store.ModHandle, path string) ([]byte, error) { + s.getFiles.Add(1) + return s.Store.GetFile(ctx, h, path) +} + +func TestGetFileContentsCache(t *testing.T) { + gs := &gitstore.Storage{GitRepo: testGitDir(t)} + addStopGitStoreCleanup(t, gs) + cs := &getFileCountingStore{Store: gs} + fs := &FS{ + Store: cs, + Client: &http.Client{Transport: testDataTransport{}}, + Logf: t.Logf, + } + ctx := t.Context() + mv := store.ModuleVersion{ + Module: "go4.org/mem", + Version: "v0.0.0-20240501181205-ae6ca9944745", + } + mh, err := fs.getZipRoot(ctx, mv) + if err != nil { + t.Fatal(err) + } + + read := func(path string) []byte { + t.Helper() + got, err := fs.getFileContents(ctx, mv, mh, path) + if err != nil { + t.Fatalf("getFileContents(%q): %v", path, err) + } + want, err := gs.GetFile(ctx, mh, path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("getFileContents(%q) = %q; want %q", path, got, want) + } + return got + } + wantGetFiles := func(want int64) { + t.Helper() + if got := cs.getFiles.Load(); got != want { + t.Errorf("store GetFile calls = %d; want %d", got, want) + } + } + + for range 3 { + read("fold.go") + } + wantGetFiles(1) + + // With room for only one of the two files, reading the other evicts it. + fs2 := &FS{Store: cs, FileCacheSize: int64(len(read("fold.go")))} + cs.getFiles.Store(0) + for _, path := range []string{"fold.go", "fold.go", "fields.go", "fold.go"} { + if _, err := fs2.getFileContents(ctx, mv, mh, path); err != nil { + t.Fatal(err) + } + } + wantGetFiles(3) +} diff --git a/gomodfs.go b/gomodfs.go index 4324db2..a80f634 100644 --- a/gomodfs.go +++ b/gomodfs.go @@ -104,6 +104,19 @@ type FS struct { entCache lru.Cache[handle, *readCacheEntry] blobCache lru.Cache[blobHash, []byte] blobCount map[blobHash]int // ref count of blobHash in entCache + + // pathContentsMu guards pathContents. + pathContentsMu sync.Mutex + // pathContents caches file contents by path, for getFileContents. Its + // sizes are in bytes, and it's lazily set up with a MaxSize of + // GetFileCacheSize. + pathContents lru.Cache[pathContentsKey, []byte] +} + +// pathContentsKey is the key of FS.pathContents. +type pathContentsKey struct { + mv store.ModuleVersion + path string // slash-separated path within the module } // GetStore returns the underlying store.Store. @@ -114,6 +127,44 @@ func (fs *FS) GetFileCacheSize() int64 { return cmp.Or(fs.FileCacheSize, defaultFileCacheSize) } +// getFileContents returns the contents of the file at path in module +// version mv, whose handle is mh. +// +// Unlike fs.Store.GetFile, it caches the contents, keeping up to +// GetFileCacheSize bytes, for filesystems that read files by path. (NFS +// caches by handle instead; see getFileContents on NFSHandler.) WinFsp +// needs it because Windows drops a file's cached data once no process has +// it open, so without this, each go command that read a file would get it +// from the store again. +func (fs *FS) getFileContents(ctx context.Context, mv store.ModuleVersion, mh store.ModHandle, path string) ([]byte, error) { + k := pathContentsKey{mv, path} + fs.pathContentsMu.Lock() + v, ok := fs.pathContents.GetOk(k) + fs.pathContentsMu.Unlock() + if ok { + fs.MetricFileContentCacheHit.Add(1) + return v, nil + } + fs.MetricFileContentCacheMiss.Add(1) + + sp := fs.Stats.StartSpan("get-file-contents-miss") + v, err := fs.Store.GetFile(ctx, mh, path) + sp.End(err) + if err != nil { + return nil, err + } + + fs.MetricFileContentCacheFill.Add(1) + fs.pathContentsMu.Lock() + defer fs.pathContentsMu.Unlock() + if fs.pathContents.EntrySize == nil { + fs.pathContents.EntrySize = func(_ pathContentsKey, v []byte) int64 { return int64(len(v)) } + fs.pathContents.MaxSize = fs.GetFileCacheSize() + } + fs.pathContents.Set(k, v) + return v, nil +} + func hashModVersion(mv store.ModuleVersion) (ret modVerHash) { s := sha256.New() io.WriteString(s, mv.Module) diff --git a/winfsp.go b/winfsp.go index dba094d..532856d 100644 --- a/winfsp.go +++ b/winfsp.go @@ -314,7 +314,7 @@ func (pfs *fspFS) OpenFile(name string, flag int, perm os.FileMode) (retFile gof fi: regFileInfo{name: name, size: fi.Size()}, getContents: func() ([]byte, error) { sp := d.fs.Stats.StartSpan("fsp.ReadAt-GetFile") - contents, err := d.fs.Store.GetFile(ctx, mh, dp.Path) + contents, err := d.fs.getFileContents(ctx, dp.ModVersion, mh, dp.Path) sp.End(err) return contents, err },