gomodfs: cache file contents in WinFsp mode

filecontents_test.go

@@ -0,0 +1,83 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+package gomodfs
+
+import (
+	"bytes"
+	"context"
+	"net/http"
+	"sync/atomic"
+	"testing"
+
+	"github.com/tailscale/gomodfs/store"
+	"github.com/tailscale/gomodfs/store/gitstore"
+)
+
+// getFileCountingStore is a store.Store that counts GetFile calls.
+type getFileCountingStore struct {
+	store.Store
+	getFiles atomic.Int64
+}
+
+func (s *getFileCountingStore) GetFile(ctx context.Context, h store.ModHandle, path string) ([]byte, error) {
+	s.getFiles.Add(1)
+	return s.Store.GetFile(ctx, h, path)
+}
+
+func TestGetFileContentsCache(t *testing.T) {
+	gs := &gitstore.Storage{GitRepo: testGitDir(t)}
+	addStopGitStoreCleanup(t, gs)
+	cs := &getFileCountingStore{Store: gs}
+	fs := &FS{
+		Store:  cs,
+		Client: &http.Client{Transport: testDataTransport{}},
+		Logf:   t.Logf,
+	}
+	ctx := t.Context()
+	mv := store.ModuleVersion{
+		Module:  "go4.org/mem",
+		Version: "v0.0.0-20240501181205-ae6ca9944745",
+	}
+	mh, err := fs.getZipRoot(ctx, mv)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	read := func(path string) []byte {
+		t.Helper()
+		got, err := fs.getFileContents(ctx, mv, mh, path)
+		if err != nil {
+			t.Fatalf("getFileContents(%q): %v", path, err)
+		}
+		want, err := gs.GetFile(ctx, mh, path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(got, want) {
+			t.Fatalf("getFileContents(%q) = %q; want %q", path, got, want)
+		}
+		return got
+	}
+	wantGetFiles := func(want int64) {
+		t.Helper()
+		if got := cs.getFiles.Load(); got != want {
+			t.Errorf("store GetFile calls = %d; want %d", got, want)
+		}
+	}
+
+	for range 3 {
+		read("fold.go")
+	}
+	wantGetFiles(1)
+
+	// With room for only one of the two files, reading the other evicts it.
+	fs2 := &FS{Store: cs, FileCacheSize: int64(len(read("fold.go")))}
+	cs.getFiles.Store(0)
+	for _, path := range []string{"fold.go", "fold.go", "fields.go", "fold.go"} {
+		if _, err := fs2.getFileContents(ctx, mv, mh, path); err != nil {
+			t.Fatal(err)
+		}
+	}
+	wantGetFiles(3)
+}

gomodfs.go

@@ -104,6 +104,19 @@ type FS struct {
 	entCache  lru.Cache[handle, *readCacheEntry]
 	blobCache lru.Cache[blobHash, []byte]
 	blobCount map[blobHash]int // ref count of blobHash in entCache
+
+	// pathContentsMu guards pathContents.
+	pathContentsMu sync.Mutex
+	// pathContents caches file contents by path, for getFileContents. Its
+	// sizes are in bytes, and it's lazily set up with a MaxSize of
+	// GetFileCacheSize.
+	pathContents lru.Cache[pathContentsKey, []byte]
+}
+
+// pathContentsKey is the key of FS.pathContents.
+type pathContentsKey struct {
+	mv   store.ModuleVersion
+	path string // slash-separated path within the module
 }
 
 // GetStore returns the underlying store.Store.
@@ -114,6 +127,44 @@ func (fs *FS) GetFileCacheSize() int64 {
 	return cmp.Or(fs.FileCacheSize, defaultFileCacheSize)
 }
 
+// getFileContents returns the contents of the file at path in module
+// version mv, whose handle is mh.
+//
+// Unlike fs.Store.GetFile, it caches the contents, keeping up to
+// GetFileCacheSize bytes, for filesystems that read files by path. (NFS
+// caches by handle instead; see getFileContents on NFSHandler.) WinFsp
+// needs it because Windows drops a file's cached data once no process has
+// it open, so without this, each go command that read a file would get it
+// from the store again.
+func (fs *FS) getFileContents(ctx context.Context, mv store.ModuleVersion, mh store.ModHandle, path string) ([]byte, error) {
+	k := pathContentsKey{mv, path}
+	fs.pathContentsMu.Lock()
+	v, ok := fs.pathContents.GetOk(k)
+	fs.pathContentsMu.Unlock()
+	if ok {
+		fs.MetricFileContentCacheHit.Add(1)
+		return v, nil
+	}
+	fs.MetricFileContentCacheMiss.Add(1)
+
+	sp := fs.Stats.StartSpan("get-file-contents-miss")
+	v, err := fs.Store.GetFile(ctx, mh, path)
+	sp.End(err)
+	if err != nil {
+		return nil, err
+	}
+
+	fs.MetricFileContentCacheFill.Add(1)
+	fs.pathContentsMu.Lock()
+	defer fs.pathContentsMu.Unlock()
+	if fs.pathContents.EntrySize == nil {
+		fs.pathContents.EntrySize = func(_ pathContentsKey, v []byte) int64 { return int64(len(v)) }
+		fs.pathContents.MaxSize = fs.GetFileCacheSize()
+	}
+	fs.pathContents.Set(k, v)
+	return v, nil
+}
+
 func hashModVersion(mv store.ModuleVersion) (ret modVerHash) {
 	s := sha256.New()
 	io.WriteString(s, mv.Module)

winfsp.go

@@ -314,7 +314,7 @@ func (pfs *fspFS) OpenFile(name string, flag int, perm os.FileMode) (retFile gof
 		fi: regFileInfo{name: name, size: fi.Size()},
 		getContents: func() ([]byte, error) {
 			sp := d.fs.Stats.StartSpan("fsp.ReadAt-GetFile")
-			contents, err := d.fs.Store.GetFile(ctx, mh, dp.Path)
+			contents, err := d.fs.getFileContents(ctx, dp.ModVersion, mh, dp.Path)
 			sp.End(err)
 			return contents, err
 		},