fix(content/oci): write the metadata files atomically

`index.json` and `oci-layout` are written with `os.WriteFile`, which opens
the file with `O_TRUNC`. The truncation succeeds even when the file system
is full, so a write that fails part-way replaces a good file with an empty
one, or with only the part of the content that was written before the
failure.

A store whose `index.json` has been emptied this way fails to open with
"failed to decode index file: EOF" for as long as the file is there, and
nothing rebuilds it, so a transient disk-full condition becomes permanent.

Write both files to a temporary file in the same directory and rename that
file over the target, which is what the package already does for blobs in
`content/oci/storage.go`, and what `registry/remote/config` does for the
configuration file. The content is flushed before the rename: with delayed
allocation a write can be accepted against space that is never allocated,
and the resulting failure is reported neither by the write nor by `Close`,
so a rename without a flush can still publish content that never reached
the disk. The directory entry is flushed afterwards on a best-effort basis,
since by then the rename has taken effect and reporting a failure would
describe an operation that did happen.

Treat a zero-length `index.json` or `oci-layout` as a missing one and
rewrite it. The store already recovers from a missing index, and an empty
file carries no information and describes the same state. The recovery is
deliberately limited to zero-length files: one that is malformed but not
empty still fails, since discarding it would silently lose the tags of the
store and hide a problem that is not a partial write. Recovering the index
does lose the tags it held, which is documented on New.

The permission that `os.WriteFile` produced is preserved. A file being
created is created with 0666, so the umask applies to it as before, and the
permission of an existing file is restored onto the temporary file before
the rename. Replacing a file rather than writing through it differs from
`os.WriteFile` in ways that are documented on the function.

Signed-off-by: Chris Wedgwood <cw@f00f.org>

Author: cwedgwood

content/oci/atomicwrite.go

@@ -0,0 +1,164 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import (
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// tempFileAttempts is the number of names tried when creating a temporary
+// file. A name carries the 128 bits of randomness of rand.Text, so a single
+// collision is already unlikely; retrying is only a guard against a caller
+// that has filled the directory with matching names.
+const tempFileAttempts = 10
+
+// tempFileSuffixMinLen is the minimum length of the random part of the name of
+// a temporary file. rand.Text returns 26 characters today and is documented to
+// possibly return more in a future release, so the recognizer accepts any name
+// at least this long: a store written by an older binary must stay
+// recognizable to a newer one.
+const tempFileSuffixMinLen = 26
+
+// fileWrite is the function used to write to a file, overridable in tests.
+var fileWrite = (*os.File).Write
+
+// createTempFile creates a new temporary file in dir whose name is base joined
+// with a random string, in the same manner as the ingest files created by
+// Storage.
+//
+// Unlike os.CreateTemp, which always creates the file with permission 0600,
+// the file is created with the permission bits perm, so that the process umask
+// is applied to it in exactly the same way as it would be by os.WriteFile.
+func createTempFile(dir, base string, perm os.FileMode) (*os.File, error) {
+	for range tempFileAttempts {
+		name := filepath.Join(dir, base+"_"+rand.Text())
+		file, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm)
+		if err == nil {
+			return file, nil
+		}
+		if !errors.Is(err, fs.ErrExist) {
+			return nil, fmt.Errorf("failed to create temporary file: %w", err)
+		}
+	}
+	return nil, fmt.Errorf("failed to create temporary file in %s: %w", dir, fs.ErrExist)
+}
+
+// writeFileAtomic writes data to the file named by path, by writing it to a
+// temporary file in the same directory and renaming that file over path.
+//
+// Unlike os.WriteFile, which truncates the file before writing it, a failed
+// write leaves any existing file untouched: the truncation performed by
+// os.WriteFile succeeds even when the file system is full, which leaves the
+// file empty, or holding only the part of the content that was written before
+// the failure.
+//
+// The content is flushed to stable storage before the file is renamed, so that
+// a write failure cannot make path visible with content that was never
+// written. The containing directory is flushed afterwards, on a best-effort
+// basis, so that the rename is not lost on a crash.
+//
+// The file is replaced rather than written through, which differs from
+// os.WriteFile in ways that matter only outside the layout of a content store.
+// path is replaced even if it is a symbolic link, a hard link, or not writable
+// by the caller; the mode bits beyond the permission bits, and the ownership,
+// access control lists and extended attributes of the file, are not carried
+// over; and on Windows the rename fails while any other process holds the file
+// open, where a write in place would have succeeded.
+func writeFileAtomic(path string, data []byte) (writeErr error) {
+	// os.WriteFile applies its permission argument only when it creates the
+	// file, leaving the permission of an existing file unchanged. Reproduce
+	// both behaviors: a file that is being created is created with 0666, so
+	// that the umask is applied to it in the same way, and the permission of
+	// an existing file is restored onto the temporary file before the rename.
+	var perm os.FileMode
+	var replacing bool
+	if fi, err := os.Stat(path); err == nil {
+		perm, replacing = fi.Mode().Perm(), true
+	} else if !errors.Is(err, fs.ErrNotExist) {
+		return fmt.Errorf("failed to stat %s: %w", path, err)
+	}
+
+	dir := filepath.Dir(path)
+	// the temporary file is created no wider than the file it replaces, so
+	// that it is never briefly more permissive than the target, and its
+	// permission is then set exactly, since the umask only clears bits.
+	createPerm := os.FileMode(0666)
+	if replacing {
+		createPerm = perm
+	}
+	tempFile, err := createTempFile(dir, filepath.Base(path), createPerm)
+	if err != nil {
+		return err
+	}
+	tempPath := tempFile.Name()
+	defer func() {
+		// remove the temporary file in case of error
+		if writeErr != nil {
+			tempFile.Close()
+			os.Remove(tempPath)
+		}
+	}()
+
+	if replacing {
+		if err := tempFile.Chmod(perm); err != nil {
+			return fmt.Errorf("failed to set permission of temporary file: %w", err)
+		}
+	}
+	if _, err := fileWrite(tempFile, data); err != nil {
+		return fmt.Errorf("failed to write temporary file: %w", err)
+	}
+	// flush the content before closing the file. With delayed allocation, a
+	// write can be accepted against space that is never allocated, and the
+	// resulting failure is reported neither by Write nor by Close.
+	if err := tempFile.Sync(); err != nil {
+		return fmt.Errorf("failed to sync temporary file: %w", err)
+	}
+	if err := tempFile.Close(); err != nil {
+		return fmt.Errorf("failed to close temporary file: %w", err)
+	}
+
+	if err := os.Rename(tempPath, path); err != nil {
+		return fmt.Errorf("failed to rename temporary file: %w", err)
+	}
+	// The rename has already made the content visible, so the durability of
+	// the directory entry is all that is left to gain here. Reporting a
+	// failure would describe an operation that did take effect.
+	syncDir(dir)
+	return nil
+}
+
+// isTempFileOf reports whether name is the name of a temporary file created by
+// createTempFile for the file named base.
+//
+// The random part of the name is matched against the alphabet and the minimum
+// length of rand.Text, so that a file that merely shares the prefix, such as a
+// copy of index.json that someone has kept, is not mistaken for one of ours.
+func isTempFileOf(name, base string) bool {
+	suffix, ok := strings.CutPrefix(name, base+"_")
+	if !ok || len(suffix) < tempFileSuffixMinLen {
+		return false
+	}
+	// rand.Text returns the RFC 4648 base32 alphabet without padding.
+	return !strings.ContainsFunc(suffix, func(r rune) bool {
+		return (r < 'A' || r > 'Z') && (r < '2' || r > '7')
+	})
+}

content/oci/dirsync_other.go

@@ -0,0 +1,37 @@
+//go:build !windows
+
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+import "os"
+
+// syncDir flushes the directory entries of dir to stable storage, so that a
+// file that has been renamed within dir is not lost on a crash.
+//
+// The operation is best-effort and reports nothing: its callers reach it only
+// once a rename has already taken effect, so a failure here describes an
+// operation that did happen and that cannot be undone. Directory
+// synchronization is also unsupported by some file systems, which report it in
+// ways that vary between them.
+func syncDir(dir string) {
+	dirFile, err := os.Open(dir)
+	if err != nil {
+		return
+	}
+	defer dirFile.Close()
+	_ = dirFile.Sync()
+}

content/oci/dirsync_windows.go

@@ -0,0 +1,23 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oci
+
+// syncDir does nothing on Windows, where flushing a directory handle is not
+// supported. The rename that its callers perform still replaces the target in
+// a single operation there, although the Go API does not promise the same
+// atomicity that it does on Unix; only the additional durability against a
+// crash is unavailable.
+func syncDir(dir string) {}

content/oci/oci.go

@@ -81,11 +81,19 @@ type Store struct {
 }
 
 // New creates a new OCI store with context.Background().
+//
+// If `index.json` is present but empty, which is what a write interrupted by a
+// full file system leaves behind, it is reinitialized rather than reported as
+// an error. The blobs of the store are kept, but the tags recorded in the lost
+// index are not recoverable, and the blobs they referenced become unreferenced
+// and are removed by the next call to GC.
 func New(root string) (*Store, error) {
 	return NewWithContext(context.Background(), root)
 }
 
 // NewWithContext creates a new OCI store.
+//
+// See New for the handling of an empty `index.json`.
 func NewWithContext(ctx context.Context, root string) (*Store, error) {
 	rootAbs, err := filepath.Abs(root)
 	if err != nil {
@@ -362,26 +370,55 @@ func (s *Store) ensureOCILayoutFile() error {
 		if !os.IsNotExist(err) {
 			return fmt.Errorf("failed to open OCI layout file: %w", err)
 		}
+		return writeOCILayoutFile(layoutFilePath)
+	}
+	defer layoutFile.Close()
 
-		layout := ocispec.ImageLayout{
-			Version: ocispec.ImageLayoutVersion,
-		}
-		layoutJSON, err := json.Marshal(layout)
-		if err != nil {
-			return fmt.Errorf("failed to marshal OCI layout file: %w", err)
+	// An empty file is what an interrupted write leaves behind. It carries no
+	// information, and it describes the same state as a missing file, so it is
+	// treated in the same way and rewritten, rather than failing the store for
+	// as long as it is there.
+	empty, err := isEmptyFile(layoutFile)
+	if err != nil {
+		return fmt.Errorf("failed to stat OCI layout file: %w", err)
+	}
+	if empty {
+		// the file is closed before it is replaced, since on Windows a file
+		// cannot be renamed over while it is open.
+		if err := layoutFile.Close(); err != nil {
+			return fmt.Errorf("failed to close OCI layout file: %w", err)
 		}
-		return os.WriteFile(layoutFilePath, layoutJSON, 0666)
+		return writeOCILayoutFile(layoutFilePath)
 	}
-	defer layoutFile.Close()
 
 	var layout ocispec.ImageLayout
-	err = json.NewDecoder(layoutFile).Decode(&layout)
-	if err != nil {
+	if err := json.NewDecoder(layoutFile).Decode(&layout); err != nil {
 		return fmt.Errorf("failed to decode OCI layout file: %w", err)
 	}
 	return validateOCILayout(&layout)
 }
 
+// writeOCILayoutFile writes the `oci-layout` file at the given path.
+func writeOCILayoutFile(layoutFilePath string) error {
+	layout := ocispec.ImageLayout{
+		Version: ocispec.ImageLayoutVersion,
+	}
+	layoutJSON, err := json.Marshal(layout)
+	if err != nil {
+		return fmt.Errorf("failed to marshal OCI layout file: %w", err)
+	}
+	return writeFileAtomic(layoutFilePath, layoutJSON)
+}
+
+// isEmptyFile reports whether the open file is zero-length.
+func isEmptyFile(file *os.File) (bool, error) {
+	fi, err := file.Stat()
+	if err != nil {
+		return false, err
+	}
+	return fi.Size() == 0, nil
+}
+
 // loadIndexFile reads index.json from the file system.
 // Create index.json if it does not exist.
 func (s *Store) loadIndexFile(ctx context.Context) error {
@@ -392,17 +429,26 @@ func (s *Store) loadIndexFile(ctx context.Context) error {
 		}
 
 		// write index.json if it does not exist
-		s.index = &ocispec.Index{
-			Versioned: specs.Versioned{
-				SchemaVersion: 2, // historical value
-			},
-			MediaType: ocispec.MediaTypeImageIndex,
-			Manifests: []ocispec.Descriptor{},
-		}
-		return s.writeIndexFile()
+		return s.resetIndexFile()
 	}
 	defer indexFile.Close()
 
+	// An empty file is what an interrupted write leaves behind. It is not a
+	// valid index, and it describes the same state as a missing index file, so
+	// it is treated in the same way and rewritten.
+	empty, err := isEmptyFile(indexFile)
+	if err != nil {
+		return fmt.Errorf("failed to stat index file: %w", err)
+	}
+	if empty {
+		// the file is closed before it is replaced, since on Windows a file
+		// cannot be renamed over while it is open.
+		if err := indexFile.Close(); err != nil {
+			return fmt.Errorf("failed to close index file: %w", err)
+		}
+		return s.resetIndexFile()
+	}
+
 	var index ocispec.Index
 	if err := json.NewDecoder(indexFile).Decode(&index); err != nil {
 		return fmt.Errorf("failed to decode index file: %w", err)
@@ -411,6 +457,19 @@ func (s *Store) loadIndexFile(ctx context.Context) error {
 	return loadIndex(ctx, s.index, s.storage, s.tagResolver, s.graph)
 }
 
+// resetIndexFile sets the index to an empty index and writes it to the file
+// system, replacing the `index.json` file if it is already present.
+func (s *Store) resetIndexFile() error {
+	s.index = &ocispec.Index{
+		Versioned: specs.Versioned{
+			SchemaVersion: 2, // historical value
+		},
+		MediaType: ocispec.MediaTypeImageIndex,
+		Manifests: []ocispec.Descriptor{},
+	}
+	return s.writeIndexFile()
+}
+
 // SaveIndex writes the `index.json` file to the file system.
 //   - If AutoSaveIndex is set to true (default value),
 //     the OCI store will automatically save the changes to `index.json`
@@ -463,7 +522,7 @@ func (s *Store) writeIndexFile() error {
 	if err != nil {
 		return fmt.Errorf("failed to marshal index file: %w", err)
 	}
-	return os.WriteFile(s.indexPath, indexJSON, 0666)
+	return writeFileAtomic(s.indexPath, indexJSON)
 }
 
 // GC removes garbage from Store. Unsaved index will be lost. To prevent unexpected