fix(storage): retry metadata writes against a late RenameData cleanup

A RenameData that fails after its caller gave up removes its empty
object directory and the empty parents (ed05d982a, 73025824a). Only
the two renames of RenameData itself survived that cleanup, through
renameDataAll. Any other write of the same key that creates the object
directory failed on the drive when the cleanup removed the directory
between the directory creation and the open: WriteMetadata for a
delete marker on an absent key or for a first version, through
openFile, returned the raw open error, and writeAllDirect returned
errFileNotFound. The version was then missing on that drive, and
DeleteObject queues only offline drives for MRF. Before ed05d982a the
failure path never removed the directory, so this window is new.

Move the retry policy of renameDataAll into the shared helpers. A write
below the volume directory retries once per path level when the
directory creation or the use of the directory reports "not found"
(retriesAgainstCleanup): a cleanup removes each directory at most once
and each failed attempt needs one such removal. reliableRename applies
it in place of its blind single rename retry, which never recreated the
directory, and only while the source still exists, so renameDataAll is
replaced by renameAll. openFileAll applies it to the directory creation
and open shared by openFile and writeAllDirect. mkdirAllErr is split
out of mkdirAll, like renameAllErr, so that the retry decision uses the
OS error: on unix, mkdirAll maps a vanished parent to
errFileAccessDenied through isSysErrPathNotFound.

Invariants kept: the cleanup still removes only empty directories and
never a committed xl.meta or data directory; the error mapping of
mkdirAll, renameAll, openFile and writeAllDirect is unchanged, including
the Windows isSysErrPathNotFound branch; retries are bounded by the
path depth; a rename whose source is gone still fails at once.
UpdateMetadata renames over an existing xl.meta, so its directory is
never empty and the cleanup cannot remove it; it needs no change.

TestXLStorageMetadataWriteSurvivesLateCleanup injects, right after the
writer's os.Mkdir of the object directory, either a late RenameData
whose staged data is gone or a cleanup that walks up the whole path one
directory per step, for a delete marker, a first version and a large
xl.meta. Before the fix all six cases failed with "no such file or
directory" or "file not found".

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/os-reliable.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"strings"
 )
 
 // Wrapper functions to os.RemoveAll, which calls reliableRemoveAll
@@ -82,20 +83,63 @@ func mkdirAll(dirPath string, mode os.FileMode, baseDir string) (err error) {
 		return err
 	}
 
-	if err = reliableMkdirAll(dirPath, mode, baseDir); err != nil {
+	return mkdirAllErr(reliableMkdirAll(dirPath, mode, baseDir))
+}
+
+// mkdirAllErr converts an error of reliableMkdirAll into the error that
+// mkdirAll returns.
+func mkdirAllErr(err error) error {
+	switch {
+	case err == nil:
+		return nil
+	case isSysErrNotDir(err):
 		// File path cannot be verified since one of the parents is a file.
-		if isSysErrNotDir(err) {
-			return errFileAccessDenied
-		} else if isSysErrPathNotFound(err) {
-			// This is a special case should be handled only for
-			// windows, because windows API does not return "not a
-			// directory" error message. Handle this specifically here.
-			return errFileAccessDenied
-		}
+		return errFileAccessDenied
+	case isSysErrPathNotFound(err):
+		// This is a special case should be handled only for
+		// windows, because windows API does not return "not a
+		// directory" error message. Handle this specifically here.
+		return errFileAccessDenied
+	default:
 		return osErrToFileErr(err)
 	}
+}
 
-	return nil
+// openFileAll is mkdirAll for the parent of filePath followed by the open
+// of filePath with flags, with direct I/O when direct is set. A cleanup of a
+// failed write of the same key can remove the directories in between, see
+// retriesAgainstCleanup; then it creates them again. An error of the
+// directory creation is the error of mkdirAll, an error of the open is the
+// error of the OS call.
+func openFileAll(filePath string, flags int, direct bool, baseDir string) (*os.File, error) {
+	parentDir := path.Dir(filePath)
+	if err := checkPathLength(parentDir); err != nil {
+		return nil, err
+	}
+	retries := retriesAgainstCleanup(filePath, baseDir)
+	for {
+		// Create top level directories if they don't exist.
+		// with mode 0777 mkdir honors system umask.
+		if err := reliableMkdirAll(parentDir, 0o777, baseDir); err != nil {
+			if retries > 0 && osIsNotExist(err) {
+				retries--
+				continue
+			}
+			return nil, mkdirAllErr(err)
+		}
+		var f *os.File
+		var err error
+		if direct {
+			f, err = OpenFileDirectIO(filePath, flags, 0o666)
+		} else {
+			f, err = OpenFile(filePath, flags, 0o666)
+		}
+		if err != nil && retries > 0 && osIsNotExist(err) && flags&os.O_CREATE != 0 {
+			retries--
+			continue
+		}
+		return f, err
+	}
 }
 
 // Reliably retries os.MkdirAll if for some reason os.MkdirAll returns
@@ -175,24 +219,41 @@ func renameAllErr(err error, srcFilePath, dstFilePath string) error {
 	}
 }
 
-// Reliably retries os.RenameAll if for some reason os.RenameAll returns
-// syscall.ENOENT (parent does not exist).
-func reliableRename(srcFilePath, dstFilePath, baseDir string) (err error) {
-	if err = reliableMkdirAll(path.Dir(dstFilePath), 0o777, baseDir); err != nil {
-		return err
-	}
-
-	i := 0
-	for {
-		// After a successful parent directory create attempt a renameAll.
-		if err = Rename(srcFilePath, dstFilePath); err != nil {
-			// Retry only for the first retryable error.
-			if osIsNotExist(err) && i == 0 {
-				i++
-				continue
-			}
-		}
-		break
-	}
-	return err
+// retriesAgainstCleanup returns how often a write to filePath is retried
+// when a directory of its path disappears between the directory creation
+// and the use of the directory: once per path level below baseDir.
+//
+// A failed RenameData, typically one that runs after its caller gave up,
+// removes its object directory and the parents that are empty, so it can
+// remove the directories that another write of the same key has just
+// created. The directory creation or the write then reports "not found"
+// although nothing is wrong with the write: retry, which creates the
+// directories again. A cleanup removes each directory of the path at most
+// once and each failed attempt needs at least one such removal, so one retry
+// per path level is enough against any single cleanup.
+func retriesAgainstCleanup(filePath, baseDir string) int {
+	return strings.Count(strings.TrimPrefix(path.Clean(filePath), path.Clean(baseDir)), SlashSeparator)
+}
+
+// Reliably retries os.Rename if for some reason the directory creation or
+// os.Rename returns syscall.ENOENT (parent does not exist) while the source
+// is still in place, see retriesAgainstCleanup. The decision uses the error
+// of the attempt itself, since another write can recreate the directories
+// in between.
+func reliableRename(srcFilePath, dstFilePath, baseDir string) (err error) {
+	retries := retriesAgainstCleanup(dstFilePath, baseDir)
+	for {
+		if err = reliableMkdirAll(path.Dir(dstFilePath), 0o777, baseDir); err == nil {
+			// After a successful parent directory create attempt a rename.
+			err = Rename(srcFilePath, dstFilePath)
+		}
+		if err == nil || retries == 0 || !osIsNotExist(err) {
+			return err
+		}
+		if _, serr := Lstat(path.Clean(srcFilePath)); serr != nil {
+			// The source is gone: a late call, which the caller cleans up.
+			return err
+		}
+		retries--
+	}
 }