astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-11-11 10:05:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(storage): retry metadata writes against a late RenameData cleanup

Browse Source 
A RenameData that fails after its caller gave up removes its empty
object directory and the empty parents (ed05d982a, 73025824a). Only
the two renames of RenameData itself survived that cleanup, through
renameDataAll. Any other write of the same key that creates the object
directory failed on the drive when the cleanup removed the directory
between the directory creation and the open: WriteMetadata for a
delete marker on an absent key or for a first version, through
openFile, returned the raw open error, and writeAllDirect returned
errFileNotFound. The version was then missing on that drive, and
DeleteObject queues only offline drives for MRF. Before ed05d982a the
failure path never removed the directory, so this window is new.

Move the retry policy of renameDataAll into the shared helpers. A write
below the volume directory retries once per path level when the
directory creation or the use of the directory reports "not found"
(retriesAgainstCleanup): a cleanup removes each directory at most once
and each failed attempt needs one such removal. reliableRename applies
it in place of its blind single rename retry, which never recreated the
directory, and only while the source still exists, so renameDataAll is
replaced by renameAll. openFileAll applies it to the directory creation
and open shared by openFile and writeAllDirect. mkdirAllErr is split
out of mkdirAll, like renameAllErr, so that the retry decision uses the
OS error: on unix, mkdirAll maps a vanished parent to
errFileAccessDenied through isSysErrPathNotFound.

Invariants kept: the cleanup still removes only empty directories and
never a committed xl.meta or data directory; the error mapping of
mkdirAll, renameAll, openFile and writeAllDirect is unchanged, including
the Windows isSysErrPathNotFound branch; retries are bounded by the
path depth; a rename whose source is gone still fails at once.
UpdateMetadata renames over an existing xl.meta, so its directory is
never empty and the cleanup cannot remove it; it needs no change.

TestXLStorageMetadataWriteSurvivesLateCleanup injects, right after the
writer's os.Mkdir of the object directory, either a late RenameData
whose staged data is gone or a cleanup that walks up the whole path one
directory per step, for a delete marker, a first version and a large
xl.meta. Before the fix all six cases failed with "no such file or
directory" or "file not found".

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-11-07 16:07:55 +08:00
parent 8cc2360c3a
commit 479a49b28b
3 changed files with 250 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/xl-storage.go
+6 -56
View File
@@ -2001,13 +2001,8 @@ func (s *xlStorage) openFile(filePath string, mode int, skipParent string) (f *o
if skipParent == "" {
skipParent = s.drivePath
}
// Create top level directories if they don't exist.
// with mode 0777 mkdir honors system umask.
if err = mkdirAll(pathutil.Dir(filePath), 0o777, skipParent); err != nil {
return nil, osErrToFileErr(err)
}
w, err := OpenFile(filePath, mode, 0o666)
// Create top level directories if they don't exist, then open the file.
w, err := openFileAll(filePath, mode, false, skipParent)
if err != nil {
// File path cannot be verified since one of the parents is a file.
switch {
@@ -2158,21 +2153,10 @@ func (s *xlStorage) writeAllDirect(ctx context.Context, filePath string, fileSiz
skipParent = s.drivePath
}
// Create top level directories if they don't exist.
// with mode 0777 mkdir honors system umask.
parentFilePath := pathutil.Dir(filePath)
if err = mkdirAll(parentFilePath, 0o777, skipParent); err != nil {
return osErrToFileErr(err)
}
odirectEnabled := globalAPIConfig.odirectEnabled() && s.oDirect && fileSize > 0
var w *os.File
if odirectEnabled {
w, err = OpenFileDirectIO(filePath, flags, 0o666)
} else {
w, err = OpenFile(filePath, flags, 0o666)
}
// Create top level directories if they don't exist, then open the file.
w, err := openFileAll(filePath, flags, odirectEnabled, skipParent)
if err != nil {
return osErrToFileErr(err)
}
@@ -2583,40 +2567,6 @@ func skipAccessChecks(volume string) (ok bool) {
return strings.HasPrefix(volume, minioMetaBucket)
}
// renameDataAll is renameAll for the two renames of RenameData. A failed
// RenameData, typically one that runs after its caller gave up, removes its
// object directory and the parents that are empty, so it can remove the
// directories that renameAll is creating for this rename. The directory
// creation or the rename then reports "not found" although the source is
// still in place: retry, which creates the directories again. A cleanup
// removes each directory of the path at most once and each failed attempt
// needs at least one such removal, so one retry per path level is enough
// against any single cleanup. The decision uses the error of the attempt
// itself, since another write can recreate the directories in between.
func renameDataAll(srcPath, dstPath, skipParent string) error {
if srcPath == "" || dstPath == "" {
return errInvalidArgument
}
if err := checkPathLength(srcPath); err != nil {
return err
}
if err := checkPathLength(dstPath); err != nil {
return err
}
retries := strings.Count(strings.TrimPrefix(pathutil.Clean(dstPath), pathutil.Clean(skipParent)), SlashSeparator)
for {
err := reliableRename(srcPath, dstPath, skipParent)
if err == nil || retries == 0 || !osIsNotExist(err) {
return renameAllErr(err, srcPath, dstPath)
}
if _, serr := Lstat(pathutil.Clean(srcPath)); serr != nil {
// The source is gone: a late call, which the caller cleans up.
return renameAllErr(err, srcPath, dstPath)
}
retries--
}
}
// RenameData - rename source path to destination path atomically, metadata and data directory.
func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (res RenameDataResp, err error) {
defer func() {
@@ -2923,7 +2873,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
if contextCanceled(ctx) {
return res, ctx.Err()
}
if err = renameDataAll(srcDataPath, dstDataPath, skipParent); err != nil {
if err = renameAll(srcDataPath, dstDataPath, skipParent); err != nil {
if legacyPreserved {
// Any failed rename calls un-roll previous transaction.
s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
@@ -2962,7 +2912,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
}
// Commit meta-file
if err = renameDataAll(srcFilePath, dstFilePath, skipParent); err != nil {
if err = renameAll(srcFilePath, dstFilePath, skipParent); err != nil {
if legacyPreserved {
// Any failed rename calls un-roll previous transaction.
s.deleteFile(dstVolumeDir, legacyDataPath, true, false)