fix(storage): also remove the empty object directory when the xl.meta commit fails

The xl.meta rename at the end of RenameData creates the object directory
the same way the data rename does, and its failure branch left that
directory behind for writes without a data directory, such as inline
and zero-byte objects. Remove it and its empty parents there too, with
the same trailing-slash call that PR #286 adds to the data branch for
issue #285.

Making the cleanup safe against concurrent writes needs two more
changes:

- deleteFile now walks up with a trailing slash, so each parent step
  removes only an empty directory. Between two steps of a walk, another
  walk can remove the rest of the chain and a writer can commit an
  object whose xl.meta sits at the path the first walk removes next,
  for example the object "parent" under the key "parent/xl.meta/child".
  The walk used to delete that file. Ordinary deletes could already do
  this; the new cleanup made it reachable with failed writes.

- The two renames of RenameData retry when the directory creation or
  the rename reports "not found" while the source is still in place,
  up to once per path level. A late, failed RenameData can now remove
  the directory another write of the same key has just created, and
  that write used to fail on the drive although its data was intact.
  The retry decides on the error of the attempt itself, because another
  write can recreate the directories in between. renameAll's error
  mapping moves unchanged into renameAllErr so that the retry can see
  the raw error.

The stagedDataGoneDisks test helper now swaps the drives behind the
set's disk list under the lock that the list takes, instead of
replacing the getDisks callback that background tasks read unlocked.

Tests:
- TestXLStorageRenameDataCommitMetaMissingRemovesEmptyDirs: zero-byte
  and inline commit failures leave no empty directories, a sibling and
  a moved data directory stay.
- TestXLStorageWalkUpKeepsCommittedObject: a committed object at a
  parent path survives the walk, through DeleteVersion and both
  cleanup branches.
- TestRenameDataSurvivesConcurrentEmptyDirCleanup: PutObject (large
  and inline) and CompleteMultipartUpload succeed when a late failed
  RenameData of the same key removes their fresh directory on all 16
  drives.
- TestXLStorageRenameDataRetriesWhileCleanupWalksUp: a write survives
  a cleanup that removes each level of its path in turn, also when a
  nested-key write recreates the directories in between.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/xl-storage.go

@@ -2508,8 +2508,11 @@ func (s *xlStorage) deleteFile(basePath, deletePath string, recursive, immediate
 	}
 
 	// Delete parent directory obviously not recursively. Errors for
-	// parent directories shouldn't trickle down.
-	s.deleteFile(basePath, pathutil.Dir(pathutil.Clean(deletePath)), false, false)
+	// parent directories shouldn't trickle down. The trailing slash
+	// removes only an empty directory: a concurrent writer can have
+	// replaced the parent with a file, for example the xl.meta of an
+	// object whose name is that parent path, and it must stay.
+	s.deleteFile(basePath, pathutil.Dir(pathutil.Clean(deletePath))+SlashSeparator, false, false)
 
 	return nil
 }
@@ -2577,6 +2580,40 @@ func skipAccessChecks(volume string) (ok bool) {
 	return strings.HasPrefix(volume, minioMetaBucket)
 }
 
+// renameDataAll is renameAll for the two renames of RenameData. A failed
+// RenameData, typically one that runs after its caller gave up, removes its
+// object directory and the parents that are empty, so it can remove the
+// directories that renameAll is creating for this rename. The directory
+// creation or the rename then reports "not found" although the source is
+// still in place: retry, which creates the directories again. A cleanup
+// removes each directory of the path at most once and each failed attempt
+// needs at least one such removal, so one retry per path level is enough
+// against any single cleanup. The decision uses the error of the attempt
+// itself, since another write can recreate the directories in between.
+func renameDataAll(srcPath, dstPath, skipParent string) error {
+	if srcPath == "" || dstPath == "" {
+		return errInvalidArgument
+	}
+	if err := checkPathLength(srcPath); err != nil {
+		return err
+	}
+	if err := checkPathLength(dstPath); err != nil {
+		return err
+	}
+	retries := strings.Count(strings.TrimPrefix(pathutil.Clean(dstPath), pathutil.Clean(skipParent)), SlashSeparator)
+	for {
+		err := reliableRename(srcPath, dstPath, skipParent)
+		if err == nil || retries == 0 || !osIsNotExist(err) {
+			return renameAllErr(err, srcPath, dstPath)
+		}
+		if _, serr := Lstat(pathutil.Clean(srcPath)); serr != nil {
+			// The source is gone: a late call, which the caller cleans up.
+			return renameAllErr(err, srcPath, dstPath)
+		}
+		retries--
+	}
+}
+
 // RenameData - rename source path to destination path atomically, metadata and data directory.
 func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (res RenameDataResp, err error) {
 	defer func() {
@@ -2883,7 +2920,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 		if contextCanceled(ctx) {
 			return res, ctx.Err()
 		}
-		if err = renameAll(srcDataPath, dstDataPath, skipParent); err != nil {
+		if err = renameDataAll(srcDataPath, dstDataPath, skipParent); err != nil {
 			if legacyPreserved {
 				// Any failed rename calls un-roll previous transaction.
 				s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
@@ -2922,7 +2959,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 	}
 
 	// Commit meta-file
-	if err = renameAll(srcFilePath, dstFilePath, skipParent); err != nil {
+	if err = renameDataAll(srcFilePath, dstFilePath, skipParent); err != nil {
 		if legacyPreserved {
 			// Any failed rename calls un-roll previous transaction.
 			s.deleteFile(dstVolumeDir, legacyDataPath, true, false)
@@ -2930,6 +2967,11 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 		// if its a partial rename() do not attempt to delete recursively.
 		// this can be healed since all parts are available.
 		s.deleteFile(dstVolumeDir, dstDataPath, false, false)
+		// As with the data rename above, renameAll() may have created the
+		// object directory; drop it and its parents while empty. A moved
+		// data directory keeps it, since the trailing slash removes only an
+		// empty directory.
+		s.deleteFile(dstVolumeDir, pathutil.Dir(dstFilePath)+SlashSeparator, false, false)
 		err = osErrToFileErr(err)
 		if errors.Is(err, errFileNotFound) {
 			// The staged xl.meta disappeared between staging and commit.