fix(restore): stage native restores outside the object lock

The unreleased native-restore fixes after RELEASE.2026-10-26T00-00-00Z
let the lock decide who owns a restore. Since fa42a0982 a single-pool
worker holds the object lock for the whole tier download, and since
7f38bf628 a busy lock means "a restore runs". 7f38bf628 also keeps the
restore's data in a marked multipart upload. Two regressions followed:

- While an old version restored, every GET, HEAD and write of the same
  key waited for the download. On a single pool this is new; with
  several pools the released code already held the lock that long.
- An ordinary upload whose completion failed after staging its metadata
  became InvalidUploadID for every caller, and could not be aborted for
  an hour. The marker that hid restore uploads could not tell them apart
  once a completion had rewritten the upload's xl.meta.

A restore now runs in two phases:

- The worker erasure-codes the tier copy into the tmp volume of the
  version's set, without the object lock, as heal does. A drive that
  fails any part, or misses a part when CheckParts runs before the
  commit, takes no part in the commit.
- A restore lock per version tells whether a restore runs. POST takes it
  before the object lock and hands it to the worker. It is a liveness
  hint: only the checks under the object lock decide what is written.
- The commit takes the object lock briefly, on one pool or several. It
  reads the version again and checks the attempt's ownership, the
  version identity, the tier copy, the part layout and the erasure
  layout. It then publishes with RenameData drive by drive, never
  through erasure renameData's undo, and needs the write quorum. The
  replaced data directory goes only when every published drive reports
  the same one.
- A publication that reached some drives but not the write quorum
  converges under the same lock hold, in bounded retries after it: back
  to the unrestored version on every drive that holds it, or to a
  committed restore when the copy turns out to be on a write quorum.
  Each pass first checks that every drive holds what the version held
  before the commit, the commit wrote, or a convergence wrote, and stops
  on anything else. It deletes no directory.

Multipart goes back to main: erasure-multipart.go and
bucket-object-lock.go are identical to 69fcf50cd. The commit lock, the
marker and the restore exceptions of multipart completion and of the
null-version Object Lock check are gone, and single-pool completion
again takes the upload lock first and the object lock right before it
commits. With several pools, abandoned-data cleanup now takes the pools
lock that writes take, as heal does, so it cannot remove a directory
that a commit moved in place and has not yet referenced. The restore
capability becomes protocol 2, and a queued worker checks it again.

Recovery after a coordinator crash during publication, and storage RPCs
that complete after their caller returned, stay excluded. Main has the
same exposure, and they are tracked separately.

Tests:
- New regression tests cover both problems. They fail on f85d4a5fa,
  and the multipart one also passes on main.
- Fault-injection tests cover a publication below the write quorum,
  including seven of twelve drives on 8+4; lost answers; convergence
  retries and stops; the old data directory; a stale inline marker; a
  recreated tmp volume; a published revision read across its expiry; a
  second POST; a target deleted during the download; a tier 404; and
  pools-locked abandoned cleanup.
- Thirteen tests of the upload marker from 7f38bf628 are dropped, and
  so is TestNativeRestoreConcurrentCompletion from fa42a0982, which
  completed the restore's upload from a client. Four tests are reframed
  for the new semantics, including the lock-domain restore test, which
  now gates the commit.
- Three self-recursive WriteMetadata test doubles are fixed, and the
  helpers of the dropped tests are removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/restore-native-generation_test.go

@@ -21,17 +21,14 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"maps"
 	"net/http"
 	"net/http/httptest"
-	"os"
 	"reflect"
 	"runtime"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -40,7 +37,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/versioning"
-	"github.com/minio/minio/internal/config/drive"
 	xhttp "github.com/minio/minio/internal/http"
 )
 
@@ -56,7 +52,7 @@ func (d restoreGenerationDisk) WriteMetadata(ctx context.Context, orig, vol, pat
 	if d.write != nil && d.write.Load() && vol == d.bucket && path == d.object {
 		return errFaultyDisk
 	}
-	return d.WriteMetadata(ctx, orig, vol, path, fi)
+	return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi)
 }
 
 func (d restoreGenerationDisk) UpdateMetadata(ctx context.Context, vol, path string, fi FileInfo, opts UpdateMetadataOpts) error {
@@ -99,7 +95,7 @@ func (d restoreGenerationOffline) WriteMetadata(ctx context.Context, orig, bucke
 	if d.fail.Load() && bucket == d.bucket && object == d.object {
 		return errFaultyDisk
 	}
-	return d.WriteMetadata(ctx, orig, bucket, object, fi)
+	return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi)
 }
 
 // restoreGenerationMetadata returns the version's metadata on each drive.
@@ -259,168 +255,6 @@ func TestRestoreAttemptGenerationFencing(t *testing.T) {
 	}
 }
 
-// restoreGenerationUploadPath is the stored path of an upload.
-func restoreGenerationUploadPath(t *testing.T, r *restoreTest, id string) string {
-	t.Helper()
-	b, err := base64.RawURLEncoding.DecodeString(id)
-	if err != nil {
-		t.Fatal(err)
-	}
-	_, uploadUUID, ok := strings.Cut(string(b), ".")
-	if !ok {
-		t.Fatalf("upload ID without UUID: %q", id)
-	}
-	return pathJoin(r.z.serverPools[0].getHashedSet(r.object).getMultipartSHADir(r.bucket, r.object), uploadUUID)
-}
-
-// The restore marker is established on a majority of the drives. One drive
-// whose copy of the upload lacks it does not open the upload to clients,
-// and the restore still completes.
-func TestRestoreUploadMarkerQuorum(t *testing.T) {
-	r := newRestoreSafetyTest(t, 2, versioning.Enabled)
-	initConfigSubsystem(t.Context(), r.z)
-	oi, body := restoreIsolationSeed(t, r, 0, false, true)
-	tier := newRestoreIsolationGateTier(5<<20, nil)
-	globalTierConfigMgr.drivercache["WARM"] = tier
-	defer tier.unblock()
-	layer := newRestoreIsolationLayer(r)
-	if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
-		t.Fatal(rec.Body.String())
-	}
-	restoreIsolationEntered(t, tier)
-	id, _ := restoreIsolationUpload(t, r, 0)
-	disk := r.z.serverPools[0].getHashedSet(r.object).getDisks()[0]
-	path := restoreGenerationUploadPath(t, r, id)
-	fi, err := disk.ReadVersion(t.Context(), "", minioMetaMultipartBucket, path, "", ReadOptions{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	delete(fi.Metadata, restoreUploadKey)
-	if err := disk.WriteMetadata(t.Context(), "", minioMetaMultipartBucket, path, fi); err != nil {
-		t.Fatal(err)
-	}
-	poison := bytes.Repeat([]byte("p"), 5<<20)
-	if _, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, id, 1,
-		mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", ""), ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) {
-		t.Errorf("public part with one unmarked copy of the upload: %v, want NoSuchUpload", err)
-	}
-	for _, op := range []func() error{
-		func() error {
-			_, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{})
-			return err
-		},
-		func() error {
-			_, err := r.z.GetMultipartInfo(t.Context(), r.bucket, r.object, id, ObjectOptions{})
-			return err
-		},
-	} {
-		if err := op(); !errors.As(err, &InvalidUploadID{}) {
-			t.Errorf("public read with one unmarked copy of the upload: %v, want NoSuchUpload", err)
-		}
-	}
-	if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10); err != nil || len(uploads.Uploads) != 0 {
-		t.Errorf("listing with one unmarked copy of the upload: %+v, %v", uploads.Uploads, err)
-	}
-	tier.unblock()
-	if err := layer.wait(t); err != nil {
-		t.Fatal(err)
-	}
-	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
-		t.Fatalf("restore contents: %v", err)
-	}
-}
-
-// When the drives cannot tell an upload's marker, no client lists it, no
-// abort removes it and stale cleanup keeps it; the restore completes once
-// the drives answer again.
-func TestRestoreUploadUnreadableFailsClosed(t *testing.T) {
-	for _, mode := range []string{"list", "abort", "stale"} {
-		t.Run(mode, func(t *testing.T) {
-			r := newRestoreSafetyTest(t, 2, versioning.Enabled)
-			initConfigSubsystem(t.Context(), r.z)
-			oi, body := restoreIsolationSeed(t, r, 0, false, true)
-			er := r.z.serverPools[0].getHashedSet(r.object)
-			disks := er.getDisks()
-			var fail atomic.Bool
-			// Stale cleanup with a short age also removes in-flight temporary
-			// parts: hold that restore in completion, after its last part and
-			// before its commit, and the others in their download, outside
-			// the upload lock.
-			gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})}
-			defer gate.unblock()
-			tier := newRestoreIsolationGateTier(5<<20, nil)
-			defer tier.unblock()
-			wrapped := make([]StorageAPI, len(disks))
-			for i, d := range disks {
-				disk := d
-				if mode == "stale" {
-					disk = restoreIsolationValidationDisk{d, gate}
-				}
-				wrapped[i] = restoreGenerationDisk{StorageAPI: disk, uploadRead: &fail}
-			}
-			er.getDisks = func() []StorageAPI { return wrapped }
-			if mode != "stale" {
-				globalTierConfigMgr.drivercache["WARM"] = tier
-			}
-			layer := newRestoreIsolationLayer(r)
-			if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
-				t.Fatal(rec.Body.String())
-			}
-			if mode == "stale" {
-				select {
-				case <-gate.entered:
-				case <-time.After(restoreTestWait):
-					t.Fatal("restore never reached completion")
-				}
-			} else {
-				restoreIsolationEntered(t, tier)
-			}
-			stored := restoreIsolationUploads(t, r, 0)
-			if len(stored) != 1 {
-				t.Fatalf("restore upload on disk: %v", stored)
-			}
-			var id string
-			for uploadUUID := range stored {
-				id = base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID))
-			}
-			fail.Store(true)
-			switch mode {
-			case "list":
-				if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 100); len(uploads.Uploads) != 0 {
-					t.Errorf("an upload with an unknown marker is listed: %+v, %v", uploads.Uploads, err)
-				}
-			case "abort":
-				if err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); err == nil {
-					t.Error("abort of an upload with an unknown marker succeeded")
-				}
-			case "stale":
-				globalAPIConfig.mu.Lock()
-				prev := globalAPIConfig.staleUploadsExpiry
-				globalAPIConfig.staleUploadsExpiry = time.Nanosecond
-				globalAPIConfig.mu.Unlock()
-				for _, disk := range wrapped {
-					er.cleanupStaleUploadsOnDisk(t.Context(), disk)
-				}
-				globalAPIConfig.mu.Lock()
-				globalAPIConfig.staleUploadsExpiry = prev
-				globalAPIConfig.mu.Unlock()
-			}
-			fail.Store(false)
-			if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 1 {
-				t.Errorf("%s removed an upload with an unknown marker: %v", mode, uploads)
-			}
-			gate.unblock()
-			tier.unblock()
-			if err := layer.wait(t); err != nil {
-				t.Fatalf("restore after %s: %v", mode, err)
-			}
-			if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
-				t.Fatalf("restore contents: %v", err)
-			}
-		})
-	}
-}
-
 // POST restores the version it resolved with that version's identity. A
 // replacement under the same version ID between its read and its status
 // write is refused, and nothing restores it.
@@ -633,190 +467,6 @@ func TestRestoreGenerationSplitNeedsQuorum(t *testing.T) {
 	}
 }
 
-// restoreGenerationStageDisk holds a completion that is staging: as
-// xlStorage.RenameData does, a drive that stages writes the committed
-// object metadata into the upload before it renames the upload into the
-// object.
-type restoreGenerationStageDisk struct {
-	StorageAPI
-	bucket, object string
-	stage          bool
-	arrived        *atomic.Int32
-	entered        chan struct{}
-	release        chan struct{}
-	disks          int32
-}
-
-func (d restoreGenerationStageDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
-	if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object {
-		if d.stage {
-			if err := d.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil {
-				return RenameDataResp{}, err
-			}
-		}
-		if d.arrived.Add(1) == d.disks {
-			close(d.entered)
-		}
-		select {
-		case <-d.release:
-		case <-ctx.Done():
-			return RenameDataResp{}, ctx.Err()
-		}
-	}
-	return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
-}
-
-// While a completion stages its commit, the upload holds object metadata
-// without the marker on the drives that staged. Nobody lists, aborts or
-// removes it then, also when the copies not staged yet are old.
-func TestRestoreUploadStagingFailsClosed(t *testing.T) {
-	for _, mode := range []string{"staged", "partly-staged", "stalled-commit"} {
-		t.Run(mode, func(t *testing.T) { testRestoreUploadStaging(t, mode) })
-	}
-}
-
-func testRestoreUploadStaging(t *testing.T, mode string) {
-	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
-	initConfigSubsystem(t.Context(), r.z)
-	oi, body := restoreIsolationSeed(t, r, 0, false, true)
-	er := r.z.serverPools[0].getHashedSet(r.object)
-	disks := er.getDisks()
-	var arrived atomic.Int32
-	entered, release := make(chan struct{}), make(chan struct{})
-	released := false
-	defer func() {
-		if !released {
-			close(release)
-		}
-	}()
-	wrapped := make([]StorageAPI, len(disks))
-	for i, d := range disks {
-		wrapped[i] = restoreGenerationStageDisk{
-			StorageAPI: d, bucket: r.bucket, object: r.object, stage: mode != "partly-staged" || i == 0,
-			arrived: &arrived, entered: entered, release: release, disks: int32(len(disks)),
-		}
-	}
-	er.getDisks = func() []StorageAPI { return wrapped }
-	layer := newRestoreIsolationLayer(r)
-	if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
-		t.Fatal(rec.Body.String())
-	}
-	select {
-	case <-entered:
-	case <-time.After(restoreTestWait):
-		t.Fatal("completion never staged its commit")
-	}
-	stored := restoreIsolationUploads(t, r, 0)
-	var id string
-	for uploadUUID := range stored {
-		id = base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID))
-		if mode != "staged" {
-			// partly-staged: a long restore, the copies not staged yet are
-			// old. stalled-commit: drives that stall the running commit
-			// past the grace make its staged copies look abandoned.
-			old := UTCNow().Add(-48 * time.Hour)
-			for _, d := range disks[1:] {
-				if err := os.Chtimes(pathJoin(d.Endpoint().Path, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), uploadUUID, xlStorageFormatFile), old, old); err != nil {
-					t.Fatal(err)
-				}
-			}
-		}
-	}
-	if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10); err != nil || len(uploads.Uploads) != 0 {
-		t.Errorf("a staged restore upload is listed: %+v, %v", uploads.Uploads, err)
-	}
-	// The set's own abort: the pools layer would wait for the upload lock
-	// the completion holds.
-	if err := er.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); err == nil {
-		t.Error("abort of a staged restore upload succeeded")
-	}
-	globalAPIConfig.mu.Lock()
-	prev := globalAPIConfig.staleUploadsExpiry
-	globalAPIConfig.staleUploadsExpiry = time.Nanosecond
-	globalAPIConfig.mu.Unlock()
-	for _, disk := range wrapped {
-		er.cleanupStaleUploadsOnDisk(t.Context(), disk)
-	}
-	globalAPIConfig.mu.Lock()
-	globalAPIConfig.staleUploadsExpiry = prev
-	globalAPIConfig.mu.Unlock()
-	if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) {
-		t.Errorf("a staged restore upload was removed: %v -> %v", stored, after)
-	}
-	released = true
-	close(release)
-	if err := layer.wait(t); err != nil {
-		t.Fatalf("restore after its staging was attacked: %v", err)
-	}
-	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
-		t.Fatalf("restore contents: %v", err)
-	}
-}
-
-// When a drive of an acknowledged attempt becomes unreadable, the drives that
-// answer disagree on the restore status. That the attempt ended is then not
-// established, and stale cleanup keeps its upload.
-func TestRestoreSplitKeepsRunningUpload(t *testing.T) {
-	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
-	initConfigSubsystem(t.Context(), r.z)
-	oi, body := restoreIsolationSeed(t, r, 0, false, true)
-	er := r.z.serverPools[0].getHashedSet(r.object)
-	disks := er.getDisks()
-	var missed, offline atomic.Bool
-	gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})}
-	defer gate.unblock()
-	wrapped := make([]StorageAPI, len(disks))
-	for i, d := range disks {
-		var disk StorageAPI = restoreIsolationValidationDisk{d, gate}
-		switch i {
-		case 0:
-			disk = restoreGenerationDisk{StorageAPI: disk, bucket: r.bucket, object: r.object, write: &missed}
-		case 1:
-			disk = restoreGenerationOffline{StorageAPI: disk, bucket: r.bucket, object: r.object, fail: &offline}
-		}
-		wrapped[i] = disk
-	}
-	er.getDisks = func() []StorageAPI { return wrapped }
-	layer := newRestoreIsolationLayer(r)
-	missed.Store(true)
-	rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
-	missed.Store(false)
-	if rec.Code != http.StatusOK {
-		t.Fatal(rec.Body.String())
-	}
-	select {
-	case <-gate.entered:
-	case <-time.After(restoreTestWait):
-		t.Fatal("restore never reached completion")
-	}
-	stored := restoreIsolationUploads(t, r, 0)
-	offline.Store(true)
-	if gen, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); err != nil || !gen.split {
-		t.Fatalf("fixture: no split generation: %v", err)
-	}
-	globalAPIConfig.mu.Lock()
-	prev := globalAPIConfig.staleUploadsExpiry
-	globalAPIConfig.staleUploadsExpiry = time.Nanosecond
-	globalAPIConfig.mu.Unlock()
-	for _, disk := range wrapped {
-		er.cleanupStaleUploadsOnDisk(t.Context(), disk)
-	}
-	globalAPIConfig.mu.Lock()
-	globalAPIConfig.staleUploadsExpiry = prev
-	globalAPIConfig.mu.Unlock()
-	offline.Store(false)
-	if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) {
-		t.Errorf("stale cleanup removed a running restore's upload: %v -> %v", stored, after)
-	}
-	gate.unblock()
-	if err := layer.wait(t); err != nil {
-		t.Fatalf("restore after the split: %v", err)
-	}
-	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
-		t.Fatalf("restore contents: %v", err)
-	}
-}
-
 // restoreGenerationCleanupDisk misses restored-copy cleanups and metadata
 // writes of the object while its toggles are set.
 type restoreGenerationCleanupDisk struct {
@@ -836,7 +486,7 @@ func (d restoreGenerationCleanupDisk) WriteMetadata(ctx context.Context, orig, b
 	if bucket == d.bucket && object == d.object && d.metaWrites.Load() {
 		return errFaultyDisk
 	}
-	return d.WriteMetadata(ctx, orig, bucket, object, fi)
+	return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi)
 }
 
 // Restored-copy expiry decides on the version's authoritative generation. A
@@ -1033,268 +683,6 @@ func TestRestoreHealNeedsRestoreQuorum(t *testing.T) {
 	}
 }
 
-// restoreGenerationStageFails fails the commit before it stages.
-type restoreGenerationStageFails struct {
-	StorageAPI
-	bucket, object string
-}
-
-func (d restoreGenerationStageFails) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
-	if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object {
-		return RenameDataResp{}, errFaultyDisk
-	}
-	return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
-}
-
-// restoreGenerationFailedCommit stages the committed metadata into the
-// upload, as RenameData does, and then fails the commit.
-type restoreGenerationFailedCommit struct {
-	StorageAPI
-	bucket, object string
-}
-
-func (d restoreGenerationFailedCommit) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
-	if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object {
-		if err := d.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil {
-			return RenameDataResp{}, err
-		}
-		return RenameDataResp{}, errFaultyDisk
-	}
-	return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
-}
-
-// restoreGenerationLeftover runs a restore whose commit fails after it
-// staged on every drive, or on every drive but drive 0, which fails before
-// it stages. It returns the upload left behind and its on-disk UUID.
-func restoreGenerationLeftover(t *testing.T, r *restoreTest, unstaged bool) (map[string]int, string) {
-	t.Helper()
-	initConfigSubsystem(t.Context(), r.z)
-	oi, _ := restoreIsolationSeed(t, r, 0, false, true)
-	er := r.z.serverPools[0].getHashedSet(r.object)
-	disks := er.getDisks()
-	wrapped := make([]StorageAPI, len(disks))
-	for i, d := range disks {
-		if unstaged && i == 0 {
-			wrapped[i] = restoreGenerationStageFails{StorageAPI: d, bucket: r.bucket, object: r.object}
-			continue
-		}
-		wrapped[i] = restoreGenerationFailedCommit{StorageAPI: d, bucket: r.bucket, object: r.object}
-	}
-	er.getDisks = func() []StorageAPI { return wrapped }
-	layer := newRestoreIsolationLayer(r)
-	if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
-		t.Fatal(rec.Body.String())
-	}
-	if err := layer.wait(t); err == nil {
-		t.Fatal("fixture: the failed commit succeeded")
-	}
-	er.getDisks = func() []StorageAPI { return disks }
-	stored := restoreIsolationUploads(t, r, 0)
-	if len(stored) != 1 {
-		t.Fatalf("fixture: no leftover of the failed commit: %v", stored)
-	}
-	for uploadUUID := range stored {
-		return stored, uploadUUID
-	}
-	return nil, ""
-}
-
-// restoreGenerationAgeUpload makes every drive's copy of an upload's
-// metadata look written d ago.
-func restoreGenerationAgeUpload(t *testing.T, r *restoreTest, uploadUUID string, d time.Duration) {
-	t.Helper()
-	er := r.z.serverPools[0].getHashedSet(r.object)
-	old := UTCNow().Add(-d)
-	for _, disk := range er.getDisks() {
-		if err := os.Chtimes(pathJoin(disk.Endpoint().Path, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), uploadUUID, xlStorageFormatFile), old, old); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-// What a failed commit leaves in an upload can always be removed once every
-// copy is staged and older than the commit grace: abort removes it, also
-// when stale cleanup could not. A recent staging, a copy that cannot be read
-// and a copy not staged yet are refused, and nothing is removed then.
-func TestRestoreAbandonedCommitAbort(t *testing.T) {
-	for _, age := range []string{"recent", "abandoned", "unreadable-copy", "unstaged-copy"} {
-		t.Run(age, func(t *testing.T) {
-			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
-			stored, uploadUUID := restoreGenerationLeftover(t, r, age == "unstaged-copy")
-			if age != "recent" {
-				restoreGenerationAgeUpload(t, r, uploadUUID, 2*restoreCommitGrace)
-			}
-			id := base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID))
-			er := r.z.serverPools[0].getHashedSet(r.object)
-			disks := er.getDisks()
-			var unreadable atomic.Bool
-			if age == "unreadable-copy" {
-				wrapped := append([]StorageAPI(nil), disks...)
-				wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], uploadRead: &unreadable}
-				er.getDisks = func() []StorageAPI { return wrapped }
-				unreadable.Store(true)
-			}
-			err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{})
-			unreadable.Store(false)
-			left := restoreIsolationUploads(t, r, 0)
-			if age != "abandoned" {
-				if err == nil || !reflect.DeepEqual(left, stored) {
-					t.Errorf("abort of an upload that may still commit (%s): %v, left %v", age, err, left)
-				}
-				return
-			}
-			if err != nil || len(left) != 0 {
-				t.Errorf("abort of an abandoned commit: %v, left %v", err, left)
-			}
-		})
-	}
-}
-
-// With no commit running, stale cleanup removes what a failed commit left
-// drive by drive: a staged copy older than the commit grace. A recent
-// staging and a copy that never staged stay.
-func TestRestoreFailedCommitStaleCleanup(t *testing.T) {
-	for _, mode := range []string{"recent", "abandoned", "unstaged-copy"} {
-		t.Run(mode, func(t *testing.T) {
-			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
-			stored, uploadUUID := restoreGenerationLeftover(t, r, mode == "unstaged-copy")
-			if mode != "recent" {
-				// The copy that never staged is as old as the others.
-				restoreGenerationAgeUpload(t, r, uploadUUID, 2*restoreCommitGrace)
-			}
-			er := r.z.serverPools[0].getHashedSet(r.object)
-			globalAPIConfig.mu.Lock()
-			prev := globalAPIConfig.staleUploadsExpiry
-			globalAPIConfig.staleUploadsExpiry = time.Nanosecond
-			globalAPIConfig.mu.Unlock()
-			for _, disk := range er.getDisks() {
-				er.cleanupStaleUploadsOnDisk(t.Context(), disk)
-			}
-			globalAPIConfig.mu.Lock()
-			globalAPIConfig.staleUploadsExpiry = prev
-			globalAPIConfig.mu.Unlock()
-			want := map[string]int{}
-			switch mode {
-			case "recent":
-				want = stored
-			case "unstaged-copy":
-				want = map[string]int{uploadUUID: 1}
-			}
-			if left := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(left, want) {
-				t.Errorf("stale cleanup of a failed commit's leftover: %v -> %v, want %v", stored, left, want)
-			}
-		})
-	}
-}
-
-// restoreGenerationSnapshotKey marks the context of a stale cleanup whose
-// reads of upload metadata restoreGenerationSnapshotDisk holds.
-type restoreGenerationSnapshotKey struct{}
-
-// restoreGenerationSnapshotDisk reads upload metadata at once, and hands the
-// reply to a marked stale cleanup only on release.
-type restoreGenerationSnapshotDisk struct {
-	StorageAPI
-	reads   *atomic.Int32
-	release chan struct{}
-}
-
-func (d restoreGenerationSnapshotDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) {
-	fi, err := d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
-	if volume == minioMetaMultipartBucket && ctx.Value(restoreGenerationSnapshotKey{}) != nil {
-		d.reads.Add(1)
-		select {
-		case <-d.release:
-		case <-ctx.Done():
-			return FileInfo{}, ctx.Err()
-		}
-	}
-	return fi, err
-}
-
-// Stale cleanup decides on a removal under the commit lock. A completion
-// that starts while cleanup reads an upload past the stale age waits for
-// it, and a cleanup that comes later skips the running commit: the upload,
-// an ordinary one, still commits with its contents.
-func TestCleanupStaleUploadsSkipsRunningCommit(t *testing.T) {
-	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
-	initConfigSubsystem(t.Context(), r.z)
-	er := r.z.serverPools[0].getHashedSet(r.object)
-	mp, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true, MTime: UTCNow().Add(-48 * time.Hour)})
-	if err != nil {
-		t.Fatal(err)
-	}
-	body := []byte("an ordinary upload past the stale age")
-	part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	disks := er.getDisks()
-	var reads, staged atomic.Int32
-	readRelease := make(chan struct{})
-	stageEntered, stageRelease := make(chan struct{}), make(chan struct{})
-	var readOnce, stageOnce sync.Once
-	defer readOnce.Do(func() { close(readRelease) })
-	defer stageOnce.Do(func() { close(stageRelease) })
-	wrapped := make([]StorageAPI, len(disks))
-	for i, d := range disks {
-		stage := restoreGenerationStageDisk{
-			StorageAPI: d, bucket: r.bucket, object: r.object, stage: true,
-			arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks)),
-		}
-		wrapped[i] = restoreGenerationSnapshotDisk{StorageAPI: stage, reads: &reads, release: readRelease}
-	}
-	er.getDisks = func() []StorageAPI { return wrapped }
-	globalAPIConfig.mu.Lock()
-	prev := globalAPIConfig.staleUploadsExpiry
-	globalAPIConfig.staleUploadsExpiry = time.Nanosecond
-	globalAPIConfig.mu.Unlock()
-	defer func() {
-		globalAPIConfig.mu.Lock()
-		globalAPIConfig.staleUploadsExpiry = prev
-		globalAPIConfig.mu.Unlock()
-	}()
-	// Every drive's cleanup at once, each reading the upload on every drive.
-	ctx := context.WithValue(t.Context(), restoreGenerationSnapshotKey{}, true)
-	var cleanups sync.WaitGroup
-	for _, disk := range wrapped {
-		cleanups.Go(func() { er.cleanupStaleUploadsOnDisk(ctx, disk) })
-	}
-	cleanupDone := make(chan struct{})
-	go func() { cleanups.Wait(); close(cleanupDone) }()
-	// The completion starts once every cleanup that may decide holds its
-	// read of the upload from before the commit.
-	for deadline := time.Now().Add(2 * time.Second); reads.Load() < int32(len(disks)*len(disks)) && time.Now().Before(deadline); {
-		time.Sleep(10 * time.Millisecond)
-	}
-	completion := make(chan ObjectInfo, 1)
-	completionErr := make(chan error, 1)
-	go func() {
-		oi, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true})
-		completion <- oi
-		completionErr <- err
-	}()
-	select {
-	case <-stageEntered:
-	case <-time.After(3 * time.Second):
-	}
-	readOnce.Do(func() { close(readRelease) })
-	select {
-	case <-cleanupDone:
-	case <-time.After(restoreTestWait):
-		t.Fatal("stale cleanup never returned")
-	}
-	stageOnce.Do(func() { close(stageRelease) })
-	oi := <-completion
-	if err := <-completionErr; err != nil {
-		t.Fatalf("completion of an upload stale cleanup read: %v", err)
-	}
-	er.getDisks = func() []StorageAPI { return disks }
-	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
-		t.Fatalf("completed contents: %q, %v", got, err)
-	}
-}
-
 // restoreGenerationRand draws UUIDs from crypto/rand for the whole test
 // binary, installed before any goroutine runs. While a pause is set, it holds
 // the UUID that stale cleanup draws in its deadline-bound worker, for the
@@ -1334,91 +722,3 @@ func restoreGenerationInCleanupRename() bool {
 		}
 	}
 }
-
-// Stale cleanup holds the commit lock until its rename ends, also when the
-// rename outlives the drive deadline and cleanup has moved on: a completion
-// does not stage while that rename can still remove its upload.
-func TestCleanupStaleUploadsHoldsLockThroughRename(t *testing.T) {
-	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
-	initConfigSubsystem(t.Context(), r.z)
-	er := r.z.serverPools[0].getHashedSet(r.object)
-	mp, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true, MTime: UTCNow().Add(-48 * time.Hour)})
-	if err != nil {
-		t.Fatal(err)
-	}
-	body := []byte("an upload whose stale cleanup outlives the drive deadline")
-	part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
-	if err != nil {
-		t.Fatal(err)
-	}
-	disks := er.getDisks()
-	stored := restoreIsolationUploads(t, r, 0)
-	prev := globalDriveConfig.GetMaxTimeout()
-	// The shortest drive deadline the configuration takes.
-	globalDriveConfig.Update(drive.Config{MaxTimeout: 1100 * time.Millisecond})
-	defer globalDriveConfig.Update(drive.Config{MaxTimeout: prev})
-	pause := &restoreGenerationPause{entered: make(chan struct{}), release: make(chan struct{})}
-	var paused sync.Once
-	defer paused.Do(func() { close(pause.release) })
-	restoreGenerationRand.pause.Store(pause)
-	defer restoreGenerationRand.pause.CompareAndSwap(pause, nil)
-
-	cleanupDone := make(chan struct{})
-	go func() {
-		er.cleanupStaleUploadsOnDisk(t.Context(), disks[0])
-		close(cleanupDone)
-	}()
-	select {
-	case <-pause.entered:
-	case <-time.After(5 * time.Second):
-		t.Fatal("stale cleanup never reached its rename")
-	}
-	select {
-	case <-cleanupDone:
-	case <-time.After(5 * time.Second):
-		t.Fatal("stale cleanup did not move on after the drive deadline")
-	}
-	if left := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(left, stored) {
-		t.Fatalf("fixture: the held rename already ran: %v -> %v", stored, left)
-	}
-
-	var staged atomic.Int32
-	stageEntered, stageRelease := make(chan struct{}), make(chan struct{})
-	var stageOnce sync.Once
-	defer stageOnce.Do(func() { close(stageRelease) })
-	wrapped := make([]StorageAPI, len(disks))
-	for i, d := range disks {
-		wrapped[i] = restoreGenerationStageDisk{
-			StorageAPI: d, bucket: r.bucket, object: r.object, stage: true,
-			arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks)),
-		}
-	}
-	er.getDisks = func() []StorageAPI { return wrapped }
-	completion := make(chan ObjectInfo, 1)
-	completionErr := make(chan error, 1)
-	go func() {
-		oi, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true})
-		completion <- oi
-		completionErr <- err
-	}()
-	select {
-	case <-stageEntered:
-		t.Error("a completion staged while stale cleanup's rename could still run")
-	case <-time.After(2 * time.Second):
-	}
-	paused.Do(func() { close(pause.release) })
-	select {
-	case <-stageEntered:
-	case <-time.After(restoreTestWait):
-		t.Fatal("completion never staged after the rename ended")
-	}
-	stageOnce.Do(func() { close(stageRelease) })
-	oi := <-completion
-	if err := <-completionErr; err != nil {
-		t.Fatalf("completion after the rename ended: %v", err)
-	}
-	er.getDisks = func() []StorageAPI { return disks }
-	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
-		t.Fatalf("completed contents: %q, %v", got, err)
-	}
-}