Merge lifecycle expiry, native restore and lock-domain fixes (#278, #281, #284)

Supersedes PR #279 and PR #282 and keeps their contributor commits. Native restores stage outside the object lock and commit under a short object lock (#284); ordinary multipart completion returns to main's behavior.

.github/workflows/go.yml

@@ -96,10 +96,15 @@ jobs:
       - name: Run multi-pool delete consistency tests under race detector
         run: go test -race ./cmd -run '^Test(Pools(Delete|BulkDelete|RetiringCopyPreservesSharedTierObject)|BulkDeleteQuorumEvidence)' -count=1 -timeout=5m
 
+      - name: Run lifecycle expiry and native restore tests under race detector
+        env:
+          MINIO_API_REQUESTS_MAX: "10000"
+        run: go test -race -tags kqueue,dev ./cmd -run '^Test(PoolsLifecycle|PoolsMigrationLifecycle|ExpireRestored|NativeRestore)' -count=1 -timeout=5m
+
       - name: Run multipart listing, cancellation and cleanup tests under race detector
         env:
           MINIO_API_REQUESTS_MAX: "10000"
-        run: go test -race ./cmd -run '^Test(Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m
+        run: go test -race ./cmd -run '^Test(ObjectCompleteMultipartUpload|Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m
 
       - name: Run listing metadata recovery, pagination and cancellation tests under race detector
         run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity|ListObjectVersionsCrossParityNull|MigrationCrossParity)' -count=1 -timeout=5m

buildscripts/rebrand-guard/compat-baseline.json

@@ -594,11 +594,13 @@
     "x-minio-from-peer",
     "x-minio-healing-drives",
     "x-minio-internal",
+    "x-minio-internal-a",
     "x-minio-internal-actual-object-size",
     "x-minio-internal-actual-size",
     "x-minio-internal-archive-info",
     "x-minio-internal-archive-type",
     "x-minio-internal-b",
+    "x-minio-internal-c",
     "x-minio-internal-compression",
     "x-minio-internal-encrypted-multipart",
     "x-minio-internal-encryptedmultipart",
@@ -611,6 +613,8 @@
     "x-minio-internal-replication-reset-arn",
     "x-minio-internal-replication-status",
     "x-minio-internal-replication-timestamp",
+    "x-minio-internal-restore-attempt",
+    "x-minio-internal-restore-upload",
     "x-minio-internal-server-side-encryption",
     "x-minio-internal-server-side-encryption-context",
     "x-minio-internal-server-side-encryption-iv",

cmd/batch-rotate.go

@@ -180,7 +180,7 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
 	versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
 
-	lock := api.NewNSLock(r.Bucket, objInfo.Name)
+	lock := api.NewObjectNSLock(r.Bucket, objInfo.Name)
 	lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return err

cmd/bucket-lifecycle-restore-boundary_test.go

@@ -0,0 +1,277 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+)
+
+// Read the actual metadata from every disk without object-layer read repair.
+func restoreBoundarySnapshot(t *testing.T, r *restoreTest) [][]byte {
+	t.Helper()
+	var result [][]byte
+	for _, pool := range r.z.serverPools {
+		for _, disk := range pool.getHashedSet(r.object).getDisks() {
+			raw, err := disk.ReadXL(t.Context(), r.bucket, r.object, true)
+			if err != nil && err != errFileNotFound {
+				t.Fatal(err)
+			}
+			result = append(result, bytes.Clone(raw.Buf))
+		}
+	}
+	return result
+}
+
+func TestExpireRestoredKeepsReplacementDeleteMarker(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, pools, versioning.Suspended)
+			r.put("old", false, false)
+			oi := r.put("null", true, true)
+			r.restore(oi)
+			stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(nullVersionID))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, r.opts("")); err != nil {
+				t.Fatal(err)
+			}
+			if !r.fileInfo(nullVersionID).Deleted {
+				t.Fatal("ordinary DELETE did not replace null with a delete marker")
+			}
+			before := restoreBoundarySnapshot(t, r)
+			err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+			if !isErrVersionNotFound(err) {
+				t.Errorf("stale cleanup should reject the replacement marker: %v", err)
+			}
+			if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+				t.Error("stale cleanup changed stored versions or their data")
+			}
+			if _, _, err := r.read(""); !isErrObjectNotFound(err) {
+				t.Errorf("stale cleanup exposed the old version: %v", err)
+			}
+			// An explicit user purge must still remove that same marker.
+			if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, r.opts(nullVersionID)); err != nil {
+				t.Fatal(err)
+			}
+			if label, data, err := r.read(""); err != nil || label != "old" || !bytes.Equal(data, r.bodies["old"]) {
+				t.Fatalf("ordinary marker purge: label=%s err=%v", label, err)
+			}
+		})
+	}
+}
+
+func TestExpireRestoredMetadataReadKeepsMinorityVersion(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, operation := range []string{"cleanup", "restore", "restore-cleanup", "lookup-cleanup", "lookup-restore", "get", "delete"} {
+			t.Run(fmt.Sprintf("pools=%d/%s", pools, operation), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+				oi := r.put("target", false, true)
+				r.restore(oi)
+				fi := r.fileInfo(oi.VersionID)
+				// A readable latest version lets pools lookup finish before the
+				// set reads the specifically addressed, incomplete old version.
+				r.put("newer", false, false)
+				er := r.z.serverPools[0].getHashedSet(r.object)
+				disks := er.getDisks()
+				for _, disk := range disks[1:] {
+					raw, err := disk.ReadXL(t.Context(), r.bucket, r.object, true)
+					if err != nil {
+						t.Fatal(err)
+					}
+					var meta xlMetaV2
+					if err := meta.Load(raw.Buf); err != nil {
+						t.Fatal(err)
+					}
+					del := FileInfo{VersionID: oi.VersionID}
+					del.SetSkipTierFreeVersion()
+					if _, err := meta.DeleteVersion(del); err != nil {
+						t.Fatal(err)
+					}
+					buf, err := meta.AppendTo(nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+					// Remove only this metadata entry; retain every real shard.
+					if err := disk.WriteAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile), buf); err != nil {
+						t.Fatal(err)
+					}
+				}
+				before := restoreBoundarySnapshot(t, r)
+				ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+				defer cancel()
+				opts := r.opts(oi.VersionID)
+				var err error
+				switch operation {
+				case "cleanup":
+					err = expireTransitionedObject(ctx, r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredVersionAction}, lcEventSrc_Scanner)
+				case "restore":
+					opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+					err = r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, opts)
+				case "restore-cleanup":
+					// The initial read succeeded before remote download failed;
+					// cleanup must not purge a now-incomplete target on reread.
+					err = er.clearRestoreStatus(ctx, r.bucket, r.object, oi.VersionID, nil)
+				case "lookup-cleanup", "lookup-restore":
+					// Version-sensitive lookup also covers native restore routing.
+					opts.MetadataChg = true
+					if operation == "lookup-cleanup" {
+						opts.Transition.ExpireRestored = true
+					} else {
+						opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+					}
+					_, _, err = r.z.getPoolInfoExistingWithOpts(ctx, r.bucket, r.object, opts)
+				case "get":
+					_, err = r.z.GetObjectInfo(ctx, r.bucket, r.object, opts)
+				case "delete":
+					_, err = r.z.DeleteObject(ctx, r.bucket, r.object, opts)
+				}
+				if ctx.Err() != nil {
+					t.Fatalf("%s timed out instead of returning its read error: %v", operation, err)
+				}
+				if err == nil {
+					t.Fatal("incomplete target unexpectedly readable")
+				}
+				if operation != "get" && operation != "delete" {
+					if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+						t.Error("failed restore read changed metadata, inline data or free versions")
+					}
+					if n := r.dataDirs(fi.DataDir); n != len(disks) {
+						t.Errorf("failed restore read removed shards: %d/%d directories remain", n, len(disks))
+					}
+					return
+				}
+				// Ordinary GET/DELETE retain their existing dangling cleanup.
+				raw, err := disks[0].ReadXL(t.Context(), r.bucket, r.object, false)
+				if err != nil {
+					t.Fatal(err)
+				}
+				versions, err := getFileInfoVersions(raw.Buf, r.bucket, r.object, false)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if len(versions.Versions) != 1 || len(versions.FreeVersions) != 1 || r.dataDirs(fi.DataDir) != len(disks)-1 {
+					t.Fatal("ordinary dangling cleanup no longer removed the minority version")
+				}
+			})
+		}
+	}
+}
+
+func TestNativeRestoreInitialMissingReadDoesNotWrite(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	r.put("live", false, false)
+	before := restoreBoundarySnapshot(t, r)
+	opts := r.opts(mustGetUUID())
+	opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+	if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, opts); !isErrVersionNotFound(err) {
+		t.Fatalf("missing restore target: %v", err)
+	}
+	if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+		t.Fatal("initial restore failure wrote metadata without a resolved version")
+	}
+}
+
+func TestExpireRestoredStorageRejectsOtherTypes(t *testing.T) {
+	for _, kind := range []string{"null-marker", "uuid-marker", "legacy", "legacy-json"} {
+		t.Run(kind, func(t *testing.T) {
+			disk, _, err := newXLStorageTestSetup(t)
+			if err != nil {
+				t.Fatal(err)
+			}
+			ctx := t.Context()
+			const bucket, object = "restore-types", "obj"
+			if err := disk.MakeVol(ctx, bucket); err != nil {
+				t.Fatal(err)
+			}
+			var meta xlMetaV2
+			versionID, metaName, dataDir := nullVersionID, xlStorageFormatFile, ""
+			var buf []byte
+			switch kind {
+			case "null-marker", "uuid-marker":
+				if kind == "uuid-marker" {
+					versionID = mustGetUUID()
+				}
+				err = meta.AddVersion(FileInfo{VersionID: versionID, Deleted: true, ModTime: time.Now().UTC()})
+			case "legacy", "legacy-json":
+				legacy := getSampleXLMeta(1)
+				legacy.DataDir, dataDir = "legacy", "legacy"
+				if kind == "legacy-json" {
+					metaName = xlStorageFormatFileV1
+					buf, err = json.Marshal(legacy)
+					disk.storage.Lock()
+					disk.storage.formatLegacy = true
+					disk.storage.Unlock()
+				} else {
+					err = meta.AddLegacy(&legacy)
+				}
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if buf == nil {
+				buf, err = meta.AppendTo(nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			if err := disk.WriteAll(ctx, bucket, pathJoin(object, metaName), buf); err != nil {
+				t.Fatal(err)
+			}
+			payload := []byte("sole legacy contents")
+			dataPath := filepath.Join(disk.Endpoint().Path, bucket, object, dataDir, "part.1")
+			if dataDir != "" {
+				if err := os.MkdirAll(filepath.Dir(dataPath), 0o700); err != nil {
+					t.Fatal(err)
+				}
+				if err := os.WriteFile(dataPath, payload, 0o600); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if err := disk.DeleteVersion(ctx, bucket, object, FileInfo{VersionID: versionID, ExpireRestored: true, TransitionStatus: lifecycle.TransitionComplete}, false, DeleteOptions{}); err != nil {
+				t.Fatal(err)
+			}
+			after, err := disk.ReadAll(ctx, bucket, pathJoin(object, metaName))
+			if err != nil || !bytes.Equal(buf, after) {
+				t.Errorf("restore cleanup changed %s metadata: %v", kind, err)
+			}
+			if dataDir != "" {
+				if data, err := os.ReadFile(dataPath); err != nil || !bytes.Equal(data, payload) {
+					t.Errorf("restore cleanup removed legacy contents: %v", err)
+				}
+			}
+			if err := disk.DeleteVersion(ctx, bucket, object, FileInfo{VersionID: versionID}, false, DeleteOptions{}); err != nil {
+				t.Fatal(err)
+			}
+			if _, err := disk.ReadAll(ctx, bucket, pathJoin(object, metaName)); err != errFileNotFound {
+				t.Errorf("ordinary purge left %s: %v", kind, err)
+			}
+		})
+	}
+}

cmd/bucket-lifecycle-restore-safety_test.go

@@ -0,0 +1,284 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+const restoreSafetyRules = `<LifecycleConfiguration><Rule><ID>tier</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Transition><Days>1000</Days><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>`
+
+// Reuse the original restore-copy fixture with a small, isolated real cluster.
+func newRestoreSafetyTest(t *testing.T, pools int, state versioning.State) *restoreTest {
+	t.Helper()
+	prevTier := globalTierConfigMgr
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+	dirs, err := getRandomDisks(4 * pools)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { removeRoots(dirs) })
+	var endpoints EndpointServerPools
+	for i := range pools {
+		endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
+	}
+	obj, _, err := initObjectLayer(t.Context(), endpoints)
+	if err != nil {
+		t.Fatal(err)
+	}
+	z := obj.(*erasureServerPools)
+	t.Cleanup(func() { z.Shutdown(context.Background()) })
+	globalTierConfigMgr = NewTierConfigMgr()
+	globalTierConfigMgr.drivercache["WARM"] = &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
+	r := &restoreTest{t: t, z: z, bucket: getRandomBucketName(), object: "obj", labels: map[string]string{}, bodies: map[string][]byte{}}
+	if err := z.MakeBucket(t.Context(), r.bucket, MakeBucketOptions{VersioningEnabled: state != ""}); err != nil {
+		t.Fatal(err)
+	}
+	meta, err := globalBucketMetadataSys.Get(r.bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if state == versioning.Suspended {
+		meta.VersioningConfigXML = []byte(`<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>`)
+	}
+	meta.LifecycleConfigXML = []byte(restoreSafetyRules)
+	if err := meta.Save(t.Context(), z); err != nil {
+		t.Fatal(err)
+	}
+	globalBucketMetadataSys.Set(r.bucket, meta)
+	return r
+}
+
+type restoreCleanupFaultDisk struct {
+	consistencyReadFaultDisk
+	deletes *atomic.Int64
+}
+
+func (d restoreCleanupFaultDisk) DeleteVersion(ctx context.Context, bucket, object string, fi FileInfo, force bool, opts DeleteOptions) error {
+	if bucket == d.bucket && object == d.object {
+		d.deletes.Add(1)
+	}
+	return d.StorageAPI.DeleteVersion(ctx, bucket, object, fi, force, opts)
+}
+
+func TestExpireRestoredRequiresFreshMetadata(t *testing.T) {
+	for _, restored := range []bool{true, false} {
+		t.Run(fmt.Sprintf("expire-restored=%v", restored), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, "")
+			oi := r.put("v1", true, true)
+			r.restore(oi)
+			before := r.fileInfo(oi.VersionID)
+			er := r.z.serverPools[0].getHashedSet(r.object)
+			original := er.getDisks
+			disks := original()
+			var deletes atomic.Int64
+			faulty := make([]StorageAPI, len(disks))
+			for i, d := range disks {
+				faulty[i] = restoreCleanupFaultDisk{consistencyReadFaultDisk{d, r.bucket, r.object}, &deletes}
+			}
+			er.getDisks = func() []StorageAPI { return faulty }
+			// Restore the callback before any fixture reads and on a fatal assertion.
+			defer func() { er.getDisks = original }()
+			opts := ObjectOptions{VersionID: nullVersionID, Transition: TransitionOptions{ExpireRestored: restored}, Expiration: ExpirationOptions{Expire: restored}}
+			_, err := er.DeleteObject(t.Context(), r.bucket, r.object, opts)
+			er.getDisks = original
+			if restored {
+				var quorum InsufficientWriteQuorum
+				if !errors.As(err, &quorum) {
+					t.Errorf("cleanup without fresh metadata: got %v, want write-quorum error", err)
+				}
+				if n := deletes.Load(); n != 0 {
+					t.Errorf("cleanup attempted %d blind disk deletes", n)
+				}
+				if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before, after) {
+					t.Error("failed metadata read changed the stored version")
+				}
+				if n := r.dataDirs(before.DataDir); n != len(disks) {
+					t.Errorf("cleanup removed data: %d/%d directories remain", n, len(disks))
+				}
+			} else {
+				// Ordinary unversioned DELETE retains the existing tryDel behavior.
+				if err != nil || deletes.Load() != int64(len(disks)) {
+					t.Fatalf("ordinary DELETE: err=%v, deletes=%d", err, deletes.Load())
+				}
+				if _, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{}); !isErrObjectNotFound(err) {
+					t.Fatalf("ordinary DELETE left the object: %v", err)
+				}
+			}
+		})
+	}
+}
+
+func TestExpireRestoredRenewedOrMissing(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	oi := r.put("v1", false, true)
+	r.restore(oi)
+	stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+	if err != nil {
+		t.Fatal(err)
+	}
+	before := r.fileInfo(oi.VersionID)
+	renewed := completedRestoreObj(time.Now().Add(48 * time.Hour)).String()
+	opts := r.opts(oi.VersionID)
+	opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+		o.UserDefined[xhttp.AmzRestore] = renewed
+		return ReplicateDecision{}, nil
+	}
+	if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil {
+		t.Fatal(err)
+	}
+	err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+	if !errors.Is(err, errLifecycleNotApplicable) {
+		t.Errorf("stale task should be rejected, got %v", err)
+	}
+	if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed || got.DataDir != before.DataDir {
+		t.Error("stale task changed renewed restore")
+	}
+	if n := r.dataDirs(before.DataDir); n != 4 {
+		t.Errorf("renewed copy has %d directories, want 4", n)
+	}
+	if _, data, err := r.read(oi.VersionID); err != nil || !bytes.Equal(data, r.bodies["v1"]) {
+		t.Fatalf("renewed copy not readable: %v", err)
+	}
+
+	for _, id := range []string{mustGetUUID(), nullVersionID} {
+		opts := r.opts(id)
+		opts.Expiration.Expire = true
+		opts.Transition.ExpireRestored = true
+		if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, opts); !isErrVersionNotFound(err) {
+			t.Errorf("missing version %s: %v", id, err)
+		}
+	}
+	missing := r.opts(nullVersionID)
+	missing.Expiration.Expire = true
+	missing.Transition.ExpireRestored = true
+	if _, err := r.z.DeleteObject(t.Context(), r.bucket, "missing-object", missing); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+		t.Errorf("missing object: %v", err)
+	}
+	if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed {
+		t.Error("missing-target cleanup changed existing restore")
+	}
+}
+
+func TestExpireRestoredStorageBoundary(t *testing.T) {
+	for _, status := range []string{"", lifecycle.TransitionPending, lifecycle.TransitionComplete} {
+		for _, inline := range []bool{false, true} {
+			for _, requestStatus := range []string{"", lifecycle.TransitionComplete} {
+				t.Run(fmt.Sprintf("stored=%s/inline=%v/request=%s", status, inline, requestStatus), func(t *testing.T) {
+					disk, _, err := newXLStorageTestSetup(t)
+					if err != nil {
+						t.Fatal(err)
+					}
+					const bucket, object = "restore-storage", "obj"
+					ctx := t.Context()
+					if err := disk.MakeVol(ctx, bucket); err != nil {
+						t.Fatal(err)
+					}
+					fi := newFileInfo(object, 2, 2)
+					fi.VersionID, fi.DataDir = mustGetUUID(), mustGetUUID()
+					fi.ModTime, fi.Size = time.Now().UTC(), 1024
+					fi.TransitionStatus = status
+					fi.AddObjectPart(1, "", fi.Size, fi.Size, fi.ModTime, nil, nil)
+					fi.Metadata = map[string]string{xhttp.AmzRestore: completedRestoreObj(time.Now().Add(-time.Hour)).String(), "x-amz-meta-preserve": "keep"}
+					payload := []byte("sole local contents")
+					if inline {
+						fi.Data = payload
+						fi.SetInlineData()
+					}
+					if err := disk.WriteMetadata(ctx, "", bucket, object, fi); err != nil {
+						t.Fatal(err)
+					}
+					dir := filepath.Join(disk.Endpoint().Path, bucket, object, fi.DataDir)
+					if err := os.MkdirAll(dir, 0o700); err != nil {
+						t.Fatal(err)
+					}
+					if err := os.WriteFile(filepath.Join(dir, "part.1"), payload, 0o600); err != nil {
+						t.Fatal(err)
+					}
+					raw, err := disk.ReadXL(ctx, bucket, object, true)
+					if err != nil {
+						t.Fatal(err)
+					}
+					var xl xlMetaV2
+					if err := xl.Load(raw.Buf); err != nil {
+						t.Fatal(err)
+					}
+					del := FileInfo{VersionID: fi.VersionID, ExpireRestored: true, TransitionStatus: requestStatus}
+					dataDir, err := xl.DeleteVersion(del)
+					if err != nil {
+						t.Fatal(err)
+					}
+					wantDir := ""
+					if status == lifecycle.TransitionComplete {
+						wantDir = fi.DataDir
+					}
+					if dataDir != wantDir {
+						t.Errorf("format authorized directory %q, want %q", dataDir, wantDir)
+					}
+					before, err := getFileInfo(raw.Buf, bucket, object, fi.VersionID, fileInfoOpts{Data: true})
+					if err != nil {
+						t.Fatal(err)
+					}
+					if err := disk.DeleteVersion(ctx, bucket, object, del, false, DeleteOptions{}); err != nil {
+						t.Fatal(err)
+					}
+					afterRaw, err := disk.ReadXL(ctx, bucket, object, true)
+					if err != nil {
+						t.Fatal(err)
+					}
+					after, err := getFileInfo(afterRaw.Buf, bucket, object, fi.VersionID, fileInfoOpts{Data: true})
+					if err != nil {
+						t.Fatal(err)
+					}
+					data, dirErr := os.ReadFile(filepath.Join(dir, "part.1"))
+					if status != lifecycle.TransitionComplete {
+						if !reflect.DeepEqual(before, after) {
+							t.Error("non-complete cleanup changed metadata or inline data")
+						}
+						if dirErr != nil || !bytes.Equal(data, payload) {
+							t.Errorf("sole local contents removed: %v", dirErr)
+						}
+					} else {
+						if !os.IsNotExist(dirErr) {
+							t.Errorf("completed restore directory still present: %v", dirErr)
+						}
+						if len(after.Data) != 0 || after.Metadata[xhttp.AmzRestore] != "" {
+							t.Error("completed restore kept cached data or restore headers")
+						}
+						if after.TransitionStatus != status || after.VersionID != before.VersionID || after.Metadata["x-amz-meta-preserve"] != "keep" {
+							t.Error("cleanup changed the transitioned version identity or user metadata")
+						}
+					}
+				})
+			}
+		}
+	}
+}

cmd/bucket-lifecycle-restore_test.go

@@ -0,0 +1,453 @@
+// Copyright (c) 2026 Denis Babchuk
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// restoreTestTier is a remote tier held in memory that counts its reads.
+type restoreTestTier struct {
+	*tierExpiryTestBackend
+	gets atomic.Int64
+}
+
+func (b *restoreTestTier) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+	b.gets.Add(1)
+	return b.tierExpiryTestBackend.Get(ctx, object, rv, opts)
+}
+
+// restoreTest is a cluster with a remote tier WARM and a bucket whose
+// versions are transitioned to it and restored.
+type restoreTest struct {
+	t      *testing.T
+	z      *erasureServerPools
+	bucket string
+	object string
+	queue  chan expiryOp
+	labels map[string]string // version ID ("null" for the null version) -> label
+	bodies map[string][]byte // label -> data
+}
+
+func restoreTestVersionKey(versionID string) string {
+	if versionID == "" {
+		return nullVersionID
+	}
+	return versionID
+}
+
+// put writes a version of the object to the first pool. A restored version
+// is 1 MiB, so that its restored copy has a data directory of its own.
+func (r *restoreTest) put(label string, null, restored bool) ObjectInfo {
+	r.t.Helper()
+	body := []byte(r.object + " " + label)
+	if restored {
+		body = bytes.Repeat(body, (1<<20)/len(body)+1)
+	}
+	opts := ObjectOptions{Versioned: !null}
+	if null {
+		opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object)
+	}
+	oi, err := r.z.serverPools[0].PutObject(r.t.Context(), r.bucket, r.object,
+		mustGetPutObjReader(r.t, bytes.NewReader(body), int64(len(body)), "", ""), opts)
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	r.labels[restoreTestVersionKey(oi.VersionID)] = label
+	r.bodies[label] = body
+	return oi
+}
+
+// opts returns the options of a request on a version of the object, as the
+// S3 handlers set them.
+func (r *restoreTest) opts(versionID string) ObjectOptions {
+	return ObjectOptions{
+		VersionID:        versionID,
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(r.bucket, r.object),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object),
+	}
+}
+
+// restore transitions a version to the tier, restores it, and moves the
+// expiry date of the restore into the past.
+func (r *restoreTest) restore(oi ObjectInfo) {
+	r.t.Helper()
+	ctx := r.t.Context()
+	vopts := r.opts(oi.VersionID)
+	topts := vopts
+	topts.MTime = oi.ModTime
+	topts.Transition = TransitionOptions{Tier: "WARM", ETag: oi.ETag}
+	if err := r.z.TransitionObject(ctx, r.bucket, r.object, topts); err != nil {
+		r.t.Fatal(err)
+	}
+	ropts := vopts
+	ropts.Transition = TransitionOptions{RestoreRequest: &RestoreObjectRequest{Days: 1}}
+	// Through the pool, as with a single pool: with more than one pool the
+	// pools layer holds the object lock that the write of the restored copy
+	// waits for.
+	if err := r.z.serverPools[0].RestoreTransitionedObject(ctx, r.bucket, r.object, ropts); err != nil {
+		r.t.Fatal(err)
+	}
+	expired := completedRestoreObj(time.Now().Add(-24 * time.Hour).UTC()).String()
+	mopts := vopts
+	mopts.MTime = oi.ModTime
+	mopts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+		o.UserDefined[xhttp.AmzRestore] = expired
+		return ReplicateDecision{}, nil
+	}
+	if _, err := r.z.PutObjectMetadata(ctx, r.bucket, r.object, mopts); err != nil {
+		r.t.Fatal(err)
+	}
+}
+
+// fileInfo returns the version of the object from a drive of the first pool.
+func (r *restoreTest) fileInfo(versionID string) FileInfo {
+	r.t.Helper()
+	infos, errs := readAllFileInfo(r.t.Context(), r.z.serverPools[0].getHashedSet(r.object).getDisks(), "", r.bucket, r.object, restoreTestVersionKey(versionID), false, false)
+	if errs[0] != nil {
+		r.t.Fatal(errs[0])
+	}
+	return infos[0]
+}
+
+// dataDirs returns how many drives of the first pool hold the data
+// directory of a version.
+func (r *restoreTest) dataDirs(dataDir string) int {
+	r.t.Helper()
+	n := 0
+	for _, disk := range r.z.serverPools[0].getHashedSet(r.object).getDisks() {
+		_, err := os.Stat(filepath.Join(disk.Endpoint().Path, r.bucket, r.object, dataDir))
+		if err == nil {
+			n++
+		} else if !os.IsNotExist(err) {
+			r.t.Fatal(err)
+		}
+	}
+	return n
+}
+
+// versions returns the versions of the object in each pool, newest first,
+// and its free versions in all pools.
+func (r *restoreTest) versions() (views []string, free int) {
+	r.t.Helper()
+	for _, pool := range r.z.serverPools {
+		rf, err := pool.getHashedSet(r.object).getDisks()[0].ReadXL(r.t.Context(), r.bucket, r.object, false)
+		if errors.Is(err, errFileNotFound) {
+			views = append(views, "")
+			continue
+		}
+		if err != nil {
+			r.t.Fatal(err)
+		}
+		fivs, err := getFileInfoVersions(rf.Buf, r.bucket, r.object, false)
+		if err != nil {
+			r.t.Fatal(err)
+		}
+		var labels []string
+		for _, fi := range fivs.Versions {
+			if fi.Deleted {
+				labels = append(labels, "DM")
+			} else {
+				labels = append(labels, r.labels[restoreTestVersionKey(fi.VersionID)])
+			}
+		}
+		views = append(views, strings.Join(labels, " "))
+		free += len(fivs.FreeVersions)
+	}
+	return views, free
+}
+
+// read returns the label and the data of a version, or of the latest version
+// when versionID is empty, or the error of the read.
+func (r *restoreTest) read(versionID string) (string, []byte, error) {
+	r.t.Helper()
+	gr, err := r.z.GetObjectNInfo(r.t.Context(), r.bucket, r.object, nil, nil, ObjectOptions{VersionID: versionID})
+	if err != nil {
+		return "", nil, err
+	}
+	defer gr.Close()
+	data, err := io.ReadAll(gr)
+	if err != nil {
+		return "", nil, err
+	}
+	return r.labels[restoreTestVersionKey(gr.ObjInfo.VersionID)], data, nil
+}
+
+// runQueue runs the queued expiries with an expiry worker.
+func (r *restoreTest) runQueue() {
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		globalExpiryState.Worker(r.queue)
+	}()
+	r.queue <- nil // the worker returns once it ran every expiry queued before
+	<-done
+}
+
+// objInfos returns the versions of the object as a drive of the
+// first pool holds them, as the scanner and the walks of decommission and
+// the drive heal see them.
+func (r *restoreTest) objInfos() []ObjectInfo {
+	r.t.Helper()
+	rf, err := r.z.serverPools[0].getHashedSet(r.object).getDisks()[0].ReadXL(r.t.Context(), r.bucket, r.object, false)
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	fivs, err := getFileInfoVersions(rf.Buf, r.bucket, r.object, false)
+	if err != nil {
+		r.t.Fatal(err)
+	}
+	versioned := globalBucketVersioningSys.PrefixEnabled(r.bucket, r.object) || globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object)
+	objInfos := make([]ObjectInfo, len(fivs.Versions))
+	for i, fi := range fivs.Versions {
+		objInfos[i] = fi.ToObjectInfo(r.bucket, r.object, versioned)
+	}
+	return objInfos
+}
+
+// The restored copy of a transitioned version expires on the expiry date of
+// its restore. The expiry removes the copy and nothing else: the version
+// stays where it is in the version stack, transitioned, and is read from the
+// tier again. Every path that applies ILM runs the expiry: the scanner and a
+// GET or HEAD queue it for an expiry worker, decommission and the drive heal
+// run it at once. The expiry of the restored copy of the latest version used
+// to delete the object instead: a versioned bucket got a delete marker that
+// hid it. With versioning suspended it went to the null version, removed its
+// data if there was one and failed if not, and the restored copy stayed.
+func TestExpireRestoredCopy(t *testing.T) {
+	prevTier := globalTierConfigMgr
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+	prevExpiry := globalExpiryState
+	t.Cleanup(func() { globalExpiryState = prevExpiry })
+
+	// Nothing here is old enough to transition; the rule makes the bucket
+	// one with a tier, as restored versions are in.
+	const rules = `<LifecycleConfiguration><Rule><ID>tier</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter>` +
+		`<Transition><Days>1000</Days><StorageClass>WARM</StorageClass></Transition></Rule></LifecycleConfiguration>`
+
+	type version struct {
+		label    string
+		null     bool // the null version
+		restored bool // transitioned and restored, the restore has expired
+	}
+	testCases := []struct {
+		name       string
+		versioning versioning.State // "", Enabled or Suspended
+		versions   []version        // oldest first
+	}{
+		{name: "versioned", versioning: versioning.Enabled, versions: []version{{label: "v1"}, {label: "v2", restored: true}}},
+		{name: "versioned-noncurrent", versioning: versioning.Enabled, versions: []version{{label: "v1", restored: true}, {label: "v2"}}},
+		{name: "versioned-null", versioning: versioning.Enabled, versions: []version{{label: "v1", null: true, restored: true}}},
+		{name: "suspended", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true}, {label: "v2", restored: true}}},
+		{name: "suspended-no-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", restored: true}}},
+		{name: "suspended-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", null: true, restored: true}}},
+		{name: "suspended-null-noncurrent", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true, restored: true}, {label: "v2"}}},
+		{name: "unversioned", versions: []version{{label: "v1", null: true, restored: true}}},
+	}
+
+	sources := []struct {
+		name   string
+		expire func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event)
+	}{
+		{"scanner", func(r *restoreTest, _ ObjectInfo, _ lifecycle.Event) {
+			lc, err := globalLifecycleSys.Get(r.bucket)
+			if err != nil {
+				r.t.Fatal(err)
+			}
+			lr, err := globalBucketObjectLockSys.Get(r.bucket)
+			if err != nil {
+				r.t.Fatal(err)
+			}
+			item := scannerItem{bucket: r.bucket, objectName: r.object, lifeCycle: lc}
+			item.applyActions(r.t.Context(), r.z, r.objInfos(), lr, &sizeSummary{}, nil)
+			r.runQueue()
+		}},
+		{"head", func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) {
+			applyExpiryRule(evt, lcEventSrc_s3HeadObject, oi)
+			r.runQueue()
+		}},
+		{"list", func(r *restoreTest, _ ObjectInfo, _ lifecycle.Event) {
+			// ListObjects of the key with max-keys=1 reads its latest
+			// version without the versioning of the bucket, a listing of
+			// versions reads them all.
+			if _, err := r.z.ListObjects(r.t.Context(), r.bucket, r.object, "", "", 1); err != nil {
+				r.t.Fatal(err)
+			}
+			if _, err := r.z.ListObjectVersions(r.t.Context(), r.bucket, r.object, "", "", "", 100); err != nil {
+				r.t.Fatal(err)
+			}
+			r.runQueue()
+		}},
+		{"decom", func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) {
+			for _, o := range r.objInfos() {
+				if o.VersionID == oi.VersionID {
+					applyExpiryOnTransitionedObject(r.t.Context(), r.z, o, evt, lcEventSrc_Decom)
+					return
+				}
+			}
+			r.t.Fatal("version not found on the drive")
+		}},
+	}
+
+	for _, pools := range []int{1, 2} {
+		t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+			ctx := t.Context()
+			dirs, err := getRandomDisks(4 * pools)
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() { removeRoots(dirs) })
+			var endpoints EndpointServerPools
+			for i := range pools {
+				endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
+			}
+			obj, _, err := initObjectLayer(ctx, endpoints)
+			if err != nil {
+				t.Fatal(err)
+			}
+			z := obj.(*erasureServerPools)
+			t.Cleanup(func() { z.Shutdown(context.Background()) })
+			// After initObjectLayer, which sets up a new tier config.
+			tier := &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
+			globalTierConfigMgr = NewTierConfigMgr()
+			globalTierConfigMgr.drivercache["WARM"] = tier
+
+			for _, tc := range testCases {
+				for _, src := range sources {
+					t.Run(tc.name+"/"+src.name, func(t *testing.T) {
+						r := &restoreTest{
+							t: t, z: z,
+							bucket: "restore-" + tc.name + "-" + src.name, object: "obj",
+							queue:  make(chan expiryOp, 100),
+							labels: map[string]string{}, bodies: map[string][]byte{},
+						}
+						if err := z.MakeBucket(ctx, r.bucket, MakeBucketOptions{VersioningEnabled: tc.versioning != ""}); err != nil {
+							t.Fatal(err)
+						}
+						meta, err := globalBucketMetadataSys.Get(r.bucket)
+						if err != nil {
+							t.Fatal(err)
+						}
+						if tc.versioning == versioning.Suspended {
+							meta.VersioningConfigXML = []byte("<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
+						}
+						meta.LifecycleConfigXML = []byte(rules)
+						if err := meta.Save(ctx, z); err != nil {
+							t.Fatal(err)
+						}
+						globalBucketMetadataSys.Set(r.bucket, meta)
+						es := &expiryState{ctx: ctx, objAPI: z}
+						workers := []chan expiryOp{r.queue}
+						es.workers.Store(&workers)
+						globalExpiryState = es
+
+						var restored ObjectInfo
+						for _, v := range tc.versions {
+							oi := r.put(v.label, v.null, v.restored)
+							if v.restored {
+								r.restore(oi)
+								restored = oi
+							}
+						}
+						rlabel := r.labels[restoreTestVersionKey(restored.VersionID)]
+
+						// The fixture: the restored copy is on the drives,
+						// its restore has expired, ILM expires it.
+						fi := r.fileInfo(restored.VersionID)
+						if fi.TransitionStatus != lifecycle.TransitionComplete || fi.DataDir == "" || r.dataDirs(fi.DataDir) == 0 {
+							t.Fatalf("fixture: %s transition status %q, restored copy on %d drives", rlabel, fi.TransitionStatus, r.dataDirs(fi.DataDir))
+						}
+						oi, err := z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(restored.VersionID))
+						if err != nil {
+							t.Fatal(err)
+						}
+						lc, err := globalLifecycleSys.Get(r.bucket)
+						if err != nil {
+							t.Fatal(err)
+						}
+						lr, err := globalBucketObjectLockSys.Get(r.bucket)
+						if err != nil {
+							t.Fatal(err)
+						}
+						evt := evalActionFromLifecycle(ctx, *lc, lr, nil, oi)
+						if !evt.Action.DeleteRestored() || oi.RestoreExpires.IsZero() {
+							t.Fatalf("fixture: %s restore expires %v, action %v", rlabel, oi.RestoreExpires, evt.Action)
+						}
+						before, _ := r.versions()
+						for key, label := range r.labels {
+							if _, data, err := r.read(key); err != nil || !bytes.Equal(data, r.bodies[label]) {
+								t.Fatalf("fixture: GET %s: %v", label, err)
+							}
+						}
+
+						src.expire(r, oi, evt)
+
+						after, free := r.versions()
+						if fmt.Sprint(after) != fmt.Sprint(before) || free != 0 {
+							t.Errorf("versions in each pool: before %q, after %q with %d free versions", before, after, free)
+						}
+						if label, _, err := r.read(""); err != nil || label != tc.versions[len(tc.versions)-1].label {
+							t.Errorf("GET reads %q, %v; want %q", label, err, tc.versions[len(tc.versions)-1].label)
+						}
+						if n := r.dataDirs(fi.DataDir); n != 0 {
+							t.Errorf("restored copy of %s still on %d drives", rlabel, n)
+						}
+						oi, err = z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(restored.VersionID))
+						if err != nil {
+							t.Fatalf("HEAD %s: %v", rlabel, err)
+						}
+						if oi.TransitionedObject.Status != lifecycle.TransitionComplete || oi.UserDefined[xhttp.AmzRestore] != "" || !oi.RestoreExpires.IsZero() {
+							t.Errorf("HEAD %s: transition status %q, x-amz-restore %q", rlabel, oi.TransitionedObject.Status, oi.UserDefined[xhttp.AmzRestore])
+						}
+						for _, v := range tc.versions {
+							for key, label := range r.labels {
+								if label != v.label {
+									continue
+								}
+								gets := tier.gets.Load()
+								_, data, err := r.read(key)
+								switch {
+								case err != nil:
+									t.Errorf("GET %s: %v", v.label, err)
+								case !bytes.Equal(data, r.bodies[v.label]):
+									t.Errorf("GET %s reads %d bytes that differ", v.label, len(data))
+								case v.restored && tier.gets.Load() == gets:
+									t.Errorf("GET %s does not read the tier", v.label)
+								}
+							}
+						}
+					})
+				}
+			}
+		})
+	}
+}

cmd/bucket-lifecycle.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/event"
 	xhttp "github.com/minio/minio/internal/http"
@@ -611,8 +612,9 @@ func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
 func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) error {
 	traceFn := globalLifecycleSys.trace(*oi)
 	opts := ObjectOptions{
-		Versioned:  globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
-		Expiration: ExpirationOptions{Expire: true},
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+		Expiration:       ExpirationOptions{Expire: true},
 	}
 	if lcEvent.Action.DeleteVersioned() {
 		opts.VersionID = oi.VersionID
@@ -622,6 +624,17 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		// delete locally restored copy of object or object version
 		// from the source, while leaving metadata behind. The data on
 		// transitioned tier lies untouched and still accessible
+		//
+		// The restored copy belongs to this version, also when it is the
+		// latest one (DeleteRestoredAction). Without its version ID the
+		// delete applies to the object: a versioned bucket gets a delete
+		// marker that hides it, a suspended one strips the null version.
+		// An empty version ID is the null version, as callers that read
+		// it without the versioning of the bucket leave it.
+		opts.VersionID = oi.VersionID
+		if opts.VersionID == "" {
+			opts.VersionID = nullVersionID
+		}
 		opts.Transition.ExpireRestored = true
 		_, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
 		if err == nil {
@@ -636,7 +649,6 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 	// touching the remote tier. If the lifecycle configuration changed after
 	// the scanner queued this task, DeleteObject rejects the stale action while
 	// the remote object is still protected.
-	opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name)
 	keepRemote := opts.VersionID == "" &&
 		(opts.Versioned || (opts.VersionSuspended && oi.VersionID != "" && oi.VersionID != nullVersionID))
 	if lcEvent.Action == lifecycle.DeleteAction || lcEvent.Action == lifecycle.DeleteVersionAction {
@@ -976,8 +988,10 @@ func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string
 	}, nil
 }
 
-// set ObjectOptions for PUT call to restore temporary copy of transitioned data
-func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) {
+// set ObjectOptions for PUT call to restore temporary copy of transitioned data.
+// opts are the restore's options, with the versioning POST captured: the
+// bucket's versioning may have changed since, and must not mint a new version.
+func putRestoreOpts(rreq *RestoreObjectRequest, objInfo ObjectInfo, opts ObjectOptions) (putOpts ObjectOptions) {
 	meta := make(map[string]string)
 	sc := rreq.OutputLocation.S3.StorageClass
 	if sc == "" {
@@ -1000,8 +1014,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 			meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
 		}
 		return ObjectOptions{
-			Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
-			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+			Versioned:        opts.Versioned,
+			VersionSuspended: opts.VersionSuspended,
 			UserDefined:      meta,
 		}
 	}
@@ -1009,19 +1023,37 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 	if len(objInfo.UserTags) != 0 {
 		meta[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
-	// Set restore object status
+	// Set restore object status; a completed restore has no attempt.
 	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
 	meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
+	delete(meta, restoreAttemptKey)
 	return ObjectOptions{
-		Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
-		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+		Versioned:        opts.Versioned,
+		VersionSuspended: opts.VersionSuspended,
 		UserDefined:      meta,
-		VersionID:        objInfo.VersionID,
+		VersionID:        restoreVersionID(objInfo.VersionID),
 		MTime:            objInfo.ModTime,
 		Expires:          objInfo.Expires,
 	}
 }
 
+// restoredCopyExpires reports whether the restored copy of oi, as read now,
+// expires by its own reason: lifecycle expires the restored copy of this
+// version, its restore deadline is valid and has passed, its transition is
+// complete and no restore is running. Another rule falling due is no reason.
+func restoredCopyExpires(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, oi ObjectInfo) bool {
+	if !evalActionFromLifecycle(ctx, lc, lr, rcfg, oi).Action.DeleteRestored() {
+		return false
+	}
+	status, err := parseRestoreObjStatus(oi.UserDefined[xhttp.AmzRestore])
+	if err != nil {
+		return false
+	}
+	expiry, completed := status.Expiry()
+	return completed && !expiry.IsZero() && UTCNow().After(expiry) &&
+		oi.TransitionedObject.Status == lifecycle.TransitionComplete && !oi.RestoreOngoing
+}
+
 var errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
 
 // IsRemote returns true if this object version's contents are in its remote

cmd/data-scanner.go

@@ -1204,7 +1204,7 @@ func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
 func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) (ok bool) {
 	timeILM := globalScannerMetrics.timeILM(lcEvent.Action)
 	if err := expireTransitionedObject(ctx, objLayer, &obj, lcEvent, src); err != nil {
-		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+		if errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
 		ilmLogIf(ctx, fmt.Errorf("expireTransitionedObject(%s, %s): %w", obj.Bucket, obj.Name, err))
@@ -1271,7 +1271,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 
 	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
 	if err != nil {
-		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+		if errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			traceFn(ILMExpiry, nil, nil)
 			return false
 		}

cmd/erasure-healing.go

@@ -373,6 +373,13 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if err != nil {
 		return result, err
 	}
+	// The restore fields fence native restores and restored-copy cleanup.
+	// Heal copies them only as a quorum of the drives holds them; while the
+	// drives disagree, it waits rather than create a quorum for one side.
+	latestMeta, err = restoreHealMeta(partsMetadata, latestMeta, max(len(storageDisks)/2+1, readQuorum))
+	if err != nil {
+		return result, toObjectErr(err, bucket, object, versionID)
+	}
 
 	// No modtime quorum
 	filterDisksByETag := quorumETag != ""

cmd/erasure-object.go

@@ -45,7 +45,6 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/grid"
-	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
@@ -509,6 +508,11 @@ func joinErrs(errs []error) string {
 }
 
 func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int, opts ObjectOptions) (FileInfo, error) {
+	// Restore and restored-copy cleanup must preserve the version on read
+	// failure, including lookups and failure cleanup that reread metadata.
+	if opts.Transition.ExpireRestored || opts.Transition.RestoreRequest != nil {
+		return FileInfo{}, errErasureReadQuorum
+	}
 	m, ok := isObjectDangling(metaArr, errs, dataErrsByPart)
 	if !ok {
 		// We only come here if we cannot figure out if the object
@@ -2045,6 +2049,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	// expiration attempted on a bucket with no lifecycle
 	// rules shall be rejected.
 	if lc == nil && opts.Expiration.Expire {
+		if !opts.Transition.ExpireRestored {
+			return objInfo, errLifecycleNotApplicable
+		}
 		if opts.VersionID != "" {
 			return objInfo, VersionNotFound{
 				Bucket:    bucket,
@@ -2089,6 +2096,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 					isErr = true
 				}
 				if isErr {
+					if !opts.Transition.ExpireRestored {
+						return goi, errLifecycleNotApplicable
+					}
 					if goi.VersionID != "" {
 						return goi, VersionNotFound{
 							Bucket:    bucket,
@@ -2107,6 +2117,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object)
 	}
 
+	// An ILM version expiry that the pools checked with the versions in all
+	// pools; the set still guards the protection it reads, see
+	// checkedExpiryRefused.
+	checkedExpiry := opts.Expiration.Expire && opts.Expiration.CheckedAllPools && opts.VersionID != "" && !opts.Transition.ExpireRestored
+
 	storageDisks := er.getDisks()
 	versionFound := true
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
@@ -2114,7 +2129,10 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	tryDel := false
 	if gerr != nil && goi.Name == "" {
 		if _, ok := gerr.(InsufficientReadQuorum); ok {
-			if opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
+			// A restored-copy expiry needs fresh metadata to confirm its target
+			// and deadline, a checked expiry to guard its protection; ordinary
+			// unversioned DELETE may still try a blind delete.
+			if opts.Transition.ExpireRestored || checkedExpiry || opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
 				// Add an MRF heal for next time.
 				er.addPartial(bucket, object, opts.VersionID)
 				return objInfo, InsufficientWriteQuorum{}
@@ -2134,6 +2152,26 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
+	// A queued null restore may now address a replacement delete marker.
+	// Such errors carry ObjectInfo, but never authorize cache cleanup.
+	if opts.Transition.ExpireRestored && (gerr != nil || goi.DeleteMarker) {
+		return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
+	}
+	// A restored copy expires only by its own reason, decided on the
+	// version's authoritative generation: a drive that missed a renewal or a
+	// later POST must not authorize removing the cache, nor drives that
+	// disagree on the restore status. Another lifecycle deletion falling due,
+	// or the all-pools check of an ordinary expiry, does not stand in for it.
+	if opts.Transition.ExpireRestored {
+		gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
+		if err != nil || gen.split || gen.fi.Deleted || lc == nil {
+			return goi, errLifecycleNotApplicable
+		}
+		if !restoredCopyExpires(ctx, *lc, rcfg, replcfg, gen.fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)) {
+			return goi, errLifecycleNotApplicable
+		}
+	}
+
 	if opts.EvalMetadataFn != nil {
 		dsc, err := opts.EvalMetadataFn(&goi, gerr)
 		if err != nil {
@@ -2151,7 +2189,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
-	if opts.Expiration.Expire {
+	if opts.Expiration.Expire && !opts.Expiration.CheckedAllPools {
 		if gerr == nil {
 			evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
 			var isErr bool
@@ -2162,6 +2200,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 				isErr = true
 			}
 			if isErr {
+				if !opts.Transition.ExpireRestored {
+					return goi, errLifecycleNotApplicable
+				}
 				if goi.VersionID != "" {
 					return goi, VersionNotFound{
 						Bucket:    bucket,
@@ -2175,6 +2216,8 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 				}
 			}
 		}
+	} else if checkedExpiry && checkedExpiryRefused(*lc, rcfg, replcfg, goi, opts.Expiration.ApprovedProtection) {
+		return goi, errLifecycleNotApplicable
 	}
 
 	//  Determine whether to mark object deleted for replication
@@ -2621,107 +2664,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	return err
 }
 
-// RestoreTransitionedObject - restore transitioned object content locally on this cluster.
-// This is similar to PostObjectRestore from AWS GLACIER
-// storage class. When PostObjectRestore API is called, a temporary copy of the object
-// is restored locally to the bucket on source cluster until the restore expiry date.
-// The copy that was transitioned continues to reside in the transitioned tier.
-func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
-	return er.restoreTransitionedObject(ctx, bucket, object, opts)
-}
-
-// update restore status header in the metadata
-func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error {
-	oi := objInfo.Clone()
-	oi.metadataOnly = true // Perform only metadata updates.
-
-	// allow retry in the case of failure to restore
-	delete(oi.UserDefined, xhttp.AmzRestore)
-
-	if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{
-		VersionID: oi.VersionID,
-	}, ObjectOptions{
-		VersionID: oi.VersionID,
-	}); err != nil {
-		storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
-		return err
-	}
-	return nil
-}
-
-// restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts
-// as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta
-func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error {
-	setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error {
-		if rerr == nil {
-			return nil // nothing to do; restore object was successful
-		}
-		er.updateRestoreMetadata(ctx, bucket, object, oi, opts)
-		return rerr
-	}
-	var oi ObjectInfo
-	// get the file info on disk for transitioned object
-	actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
-	if err != nil {
-		return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
-	}
-
-	oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
-	ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi)
-	if len(oi.Parts) == 1 {
-		var rs *HTTPRangeSpec
-		gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
-		if err != nil {
-			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
-		}
-		defer gr.Close()
-		hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size)
-		if err != nil {
-			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
-		}
-		pReader := NewPutObjReader(hashReader)
-		_, err = er.PutObject(ctx, bucket, object, pReader, ropts)
-		return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
-	}
-
-	res, err := er.NewMultipartUpload(ctx, bucket, object, ropts)
-	if err != nil {
-		return setRestoreHeaderFn(oi, err)
-	}
-
-	var uploadedParts []CompletePart
-	var rs *HTTPRangeSpec
-	// get reader from the warm backend - note that even in the case of encrypted objects, this stream is still encrypted.
-	gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
-	if err != nil {
-		return setRestoreHeaderFn(oi, err)
-	}
-	defer gr.Close()
-
-	// rehydrate the parts back on disk as per the original xl.meta prior to transition
-	for _, partInfo := range oi.Parts {
-		hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size)
-		if err != nil {
-			return setRestoreHeaderFn(oi, err)
-		}
-		pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{})
-		if err != nil {
-			return setRestoreHeaderFn(oi, err)
-		}
-		if pInfo.Size != partInfo.Size {
-			return setRestoreHeaderFn(oi, InvalidObjectState{Bucket: bucket, Object: object})
-		}
-		uploadedParts = append(uploadedParts, CompletePart{
-			PartNumber: pInfo.PartNumber,
-			ETag:       pInfo.ETag,
-		})
-	}
-	_, err = er.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{
-		MTime: oi.ModTime,
-	})
-	return setRestoreHeaderFn(oi, err)
-}
-
 // DecomTieredObject - moves tiered object to another pool during decommissioning.
 func (er erasureObjects) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
 	if opts.UserDefined == nil {

cmd/erasure-server-pool-consistency.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"maps"
 	"sort"
 	"strings"
@@ -670,7 +671,9 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
 // deleteObjectsReconciled removes data while retaining each item's authoritative
 // copy until every other pool has completed its cleanup. The caller holds the
 // pools-layer write locks and separates marker creation from this batch.
-func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
+// approved holds the Object Lock state each checked ILM expiry was approved
+// with, see ExpirationOptions.ApprovedProtection.
+func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket string, objects []ObjectToDelete, approved []objectLockState, opts ObjectOptions) ([]DeletedObject, []error) {
 	deleted := make([]DeletedObject, len(objects))
 	errs := make([]error, len(objects))
 	primary := make([]int, len(objects))
@@ -760,11 +763,23 @@ func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket
 		}
 		deleteOpts := opts
 		deleteOpts.VersionID, deleteOpts.NoLock = obj.VersionID, true
+		// DeleteObjects already checked ordinary ILM UUID/explicit null
+		// versions against all pools. Empty IDs and user deletes have no such
+		// authorization; restored-copy expiry uses the single-object path.
+		deleteOpts.Expiration.CheckedAllPools = opts.Expiration.Expire && obj.VersionID != ""
+		if deleteOpts.Expiration.CheckedAllPools {
+			deleteOpts.Expiration.ApprovedProtection = approved[i]
+		}
 		deleteOpts.DeleteReplication = obj.ReplicationState()
 		if opts.PrefixEnabledFn != nil {
 			deleteOpts.Versioned = opts.PrefixEnabledFn(obj.ObjectName)
 		}
 		_, err := z.deleteObjectReconciled(ctx, bucket, obj.ObjectName, deleteOpts)
+		if errors.Is(err, errLifecycleNotApplicable) {
+			// A later lifecycle rejection did not commit a local purge. Do
+			// not let the worker replicate the request's pending decision.
+			deleted[i].ReplicationState = ReplicationState{}
+		}
 		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 			errs[i] = err
 		} else {

cmd/erasure-server-pool-consistency_test.go

@@ -522,10 +522,20 @@ func TestPoolsDeleteVersionSpecialCalls(t *testing.T) {
 			opts := ObjectOptions{Versioned: true, VersionID: oi.VersionID, InclFreeVersions: !expiration, Expiration: ExpirationOptions{Expire: expiration}}
 			_, err := z.DeleteObject(t.Context(), bucket, object, opts)
 			if expiration {
-				// No lifecycle rule authorizes expiration. Preserve its existing
-				// version-not-found result, even with an unrelated unreadable pool.
-				if !isErrVersionNotFound(err) {
-					t.Errorf("expiration changed its scanner contract: %v", err)
+				// No rule authorizes expiration. Distinguish refusal from real
+				// absence, even with an unrelated unreadable pool, so a worker
+				// cannot acknowledge or replicate a purge that never happened.
+				if !errors.Is(err, errLifecycleNotApplicable) {
+					t.Errorf("expiration did not preserve its rejection: %v", err)
+				}
+				gr, err := z.serverPools[0].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: oi.VersionID})
+				if err != nil {
+					t.Fatal(err)
+				}
+				data, err := io.ReadAll(gr)
+				gr.Close()
+				if err != nil || string(data) != "payload" || gr.ObjInfo.VersionID != oi.VersionID {
+					t.Errorf("rejected expiration changed the stored version: %v", err)
 				}
 			} else if err != nil {
 				t.Errorf("free-version cleanup was forced through all-pool resolution: %v", err)

cmd/erasure-server-pool-decom.go

@@ -819,6 +819,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				// a noncurrent version and has to be moved like any other. The
 				// scanner applies the expiry once it lives in another pool.
 				return lifecycle.NoneAction
+			case evt.Action.Delete() && !z.migrationExpiryApplies(ctx, objInfo, evt):
+				// Another pool holds versions with which the rule does not
+				// apply. The version moves like any other.
+				return lifecycle.NoneAction
 			case evt.Action.Delete():
 				return evt.Action
 			default:

cmd/erasure-server-pool-lifecycle-batch_test.go

@@ -0,0 +1,270 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"testing"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/pubsub"
+)
+
+// Reuse the author's scanner/worker fixture. A second configured pool is
+// sufficient to trigger the tiered NNV regression; no versions need move there.
+func TestPoolsLifecycleTieredNoncurrentBatch(t *testing.T) {
+	for _, tiered := range []bool{false, true} {
+		name := "plain"
+		if tiered {
+			name = "tiered"
+		}
+		t.Run(name, func(t *testing.T) {
+			s := newSplitILM(t, versioning.Enabled, splitILMRules)
+			old := func(label string, days float64) splitVersion {
+				v := splitAt(label, days, 0)
+				v.tiered = tiered
+				return v
+			}
+			s.run([]splitKey{
+				{"nnv-all-in-0", []splitVersion{old("v1", 5), old("v2", 4), splitAt("v3", 3, 0), splitAt("v4", 2, 0)}, "v4", "v4", [][]string{{"v4", "v3"}, nil}},
+				{"ncv-all-in-0", []splitVersion{old("v1", 5), old("v2", 4), splitAt("v3", 0.01, 0)}, "v3", "v3", [][]string{{"v3", "v2"}, nil}},
+			})
+		})
+	}
+}
+
+// Observe the actual pool results consumed by the expiry worker, without
+// replacing the storage deletion or the replication enqueue path.
+type lifecycleBatchResult struct {
+	ObjectLayer
+	deleted []DeletedObject
+	errs    []error
+}
+
+func (o *lifecycleBatchResult) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
+	o.deleted, o.errs = o.ObjectLayer.DeleteObjects(ctx, bucket, objects, opts)
+	return o.deleted, o.errs
+}
+
+func TestPoolsLifecycleTieredBatchReplication(t *testing.T) {
+	for _, identity := range []string{"uuid", "null"} {
+		t.Run(identity, func(t *testing.T) {
+			s := newSplitILM(t, versioning.Enabled, splitILMRules)
+			const key = "nnv-replicated"
+			oldest := splitTiered("v1", 5, 0)
+			oldest.null = identity == "null"
+			for _, v := range []splitVersion{oldest, splitTiered("v2", 4, 0), splitTiered("v3", 3, 0), splitAt("v4", 2, 0)} {
+				s.write(key, v)
+			}
+			target := newILMTestTarget(t)
+			registerILMTestTarget(t, s.z, s.bucket, target, func(w http.ResponseWriter, r *http.Request) {
+				if r.Method != http.MethodDelete {
+					t.Errorf("unexpected target method: %s", r.Method)
+					w.WriteHeader(500)
+					return
+				}
+				target.deletes.Add(1)
+				opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true, ReplicationRequest: true}
+				opts.SetReplicaStatus(replication.Replica)
+				if _, err := s.z.DeleteObject(r.Context(), target.bucket, key, opts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+					t.Errorf("target purge: %v", err)
+					w.WriteHeader(500)
+					return
+				}
+				w.WriteHeader(http.StatusNoContent)
+			})
+			setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket)
+			stats := NewReplicationStats(t.Context(), nil)
+			previous := globalReplicationStats.Swap(stats)
+			t.Cleanup(func() { globalReplicationStats.Store(previous) })
+			restore, _ := newILMTestReplicationPool(t.Context(), s.z, stats)
+			t.Cleanup(restore)
+			var entries []ObjectToDelete
+			for _, label := range []string{"v1", "v3"} {
+				vid := s.ids[key+" "+label]
+				if vid == "" {
+					vid = nullVersionID
+				}
+				oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: true})
+				if err != nil {
+					t.Fatal(err)
+				}
+				entry := ilmPurgeToDelete(t.Context(), s.bucket, oi, true)
+				if entry.VersionPurgeStatus != replication.VersionPurgePending {
+					t.Fatalf("fixture lacks purge decision: %+v", entry)
+				}
+				entries = append(entries, entry)
+				putConsistencyObject(t, s.z, target.bucket, key, 0, label, ObjectOptions{Versioned: identity != "null" || label != "v1", VersionID: vid})
+			}
+			result := &lifecycleBatchResult{ObjectLayer: s.z}
+			deleteObjectVersions(t.Context(), result, s.bucket, entries, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}})
+			if result.errs[0] != nil {
+				t.Errorf("eligible delete: %v", result.errs[0])
+			}
+			if !errors.Is(result.errs[1], errLifecycleNotApplicable) {
+				t.Errorf("retained version result: %v", result.errs[1])
+			}
+			if result.deleted[1].VersionPurgeStatus() != "" || result.deleted[1].ReplicationState.ReplicateDecisionStr != "" {
+				t.Errorf("rejected entry retained replication state: %+v", result.deleted[1])
+			}
+			source, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: entries[0].VersionID, Versioned: true})
+			if !isErrMethodNotAllowed(err) || source.VersionPurgeStatus != replication.VersionPurgePending {
+				t.Errorf("source must be a pending purge tombstone before enqueue: status=%q err=%v", source.VersionPurgeStatus, err)
+			}
+			retained, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: entries[1].VersionID, Versioned: true})
+			if err != nil || retained.VersionPurgeStatus != "" {
+				t.Errorf("rejected source changed: %+v, %v", retained, err)
+			}
+			queue := globalReplicationPool.Get().workers[0]
+			if len(queue) != 1 {
+				t.Fatalf("queued %d purges, want only eligible v1", len(queue))
+			}
+			deletion, ok := (<-queue).(DeletedObjectReplicationInfo)
+			if !ok || deletion.VersionID != entries[0].VersionID || deletion.EventType != ReplicateILMDelete || deletion.VersionPurgeStatus() != replication.VersionPurgePending {
+				t.Fatalf("wrong purge queued: %+v", deletion)
+			}
+			if result := replicateDelete(t.Context(), deletion, s.z); result.VersionPurgeStatus() != replication.VersionPurgeComplete {
+				t.Fatalf("replicate purge: %+v", result)
+			}
+			if target.deletes.Load() != 1 {
+				t.Errorf("target delete count=%d", target.deletes.Load())
+			}
+			for _, bucket := range []string{s.bucket, target.bucket} {
+				if _, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: entries[0].VersionID}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+					t.Errorf("purged v1 remains in %s: %v", bucket, err)
+				}
+				if _, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: entries[1].VersionID}); err != nil {
+					t.Errorf("retained v3 missing from %s: %v", bucket, err)
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsLifecycleTieredBatchIdentity(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		versioned bool
+		id        string
+		expire    bool
+		wantGone  bool
+	}{
+		{"uuid-ilm", true, "uuid", true, true},
+		{"null-ilm", true, "null", true, true},
+		{"empty-unversioned-ilm", false, "", true, false},
+		{"uuid-user-delete", true, "uuid", false, true},
+		{"empty-user-delete", false, "", false, true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			state := versioning.State("")
+			if tc.versioned {
+				state = versioning.Enabled
+			}
+			s := newSplitILM(t, state, splitILMRules)
+			const key = "nnv-identity"
+			v := splitTiered("v1", 5, 0)
+			v.null = tc.id != "uuid"
+			s.write(key, v)
+			if tc.versioned {
+				s.write(key, splitAt("v2", 4, 0))
+				s.write(key, splitAt("v3", 3, 0))
+			}
+			vid := tc.id
+			if vid == "uuid" {
+				vid = s.ids[key+" v1"]
+			}
+			vc, _ := globalBucketVersioningSys.Get(s.bucket)
+			_, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: vid}}}, ObjectOptions{PrefixEnabledFn: vc.PrefixEnabled, Expiration: ExpirationOptions{Expire: tc.expire}})
+			if tc.expire && !tc.wantGone {
+				if !errors.Is(errs[0], errLifecycleNotApplicable) {
+					t.Fatalf("ineligible expiry: %v", errs[0])
+				}
+			} else if errs[0] != nil {
+				t.Fatalf("delete: %v", errs[0])
+			}
+			query := vid
+			if query == "" {
+				query = nullVersionID
+			}
+			_, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: query})
+			gone := isErrVersionNotFound(err) || isErrObjectNotFound(err)
+			if gone != tc.wantGone || (!gone && err != nil) {
+				t.Fatalf("gone=%t err=%v, want gone=%t", gone, err, tc.wantGone)
+			}
+		})
+	}
+}
+
+// A batch entry whose expiry the all-pools recheck rejects is kept. The worker
+// must not report it as removed, by event, ILM trace or audit, while eligible
+// entries of the same batch are still reported.
+func TestPoolsLifecycleBatchRejectionNotAnnounced(t *testing.T) {
+	s := newSplitILM(t, versioning.Enabled, splitILMRules)
+	const key = "nnv-announce"
+	for _, v := range []splitVersion{splitTiered("v1", 5, 0), splitTiered("v2", 4, 0), splitTiered("v3", 3, 0), splitAt("v4", 2, 0)} {
+		s.write(key, v)
+	}
+	eligible, retained := s.ids[key+" v1"], s.ids[key+" v3"]
+	done := make(chan struct{})
+	defer close(done)
+	events := make(chan event.Event, 8)
+	if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedDelete), events, done, nil); err != nil {
+		t.Fatal(err)
+	}
+	traces := make(chan madmin.TraceInfo, 8)
+	if err := globalTrace.Subscribe(madmin.TraceILM, traces, done, nil); err != nil {
+		t.Fatal(err)
+	}
+	result := &lifecycleBatchResult{ObjectLayer: s.z}
+	deleteObjectVersions(t.Context(), result, s.bucket, []ObjectToDelete{
+		{ObjectV: ObjectV{ObjectName: key, VersionID: eligible}},
+		{ObjectV: ObjectV{ObjectName: key, VersionID: retained}},
+	}, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}})
+	if result.errs[0] != nil || !errors.Is(result.errs[1], errLifecycleNotApplicable) {
+		t.Fatalf("fixture results: eligible=%v retained=%v", result.errs[0], result.errs[1])
+	}
+	if _, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: retained, Versioned: true}); err != nil {
+		t.Fatalf("rejected version must remain: %v", err)
+	}
+	// Both publishers deliver synchronously, so everything sent is buffered.
+	var notified, traced []string
+	for drained := false; !drained; {
+		select {
+		case ev := <-events:
+			notified = append(notified, ev.S3.Object.VersionID)
+		case tr := <-traces:
+			if tr.FuncName == ILMExpiry {
+				traced = append(traced, tr.Custom["version-id"])
+			}
+		default:
+			drained = true
+		}
+	}
+	if len(notified) != 1 || notified[0] != eligible {
+		t.Errorf("removal events for %v, want only eligible %s (retained %s)", notified, eligible, retained)
+	}
+	if len(traced) != 1 || traced[0] != eligible {
+		t.Errorf("ILM expiry traces for %v, want only eligible %s (retained %s)", traced, eligible, retained)
+	}
+}

cmd/erasure-server-pool-lifecycle-safety_test.go

@@ -0,0 +1,361 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"maps"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// A partial metadata update can leave the newest lock/tag revision on a copy
+// other than the one selected by content ModTime. Keep the data identity intact.
+func setLifecycleCopyMetadata(t *testing.T, s *splitILM, key, id string, pool int, metadata map[string]string, advance bool) {
+	t.Helper()
+	oi, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id})
+	if err != nil {
+		t.Fatal(err)
+	}
+	opts := ObjectOptions{VersionID: id, Versioned: true, MTime: oi.ModTime}
+	if advance {
+		opts.MTime = opts.MTime.Add(time.Hour)
+	}
+	opts.EvalMetadataFn = func(oi *ObjectInfo, _ error) (ReplicateDecision, error) {
+		maps.Copy(oi.UserDefined, metadata)
+		return ReplicateDecision{}, nil
+	}
+	if _, err := s.z.serverPools[pool].PutObjectMetadata(t.Context(), s.bucket, key, opts); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestPoolsLifecycleDuplicateLockMetadata(t *testing.T) {
+	old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano)
+	recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+	for _, tc := range []struct {
+		name       string
+		copies     [2]map[string]string
+		mode, hold string
+		delete     bool
+		newerPool1 bool
+	}{
+		{name: "hold-on", copies: [2]map[string]string{poolLockMetadata("", "OFF", old, old), poolLockMetadata("", "ON", old, recent)}, hold: "ON"},
+		{name: "held-earlier-pool", copies: [2]map[string]string{poolLockMetadata("", "ON", old, recent), poolLockMetadata("", "OFF", old, old)}, hold: "ON", newerPool1: true},
+		{name: "retention-on", copies: [2]map[string]string{poolLockMetadata("", "OFF", old, old), poolLockMetadata("GOVERNANCE", "OFF", recent, old)}, mode: "GOVERNANCE", hold: "OFF"},
+		{name: "independent-fields", copies: [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", recent, old), poolLockMetadata("", "ON", old, recent)}, mode: "GOVERNANCE", hold: "ON"},
+		{name: "hold-released", copies: [2]map[string]string{poolLockMetadata("", "ON", old, old), poolLockMetadata("", "OFF", old, recent)}, hold: "OFF", delete: true},
+		{name: "retention-released", copies: [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", old, old), poolLockMetadata("", "OFF", recent, old)}, hold: "OFF", delete: true},
+	} {
+		for _, batch := range []bool{false, true} {
+			path := "single"
+			if batch {
+				path = "batch"
+			}
+			t.Run(tc.name+"/"+path, func(t *testing.T) {
+				s := &splitILM{migrationILM: newMigrationILM(t, versioning.Enabled, true, splitILMRules), ids: map[string]string{}, bodies: map[string]string{}}
+				const key = "ncv-copy-lock"
+				v := splitAt("old", 5, 0, 1)
+				v.tiered = batch
+				s.write(key, v)
+				s.write(key, splitAt("new", 3, 0, 1))
+				id := s.ids[key+" old"]
+				for pool, metadata := range tc.copies {
+					metadata = maps.Clone(metadata)
+					if metadata["x-amz-object-lock-mode"] != "" {
+						metadata["x-amz-object-lock-retain-until-date"] = UTCNow().AddDate(1, 0, 0).Format(time.RFC3339)
+					} else {
+						metadata["x-amz-object-lock-retain-until-date"] = ""
+					}
+					setLifecycleCopyMetadata(t, s, key, id, pool, metadata, tc.newerPool1 && pool == 1)
+				}
+				versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key)
+				switch {
+				case tc.newerPool1:
+					// Another ModTime makes the copies separate writes. The
+					// held older one refuses the object instead of merging.
+					if !errors.Is(err, errLifecycleNotApplicable) {
+						t.Fatalf("held older write: %v", err)
+					}
+				case err != nil:
+					t.Fatal(err)
+				default:
+					if len(versions) != 2 || versions[1].VersionID != id || versions[1].NumVersions != 2 {
+						t.Fatalf("wrong version history: %+v", versions)
+					}
+					state := storedObjectLockState(versions[1].UserDefined)
+					if state.mode != tc.mode || state.legalHold != tc.hold {
+						t.Errorf("merged state=%+v, want mode=%s hold=%s", state, tc.mode, tc.hold)
+					}
+					want := lifecycle.NoneAction
+					if tc.delete {
+						want = lifecycle.DeleteVersionAction
+					}
+					// Read-side expiry rechecks consume the same event producer.
+					if events[1].Action != want {
+						t.Errorf("expiry event=%s want=%s", events[1].Action, want)
+					}
+				}
+				opts := ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}}
+				if batch {
+					_, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: id}}}, opts)
+					err = errs[0]
+				} else {
+					opts.VersionID = id
+					_, err = s.z.DeleteObject(t.Context(), s.bucket, key, opts)
+				}
+				if tc.delete && err != nil || !tc.delete && !errors.Is(err, errLifecycleNotApplicable) {
+					t.Errorf("delete error=%v, wantDelete=%t", err, tc.delete)
+				}
+				for pool := range s.z.serverPools {
+					_, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id})
+					// Single DeleteObject retains its existing first-copy dispatch.
+					gone := tc.delete && (batch || pool == 0)
+					if gone && !isErrVersionNotFound(err) || !gone && err != nil {
+						t.Errorf("pool=%d err=%v wantGone=%t", pool, err, gone)
+					}
+				}
+			})
+		}
+	}
+}
+
+func TestPoolsLifecycleDuplicateTagMetadata(t *testing.T) {
+	const rule = `<Rule><ID>tagged</ID><Status>Enabled</Status><Filter><Tag><Key>expire</Key><Value>yes</Value></Tag></Filter><NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration></Rule>`
+	for _, tc := range []struct {
+		name, stale, fresh string
+		expire             bool
+	}{
+		{"removed", "expire=yes", "", false},
+		{"changed", "expire=yes", "expire=no", false},
+		{"enabled", "expire=no", "expire=yes", true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			s := newSplitILM(t, versioning.Enabled, rule)
+			const key = "tagged-copy"
+			s.write(key, splitTiered("old", 5, 0, 1))
+			s.write(key, splitAt("new", 3, 0, 1))
+			id := s.ids[key+" old"]
+			old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano)
+			recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+			for pool, tags := range []string{tc.stale, tc.fresh} {
+				stamp := old
+				if pool == 1 {
+					stamp = recent
+				}
+				setLifecycleCopyMetadata(t, s, key, id, pool, map[string]string{xhttp.AmzObjectTagging: tags, ReservedMetadataPrefixLower + TaggingTimestamp: stamp}, false)
+			}
+			versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if len(versions) != 2 || versions[1].UserTags != tc.fresh || versions[1].UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] != recent {
+				t.Fatalf("lost newest tag revision: %+v", versions)
+			}
+			want := lifecycle.NoneAction
+			if tc.expire {
+				want = lifecycle.DeleteVersionAction
+			}
+			if events[1].Action != want {
+				t.Errorf("shared read/delete expiry event=%s want=%s", events[1].Action, want)
+			}
+			_, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: id}}}, ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}})
+			if tc.expire && errs[0] != nil || !tc.expire && !errors.Is(errs[0], errLifecycleNotApplicable) {
+				t.Errorf("delete error=%v", errs[0])
+			}
+			for pool := range s.z.serverPools {
+				_, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id})
+				if tc.expire && !isErrVersionNotFound(err) || !tc.expire && err != nil {
+					t.Errorf("pool=%d err=%v wantGone=%t", pool, err, tc.expire)
+				}
+			}
+		})
+	}
+}
+
+// All-pool validation uses ReadXL. Run once at the first addressed read in
+// batch dispatch, after validation and before the set's lifecycle check.
+type lifecycleDispatchDisk struct {
+	StorageAPI
+	bucket, key string
+	once        *sync.Once
+	scans       *atomic.Int64
+	beforeRead  func()
+}
+
+func (d lifecycleDispatchDisk) ReadXL(ctx context.Context, bucket, key string, data bool) (RawFileInfo, error) {
+	if bucket == d.bucket && key == d.key {
+		d.scans.Add(1)
+	}
+	return d.StorageAPI.ReadXL(ctx, bucket, key, data)
+}
+
+func (d lifecycleDispatchDisk) ReadVersion(ctx context.Context, orig, bucket, key, id string, opts ReadOptions) (FileInfo, error) {
+	if bucket == d.bucket && key == d.key {
+		d.once.Do(d.beforeRead)
+	}
+	return d.StorageAPI.ReadVersion(ctx, orig, bucket, key, id, opts)
+}
+
+func TestPoolsLifecycleBatchRejectionReplication(t *testing.T) {
+	for _, mode := range []string{"withdraw-before-validation", "withdraw-after-validation", "already-absent", "normal"} {
+		t.Run(mode, func(t *testing.T) {
+			s := newSplitILM(t, versioning.Enabled, splitILMRules)
+			const key = "ncv-dispatch-recheck"
+			s.write(key, splitTiered("old", 5, 0))
+			s.write(key, splitAt("new", 3, 0))
+			id := s.ids[key+" old"]
+			target := newILMTestTarget(t)
+			registerILMTestTarget(t, s.z, s.bucket, target, func(w http.ResponseWriter, r *http.Request) {
+				if r.Method != http.MethodDelete || r.URL.Query().Get("versionId") != id {
+					t.Errorf("unexpected purge: %s %s", r.Method, r.URL)
+					w.WriteHeader(500)
+					return
+				}
+				target.deletes.Add(1)
+				opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true, ReplicationRequest: true}
+				opts.SetReplicaStatus(replication.Replica)
+				if _, err := s.z.DeleteObject(r.Context(), target.bucket, key, opts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+					t.Error(err)
+					w.WriteHeader(500)
+					return
+				}
+				w.WriteHeader(http.StatusNoContent)
+			})
+			setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket)
+			putConsistencyObject(t, s.z, target.bucket, key, 0, "replica", ObjectOptions{Versioned: true, VersionID: id})
+			stats := NewReplicationStats(t.Context(), nil)
+			previous := globalReplicationStats.Swap(stats)
+			t.Cleanup(func() { globalReplicationStats.Store(previous) })
+			restore, _ := newILMTestReplicationPool(t.Context(), s.z, stats)
+			t.Cleanup(restore)
+			oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id})
+			if err != nil {
+				t.Fatal(err)
+			}
+			entry := ilmPurgeToDelete(t.Context(), s.bucket, oi, true)
+			if entry.VersionPurgeStatus != replication.VersionPurgePending {
+				t.Fatal("fixture lacks pending purge")
+			}
+			withdraw := func() {
+				meta, err := globalBucketMetadataSys.Get(s.bucket)
+				if err != nil {
+					t.Error(err)
+					return
+				}
+				meta.lifecycleConfig = nil
+				meta.LifecycleConfigXML = nil
+				globalBucketMetadataSys.Set(s.bucket, meta)
+			}
+			if mode == "withdraw-before-validation" {
+				withdraw()
+			}
+			originals := s.z.serverPools[0].getHashedSet(key).getDisks()
+			var once sync.Once
+			var scans atomic.Int64
+			var dispatched atomic.Bool
+			for _, pool := range s.z.serverPools {
+				pool.erasureDisksMu.Lock()
+				prior := pool.erasureDisks
+				wrapped := make([][]StorageAPI, len(prior))
+				for i, disks := range prior {
+					wrapped[i] = append([]StorageAPI(nil), disks...)
+					for j, disk := range disks {
+						wrapped[i][j] = lifecycleDispatchDisk{StorageAPI: disk, bucket: s.bucket, key: key, once: &once, scans: &scans, beforeRead: func() {
+							dispatched.Store(true)
+							if mode != "withdraw-before-validation" && scans.Load() == 0 {
+								t.Error("dispatch injection ran before all-pool validation")
+							}
+							switch mode {
+							case "withdraw-after-validation":
+								withdraw()
+							case "already-absent":
+								fi := FileInfo{VersionID: id}
+								fi.SetTierFreeVersionID(mustGetUUID())
+								for _, disk := range originals {
+									if err := disk.DeleteVersion(t.Context(), s.bucket, key, fi, false, DeleteOptions{}); err != nil {
+										t.Error(err)
+									}
+								}
+							}
+						}}
+					}
+				}
+				pool.erasureDisks = wrapped
+				pool.erasureDisksMu.Unlock()
+				t.Cleanup(func() { pool.erasureDisksMu.Lock(); pool.erasureDisks = prior; pool.erasureDisksMu.Unlock() })
+			}
+			result := &lifecycleBatchResult{ObjectLayer: s.z}
+			deleteObjectVersions(t.Context(), result, s.bucket, []ObjectToDelete{entry}, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}})
+			if mode != "withdraw-before-validation" && !dispatched.Load() {
+				t.Fatal("dispatch injection did not run")
+			}
+			queue := globalReplicationPool.Get().workers[0]
+			reject := mode == "withdraw-before-validation" || mode == "withdraw-after-validation"
+			if reject {
+				if !errors.Is(result.errs[0], errLifecycleNotApplicable) || result.deleted[0].found || result.deleted[0].VersionPurgeStatus() != "" || result.deleted[0].ReplicationState.ReplicateDecisionStr != "" {
+					t.Errorf("rejection acknowledged a purge: %+v, %v", result.deleted[0], result.errs[0])
+				}
+				if len(queue) != 0 {
+					t.Fatalf("rejected delete queued %d remote purges", len(queue))
+				}
+				for _, bucket := range []string{s.bucket, target.bucket} {
+					oi, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: id})
+					if err != nil || oi.VersionPurgeStatus != "" {
+						t.Errorf("rejection changed %s: purge=%s err=%v", bucket, oi.VersionPurgeStatus, err)
+					}
+				}
+				if target.deletes.Load() != 0 {
+					t.Fatal("rejected delete reached remote target")
+				}
+				return
+			}
+			if result.errs[0] != nil || len(queue) != 1 {
+				t.Fatalf("valid/absent purge: error=%v queued=%d", result.errs[0], len(queue))
+			}
+			source, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id})
+			if mode == "normal" && (!isErrMethodNotAllowed(err) || source.VersionPurgeStatus != replication.VersionPurgePending) {
+				t.Fatalf("purge not committed locally: %+v %v", source, err)
+			}
+			if mode == "already-absent" && !isErrVersionNotFound(err) {
+				t.Fatalf("source must actually be absent: %v", err)
+			}
+			deletion := (<-queue).(DeletedObjectReplicationInfo)
+			if result := replicateDelete(t.Context(), deletion, s.z); result.VersionPurgeStatus() != replication.VersionPurgeComplete {
+				t.Fatalf("purge failed: %+v", result)
+			}
+			if target.deletes.Load() != 1 {
+				t.Fatalf("remote deletes=%d", target.deletes.Load())
+			}
+			for _, bucket := range []string{s.bucket, target.bucket} {
+				if _, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+					t.Errorf("purged version remains in %s: %v", bucket, err)
+				}
+			}
+		})
+	}
+}

cmd/erasure-server-pool-lifecycle.go

@@ -0,0 +1,785 @@
+// Copyright (c) 2026 Denis Babchuk
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"errors"
+	"maps"
+	"slices"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// An ordinary ILM rejection is not evidence that the version is absent. In
+// particular it must not acknowledge a pending replication purge.
+var errLifecycleNotApplicable = errors.New("lifecycle expiry no longer applies")
+
+// ILM decides on the versions of an object that one pool holds: the scanner
+// and the drive heal read the drives of one set, decommission and rebalance
+// walk one pool, and a GET or HEAD of a version reads the pool that holds it.
+// IsLatest, NumVersions and SuccessorModTime count only these. An object can
+// have versions in several pools: from two concurrent first writes, from
+// overwrites that went to another pool, from data movement. The expiries
+// these decisions run delete in every pool, so a version that is the latest
+// in its own pool but not of the object had a newer version hidden under a
+// delete marker or deleted with all versions, and a delete marker without
+// noncurrent versions in its pool was removed over the versions of another
+// pool. Deciding on fewer versions only delays the expiry of a noncurrent
+// version: its successor in the pool is not older than the one of the object,
+// and fewer newer versions count against NewerNoncurrentVersions.
+//
+// Before an expiry deletes in any pool, it is evaluated again with the
+// versions of the object in all pools, as the scanner would evaluate it if
+// one pool held them all. An expiry that no longer applies is not run, as the
+// recheck of a set does it: the scanner queues it again once the rule
+// applies to the object.
+
+// objectVersions returns the versions of object that the drives of the set
+// hold authoritatively, newest first, without free versions. It returns no
+// versions when the set does not hold the object, and an error when the
+// drives cannot tell its complete history.
+//
+// Expiry deletes by this history, so a revision of a version, its header
+// and its metadata (versionRevision), counts only when more than half of
+// the configured drives and the read quorum of its metadata hold it. A
+// version that some drive shows without such a revision, or too many
+// unreadable drives, could hide what an expiry must not delete. Free
+// versions are tier GC records and do not count.
+func (er erasureObjects) objectVersions(ctx context.Context, bucket, object string) ([]FileInfo, error) {
+	disks := er.getDisks()
+	rawFileInfos, errs := readAllRawFileInfo(ctx, disks, bucket, object, false)
+	majority := len(disks)/2 + 1
+
+	var (
+		metas   [][]xlMetaV2ShallowVersion
+		metaVs  []uint8
+		metaV   uint8
+		unknown int
+	)
+	for i, rf := range rawFileInfos {
+		if errs[i] != nil {
+			if !errors.Is(errs[i], errFileNotFound) && !errors.Is(errs[i], errVolumeNotFound) {
+				unknown++
+			}
+			continue
+		}
+		var xl xlMetaV2
+		if err := xl.LoadOrConvert(rf.Buf); err != nil {
+			unknown++
+			continue
+		}
+		metas = append(metas, xl.versions)
+		metaVs = append(metaVs, xl.metaV)
+		metaV = xl.metaV
+	}
+	// Without an unknown drive the history is complete, also on one drive.
+	if unknown > 0 && unknown >= len(disks)/2 {
+		return nil, toObjectErr(errErasureReadQuorum, bucket, object)
+	}
+
+	quorumMetas, quorumErrs := make([]FileInfo, len(disks)), make([]error, len(disks))
+	merged, err := authoritativeHistory(ctx, metas, metaVs, metaV, majority, func(ver xlMetaV2ShallowVersion, metaV uint8) int {
+		return er.versionReadQuorum(ctx, bucket, object, ver, metaV, quorumMetas, quorumErrs)
+	})
+	if err != nil {
+		return nil, toObjectErr(err, bucket, object)
+	}
+	if merged == nil {
+		return nil, nil // not in this set
+	}
+	all, err := xlMetaV2{versions: merged, metaV: metaV}.ListVersions(bucket, object, false)
+	if err != nil {
+		return nil, toObjectErr(err, bucket, object)
+	}
+	versions := all[:0]
+	for _, fi := range all {
+		if !fi.TierFreeVersion() {
+			versions = append(versions, fi)
+		}
+	}
+	return versions, nil
+}
+
+// revisionKey identifies a revision of a version: its header and a strong
+// digest of its metadata. The header Signature is a 32-bit fold of weak
+// hashes; two revisions can share it.
+type revisionKey struct {
+	header xlMetaV2VersionHeader
+	digest [sha256.Size]byte
+}
+
+// versionRevision returns the revision of ver, whose metadata is encoded as
+// metaV describes. As Signature does, it leaves out the fields that differ
+// between the drives of one write: the erasure index, empty part lists and,
+// for legacy versions, the release and the per-shard checksums. The digest
+// input is canonical: each map, including the part checksums of a legacy
+// version, is its own section with its entry count and sorted,
+// length-prefixed entries, since msgp encodes maps in iteration order and an
+// entry must not move between maps unnoticed. The header of a
+// legacy version carries a signature over its per-shard checksums, so the
+// key replaces it with one from the digest.
+func versionRevision(ver xlMetaV2ShallowVersion, metaV uint8) revisionKey {
+	h := sha256.New()
+	writeUint := func(n int) {
+		var b [8]byte
+		binary.LittleEndian.PutUint64(b[:], uint64(n))
+		h.Write(b[:])
+	}
+	writeField := func(b []byte) {
+		writeUint(len(b))
+		h.Write(b)
+	}
+	section := func(name string, entries int) {
+		writeField([]byte(name))
+		writeUint(entries)
+	}
+	writeStrings := func(name string, m map[string]string) {
+		section(name, len(m))
+		for _, k := range slices.Sorted(maps.Keys(m)) {
+			writeField([]byte(k))
+			writeField([]byte(m[k]))
+		}
+	}
+	writeBytes := func(name string, m map[string][]byte) {
+		section(name, len(m))
+		for _, k := range slices.Sorted(maps.Keys(m)) {
+			writeField([]byte(k))
+			writeField(m[k])
+		}
+	}
+	key := revisionKey{header: ver.header}
+	var v xlMetaV2Version
+	if _, err := v.unmarshalV(metaV, ver.meta); err == nil {
+		var (
+			bts []byte
+			err error = errFileCorrupt
+		)
+		section("type", 1)
+		writeField([]byte{byte(v.Type)})
+		switch {
+		case v.Type == ObjectType && v.ObjectV2 != nil:
+			c := *v.ObjectV2
+			c.ErasureIndex = 0
+			if !slices.ContainsFunc(c.PartETags, func(tag string) bool { return tag != "" }) {
+				c.PartETags = nil
+			}
+			if len(c.PartActualSizes) == 0 {
+				c.PartActualSizes = nil
+			}
+			writeStrings("user", c.MetaUser)
+			writeBytes("sys", c.MetaSys)
+			c.MetaUser, c.MetaSys = nil, nil
+			bts, err = c.MarshalMsg(nil)
+		case v.Type == DeleteType && v.DeleteMarker != nil:
+			c := *v.DeleteMarker
+			writeBytes("sys", c.MetaSys)
+			c.MetaSys = nil
+			bts, err = c.MarshalMsg(nil)
+		case v.Type == LegacyType && v.ObjectV1 != nil:
+			c := *v.ObjectV1
+			c.Erasure.Index = 0
+			c.Erasure.Checksums = nil
+			c.Minio.Release = ""
+			writeStrings("meta", c.Meta)
+			c.Meta = nil
+			// Part checksums are the only other map: one section per part.
+			section("parts", len(c.Parts))
+			c.Parts = slices.Clone(c.Parts)
+			for i := range c.Parts {
+				writeStrings("part-checksums", c.Parts[i].Checksums)
+				c.Parts[i].Checksums = nil
+			}
+			bts, err = c.MarshalMsg(nil)
+		}
+		if err == nil {
+			section("fields", 1)
+			writeField(bts)
+			h.Sum(key.digest[:0])
+			if v.Type == LegacyType {
+				copy(key.header.Signature[:], key.digest[:])
+			}
+			return key
+		}
+		h.Reset()
+	}
+	// Undecodable metadata only matches the same bytes.
+	section("raw", 1)
+	writeField(ver.meta)
+	h.Sum(key.digest[:0])
+	return key
+}
+
+// authoritativeHistory returns the versions of metas, the version lists of
+// the drives that answered, as merged newest first, when every version
+// that is not a free version has a revision that at least majority drives
+// and its readQuorum hold. It returns no versions when none is live,
+// errErasureReadQuorum when the drives cannot tell the history, and the
+// error of ctx once it ends. metaVs are the metadata versions of metas,
+// metaV the one the merged result is decoded with.
+func authoritativeHistory(ctx context.Context, metas [][]xlMetaV2ShallowVersion, metaVs []uint8, metaV uint8, majority int, readQuorum func(xlMetaV2ShallowVersion, uint8) int) ([]xlMetaV2ShallowVersion, error) {
+	// Count the drives that hold each revision; a drive counts once for each
+	// version.
+	type revision struct {
+		ver   xlMetaV2ShallowVersion
+		metaV uint8
+		count int
+	}
+	keys := make([][]revisionKey, len(metas))
+	revisions := make(map[[16]byte]map[revisionKey]*revision)
+	live := make(map[[16]byte]bool)
+	for i, versions := range metas {
+		keys[i] = make([]revisionKey, len(versions))
+		seen := make(map[[16]byte]bool, len(versions))
+		for j, ver := range versions {
+			// Decoding and hashing every history has a cost; stop with the
+			// request.
+			if err := ctx.Err(); err != nil {
+				return nil, err
+			}
+			id := ver.header.VersionID
+			if !ver.header.FreeVersion() {
+				live[id] = true
+			}
+			if seen[id] {
+				continue
+			}
+			seen[id] = true
+			key := versionRevision(ver, metaVs[i])
+			keys[i][j] = key
+			if revisions[id] == nil {
+				revisions[id] = make(map[revisionKey]*revision)
+			}
+			r := revisions[id][key]
+			if r == nil {
+				r = &revision{ver: ver, metaV: metaVs[i]}
+				revisions[id][key] = r
+			}
+			r.count++
+		}
+	}
+	accepted := make(map[[16]byte]revisionKey, len(live))
+	for id := range live {
+		var best *revision
+		var bestKey revisionKey
+		for key, r := range revisions[id] {
+			if !key.header.FreeVersion() && (best == nil || r.count > best.count) {
+				best, bestKey = r, key
+			}
+		}
+		// Only one revision can hold more than half of the drives.
+		if best == nil || best.count < majority || best.count < readQuorum(best.ver, best.metaV) {
+			return nil, errErasureReadQuorum
+		}
+		accepted[id] = bestKey
+	}
+
+	// The strict merge orders the history. It has to hold exactly the
+	// accepted revisions. A minority revision that sorts first would reset
+	// its count, and one that shares the header could stand in for the
+	// accepted one, so the merge only sees the accepted revision of each
+	// version and free versions as they are. It sees the headers of the
+	// keys, which do not differ between the drives of a legacy version.
+	filtered := make([][]xlMetaV2ShallowVersion, len(metas))
+	for i, versions := range metas {
+		for j, ver := range versions {
+			key, ok := accepted[ver.header.VersionID]
+			switch {
+			case ver.header.FreeVersion():
+			case ok && keys[i][j] == key:
+				ver.header = key.header
+			default:
+				continue
+			}
+			filtered[i] = append(filtered[i], ver)
+		}
+	}
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	merged := mergeXLV2Versions(majority, true, 0, filtered...)
+	emitted := make(map[[16]byte]bool, len(accepted))
+	for _, ver := range merged {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		if ver.header.FreeVersion() {
+			continue
+		}
+		id := ver.header.VersionID
+		if key, ok := accepted[id]; !ok || emitted[id] || versionRevision(ver, metaV) != key {
+			return nil, errErasureReadQuorum
+		}
+		emitted[id] = true
+	}
+	if len(emitted) != len(accepted) {
+		return nil, errErasureReadQuorum
+	}
+	if len(accepted) == 0 {
+		return nil, nil
+	}
+	return merged, nil
+}
+
+// versionReadQuorum returns the read quorum of the metadata of ver, as
+// objectQuorumFromMeta derives it when all drives of the set hold it, or
+// more than the drives when its metadata cannot be read. metaArr and errs
+// have one entry for each drive; errs stays nil.
+func (er erasureObjects) versionReadQuorum(ctx context.Context, bucket, object string, ver xlMetaV2ShallowVersion, metaV uint8, metaArr []FileInfo, errs []error) int {
+	var v xlMetaV2Version
+	if _, err := v.unmarshalV(metaV, ver.meta); err != nil {
+		return len(metaArr) + 1
+	}
+	fi, err := v.ToFileInfo(bucket, object, false)
+	if err != nil {
+		return len(metaArr) + 1
+	}
+	for i := range metaArr {
+		metaArr[i] = fi
+	}
+	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
+	if err != nil {
+		return len(metaArr) + 1
+	}
+	return readQuorum
+}
+
+// lifecycleEventsAllPools returns the versions of object in all pools,
+// newest first, and the lifecycle event of each as the scanner evaluates it
+// on them. A version with copies in several pools, as an interrupted move
+// leaves it, counts once, and so do other writes of its version ID.
+func (z *erasureServerPools) lifecycleEventsAllPools(ctx context.Context, bucket, object string) ([]ObjectInfo, []lifecycle.Event, error) {
+	lc, err := globalLifecycleSys.Get(bucket)
+	if err != nil {
+		if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
+			return nil, nil, errLifecycleNotApplicable
+		}
+		return nil, nil, err
+	}
+	lr, err := globalBucketObjectLockSys.Get(bucket)
+	if err != nil {
+		return nil, nil, err
+	}
+	rcfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	poolVersions := make([][]FileInfo, len(z.serverPools))
+	errs := make([]error, len(z.serverPools))
+	var wg sync.WaitGroup
+	for i, pool := range z.serverPools {
+		wg.Add(1)
+		go func(i int, set *erasureObjects) {
+			defer wg.Done()
+			poolVersions[i], errs[i] = set.objectVersions(ctx, bucket, object)
+		}(i, pool.getHashedSet(object))
+	}
+	wg.Wait()
+	for _, err := range errs {
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	vc, _ := globalBucketVersioningSys.Get(bucket)
+	versioned := vc != nil && vc.Versioned(decodeDirObject(object))
+	var all []PoolObjInfo
+	for pool, versions := range poolVersions {
+		for _, fi := range versions {
+			all = append(all, PoolObjInfo{Index: pool, ObjInfo: fi.ToObjectInfo(bucket, object, versioned)})
+		}
+	}
+	// Each pool lists its versions newest first. On equal times the lower
+	// pool comes first, as reads choose it.
+	sort.SliceStable(all, func(i, j int) bool { return all[i].ObjInfo.ModTime.After(all[j].ObjInfo.ModTime) })
+	seen := make(map[string]int, len(all))
+	var versions [][]PoolObjInfo
+	for _, copy := range all {
+		if i, ok := seen[copy.ObjInfo.VersionID]; ok {
+			versions[i] = append(versions[i], copy)
+			continue
+		}
+		seen[copy.ObjInfo.VersionID] = len(versions)
+		versions = append(versions, []PoolObjInfo{copy})
+	}
+
+	e := lifecycle.NewEvaluator(*lc).WithLockRetention(&lr).WithReplicationConfig(rcfg)
+	objInfos := make([]ObjectInfo, len(versions))
+	objOpts := make([]lifecycle.ObjectOpts, len(versions))
+	for i, copies := range versions {
+		// Copies with another ModTime are other writes of the version ID,
+		// such as null versions written in several pools, not duplicates of
+		// one write. The newest is the version. Data movement writes a new
+		// data dir, so only ModTime tells the duplicates of a write apart.
+		// An older write that is still protected keeps the whole object.
+		newest := 1
+		for newest < len(copies) && copies[newest].ObjInfo.ModTime.Equal(copies[0].ObjInfo.ModTime) {
+			newest++
+		}
+		for _, older := range copies[newest:] {
+			if opts := older.ObjInfo.ToLifecycleOpts(); e.IsObjectLocked(opts) || e.IsPendingReplication(opts) {
+				return nil, nil, errLifecycleNotApplicable
+			}
+		}
+		copies = copies[:newest]
+		oi := copies[0].ObjInfo
+		if len(copies) > 1 {
+			// The copies must tell which protection is the newest, and no
+			// copy may carry replication state that the merge would drop.
+			if !lockRevisionsOrdered(copies) || !replicationStatesAgree(copies) {
+				return nil, nil, errLifecycleNotApplicable
+			}
+			// Count the version once, but retain independently ordered lock
+			// and tag revisions, including explicit removals, from every copy.
+			oi = mergedPoolObjectInfo(copies)
+			setLifecycleLockState(oi.UserDefined, newestLockState(copies))
+		}
+		oi.IsLatest = i == 0
+		oi.NumVersions = len(versions)
+		oi.SuccessorModTime = time.Time{}
+		if i > 0 {
+			oi.SuccessorModTime = objInfos[i-1].ModTime
+		}
+		objInfos[i] = oi
+		objOpts[i] = oi.ToLifecycleOpts()
+	}
+	events, err := e.Eval(objOpts)
+	if err != nil {
+		return nil, nil, err
+	}
+	return objInfos, events, nil
+}
+
+// lockRevisionTime parses the timestamp of an Object Lock revision. Unlike
+// olderThan, a missing, malformed or zero timestamp orders nothing.
+func lockRevisionTime(stamp string) (time.Time, bool) {
+	t, err := time.Parse(time.RFC3339Nano, stamp)
+	return t, err == nil && !t.IsZero()
+}
+
+// lockRevisionNewer reports whether the revision stamped newer strictly
+// supersedes the one stamped older.
+func lockRevisionNewer(newer, older string) bool {
+	n, ok := lockRevisionTime(newer)
+	o, ok2 := lockRevisionTime(older)
+	return ok && ok2 && n.After(o)
+}
+
+// lifecycleLockState returns the Object Lock state of metadata as the
+// lifecycle evaluator reads it (objectlock.GetObjectRetentionMeta and
+// GetObjectLegalHoldMeta): a lowercase key before its canonical form. The
+// stored state (storedObjectLockState) only has lowercase keys.
+func lifecycleLockState(metadata map[string]string) objectLockState {
+	get := func(canonical string) string {
+		if v, ok := metadata[strings.ToLower(canonical)]; ok {
+			return v
+		}
+		return metadata[canonical]
+	}
+	return objectLockState{
+		mode:               get(objectlock.AmzObjectLockMode),
+		retainUntil:        get(objectlock.AmzObjectLockRetainUntilDate),
+		retentionTimestamp: get(ReservedMetadataPrefix + ObjectLockRetentionTimestamp),
+		legalHold:          get(objectlock.AmzObjectLockLegalHold),
+		legalHoldTimestamp: get(ReservedMetadataPrefix + ObjectLockLegalHoldTimestamp),
+	}
+}
+
+// setLifecycleLockState sets state in metadata so that the evaluator reads
+// it: lowercase keys take precedence over canonical ones.
+func setLifecycleLockState(metadata map[string]string, state objectLockState) {
+	set := func(canonical, value string) {
+		lower := strings.ToLower(canonical)
+		_, hasLower := metadata[lower]
+		_, hasCanonical := metadata[canonical]
+		if value != "" || hasLower || hasCanonical {
+			metadata[lower] = value
+		}
+	}
+	set(objectlock.AmzObjectLockMode, state.mode)
+	set(objectlock.AmzObjectLockRetainUntilDate, state.retainUntil)
+	set(ReservedMetadataPrefix+ObjectLockRetentionTimestamp, state.retentionTimestamp)
+	set(objectlock.AmzObjectLockLegalHold, state.legalHold)
+	set(ReservedMetadataPrefix+ObjectLockLegalHoldTimestamp, state.legalHoldTimestamp)
+}
+
+// newestLockState returns the newest hold and the newest retention of the
+// copies of a version, which lockRevisionsOrdered found equal or ordered.
+func newestLockState(copies []PoolObjInfo) objectLockState {
+	state := lifecycleLockState(copies[0].ObjInfo.UserDefined)
+	for _, copy := range copies[1:] {
+		next := lifecycleLockState(copy.ObjInfo.UserDefined)
+		if lockRevisionNewer(next.legalHoldTimestamp, state.legalHoldTimestamp) {
+			state.legalHold, state.legalHoldTimestamp = next.legalHold, next.legalHoldTimestamp
+		}
+		if lockRevisionNewer(next.retentionTimestamp, state.retentionTimestamp) {
+			state.mode, state.retainUntil, state.retentionTimestamp = next.mode, next.retainUntil, next.retentionTimestamp
+		}
+	}
+	return state
+}
+
+// checkedExpiryRefused reports whether a set must keep a version whose ILM
+// expiry the pools approved with the Object Lock state approved, by the
+// metadata goi the set reads under the same lock. A pending purge always
+// keeps it: its status orders nothing. The approved hold or retention only
+// replaces the one the set reads when it is strictly newer; otherwise the
+// protection the set reads stands, also when it is newer than the approval.
+func checkedExpiryRefused(lc lifecycle.Lifecycle, lr objectlock.Retention, rcfg *replication.Config, goi ObjectInfo, approved objectLockState) bool {
+	e := lifecycle.NewEvaluator(lc).WithLockRetention(&lr).WithReplicationConfig(rcfg)
+	obj := goi.ToLifecycleOpts()
+	if e.IsPendingReplication(obj) {
+		return true
+	}
+	local := lifecycleLockState(goi.UserDefined)
+	obj.UserDefined = maps.Clone(goi.UserDefined)
+	if obj.UserDefined == nil {
+		obj.UserDefined = make(map[string]string)
+	}
+	if lockRevisionNewer(approved.legalHoldTimestamp, local.legalHoldTimestamp) {
+		obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = approved.legalHold
+	}
+	if lockRevisionNewer(approved.retentionTimestamp, local.retentionTimestamp) {
+		obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = approved.mode
+		obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = approved.retainUntil
+	}
+	return e.IsObjectLocked(obj)
+}
+
+// lockRevisionsOrdered reports whether the copies of a version agree on the
+// legal hold and on the retention, each on its own, or carry revisions with
+// distinct valid timestamps, so that the newest one is known.
+func lockRevisionsOrdered(copies []PoolObjInfo) bool {
+	ordered := func(revision func(objectLockState) (value, stamp string)) bool {
+		stamps := make(map[[2]string]string, len(copies))
+		for _, copy := range copies {
+			value, stamp := revision(lifecycleLockState(copy.ObjInfo.UserDefined))
+			stamps[[2]string{value, stamp}] = stamp
+		}
+		if len(stamps) == 1 {
+			return true
+		}
+		seen := make([]time.Time, 0, len(stamps))
+		for _, stamp := range stamps {
+			t, ok := lockRevisionTime(stamp)
+			if !ok {
+				return false
+			}
+			for _, other := range seen {
+				if t.Equal(other) {
+					return false
+				}
+			}
+			seen = append(seen, t)
+		}
+		return true
+	}
+	return ordered(func(s objectLockState) (string, string) {
+		return s.legalHold, s.legalHoldTimestamp
+	}) && ordered(func(s objectLockState) (string, string) {
+		return s.mode + "\x00" + s.retainUntil, s.retentionTimestamp
+	})
+}
+
+// replicationStatesAgree reports whether the copies of a version carry the
+// same replication and purge state. The per-target states are compared as
+// parsed, regardless of their order.
+func replicationStatesAgree(copies []PoolObjInfo) bool {
+	first := copies[0].ObjInfo
+	for _, copy := range copies[1:] {
+		oi := copy.ObjInfo
+		if oi.ReplicationStatus != first.ReplicationStatus || oi.VersionPurgeStatus != first.VersionPurgeStatus ||
+			!maps.Equal(replicationStatusesMap(oi.ReplicationStatusInternal), replicationStatusesMap(first.ReplicationStatusInternal)) ||
+			!maps.Equal(versionPurgeStatusesMap(oi.VersionPurgeStatusInternal), versionPurgeStatusesMap(first.VersionPurgeStatusInternal)) {
+			return false
+		}
+	}
+	return true
+}
+
+// lifecycleAllPools holds the versions of an object in all pools and their
+// lifecycle events, see lifecycleEventsAllPools.
+type lifecycleAllPools struct {
+	versions []ObjectInfo
+	events   []lifecycle.Event
+	err      error
+}
+
+// checkExpiryAllPools returns nil when the ILM expiry that opts describe
+// still applies to object with the versions it has in all pools, see
+// lifecycleAllPools.expiryApplies, and the Object Lock state with which it
+// approved the expiry of a version.
+func (z *erasureServerPools) checkExpiryAllPools(ctx context.Context, bucket, object string, opts ObjectOptions) (objectLockState, error) {
+	var all lifecycleAllPools
+	all.versions, all.events, all.err = z.lifecycleEventsAllPools(ctx, bucket, object)
+	if err := all.expiryApplies(bucket, object, opts); err != nil {
+		return objectLockState{}, err
+	}
+	return all.protection(opts.VersionID), nil
+}
+
+// version returns the index of the version versionID, or -1.
+func (all lifecycleAllPools) version(versionID string) int {
+	for i, oi := range all.versions {
+		if oi.VersionID == versionID || oi.VersionID == "" && versionID == nullVersionID {
+			return i
+		}
+	}
+	return -1
+}
+
+// protection returns the Object Lock state of the version versionID that
+// the evaluator decided on, to be checked against the state the set reads.
+func (all lifecycleAllPools) protection(versionID string) objectLockState {
+	if versionID == "" {
+		return objectLockState{}
+	}
+	if i := all.version(versionID); i >= 0 {
+		return lifecycleLockState(all.versions[i].UserDefined)
+	}
+	return objectLockState{}
+}
+
+// expiryApplies returns nil when the ILM expiry that opts describe applies
+// to the versions. A missing version and a rejected expiry remain distinct:
+//   - an expiry of all versions (DeletePrefix) needs the latest version to
+//     expire with all versions;
+//   - a delete marker (no VersionID) hides the latest version, which has to
+//     expire;
+//   - a version (VersionID) has to expire by itself: a noncurrent version,
+//     or a delete marker without noncurrent versions.
+func (all lifecycleAllPools) expiryApplies(bucket, object string, opts ObjectOptions) error {
+	if all.err != nil {
+		if _, ok := all.err.(InsufficientReadQuorum); ok {
+			return InsufficientWriteQuorum{}
+		}
+		return all.err
+	}
+	name := decodeDirObject(object)
+	switch {
+	case opts.DeletePrefix:
+		if len(all.events) > 0 && all.events[0].Action.DeleteAll() {
+			return nil
+		}
+	case opts.VersionID == "":
+		if len(all.events) > 0 && (all.events[0].Action == lifecycle.DeleteAction || all.events[0].Action == lifecycle.DeleteAllVersionsAction) {
+			return nil
+		}
+	default:
+		if i := all.version(opts.VersionID); i >= 0 {
+			if all.events[i].Action == lifecycle.DeleteVersionAction {
+				return nil
+			}
+			return errLifecycleNotApplicable
+		}
+		return VersionNotFound{Bucket: bucket, Object: name, VersionID: opts.VersionID}
+	}
+	if len(all.versions) > 0 {
+		return errLifecycleNotApplicable
+	}
+	return ObjectNotFound{Bucket: bucket, Object: name}
+}
+
+// expiryNeedsAllPools reports whether an ILM expiry in opts deletes or hides
+// versions by a rule that depends on the other versions of the object. A
+// restored copy expires by its own restore date. Without versions each copy
+// of an object is a write of its own, and a set rechecks the rule on its
+// copy (deleteObjectFromAllPools).
+func (z *erasureServerPools) expiryNeedsAllPools(opts ObjectOptions) bool {
+	if !opts.Expiration.Expire || z.SinglePool() || opts.Transition.ExpireRestored {
+		return false
+	}
+	return opts.DeletePrefix || opts.VersionID != "" || opts.Versioned || opts.VersionSuspended
+}
+
+// recheckReadExpiry only rechecks a local ordinary-delete candidate. The
+// global latest expiring does not authorize hiding a different version that
+// GET/HEAD has already read. On an incomplete or rejected recheck, keep that
+// successful read and do not queue an expiry. Leave response metadata intact.
+func recheckReadExpiry(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, evt lifecycle.Event) (ObjectInfo, lifecycle.Event) {
+	z, ok := objectAPI.(*erasureServerPools)
+	if !ok || !evt.Action.Delete() || evt.Action.DeleteRestored() || !z.expiryNeedsAllPools(ObjectOptions{
+		Expiration:       ExpirationOptions{Expire: true},
+		VersionID:        oi.VersionID,
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+		DeletePrefix:     evt.Action.DeleteAll(),
+	}) {
+		return oi, evt
+	}
+	versions, events, err := z.lifecycleEventsAllPools(ctx, oi.Bucket, encodeDirObject(oi.Name))
+	if err == nil {
+		versionID := oi.VersionID
+		if versionID == "" {
+			versionID = nullVersionID
+		}
+		for i, version := range versions {
+			id := version.VersionID
+			if id == "" {
+				id = nullVersionID
+			}
+			if id == versionID {
+				if events[i].Action.Delete() && !events[i].Action.DeleteRestored() {
+					return version, events[i]
+				}
+				break
+			}
+		}
+	}
+	return oi, lifecycle.Event{Action: lifecycle.NoneAction}
+}
+
+// migrationExpiryApplies reports whether the ILM expiry evt, which a
+// decommission or rebalance decided for oi on the versions of the pool it
+// walks, applies with the versions of the object in all pools too. Only then
+// may the walk leave the version to the expiry instead of moving it. A
+// version that is the latest in the walked pool can be a noncurrent version
+// of the object, and a delete marker alone in that pool can hide versions in
+// another one. Such a version is moved like any other, and the scanner
+// applies the rule in the pool that holds the object then.
+func (z *erasureServerPools) migrationExpiryApplies(ctx context.Context, oi ObjectInfo, evt lifecycle.Event) bool {
+	// The options the expiry deletes with, see
+	// applyExpiryOnNonTransitionedObjects and expireTransitionedObject.
+	opts := ObjectOptions{
+		Expiration:       ExpirationOptions{Expire: true},
+		Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+	}
+	if evt.Action.DeleteVersioned() {
+		opts.VersionID = oi.VersionID
+	}
+	if evt.Action.DeleteAll() {
+		opts.DeletePrefix = true
+	}
+	opts.Transition.ExpireRestored = evt.Action.DeleteRestored()
+	if !z.expiryNeedsAllPools(opts) {
+		return true
+	}
+	_, err := z.checkExpiryAllPools(ctx, oi.Bucket, encodeDirObject(oi.Name), opts)
+	return err == nil
+}

cmd/erasure-server-pool-lifecycle_test.go

@@ -0,0 +1,501 @@
+// Copyright (c) 2026 Denis Babchuk
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"slices"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+)
+
+// The scanner and the walks of decommission and rebalance evaluate ILM on the
+// versions of a key in one pool: IsLatest, NumVersions and SuccessorModTime
+// count only these. A key can have versions in several pools: two concurrent
+// first writes, overwrites that went to another pool. The expiry they decide
+// runs in every pool.
+
+// splitILM lays out the versions of keys in the two pools of migrationILM.
+type splitILM struct {
+	*migrationILM
+	ids    map[string]string // object and label -> version ID
+	bodies map[string]string // body -> label
+}
+
+func newSplitILM(t *testing.T, status versioning.State, rules string) *splitILM {
+	t.Helper()
+	return &splitILM{migrationILM: newMigrationILM(t, status, false, rules), ids: map[string]string{}, bodies: map[string]string{}}
+}
+
+// splitVersion is a version of a key in a test layout.
+type splitVersion struct {
+	pools  []int // the pools that hold a copy of it
+	label  string
+	days   float64 // its age
+	marker bool
+	tiered bool // its data is in a remote tier
+	null   bool // a null version (unversioned or suspended bucket)
+	size   int  // its size, at least the size of its label
+}
+
+func (s *splitILM) write(object string, v splitVersion) {
+	s.t.Helper()
+	ctx := s.t.Context()
+	mtime := UTCNow().Add(-time.Duration(v.days * float64(24*time.Hour)))
+	versionID := mustGetUUID()
+	if v.null {
+		versionID = ""
+	}
+	for _, pool := range v.pools {
+		if v.marker {
+			_, err := s.z.serverPools[pool].DeleteObject(ctx, s.bucket, object,
+				ObjectOptions{Versioned: true, VersionID: versionID, DeleteMarker: true, MTime: mtime})
+			if err != nil {
+				s.t.Fatal(err)
+			}
+			continue
+		}
+		opts := ObjectOptions{Versioned: !v.null, VersionID: versionID, MTime: mtime}
+		if v.null && s.status() == versioning.Suspended {
+			opts = ObjectOptions{VersionSuspended: true, MTime: mtime}
+		}
+		body := object + " " + v.label
+		if len(body) < v.size {
+			body += strings.Repeat(".", v.size-len(body))
+		}
+		putConsistencyObject(s.t, s.z, s.bucket, object, pool, body, opts)
+		s.bodies[body] = v.label
+		if v.tiered {
+			vid := versionID
+			if vid == "" {
+				vid = nullVersionID
+			}
+			infos, errs := readAllFileInfo(ctx, s.z.serverPools[pool].getHashedSet(object).getDisks(), "", s.bucket, object, vid, false, false)
+			if errs[0] != nil {
+				s.t.Fatal(errs[0])
+			}
+			fi := infos[0]
+			fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", "remote-"+v.label
+			if err := s.z.serverPools[pool].DecomTieredObject(ctx, s.bucket, object, fi, ObjectOptions{}); err != nil {
+				s.t.Fatal(err)
+			}
+		}
+	}
+	if v.marker {
+		s.markers[s.key(object, versionID)] = v.label
+	} else {
+		s.labels[s.key(object, versionID)] = v.label
+	}
+	s.ids[object+" "+v.label] = versionID
+	s.track(object)
+}
+
+func (s *splitILM) status() versioning.State {
+	vc, _ := globalBucketVersioningSys.Get(s.bucket)
+	if vc == nil {
+		return ""
+	}
+	return vc.Status
+}
+
+// scan applies ILM to every object in a pool as the scanner of a drive of
+// that pool does, from the xl.meta of that drive, and runs the queued
+// expiries as an expiry worker does.
+func (s *splitILM) scan(pool int) {
+	s.t.Helper()
+	ctx := s.t.Context()
+	lc, err := globalLifecycleSys.Get(s.bucket)
+	if err != nil {
+		s.t.Fatal(err)
+	}
+	lr, err := globalBucketObjectLockSys.Get(s.bucket)
+	if err != nil {
+		s.t.Fatal(err)
+	}
+	vc, _ := globalBucketVersioningSys.Get(s.bucket)
+	for _, object := range s.objects {
+		disk := s.z.serverPools[pool].getHashedSet(object).getDisks()[0]
+		rf, err := disk.ReadXL(ctx, s.bucket, object, false)
+		if err != nil {
+			continue // not in this pool
+		}
+		fivs, err := getFileInfoVersions(rf.Buf, s.bucket, object, false)
+		if err != nil {
+			s.t.Fatal(err)
+		}
+		objInfos := make([]ObjectInfo, len(fivs.Versions))
+		for i, fi := range fivs.Versions {
+			objInfos[i] = fi.ToObjectInfo(s.bucket, object, vc != nil && vc.Versioned(object))
+		}
+		item := scannerItem{bucket: s.bucket, objectName: object, lifeCycle: lc}
+		item.applyActions(ctx, s.z, objInfos, lr, &sizeSummary{}, nil)
+	}
+	s.runQueue()
+}
+
+// head applies ILM to a version as a HEAD or GET with its versionId does.
+func (s *splitILM) head(object, label string) lifecycle.Action {
+	s.t.Helper()
+	ctx := s.t.Context()
+	oi, err := s.z.GetObjectInfo(ctx, s.bucket, object, ObjectOptions{VersionID: s.ids[object+" "+label]})
+	if err != nil {
+		s.t.Fatal(err)
+	}
+	lc, err := globalLifecycleSys.Get(s.bucket)
+	if err != nil {
+		s.t.Fatal(err)
+	}
+	lr, err := globalBucketObjectLockSys.Get(s.bucket)
+	if err != nil {
+		s.t.Fatal(err)
+	}
+	evt := evalActionFromLifecycle(ctx, *lc, lr, nil, oi)
+	if evt.Action.Delete() {
+		applyExpiryRule(evt, lcEventSrc_s3HeadObject, oi)
+	}
+	s.runQueue()
+	return evt.Action
+}
+
+// runQueue runs the queued expiries with an expiry worker.
+func (s *splitILM) runQueue() {
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		globalExpiryState.Worker(s.queue)
+	}()
+	s.queue <- nil // the worker returns once it ran every expiry queued before
+	<-done
+}
+
+// get returns the label of the version a GET of object reads, or "-" when it
+// reads as deleted.
+func (s *splitILM) get(object string) string {
+	s.t.Helper()
+	ctx := s.t.Context()
+	oi, err := s.z.GetObjectInfo(ctx, s.bucket, object, ObjectOptions{})
+	if err != nil {
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrMethodNotAllowed(err) {
+			return "-"
+		}
+		return fmt.Sprintf("<%v>", err)
+	}
+	if oi.VersionID != "" && oi.VersionID != nullVersionID {
+		return s.label(object, oi.VersionID, false)
+	}
+	// Copies of a null version in two pools are told apart by their data.
+	gr, err := s.z.GetObjectNInfo(ctx, s.bucket, object, nil, nil, ObjectOptions{})
+	if err != nil {
+		return fmt.Sprintf("<%v>", err)
+	}
+	defer gr.Close()
+	data, err := io.ReadAll(gr)
+	if err != nil {
+		s.t.Fatal(err)
+	}
+	if l, ok := s.bodies[string(data)]; ok {
+		return l
+	}
+	return "?" + string(data)
+}
+
+// views returns the versions of each object in each pool, newest first.
+func (s *splitILM) views() []map[string][]string {
+	views := make([]map[string][]string, len(s.z.serverPools))
+	for pool := range s.z.serverPools {
+		views[pool] = s.view(pool)
+	}
+	return views
+}
+
+type splitKey struct {
+	name     string
+	versions []splitVersion
+	before   string     // what a GET reads before ILM ran
+	after    string     // what a GET reads once ILM ran
+	want     [][]string // versions in each pool once ILM ran, nil to skip
+}
+
+// run lays out keys, runs two scanner cycles over all pools and checks that
+// a GET of each key never reads anything but before or after, and the
+// versions each pool holds at the end. The second cycle must not change
+// anything.
+func (s *splitILM) run(keys []splitKey) {
+	s.t.Helper()
+	for _, k := range keys {
+		for _, v := range k.versions {
+			s.write(k.name, v)
+		}
+		if got := s.get(k.name); got != k.before {
+			s.t.Fatalf("%s: fixture reads %s, want %s", k.name, got, k.before)
+		}
+	}
+	var first []map[string][]string
+	for cycle := 1; cycle <= 2; cycle++ {
+		for pool := range s.z.serverPools {
+			s.scan(pool)
+			for _, k := range keys {
+				if got := s.get(k.name); got != k.before && got != k.after {
+					s.t.Errorf("%s: reads %s after the scanner of pool %d (cycle %d), want %s", k.name, got, pool, cycle, k.after)
+				}
+			}
+		}
+		if cycle == 1 {
+			first = s.views()
+		}
+	}
+	last := s.views()
+	for _, k := range keys {
+		if got := s.get(k.name); got != k.after {
+			s.t.Errorf("%s: reads %s once ILM ran, want %s", k.name, got, k.after)
+		}
+		for pool := range last {
+			if !slices.Equal(first[pool][k.name], last[pool][k.name]) {
+				s.t.Errorf("%s: the second scanner cycle changed pool %d from %v to %v", k.name, pool, first[pool][k.name], last[pool][k.name])
+			}
+			if k.want != nil && !slices.Equal(last[pool][k.name], k.want[pool]) {
+				s.t.Errorf("%s: pool %d holds %v, want %v", k.name, pool, last[pool][k.name], k.want[pool])
+			}
+		}
+	}
+}
+
+const splitILMRules = "<Rule><ID>exp</ID><Status>Enabled</Status><Filter><Prefix>exp-</Prefix></Filter>" +
+	"<Expiration><Days>1</Days></Expiration></Rule>" +
+	"<Rule><ID>eav</ID><Status>Enabled</Status><Filter><Prefix>eav-</Prefix></Filter>" +
+	"<Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration></Rule>" +
+	"<Rule><ID>edm</ID><Status>Enabled</Status><Filter><Prefix>edm-</Prefix></Filter>" +
+	"<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration></Rule>" +
+	"<Rule><ID>dme</ID><Status>Enabled</Status><Filter><Prefix>dme-</Prefix></Filter>" +
+	"<DelMarkerExpiration><Days>1</Days></DelMarkerExpiration></Rule>" +
+	"<Rule><ID>ncv</ID><Status>Enabled</Status><Filter><Prefix>ncv-</Prefix></Filter>" +
+	"<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration></Rule>" +
+	"<Rule><ID>exs</ID><Status>Enabled</Status><Filter><And><Prefix>exs-</Prefix><ObjectSizeGreaterThan>100</ObjectSizeGreaterThan></And></Filter>" +
+	"<Expiration><Days>1</Days></Expiration></Rule>" +
+	"<Rule><ID>nnv</ID><Status>Enabled</Status><Filter><Prefix>nnv-</Prefix></Filter>" +
+	"<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays><NewerNoncurrentVersions>1</NewerNoncurrentVersions></NoncurrentVersionExpiration></Rule>"
+
+func splitAt(label string, days float64, pools ...int) splitVersion {
+	return splitVersion{pools: pools, label: label, days: days}
+}
+
+func splitMarker(label string, days float64, pools ...int) splitVersion {
+	return splitVersion{pools: pools, label: label, days: days, marker: true}
+}
+
+func splitTiered(label string, days float64, pools ...int) splitVersion {
+	return splitVersion{pools: pools, label: label, days: days, tiered: true}
+}
+
+func splitNull(label string, days float64, pools ...int) splitVersion {
+	return splitVersion{pools: pools, label: label, days: days, null: true}
+}
+
+// splitILMOnePool are keys in one pool. ILM applies to them as usual.
+func splitILMOnePool(pool int) []splitKey {
+	in := func(vs ...string) [][]string {
+		want := [][]string{nil, nil}
+		want[pool] = vs
+		return want
+	}
+	return []splitKey{
+		{"exp-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool)}, "v2", "-", in("DM", "v2", "v1")},
+		{"exp-one-tiered", []splitVersion{splitAt("v1", 5, pool), splitTiered("v2", 4, pool)}, "v2", "-", in("DM", "v2", "v1")},
+		{"eav-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool)}, "v2", "-", in()},
+		{"edm-one", []splitVersion{splitMarker("dm", 3, pool)}, "-", "-", in()},
+		{"dme-one", []splitVersion{splitAt("v1", 5, pool), splitMarker("dm", 3, pool)}, "-", "-", in()},
+		{"ncv-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool), splitAt("v3", 0.01, pool)}, "v3", "v3", in("v3", "v2")},
+		{"nnv-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool), splitAt("v3", 3, pool), splitAt("v4", 2, pool)}, "v4", "v4", in("v4", "v3")},
+	}
+}
+
+func TestPoolsLifecycleSplitObject(t *testing.T) {
+	t.Run("versioned", func(t *testing.T) {
+		s := newSplitILM(t, versioning.Enabled, splitILMRules)
+		s.run([]splitKey{
+			// Expiration with Days applies to the current version v2 only.
+			{"exp-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+			{"exp-old-in-1", []splitVersion{splitAt("v2", 0.01, 0), splitAt("v1", 5, 1)}, "v2", "v2", [][]string{{"v2"}, {"v1"}}},
+			{"exp-tiered-old-in-0", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+			// v2 is old enough: one delete marker hides the key.
+			{"exp-both-old", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1)}, "v2", "-", [][]string{{"DM", "v1"}, {"v2"}}},
+			// The rule applies to v2 only by its size.
+			{"exs-big-in-1", []splitVersion{splitAt("v1", 5, 0), {pools: []int{1}, label: "v2", days: 4, size: 200}}, "v2", "-", [][]string{{"DM", "v1"}, {"v2"}}},
+			// The marker hides v1 and is not a marker without noncurrent
+			// versions; v1 is not the current version.
+			{"exp-marker-in-0", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", [][]string{{"dm"}, {"v1"}}},
+			// ExpiredObjectAllVersions applies to the current version v2 only.
+			{"eav-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+			{"eav-old-in-1", []splitVersion{splitAt("v2", 0.01, 0), splitAt("v1", 5, 1)}, "v2", "v2", [][]string{{"v2"}, {"v1"}}},
+			{"eav-tiered-old-in-0", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+			{"eav-both-old", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1)}, "v2", "-", [][]string{nil, nil}},
+			// A copy of v1 in both pools, as an interrupted move leaves it.
+			{"eav-copies", []splitVersion{splitAt("v1", 5, 0, 1)}, "v1", "-", [][]string{nil, nil}},
+			// The marker hides v1 in the other pool.
+			{"edm-marker-in-0", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", [][]string{{"dm"}, {"v1"}}},
+			{"edm-marker-in-1", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 1)}, "-", "-", [][]string{{"v1"}, {"dm"}}},
+			{"edm-copies", []splitVersion{splitMarker("dm", 3, 0, 1)}, "-", "-", [][]string{nil, nil}},
+			// v2 in the other pool is the current version, the marker does
+			// not expire.
+			{"dme-new-in-1", []splitVersion{splitMarker("dm", 3, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"dm"}, {"v2"}}},
+			{"dme-new-in-0", []splitVersion{splitAt("v2", 0.01, 0), splitMarker("dm", 3, 1)}, "v2", "v2", [][]string{{"v2"}, {"dm"}}},
+			// The marker is the current version and expires with v1.
+			{"dme-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 1)}, "-", "-", [][]string{nil, nil}},
+			// v1 is noncurrent since v2 was written, v2 since v3 was.
+			{"ncv-split", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1), splitAt("v3", 2, 0)}, "v3", "v3", [][]string{{"v3"}, {"v2"}}},
+		})
+	})
+	t.Run("versioned/one-pool", func(t *testing.T) {
+		for pool := range 2 {
+			s := newSplitILM(t, versioning.Enabled, splitILMRules)
+			s.run(splitILMOnePool(pool))
+		}
+	})
+	t.Run("versioned/head-version", func(t *testing.T) {
+		s := newSplitILM(t, versioning.Enabled, splitILMRules)
+		keys := []splitKey{
+			{"exp-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+			{"eav-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+		}
+		for _, k := range keys {
+			for _, v := range k.versions {
+				s.write(k.name, v)
+			}
+			act := s.head(k.name, "v1")
+			t.Logf("%s: HEAD of v1 evaluates %s", k.name, act)
+			if got := s.get(k.name); got != k.after {
+				t.Errorf("%s: reads %s after a HEAD of v1, want %s", k.name, got, k.after)
+			}
+			views := s.views()
+			for pool := range views {
+				if !slices.Equal(views[pool][k.name], k.want[pool]) {
+					t.Errorf("%s: pool %d holds %v, want %v", k.name, pool, views[pool][k.name], k.want[pool])
+				}
+			}
+		}
+	})
+	t.Run("suspended", func(t *testing.T) {
+		s := newSplitILM(t, versioning.Suspended, splitILMRules)
+		s.run([]splitKey{
+			{"exp-null-old-in-0", []splitVersion{splitNull("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+		})
+	})
+	t.Run("unversioned", func(t *testing.T) {
+		s := newSplitILM(t, "", splitILMRules)
+		s.run([]splitKey{
+			{"eav-old-in-0", []splitVersion{splitNull("old", 5, 0), splitNull("new", 0.01, 1)}, "new", "new", nil},
+			{"exp-old-in-0", []splitVersion{splitNull("old", 5, 0), splitNull("new", 0.01, 1)}, "new", "new", nil},
+		})
+		views := s.views()
+		for object, want := range map[string]int{"eav-old-in-0": 1, "exp-old-in-0": 0} {
+			if len(views[0][object]) != want || len(views[1][object]) != 1 {
+				t.Errorf("%s: pools hold %v and %v, want %d and 1 copies", object, views[0][object], views[1][object], want)
+			}
+		}
+	})
+}
+
+// walk moves pool 0 out with a decommission or a rebalance pass, as move
+// does, but reports a decommission that did not complete instead of stopping
+// the test, so the reads are still checked.
+func (s *splitILM) walk(kind string) {
+	s.t.Helper()
+	ctx := s.t.Context()
+	if kind == "rebalance" {
+		migrationTestWalk(s.t, s.z, s.bucket, 0, kind)
+		s.z.rebalMu.Lock()
+		s.z.rebalMeta.PoolStats[0].Info.Status = rebalCompleted
+		s.z.rebalMu.Unlock()
+		return
+	}
+	if err := s.z.StartDecommission(ctx, 0); err != nil {
+		s.t.Fatal(err)
+	}
+	s.z.doDecommissionInRoutine(ctx, 0)
+	s.z.poolMetaMutex.RLock()
+	info := *s.z.poolMeta.Pools[0].Decommission
+	s.z.poolMetaMutex.RUnlock()
+	if !info.Complete || info.Failed || info.ItemsDecommissionFailed > 0 {
+		s.t.Errorf("decommission did not complete: complete=%t failed=%t itemsFailed=%d; pool 0 holds %v",
+			info.Complete, info.Failed, info.ItemsDecommissionFailed, s.view(0))
+	}
+}
+
+// Rules that expire a whole object (DelMarkerExpiration,
+// ExpiredObjectAllVersions) or a delete marker without noncurrent versions
+// (ExpiredObjectDeleteMarker) depend on every version of the object. A walk
+// sees the versions in its own pool only.
+func TestPoolsMigrationLifecycleWholeObject(t *testing.T) {
+	recent := 1.0 / 24
+	objects := []struct {
+		name     string
+		versions []splitVersion // oldest first
+		before   string         // what a GET reads before the walk
+		after    string         // what a GET reads once ILM caught up
+		want     []string       // versions in pool 1 once ILM caught up, newest first
+	}{
+		// The marker expires and takes v1 with it.
+		{"dme-one-pool", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 0)}, "-", "-", nil},
+		{"dme-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", nil},
+		// v2 is the current version, the marker does not expire.
+		{"dme-newer-in-pool-1", []splitVersion{splitMarker("dm", 3, 0), splitAt("v2", recent, 1)}, "v2", "v2", []string{"v2", "dm"}},
+		// v2 expires and takes v1 with it.
+		{"eav-one-pool", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 0)}, "v2", "-", nil},
+		{"eav-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitAt("v2", 4, 0)}, "v2", "-", nil},
+		// v2 is the current version and does not expire.
+		{"eav-newer-in-pool-1", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", recent, 1)}, "v2", "v2", []string{"v2", "v1"}},
+		// The marker hides v1, it is not a marker without noncurrent versions.
+		{"edm-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", []string{"dm", "v1"}},
+		{"keep-marker", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 0)}, "-", "-", []string{"dm", "v1"}},
+		{"keep-versions", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 0)}, "v2", "v2", []string{"v2", "v1"}},
+	}
+	for _, kind := range []string{"decommission", "rebalance"} {
+		t.Run(kind, func(t *testing.T) {
+			s := newSplitILM(t, versioning.Enabled, splitILMRules)
+			for _, o := range objects {
+				for _, v := range o.versions {
+					s.write(o.name, v)
+				}
+				if got := s.get(o.name); got != o.before {
+					t.Fatalf("%s: fixture reads %s, want %s", o.name, got, o.before)
+				}
+			}
+			s.walk(kind)
+			for _, o := range objects {
+				if got := s.get(o.name); got != o.before && got != o.after {
+					t.Errorf("%s: reads %s after the %s, before ILM caught up; want %s or %s", o.name, got, kind, o.before, o.after)
+				}
+			}
+			s.runQueue()
+			s.scan(1)
+			want := map[string][]string{}
+			for _, o := range objects {
+				want[o.name] = o.want
+				if got := s.get(o.name); got != o.after {
+					t.Errorf("%s: reads %s once ILM caught up, want %s", o.name, got, o.after)
+				}
+			}
+			s.check(want)
+		})
+	}
+}

cmd/erasure-server-pool-locks.go

@@ -0,0 +1,322 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"math/rand/v2"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/v7/pkg/set"
+)
+
+// Object lock domains.
+//
+// Every erasure set has its own lockers, one per host among its drives, and a
+// distributed lock is granted by a majority of them. Two locks of one name
+// exclude each other only when they ask the same lockers. In a single pool the
+// writers (PutObject, multipart commit, PutObjectPart, tags, retention, legal
+// hold, metadata, restore, transition, heal) lock in the set an object hashes
+// to. The pools layer locks an object, and its upload IDs, in the same set, so
+// that a pool whose sets use different hosts still has one lock per object.
+// With several pools every object write takes the pools lock instead, which
+// these helpers then return unchanged.
+//
+// Generic NewNSLock keeps one stable domain for names that are not objects:
+// bucket and IAM metadata locks, format and leader coordination, and the
+// /[replicate]/ serializer.
+
+// NewObjectNSLock returns the lock of one object in the lock domain that its
+// writes use: in a single pool the object's own erasure set, otherwise the
+// pools lock. Directory objects are named as storage names them.
+func (z *erasureServerPools) NewObjectNSLock(bucket, object string) RWLocker {
+	if z.SinglePool() {
+		object = encodeDirObject(object)
+	}
+	return z.objectNSLock(bucket, object, object)
+}
+
+// uploadIDNSLock returns the guard of one upload ID. It is chosen by the
+// upload's object, as PutObjectPart locks it in that object's set; the name
+// locked stays object/uploadID.
+func (z *erasureServerPools) uploadIDNSLock(bucket, object, uploadID string) RWLocker {
+	return z.objectNSLock(bucket, object, pathJoin(object, uploadID))
+}
+
+// objectNSLock returns the lock of resource, a name owned by object, in the
+// lock domain of object.
+func (z *erasureServerPools) objectNSLock(bucket, object, resource string) RWLocker {
+	if z.SinglePool() {
+		return z.serverPools[0].getHashedSet(encodeDirObject(object)).NewNSLock(bucket, resource)
+	}
+	return z.NewNSLock(bucket, resource)
+}
+
+// objectLockRequest is one object lock that lockObjects takes.
+type objectLockRequest struct {
+	bucket, object string
+	write          bool
+}
+
+// lockObjects takes the locks of several objects, such as the source and
+// destination of a copy, before any of them is read. A name that appears twice
+// gets one lock, a write lock if either asks for one. The locks are taken in
+// one order, by lock domain and then by bucket and object, and transactionally
+// (see getNSLocks), so that two copies in opposite directions, or an operation
+// that orders its locks differently, cannot hold each other forever. The
+// returned context ends when any of the locks is lost; the returned function
+// releases them all.
+func (z *erasureServerPools) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) {
+	type entry struct {
+		objectLockRequest
+		domain int
+	}
+	locks := make([]entry, 0, len(reqs))
+	for _, r := range reqs {
+		r.object = encodeDirObject(r.object)
+		i := slices.IndexFunc(locks, func(e entry) bool { return e.bucket == r.bucket && e.object == r.object })
+		if i >= 0 {
+			locks[i].write = locks[i].write || r.write
+			continue
+		}
+		domain := 0
+		if z.SinglePool() {
+			domain = z.serverPools[0].getHashedSetIndex(r.object)
+		}
+		locks = append(locks, entry{objectLockRequest: r, domain: domain})
+	}
+	slices.SortFunc(locks, func(a, b entry) int {
+		if a.domain != b.domain {
+			return a.domain - b.domain
+		}
+		return strings.Compare(pathJoin(a.bucket, a.object), pathJoin(b.bucket, b.object))
+	})
+	nsLocks := make([]nsLockRequest, len(locks))
+	for i, l := range locks {
+		nsLocks[i] = nsLockRequest{
+			newLock: func() RWLocker { return z.NewObjectNSLock(l.bucket, l.object) },
+			write:   l.write,
+			atomic:  true,
+		}
+	}
+	return getNSLocks(ctx, nsLocks, globalOperationTimeout)
+}
+
+// nsLockRequest is one lock of a transactional acquisition.
+type nsLockRequest struct {
+	// newLock returns a new instance of the lock for each attempt. An
+	// instance names its holder, and a distributed unlock retries in the
+	// background; a later attempt with the same instance could be released
+	// by such a retry of the attempt before.
+	newLock func() RWLocker
+	write   bool
+	// atomic: the lock is granted as a whole or not at all. A distributed
+	// lock is, also over several names. The local backend takes several
+	// names one at a time, holding the first while it waits for the next;
+	// its lock context is the context it was given.
+	atomic bool
+}
+
+const (
+	// nsLockTryTimeout bounds each wait for a lock while holding another.
+	nsLockTryTimeout = 2 * time.Second
+	// nsLockRetryWait is the longest random pause before a new attempt.
+	nsLockRetryWait = 250 * time.Millisecond
+)
+
+// getNSLocks acquires every lock of locks, in their order, or none. While it
+// holds a lock it waits for the next one only up to nsLockTryTimeout; then it
+// releases what it holds in reverse order, pauses, and tries again with a
+// fresh context, until the timeout of all attempts. A wait that holds nothing
+// can last for the rest of the timeout. Each lock is taken in the context of
+// the one before, so the returned context ends when any lease is lost.
+func getNSLocks(ctx context.Context, locks []nsLockRequest, timeout *dynamicTimeout) (LockContext, func(), error) {
+	type held struct {
+		lock  RWLocker
+		write bool
+		lkctx LockContext
+	}
+	release := func(hs []held) {
+		for i := len(hs) - 1; i >= 0; i-- {
+			if hs[i].write {
+				hs[i].lock.Unlock(hs[i].lkctx)
+			} else {
+				hs[i].lock.RUnlock(hs[i].lkctx)
+			}
+		}
+	}
+	// Fail as a single lock does: a canceled request with its own error,
+	// anything else as a timeout.
+	failed := func() error {
+		if err := ctx.Err(); errors.Is(err, context.Canceled) {
+			return err
+		}
+		return OperationTimedOut{}
+	}
+	start := time.Now()
+	deadline := start.Add(timeout.Timeout())
+	for {
+		hs := make([]held, 0, len(locks))
+		lctx := ctx
+		var err error
+		for _, l := range locks {
+			wait := time.Until(deadline)
+			if len(hs) > 0 || !l.atomic {
+				wait = min(wait, nsLockTryTimeout)
+			}
+			if wait <= 0 {
+				err = OperationTimedOut{}
+				break
+			}
+			lk := l.newLock()
+			try := newDynamicTimeout(wait, wait)
+			// The local backend waits up to the timeout for each name: bound
+			// the whole attempt by its context instead.
+			tctx, cancel := lctx, context.CancelFunc(func() {})
+			if !l.atomic {
+				tctx, cancel = context.WithTimeout(lctx, wait)
+			}
+			var lkctx LockContext
+			if l.write {
+				lkctx, err = lk.GetLock(tctx, try)
+			} else {
+				lkctx, err = lk.GetRLock(tctx, try)
+			}
+			cancel()
+			if err != nil {
+				break
+			}
+			if !l.atomic {
+				lkctx.ctx = lctx
+			}
+			hs = append(hs, held{lock: lk, write: l.write, lkctx: lkctx})
+			lctx = lkctx.Context()
+		}
+		if err == nil {
+			timeout.LogSuccess(time.Since(start))
+			if len(hs) == 0 {
+				return LockContext{ctx: ctx, cancel: func() {}}, func() {}, nil
+			}
+			return LockContext{ctx: lctx, cancel: func() {}}, func() { release(hs) }, nil
+		}
+		release(hs)
+		// A lost lease cancels the context of a later wait; only the
+		// request's own context ends the attempts.
+		if ctx.Err() != nil || (!errors.As(err, &OperationTimedOut{}) && !errors.Is(err, context.Canceled)) {
+			return LockContext{}, nil, err
+		}
+		pause := rand.N(nsLockRetryWait) + time.Millisecond
+		if time.Until(deadline) <= pause {
+			timeout.LogFailure()
+			return LockContext{}, nil, OperationTimedOut{}
+		}
+		t := time.NewTimer(pause)
+		select {
+		case <-ctx.Done():
+			t.Stop()
+			return LockContext{}, nil, failed()
+		case <-t.C:
+		}
+	}
+}
+
+// sharesLocalNSLock reports whether every set of the pool uses one local lock
+// table, as a single node does: there all sets are one lock domain.
+func (s *erasureSets) sharesLocalNSLock() bool {
+	first := s.sets[0].nsMutex
+	if first.isDistErasure {
+		return false
+	}
+	for _, set := range s.sets[1:] {
+		if set.nsMutex != first {
+			return false
+		}
+	}
+	return true
+}
+
+// deleteObjectsInSets is the bulk delete of a single pool whose sets do not
+// share one lock table. The names are grouped by the set they hash to, the
+// lock domain of their writers. The groups run one after another in ascending
+// set index; each takes its set's lock over all of its names in one
+// acquisition, deletes its sub-batch, and releases the lock before the next
+// group starts. No two sets' locks are held at once.
+//
+// Each group's lock is taken in the request's context. A group whose lock
+// cannot be taken fails as a whole and the next group still runs; once the
+// request is canceled the remaining groups fail without being attempted. The
+// results keep their positions in objects. Replication, events and auditing
+// stay with the callers, on the merged results.
+func (z *erasureServerPools) deleteObjectsInSets(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions, dobjects []DeletedObject, derrs []error, timeout *dynamicTimeout) {
+	pool := z.serverPools[0]
+	groups := make(map[int][]int)
+	for i, obj := range objects {
+		idx := pool.getHashedSetIndex(obj.ObjectName)
+		groups[idx] = append(groups[idx], i)
+	}
+	order := make([]int, 0, len(groups))
+	for idx := range groups {
+		order = append(order, idx)
+	}
+	slices.Sort(order)
+
+	for _, idx := range order {
+		members := groups[idx]
+		fail := func(err error) {
+			for _, i := range members {
+				derrs[i] = err
+			}
+		}
+		if err := ctx.Err(); err != nil {
+			fail(err)
+			continue
+		}
+		er := pool.sets[idx]
+		names := set.NewStringSet()
+		batch := make([]ObjectToDelete, len(members))
+		for j, i := range members {
+			names.Add(objects[i].ObjectName)
+			batch[j] = objects[i]
+		}
+		resources := names.ToSlice()
+		lkctx, unlock, err := getNSLocks(ctx, []nsLockRequest{{
+			newLock: func() RWLocker { return er.NewNSLock(bucket, resources...) },
+			write:   true,
+			atomic:  er.nsMutex.isDistErasure || len(resources) == 1,
+		}}, timeout)
+		if err != nil {
+			fail(err)
+			continue
+		}
+		deleted, errs := er.DeleteObjects(lkctx.Context(), bucket, batch, opts)
+		unlock()
+		// Merge as the pools loop of DeleteObjects does for one pool.
+		for j, i := range members {
+			if errs[j] == nil && deleted[j].found {
+				dobjects[i], derrs[i] = deleted[j], nil
+				continue
+			}
+			if derrs[i] == nil {
+				dobjects[i], derrs[i] = deleted[j], errs[j]
+			}
+		}
+	}
+}

cmd/erasure-server-pool-locks_test.go

@@ -0,0 +1,555 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/dsync"
+)
+
+// ldTry reports whether lk can be taken now, and releases it again.
+func ldTry(t *testing.T, lk RWLocker, write bool) bool {
+	t.Helper()
+	dt := newDynamicTimeout(100*time.Millisecond, 100*time.Millisecond)
+	if write {
+		lc, err := lk.GetLock(t.Context(), dt)
+		if err == nil {
+			lk.Unlock(lc)
+		}
+		return err == nil
+	}
+	lc, err := lk.GetRLock(t.Context(), dt)
+	if err == nil {
+		lk.RUnlock(lc)
+	}
+	return err == nil
+}
+
+// ldFree reports whether lk can be taken within a few seconds: a distributed
+// unlock completes in the background.
+func ldFree(t *testing.T, lk RWLocker, write bool) bool {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for !ldTry(t, lk, write) {
+		if time.Now().After(deadline) {
+			return false
+		}
+	}
+	return true
+}
+
+// ldTaken waits until someone else holds lk.
+func ldTaken(t *testing.T, lk RWLocker) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for ldTry(t, lk, false) {
+		if time.Now().After(deadline) {
+			t.Fatal("lock never taken")
+		}
+		time.Sleep(time.Millisecond)
+	}
+}
+
+func ldHoldLock(t *testing.T, lk RWLocker) func() {
+	t.Helper()
+	lc, err := lk.GetLock(t.Context(), globalOperationTimeout)
+	if err != nil {
+		t.Fatal(err)
+	}
+	done := false
+	return func() {
+		if !done {
+			done = true
+			lk.Unlock(lc)
+		}
+	}
+}
+
+// Object locks, and upload guards by their object, are in the set the
+// object's writes lock in; with several pools they stay the pools lock.
+func TestObjectNSLockRouting(t *testing.T) {
+	for _, mode := range ldModes {
+		t.Run(mode, func(t *testing.T) {
+			z, bucket := ldPool(t, mode)
+			pool := z.serverPools[0]
+			key := ldKey(t, z, "route", 1)
+			dir := ldKey(t, z, "route-dir", 0) + "/"
+			for _, name := range []string{key, dir} {
+				release := ldHoldLock(t, z.NewObjectNSLock(bucket, name))
+				encoded := encodeDirObject(name)
+				if ldTry(t, pool.getHashedSet(encoded).NewNSLock(bucket, encoded), false) {
+					t.Errorf("%s: a writer of the object's set ignores the object lock", name)
+				}
+				release()
+			}
+			id := mustGetUUID()
+			release := ldHoldLock(t, z.uploadIDNSLock(bucket, key, id))
+			if ldTry(t, pool.getHashedSet(key).NewNSLock(bucket, pathJoin(key, id)), false) {
+				t.Error("PutObjectPart's guard ignores the upload guard")
+			}
+			release()
+			if mode != ldShared && pool.sharesLocalNSLock() {
+				t.Error("split sets reported as one local table")
+			}
+			if mode == ldShared && !pool.sharesLocalNSLock() {
+				t.Error("one local table not recognized")
+			}
+		})
+	}
+	t.Run("pools", func(t *testing.T) {
+		z, bucket := bulkDeletePools(t, 2)
+		release := ldHoldLock(t, z.NewNSLock(bucket, "pooled"))
+		defer release()
+		if ldTry(t, z.NewObjectNSLock(bucket, "pooled"), false) || ldTry(t, z.uploadIDNSLock(bucket, "pooled", ""), false) {
+			t.Error("multi-pool object locks left the pools lock")
+		}
+	})
+}
+
+// lockObjects merges a name asked twice into one write lock and takes source
+// and destination in their own sets.
+func TestLockObjects(t *testing.T) {
+	for _, mode := range ldModes {
+		t.Run(mode, func(t *testing.T) {
+			z, bucket := ldPool(t, mode)
+			pool := z.serverPools[0]
+			src, dst := ldKey(t, z, "lock-src", 1), ldKey(t, z, "lock-dst", 0)
+			lkctx, unlock, err := z.lockObjects(t.Context(), objectLockRequest{bucket: bucket, object: src}, objectLockRequest{bucket: bucket, object: dst, write: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if lkctx.Context().Err() != nil {
+				t.Fatal("lock context ended")
+			}
+			if !ldTry(t, pool.getHashedSet(src).NewNSLock(bucket, src), false) {
+				t.Error("source not shared with readers")
+			}
+			if ldTry(t, pool.getHashedSet(src).NewNSLock(bucket, src), true) {
+				t.Error("source not locked against writers")
+			}
+			if ldTry(t, pool.getHashedSet(dst).NewNSLock(bucket, dst), false) {
+				t.Error("destination not locked")
+			}
+			unlock()
+			if !ldFree(t, pool.getHashedSet(dst).NewNSLock(bucket, dst), true) || !ldFree(t, pool.getHashedSet(src).NewNSLock(bucket, src), true) {
+				t.Error("locks left behind")
+			}
+
+			// A copy onto itself takes one write lock; a directory object is
+			// locked by its stored name.
+			self := ldKey(t, z, "lock-self", 1) + "/"
+			_, unlock, err = z.lockObjects(t.Context(), objectLockRequest{bucket: bucket, object: self}, objectLockRequest{bucket: bucket, object: self, write: true})
+			if err != nil {
+				t.Fatal(err)
+			}
+			encoded := encodeDirObject(self)
+			if ldTry(t, pool.getHashedSet(encoded).NewNSLock(bucket, encoded), false) {
+				t.Error("self copy not write locked")
+			}
+			unlock()
+		})
+	}
+}
+
+// ldLeaseLock is a lock whose lease the test ends.
+type ldLeaseLock struct {
+	RWLocker
+	lost chan context.CancelFunc
+}
+
+func (l ldLeaseLock) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
+	lc, err := l.RWLocker.GetLock(ctx, timeout)
+	if err != nil {
+		return lc, err
+	}
+	lease, cancel := context.WithCancel(lc.Context())
+	select {
+	case l.lost <- cancel:
+	default:
+		panic("too many lock attempts")
+	}
+	return LockContext{ctx: lease, cancel: cancel}, nil
+}
+
+// getNSLocks never holds one lock while it waits for another for long: it
+// releases what it holds and retries, also after a lost lease, until its
+// timeout or the request ends, and leaves nothing behind.
+func TestGetNSLocksTransactional(t *testing.T) {
+	ns := newNSLock(false)
+	lock := func(name string) RWLocker { return ns.NewNSLock(nil, "bucket", name) }
+	newLock := func(name string) func() RWLocker { return func() RWLocker { return lock(name) } }
+	reqs := func() []nsLockRequest {
+		return []nsLockRequest{{newLock: newLock("a"), write: true, atomic: true}, {newLock: newLock("b"), write: true, atomic: true}}
+	}
+
+	t.Run("partial", func(t *testing.T) {
+		releaseB := ldHoldLock(t, lock("b"))
+		defer releaseB()
+		type result struct {
+			unlock func()
+			err    error
+		}
+		done := make(chan result, 1)
+		go func() {
+			_, unlock, err := getNSLocks(t.Context(), reqs(), globalOperationTimeout)
+			done <- result{unlock, err}
+		}()
+		// "a" is taken while "b" is waited for, and free again between
+		// attempts.
+		ldTaken(t, lock("a"))
+		deadline := time.Now().Add(3 * nsLockTryTimeout)
+		for !ldTry(t, lock("a"), true) {
+			if time.Now().After(deadline) {
+				t.Fatal("a stays held while b is waited for")
+			}
+		}
+		releaseB()
+		res := <-done
+		if res.err != nil {
+			t.Fatal(res.err)
+		}
+		if ldTry(t, lock("a"), false) || ldTry(t, lock("b"), false) {
+			t.Error("not both locks held")
+		}
+		res.unlock()
+		if !ldTry(t, lock("a"), true) || !ldTry(t, lock("b"), true) {
+			t.Error("locks left behind")
+		}
+	})
+
+	t.Run("timeout", func(t *testing.T) {
+		releaseB := ldHoldLock(t, lock("b"))
+		defer releaseB()
+		start := time.Now()
+		_, _, err := getNSLocks(t.Context(), reqs(), newDynamicTimeout(3*time.Second, 3*time.Second))
+		if !errors.As(err, &OperationTimedOut{}) {
+			t.Fatalf("got %v, want a timeout", err)
+		}
+		if took := time.Since(start); took > 6*time.Second {
+			t.Errorf("timed out after %v", took)
+		}
+		if !ldTry(t, lock("a"), true) {
+			t.Error("a left behind")
+		}
+	})
+
+	t.Run("cancel", func(t *testing.T) {
+		releaseB := ldHoldLock(t, lock("b"))
+		defer releaseB()
+		ctx, cancel := context.WithCancel(t.Context())
+		time.AfterFunc(300*time.Millisecond, cancel)
+		_, _, err := getNSLocks(ctx, reqs(), globalOperationTimeout)
+		if !errors.Is(err, context.Canceled) {
+			t.Fatalf("got %v, want canceled", err)
+		}
+		if !ldTry(t, lock("a"), true) {
+			t.Error("a left behind")
+		}
+	})
+
+	t.Run("lease", func(t *testing.T) {
+		lost := make(chan context.CancelFunc, 64)
+		leased := []nsLockRequest{{newLock: func() RWLocker { return ldLeaseLock{lock("a"), lost} }, write: true, atomic: true}, {newLock: newLock("b"), write: true, atomic: true}}
+		// Losing the first lease while the second is waited for retries
+		// with a fresh context.
+		releaseB := ldHoldLock(t, lock("b"))
+		type result struct {
+			lkctx  LockContext
+			unlock func()
+			err    error
+		}
+		done := make(chan result, 1)
+		go func() {
+			lkctx, unlock, err := getNSLocks(t.Context(), leased, globalOperationTimeout)
+			done <- result{lkctx, unlock, err}
+		}()
+		(<-lost)()
+		releaseB()
+		res := <-done
+		if res.err != nil {
+			t.Fatal(res.err)
+		}
+		if res.lkctx.Context().Err() != nil {
+			t.Fatal("returned context ended before any lease was lost")
+		}
+		// Losing a held lease ends the returned context.
+		var cancel context.CancelFunc
+		for len(lost) > 0 {
+			cancel = <-lost
+		}
+		cancel()
+		select {
+		case <-res.lkctx.Context().Done():
+		case <-time.After(time.Second):
+			t.Error("lost lease did not end the locks' context")
+		}
+		res.unlock()
+		if !ldTry(t, lock("a"), true) || !ldTry(t, lock("b"), true) {
+			t.Error("locks left behind")
+		}
+	})
+}
+
+// A group whose lock times out fails alone and the later groups still run.
+func TestDeleteObjectsInSetsLaterGroupsRun(t *testing.T) {
+	for _, mode := range ldModes[1:] {
+		t.Run(mode, func(t *testing.T) {
+			z, bucket := ldPool(t, mode)
+			k0, k1 := ldKey(t, z, "later", 0), ldKey(t, z, "later", 1)
+			ldMustPut(t, z, bucket, k0, "zero")
+			ldMustPut(t, z, bucket, k1, "one")
+			release := ldHoldLock(t, z.serverPools[0].getHashedSet(k0).NewNSLock(bucket, k0))
+			defer release()
+			objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: k0}}, {ObjectV: ObjectV{ObjectName: k1}}}
+			dobjects := []DeletedObject{{ObjectName: k0}, {ObjectName: k1}}
+			derrs := make([]error, 2)
+			z.deleteObjectsInSets(t.Context(), bucket, objects, ObjectOptions{}, dobjects, derrs, newDynamicTimeout(time.Second, time.Second))
+			if !errors.As(derrs[0], &OperationTimedOut{}) || derrs[1] != nil {
+				t.Fatalf("results %v, want a timeout for k0 only", derrs)
+			}
+			release()
+			if body, _ := ldBody(t, z, bucket, k0); body != "zero" {
+				t.Errorf("k0 = %q", body)
+			}
+			ldAbsent(t, z, bucket, k1)
+		})
+	}
+}
+
+// The local lock takes the names of a group one by one. A group that waits
+// for a name must not keep the names it holds: a part upload holds its part
+// lock and then reads its upload ID, which the batch names too.
+func TestDeleteObjectsInSetsLocalGroupReleases(t *testing.T) {
+	z, bucket := ldPool(t, ldLocal)
+	pool := z.serverPools[0]
+	var upload, part string
+	for i := range 1000 {
+		upload = ldKey(t, z, "local-upload", 1) + fmt.Sprintf("/u%d", i)
+		part = upload + "/1"
+		if pool.getHashedSetIndex(upload) == 1 && pool.getHashedSetIndex(part) == 1 {
+			break
+		}
+	}
+	er := pool.getHashedSet(upload)
+	releasePart := ldHoldLock(t, er.NewNSLock(bucket, part))
+	defer releasePart()
+	done := make(chan []error, 1)
+	go func() {
+		_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: upload}}, {ObjectV: ObjectV{ObjectName: part}}}, ObjectOptions{})
+		done <- errs
+	}()
+	// The batch takes the upload name, waits for the part, and lets the
+	// part upload's read of its upload ID through.
+	ldTaken(t, er.NewNSLock(bucket, upload))
+	read := er.NewNSLock(bucket, upload)
+	rctx, err := read.GetRLock(t.Context(), newDynamicTimeout(4*nsLockTryTimeout, 4*nsLockTryTimeout))
+	if err != nil {
+		t.Fatalf("batch kept a name while waiting: %v", err)
+	}
+	read.RUnlock(rctx)
+	releasePart()
+	select {
+	case errs := <-done:
+		if errors.Join(errs...) != nil {
+			t.Fatal(errs)
+		}
+	case <-time.After(30 * time.Second):
+		t.Fatal("bulk delete did not finish")
+	}
+}
+
+// ldLostUnlockReply applies its first unlock but loses the reply until gate
+// opens, so dsync retries that unlock in the background.
+type ldLostUnlockReply struct {
+	dsync.NetLocker
+	gate             <-chan struct{}
+	applied, retried chan<- struct{}
+	calls            atomic.Int32
+}
+
+func (l *ldLostUnlockReply) Unlock(ctx context.Context, args dsync.LockArgs) (bool, error) {
+	n := l.calls.Add(1)
+	ok, err := l.NetLocker.Unlock(ctx, args)
+	switch n {
+	case 1:
+		l.applied <- struct{}{}
+		<-l.gate
+		return false, errors.New("unlock reply lost")
+	case 2:
+		l.retried <- struct{}{}
+	}
+	return ok, err
+}
+
+// ldFailFirst fails the first acquisition of any of its instances.
+type ldFailFirst struct {
+	RWLocker
+	calls *atomic.Int32
+}
+
+func (l ldFailFirst) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
+	if l.calls.Add(1) == 1 {
+		return LockContext{}, OperationTimedOut{}
+	}
+	return l.RWLocker.GetLock(ctx, timeout)
+}
+
+// A distributed unlock of a failed attempt that is retried late must not
+// release the lock the next attempt took.
+func TestGetNSLocksFreshLockPerAttempt(t *testing.T) {
+	gate := make(chan struct{})
+	applied, retried := make(chan struct{}, 3), make(chan struct{}, 3)
+	lockers := make([]dsync.NetLocker, 3)
+	for i := range lockers {
+		lockers[i] = &ldLostUnlockReply{NetLocker: newLocker(), gate: gate, applied: applied, retried: retried}
+	}
+	ns := newNSLock(true)
+	dst := func() RWLocker {
+		return ns.NewNSLock(func() ([]dsync.NetLocker, string) { return lockers, "test" }, "bucket", "destination")
+	}
+	local := newNSLock(false)
+	var calls atomic.Int32
+	src := func() RWLocker { return ldFailFirst{local.NewNSLock(nil, "bucket", "source"), &calls} }
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second)
+	defer cancel()
+	opened := false
+	open := func() {
+		if !opened {
+			opened = true
+			close(gate)
+		}
+	}
+	defer open()
+	_, unlock, err := getNSLocks(ctx, []nsLockRequest{{newLock: dst, write: true, atomic: true}, {newLock: src, write: true, atomic: true}}, newDynamicTimeout(10*time.Second, 10*time.Second))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer unlock()
+	for range lockers {
+		select {
+		case <-applied:
+		case <-ctx.Done():
+			t.Fatal("the first attempt's unlock never reached the lockers")
+		}
+	}
+	open()
+	for range lockers {
+		select {
+		case <-retried:
+		case <-ctx.Done():
+			t.Fatal("the lost unlock was not retried")
+		}
+	}
+	competitor := dst()
+	lc, err := competitor.GetLock(ctx, newDynamicTimeout(300*time.Millisecond, 300*time.Millisecond))
+	if err == nil {
+		competitor.Unlock(lc)
+		t.Fatal("a late unlock retry released the held lock")
+	}
+}
+
+// One attempt at a local lock over several names is bounded as a whole, and so
+// is the acquisition.
+func TestGetNSLocksLocalAttemptBound(t *testing.T) {
+	ns := newNSLock(false)
+	a, b := ns.NewNSLock(nil, "bucket", "a"), ns.NewNSLock(nil, "bucket", "b")
+	releaseA, releaseB := ldHoldLock(t, a), ldHoldLock(t, b)
+	defer releaseA()
+	defer releaseB()
+	time.AfterFunc(900*time.Millisecond, releaseA)
+	time.AfterFunc(1800*time.Millisecond, releaseB)
+	start := time.Now()
+	_, unlock, err := getNSLocks(t.Context(), []nsLockRequest{{
+		newLock: func() RWLocker { return ns.NewNSLock(nil, "bucket", "a", "b") }, write: true,
+	}}, newDynamicTimeout(1500*time.Millisecond, 1500*time.Millisecond))
+	if err == nil {
+		unlock()
+		t.Fatalf("acquired after %v with a deadline of 1.5s", time.Since(start))
+	}
+	if !errors.As(err, &OperationTimedOut{}) {
+		t.Fatalf("got %v, want a timeout", err)
+	}
+}
+
+// ldLeaseLayer gives the COPY handler locks whose lease the test ends, and
+// holds its destination lookup until the lookup's context ends.
+type ldLeaseLayer struct {
+	ObjectLayer
+	destination string
+	lost        chan context.CancelFunc
+	lookup      chan context.Context
+}
+
+func (l *ldLeaseLayer) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) {
+	lc, unlock, err := l.ObjectLayer.lockObjects(ctx, reqs...)
+	if err != nil {
+		return lc, unlock, err
+	}
+	lease, cancel := context.WithCancel(lc.Context())
+	l.lost <- cancel
+	return LockContext{ctx: lease, cancel: cancel}, func() { cancel(); unlock() }, nil
+}
+
+func (l *ldLeaseLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	if object == l.destination && opts.NoLock {
+		l.lookup <- ctx
+		<-ctx.Done()
+		return ObjectInfo{}, ctx.Err()
+	}
+	return l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+}
+
+// The COPY handler's work under its locks ends when a lease is lost, the
+// destination lookups included.
+func TestLockDomainCopyLookupEndsWithLease(t *testing.T) {
+	z, bucket, _ := ldAPI(t, ldShared)
+	ldMustPut(t, z, bucket, "source", "")
+	prevTier := globalTierConfigMgr
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+	globalTierConfigMgr = NewTierConfigMgr()
+	globalTierConfigMgr.Tiers["WARM"] = madmin.TierConfig{Version: madmin.TierConfigVer, Type: madmin.S3, Name: "WARM", S3: &madmin.TierS3{Bucket: "warm"}}
+	layer := &ldLeaseLayer{ObjectLayer: z, destination: "destination", lost: make(chan context.CancelFunc, 1), lookup: make(chan context.Context, 1)}
+	router := initTestAPIEndPoints(layer, []string{"CopyObject"})
+	ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer cancel()
+	code := make(chan int, 1)
+	go func() { code <- ldCopyRequest(ctx, t, router, bucket, "source", "destination", nil).Code }()
+	var lookup context.Context
+	select {
+	case lookup = <-layer.lookup:
+	case <-ctx.Done():
+		t.Fatal("COPY did not look up its destination")
+	}
+	(<-layer.lost)()
+	select {
+	case <-lookup.Done():
+	case <-time.After(5 * time.Second):
+		t.Fatal("the destination lookup outlived the lost lease")
+	}
+	if c := <-code; c == http.StatusOK {
+		t.Error("COPY succeeded after losing its lock")
+	}
+}

cmd/erasure-server-pool-rebalance.go

@@ -605,6 +605,10 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 				// a noncurrent version and has to be moved like any other. The
 				// scanner applies the expiry once it lives in another pool.
 				return lifecycle.NoneAction
+			case evt.Action.Delete() && !z.migrationExpiryApplies(ctx, objInfo, evt):
+				// Another pool holds versions with which the rule does not
+				// apply. The version moves like any other.
+				return lifecycle.NoneAction
 			case evt.Action.Delete():
 				return evt.Action
 			}

cmd/erasure-server-pool.go

@@ -1239,8 +1239,26 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		object = encodeDirObject(object)
 	}
 
-	// Serialize logical deletion with pooled writes and metadata updates.
-	lk := z.NewNSLock(bucket, object)
+	// Serialize logical deletion with the writes and metadata updates of the
+	// object: take the lock they take (see NewObjectNSLock). With one pool
+	// that is the object's own erasure set, which is where PutObject,
+	// multipart commit and the metadata updates lock, so a pool whose sets
+	// use different hosts still excludes them. With several pools it is the
+	// pools lock, as before. Exact-key deletes that use prefix deletion
+	// (DeletePrefixObject) are deletes of one object. A true prefix deletion
+	// keeps the pools lock on the prefix name, which never covered the
+	// objects below it.
+	//
+	// Nodes that still run a release before this routing lock single-pool
+	// deletes and copies in the first set. In a pool whose sets use different
+	// hosts they do not exclude the deletes and copies of updated nodes, nor
+	// their writes, so such a pool must not run mixed versions.
+	var lk RWLocker
+	if opts.DeletePrefix && !opts.DeletePrefixObject {
+		lk = z.NewNSLock(bucket, object)
+	} else {
+		lk = z.NewObjectNSLock(bucket, object)
+	}
 	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -1248,6 +1266,19 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	ctx = lkctx.Context()
 	defer lk.Unlock(lkctx)
 
+	// An ILM expiry is decided on the versions in one pool. Under the lock,
+	// check it with the versions in all pools; the sets then take it as
+	// decided, without a recheck by the versions each one holds. They only
+	// keep a version whose protection they read differently.
+	if z.expiryNeedsAllPools(opts) {
+		var approved objectLockState
+		if approved, err = z.checkExpiryAllPools(ctx, bucket, object, opts); err != nil {
+			return ObjectInfo{Name: decodeDirObject(object), VersionID: opts.VersionID}, err
+		}
+		opts.Expiration.CheckedAllPools = true
+		opts.Expiration.ApprovedProtection = approved
+	}
+
 	if opts.DeletePrefix {
 		return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
 	}
@@ -1309,10 +1340,10 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 
 	// Evaluate the conditional (If-Match) precondition while the write lock
 	// acquired above is held, before the delete-marker short-circuit and before
-	// any version is removed, so the object cannot change between the check and
-	// the delete. This is scoped to a single erasure set (see the note at the
-	// lock above): only there do the delete lock and the write path share the
-	// same lock namespace, making the check-then-delete atomic.
+	// any version is removed. The object's writers take the same lock (see the
+	// note at the lock above), so the object cannot change between the check
+	// and the delete. Multi-pool conditional deletes return above, through
+	// deleteObjectReconciled, under the same pools lock.
 	if opts.CheckPrecondFn != nil {
 		// pinfo.ObjInfo is the current latest version. getPoolInfoExistingWithOpts
 		// intentionally strips VersionID, so for a version-scoped delete read the
@@ -1496,6 +1527,18 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		objSets.Add(objects[i].ObjectName)
 	}
 
+	// A single pool locks each name in the set its writers lock it in, one
+	// set at a time. A single node keeps one lock table for all sets, so it
+	// keeps the one bulk lock below: its local lock takes the names one by
+	// one and holds the earlier names while it waits.
+	if z.SinglePool() && !z.serverPools[0].sharesLocalNSLock() {
+		z.deleteObjectsInSets(ctx, bucket, objects, opts, dobjects, derrs, globalOperationTimeout)
+		for i := range dobjects {
+			dobjects[i].ObjectName = decodeDirObject(dobjects[i].ObjectName)
+		}
+		return dobjects, derrs
+	}
+
 	// Acquire a bulk write lock across 'objects'
 	multiDeleteLock := z.NewNSLock(bucket, objSets.ToSlice()...)
 	lkctx, err := multiDeleteLock.GetLock(ctx, globalOperationTimeout)
@@ -1517,10 +1560,31 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		batch = make([]ObjectToDelete, 0, len(objects))
 		batchIdx = make([]int, 0, len(objects))
 		data, dataIdx := make([]ObjectToDelete, 0, len(objects)), make([]int, 0, len(objects))
+		approved := make([]objectLockState, 0, len(objects))
+		ilm := make(map[string]lifecycleAllPools)
 		for i, obj := range objects {
 			if derrs[i] != nil {
 				continue
 			}
+			// A version that ILM expires is checked with the versions of
+			// its object in all pools, see checkExpiryAllPools. When the
+			// expiry no longer applies, nothing is deleted and nothing is
+			// left to replicate.
+			var protection objectLockState
+			vopts := ObjectOptions{Expiration: opts.Expiration, VersionID: obj.VersionID}
+			if obj.VersionID != "" && z.expiryNeedsAllPools(vopts) {
+				all, ok := ilm[obj.ObjectName]
+				if !ok {
+					all.versions, all.events, all.err = z.lifecycleEventsAllPools(ctx, bucket, obj.ObjectName)
+					ilm[obj.ObjectName] = all
+				}
+				if err := all.expiryApplies(bucket, obj.ObjectName, vopts); err != nil {
+					dobjects[i] = DeletedObject{ObjectName: obj.ObjectName, VersionID: obj.VersionID}
+					derrs[i] = err
+					continue
+				}
+				protection = all.protection(obj.VersionID)
+			}
 			if deletesData(obj, opts) {
 				if z.copyWithoutDeleteQuorum(ctx, bucket, obj.ObjectName, obj.VersionID) {
 					derrs[i] = InsufficientWriteQuorum{}
@@ -1528,13 +1592,14 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 				}
 				data = append(data, obj)
 				dataIdx = append(dataIdx, i)
+				approved = append(approved, protection)
 				continue
 			}
 			batch = append(batch, obj)
 			batchIdx = append(batchIdx, i)
 		}
 		if len(data) > 0 {
-			deleted, errs := z.deleteObjectsReconciled(ctx, bucket, data, opts)
+			deleted, errs := z.deleteObjectsReconciled(ctx, bucket, data, approved, opts)
 			for j, i := range dataIdx {
 				dobjects[i], derrs[i] = deleted[j], errs[j]
 			}
@@ -1598,8 +1663,10 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 
+	// A caller that also locks the source takes both with lockObjects, before
+	// it reads the source, and passes NoLock.
 	if !dstOpts.NoLock {
-		ns := z.NewNSLock(dstBucket, dstObject)
+		ns := z.NewObjectNSLock(dstBucket, dstObject)
 		lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
 			return ObjectInfo{}, err
@@ -1625,7 +1692,21 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 		return oi, err
 	}
 
-	poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
+	var poolIdx int
+	if dstOpts.MetadataChg && cpSrcDstSame && srcInfo.metadataOnly && srcOpts.VersionID == dstOpts.VersionID {
+		// Restore status updates address an existing version. Keep ordinary
+		// copy/write allocation unchanged, including its latest-version lookup.
+		poolIdx, err = z.getPoolIdxExistingWithOpts(ctx, dstBucket, dstObject, ObjectOptions{
+			VersionID:          dstOpts.VersionID,
+			MetadataChg:        true,
+			NoLock:             true,
+			Transition:         dstOpts.Transition,
+			SkipDecommissioned: true,
+			SkipRebalancing:    true,
+		})
+	} else {
+		poolIdx, err = z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
+	}
 	if err != nil {
 		return objInfo, err
 	}
@@ -2182,7 +2263,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 		return MultipartInfo{}, err
 	}
 
-	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID)
 	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return MultipartInfo{}, err
@@ -2223,7 +2304,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 		return ListPartsInfo{}, err
 	}
 
-	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID)
 	lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return ListPartsInfo{}, err
@@ -2274,7 +2355,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 		}
 	}(ctx)
 
-	lk := z.NewNSLock(bucket, pathJoin(object, uploadID))
+	lk := z.uploadIDNSLock(bucket, object, uploadID)
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return err
@@ -2312,9 +2393,12 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 		}
 	}()
 
+	// Several pools take the object lock first, then the upload lock below.
+	// A single pool takes the upload lock here and the set takes the object
+	// lock, in the object's own erasure set, only right before it commits.
 	if !z.SinglePool() {
 		if !opts.NoLock {
-			lk := z.NewNSLock(bucket, encodeDirObject(object))
+			lk := z.NewObjectNSLock(bucket, encodeDirObject(object))
 			lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 			if err != nil {
 				return objInfo, err
@@ -2375,13 +2459,18 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 
 	// Hold write locks to verify uploaded parts, also disallows any
 	// parallel PutObjectPart() requests.
-	uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
-	wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return objInfo, err
+	// An empty upload ID would name the object lock itself, which this
+	// completion holds or takes later: let the storage layer return its
+	// normal missing bucket/upload error instead.
+	if uploadID != "" {
+		uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID)
+		wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return objInfo, err
+		}
+		ctx = wlkctx.Context()
+		defer uploadIDLock.Unlock(wlkctx)
 	}
-	ctx = wlkctx.Context()
-	defer uploadIDLock.Unlock(wlkctx)
 
 	if z.SinglePool() {
 		return z.serverPools[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
@@ -3309,39 +3398,25 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
 	return z.serverPools[idx].TransitionObject(ctx, bucket, object, opts)
 }
 
-// RestoreTransitionedObject - restore transitioned object content locally on this cluster.
-func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+func (z *erasureServerPools) CheckAbandonedParts(ctx context.Context, bucket, object string, opts madmin.HealOpts) error {
 	object = encodeDirObject(object)
 	if z.SinglePool() {
-		return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
+		return z.serverPools[0].CheckAbandonedParts(ctx, bucket, object, opts)
 	}
-
-	if !opts.NoLock {
-		// Acquire write lock before restoring transitioned object
-		lk := z.NewNSLock(bucket, object)
-		lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
+	// Several pools: remove abandoned data under the pools lock, which every
+	// write of the object takes, as healObjectInPool does. The lock of one
+	// pool's set would not exclude a write that has moved its data directory
+	// in place but not yet published the metadata that references it, such
+	// as a native restore's commit.
+	if opts.Remove && !opts.DryRun && !opts.NoLock {
+		lk := z.NewObjectNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
 			return err
 		}
 		ctx = lkctx.Context()
 		defer lk.Unlock(lkctx)
-	}
-
-	// Avoid restoring object from a pool being decommissioned.
-	opts.SkipDecommissioned = true
-	opts.NoLock = true
-	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
-	if err != nil {
-		return err
-	}
-
-	return z.serverPools[idx].RestoreTransitionedObject(ctx, bucket, object, opts)
-}
-
-func (z *erasureServerPools) CheckAbandonedParts(ctx context.Context, bucket, object string, opts madmin.HealOpts) error {
-	object = encodeDirObject(object)
-	if z.SinglePool() {
-		return z.serverPools[0].CheckAbandonedParts(ctx, bucket, object, opts)
+		opts.NoLock = true
 	}
 	errs := make([]error, len(z.serverPools))
 	var wg sync.WaitGroup

cmd/globals.go

@@ -319,6 +319,8 @@ var (
 
 	globalOperationTimeout       = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops
 	globalDeleteOperationTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute)  // default time for delete ops
+	// Queued restores retry this wait; keep their failures out of delete timeouts.
+	globalRestoreLockTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute)
 	// Listing recovery has its own failure history and a five-second caller budget.
 	globalListingOperationTimeout = newDynamicTimeout(5*time.Second, 5*time.Second)
 

cmd/metacache-server-pool.go

@@ -265,9 +265,12 @@ func (z *erasureServerPools) listPathShortcutEOF(ctx context.Context, bucket str
 }
 
 // listingRLock covers the lock domains used by normal same-key mutations.
-// Multi-pool writes use the pools lock. In a single pool, DELETE/COPY use the
-// first set while PUT and metadata updates use the destination set, so recovery
-// takes both in that order. No writer routing or wire protocol changes here.
+// Multi-pool writes use the pools lock. In a single pool every object write,
+// DELETE and COPY included, locks the object's own set (see NewObjectNSLock),
+// which is the source set here. Releases before that routing locked
+// single-pool DELETE and COPY in the first set, so recovery still takes the
+// first set too, in that order, while nodes of those releases may run. It can
+// drop that lock only behind a version boundary that keeps such nodes out.
 func (z *erasureServerPools) listingRLock(ctx context.Context, bucket, object string, source *erasureObjects) (LockContext, func(), error) {
 	firstSet := z.serverPools[0].sets[0]
 	first := z.NewNSLock(bucket, object)

cmd/object-api-interface.go

@@ -159,6 +159,14 @@ type WalkOptions struct {
 // ExpirationOptions represents object options for object expiration at objectLayer.
 type ExpirationOptions struct {
 	Expire bool
+	// CheckedAllPools: the expiry was checked with the versions of the
+	// object in all pools (checkExpiryAllPools). A set does not check it
+	// again with the versions it holds alone.
+	CheckedAllPools bool
+	// ApprovedProtection: the Object Lock state of the version with which
+	// the check approved its expiry. The set compares it with the state it
+	// reads (checkedExpiryRefused). In memory only.
+	ApprovedProtection objectLockState `msg:"-"`
 }
 
 // TransitionOptions represents object options for transition ObjectLayer operation
@@ -169,6 +177,15 @@ type TransitionOptions struct {
 	RestoreRequest *RestoreObjectRequest
 	RestoreExpiry  time.Time
 	ExpireRestored bool
+
+	// attempt is the native restore a call belongs to: the attempt that
+	// POST marked on the version, see restoreRequest.
+	attempt *restoreAttempt
+	// owner is the restore lock of the version, which POST hands to the
+	// worker that runs attempt, see erasureServerPools.RestoreTransitionedObject.
+	owner *restoreOwner
+	// restoreDate is the request date POST recorded with attempt.
+	restoreDate string
 }
 
 // MakeBucketOptions represents bucket options for ObjectLayer bucket operations
@@ -250,7 +267,13 @@ func (o *ObjectOptions) SetEvalRetentionBypassFn(f EvalRetentionBypassFn) {
 // ObjectLayer implements primitives for object API layer.
 type ObjectLayer interface {
 	// Locking operations on object.
+	// NewNSLock locks names in one stable domain. It is for names that are
+	// not objects, such as bucket and IAM metadata locks.
 	NewNSLock(bucket string, objects ...string) RWLocker
+	// NewObjectNSLock locks one object in the lock domain of its writes.
+	NewObjectNSLock(bucket, object string) RWLocker
+	// lockObjects locks several objects together, see erasureServerPools.
+	lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error)
 
 	// Storage operations.
 	Shutdown(context.Context) error
@@ -287,6 +310,8 @@ type ObjectLayer interface {
 	DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error)
 	TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error
 	RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error
+	// restoreRequest records a POST restore on a version, see erasureServerPools.
+	restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error)
 
 	// Multipart operations.
 	ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)

cmd/object-api-interface_gen.go

@@ -79,10 +79,13 @@ func (z BucketOptions) Msgsize() (s int) {
 // MarshalMsg implements msgp.Marshaler
 func (z ExpirationOptions) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 1
+	// map header, size 2
 	// string "Expire"
-	o = append(o, 0x81, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65)
+	o = append(o, 0x82, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65)
 	o = msgp.AppendBool(o, z.Expire)
+	// string "CheckedAllPools"
+	o = append(o, 0xaf, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x41, 0x6c, 0x6c, 0x50, 0x6f, 0x6f, 0x6c, 0x73)
+	o = msgp.AppendBool(o, z.CheckedAllPools)
 	return
 }
 
@@ -110,6 +113,12 @@ func (z *ExpirationOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Expire")
 				return
 			}
+		case "CheckedAllPools":
+			z.CheckedAllPools, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "CheckedAllPools")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -124,7 +133,7 @@ func (z *ExpirationOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z ExpirationOptions) Msgsize() (s int) {
-	s = 1 + 7 + msgp.BoolSize
+	s = 1 + 7 + msgp.BoolSize + 16 + msgp.BoolSize
 	return
 }
 

cmd/object-copy-metadata_test.go

@@ -340,9 +340,9 @@ func TestCopyRewritesObjectData(t *testing.T) {
 			name: "data copy always rewrites",
 			want: true,
 		},
-		// PostRestoreObjectHandler, updateRestoreMetadata and batchKeyRotate all
-		// address the same version on both sides and never set Versioned, so they
-		// only ever reach these two cases.
+		// In-place metadata callers such as batchKeyRotate address the same
+		// version on both sides and never set Versioned, so they only ever
+		// reach these two cases.
 		{
 			name:         "unversioned in-place metadata update",
 			metadataOnly: true,

cmd/object-handlers-common.go

@@ -467,9 +467,17 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
 		deletedObjs, errs := o.DeleteObjects(ctx, bucket, toDel, ObjectOptions{
 			PrefixEnabledFn:  vc.PrefixEnabled,
 			VersionSuspended: vc.Suspended(),
+			Expiration:       ExpirationOptions{Expire: true},
 		})
 
 		for i, dobj := range deletedObjs {
+			if errs[i] != nil && !isErrObjectNotFound(errs[i]) && !isErrVersionNotFound(errs[i]) {
+				// Only a completed delete, or one confirmed already absent, is
+				// a removal. After a refusal, a quorum, configuration or any
+				// other error the version may remain; do not audit or announce
+				// it. The next scan re-evaluates it.
+				continue
+			}
 			oi := ObjectInfo{
 				Bucket:    bucket,
 				Name:      dobj.ObjectName,

cmd/object-handlers.go

@@ -34,6 +34,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -183,7 +184,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 
 	// Take read lock on object, here so subsequent lower-level
 	// calls do not need to.
-	lock := objectAPI.NewNSLock(bucket, object)
+	lock := objectAPI.NewObjectNSLock(bucket, object)
 	lkctx, err := lock.GetRLock(ctx, globalOperationTimeout)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -492,9 +493,10 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 				return
 			}
 			event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo)
+			expiryInfo, event := recheckReadExpiry(ctx, objectAPI, objInfo, event)
 			if event.Action.Delete() {
 				// apply whatever the expiry rule is.
-				applyExpiryRule(event, lcEventSrc_s3GetObject, objInfo)
+				applyExpiryRule(event, lcEventSrc_s3GetObject, expiryInfo)
 				if !event.Action.DeleteRestored() {
 					// If the ILM action is not on restored object return error.
 					writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchKey), r.URL)
@@ -972,9 +974,10 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 				return
 			}
 			event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo)
+			expiryInfo, event := recheckReadExpiry(ctx, objectAPI, objInfo, event)
 			if event.Action.Delete() {
 				// apply whatever the expiry rule is.
-				applyExpiryRule(event, lcEventSrc_s3HeadObject, objInfo)
+				applyExpiryRule(event, lcEventSrc_s3HeadObject, expiryInfo)
 				if !event.Action.DeleteRestored() {
 					// If the ILM action is not on restored object return error.
 					writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
@@ -1468,12 +1471,35 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return checkCopyObjectPreconditions(ctx, w, r, o)
 	}
 	getOpts.CheckPrecondFn = checkCopyPrecondFn
-	if cpSrcDstSame {
-		getOpts.NoLock = true
+
+	// Lock the source for reading and a local destination for writing before
+	// the source is opened, both in the lock domain of their writers. Taking
+	// them together in one order keeps copies in opposite directions from
+	// holding each other; a copy onto itself takes one write lock. The source
+	// is then read, preconditions and inline data included, and the
+	// destination looked up and written, all under these locks and without
+	// taking them again. That work runs in lockCtx, which ends when either
+	// lock is lost and so aborts the copy; errors are still reported in the
+	// request's context. Both locks are released once the copy has been
+	// written: a synchronous replication below reads the destination with its
+	// own lock.
+	locks := []objectLockRequest{{bucket: srcBucket, object: srcObject}}
+	if !remoteCallRequired {
+		locks = append(locks, objectLockRequest{bucket: dstBucket, object: dstObject, write: true})
 	}
+	lkctx, unlock, err := objectAPI.lockObjects(ctx, locks...)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	unlockCopy := sync.OnceFunc(unlock)
+	defer unlockCopy()
+	lockCtx := lkctx.Context()
+	getOpts.NoLock = true
+	dstOpts.NoLock = true
 
 	var rs *HTTPRangeSpec
-	gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, getOpts)
+	gr, err := getObjectNInfo(lockCtx, srcBucket, srcObject, rs, r.Header, getOpts)
 	if err != nil {
 		if isErrPreconditionFailed(err) {
 			return
@@ -1540,7 +1566,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	length := actualSize
 
 	if !cpSrcDstSame {
-		if err := enforceBucketQuotaHard(ctx, dstBucket, actualSize); err != nil {
+		if err := enforceBucketQuotaHard(lockCtx, dstBucket, actualSize); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
@@ -1558,12 +1584,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
 		compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
 
-		reader = etag.NewReader(ctx, reader, nil, nil)
+		reader = etag.NewReader(lockCtx, reader, nil, nil)
 	} else {
 		reader = gr
 	}
 
-	srcInfo.Reader, err = hash.NewReader(ctx, reader, length, "", "", actualSize)
+	srcInfo.Reader, err = hash.NewReader(lockCtx, reader, length, "", "", actualSize)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1658,7 +1684,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			}
 		}
 
-		if err = rotateKey(ctx, oldKey, newKeyID, newKey, srcBucket, srcObject, encMetadata, kmsCtx); err != nil {
+		if err = rotateKey(lockCtx, oldKey, newKeyID, newKey, srcBucket, srcObject, encMetadata, kmsCtx); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
@@ -1743,7 +1769,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			dstOpts.IndexCB = cb
 			defer s2c.Close()
 			reader = etag.Wrap(s2c, checksumReader)
-			srcInfo.Reader, err = hash.NewReader(ctx, reader, -1, "", "", actualSize)
+			srcInfo.Reader, err = hash.NewReader(lockCtx, reader, -1, "", "", actualSize)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
@@ -1756,7 +1782,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		if isTargetEncrypted && !remoteCallRequired {
 			var encReader io.Reader
 			kind, _ := crypto.IsRequested(r.Header)
-			encReader, objEncKey, err = newEncryptReader(ctx, srcInfo.Reader, kind, newKeyID, newKey, dstBucket, dstObject, encMetadata, kmsCtx)
+			encReader, objEncKey, err = newEncryptReader(lockCtx, srcInfo.Reader, kind, newKeyID, newKey, dstBucket, dstObject, encMetadata, kmsCtx)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
@@ -1771,7 +1797,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 
 		// do not try to verify encrypted content
-		srcInfo.Reader, err = hash.NewReader(ctx, reader, targetSize, "", "", actualSize)
+		srcInfo.Reader, err = hash.NewReader(lockCtx, reader, targetSize, "", "", actualSize)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -1843,7 +1869,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
 	retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectRetentionAction)
 	holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectLegalHoldAction)
-	getObjectInfo := objectAPI.GetObjectInfo
+	// The destination is locked above: look it up without its lock, in the
+	// locks' context.
+	getObjectInfo := func(_ context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+		opts.NoLock = true
+		return objectAPI.GetObjectInfo(lockCtx, bucket, object, opts)
+	}
 
 	// apply default bucket configuration/governance headers for dest side.
 	retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms, replicaTrusted)
@@ -2029,9 +2060,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		// For an ordinary federated copy, actualSize is the logical plaintext
 		// length to declare. srcInfo.Size is not a reliable wire length because
 		// the read path may already have adjusted it.
-		writeCtx, modified := withFederatedWriteTime(ctx)
+		writeCtx, modified := withFederatedWriteTime(lockCtx)
 		remoteObjInfo, rerr := core.PutObject(writeCtx, dstBucket, dstObject, srcInfo.Reader,
 			actualSize, "", "", opts)
+		gr.Close()
+		unlockCopy()
 		if rerr != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL)
 			return
@@ -2078,7 +2111,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 		// Copy source object to destination, if source and destination
 		// object is same then only metadata is updated.
-		objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts)
+		objInfo, err = copyObjectFn(lockCtx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts)
+		gr.Close()
+		unlockCopy()
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -3951,7 +3986,13 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
+	// Preserve a failed restore read even before the request body is parsed.
+	opts.Transition.RestoreRequest = &RestoreObjectRequest{}
+	// Resolve the target without the object lock; the status step below
+	// takes the restore lock and then the object lock.
+	preOpts := opts
+	preOpts.NoLock = true
+	objInfo, err := getObjectInfo(ctx, bucket, object, preOpts)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -3976,46 +4017,56 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
+	opts.Transition.RestoreRequest = rreq
 	statusCode := http.StatusOK
-	alreadyRestored := false
-	if err == nil {
-		if objInfo.RestoreOngoing && rreq.Type != SelectRestoreRequest {
+	// set or upgrade restore expiry
+	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
+
+	// update self with restore metadata
+	var (
+		attempt     *restoreAttempt
+		owner       *restoreOwner
+		restoreDate string
+	)
+	if rreq.Type != SelectRestoreRequest {
+		// Native restore needs every server to follow the restore lock and
+		// the staged commit; refuse it, writing nothing, while that is not
+		// established.
+		if err := nativeRestoreAvailable(ctx); err != nil {
+			apiErr := errorCodes.ToAPIErr(ErrBusy)
+			apiErr.Code, apiErr.Description = "ServiceUnavailable", err.Error()
+			writeErrorResponse(ctx, w, apiErr, r.URL)
+			return
+		}
+		// Decide on a fresh read under the object lock: renew a restored copy
+		// that is still there, or mark the version ongoing for a new attempt.
+		// Only the version resolved above, with its identity, is restored.
+		statusOpts := opts
+		statusOpts.Transition.RestoreExpiry = restoreExpiry
+		status, err := objectAPI.restoreRequest(GlobalContext, bucket, object, statusOpts, objInfo)
+		if errors.Is(err, errRestoreInProgress) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectRestoreAlreadyInProgress), r.URL)
 			return
 		}
-		if !objInfo.RestoreOngoing && !objInfo.RestoreExpires.IsZero() {
-			statusCode = http.StatusAccepted
-			alreadyRestored = true
-		}
-	}
-	// set or upgrade restore expiry
-	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
-	metadata := cloneMSS(objInfo.UserDefined)
-
-	// update self with restore metadata
-	if rreq.Type != SelectRestoreRequest {
-		objInfo.metadataOnly = true // Perform only metadata updates.
-		metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days)
-		metadata[xhttp.AmzRestoreRequestDate] = time.Now().UTC().Format(http.TimeFormat)
-		if alreadyRestored {
-			metadata[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
-		} else {
-			metadata[xhttp.AmzRestore] = ongoingRestoreObj().String()
-		}
-		objInfo.UserDefined = metadata
-		if _, err := objectAPI.CopyObject(GlobalContext, bucket, object, bucket, object, objInfo, ObjectOptions{
-			VersionID: objInfo.VersionID,
-		}, ObjectOptions{
-			VersionID: objInfo.VersionID,
-			MTime:     objInfo.ModTime,
-		}); err != nil {
+		if errors.As(err, &InvalidObjectState{}) || errors.As(err, &MethodNotAllowed{}) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL)
 			return
 		}
-		// for previously restored object, just update the restore expiry
-		if alreadyRestored {
+		if err != nil {
+			// Such as a read quorum error while telling whether it restores.
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
+		// Events keep the version ID as this request resolved it.
+		status.objInfo.VersionID = objInfo.VersionID
+		objInfo = status.objInfo
+		// for previously restored object, just update the restore expiry
+		if status.attempt == nil {
+			return
+		}
+		attempt = status.attempt
+		// The restore lock goes to the worker below, which releases it.
+		owner, restoreDate = status.owner, status.date
 	}
 
 	restoreObject := mustGetUUID()
@@ -4035,6 +4086,9 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 	// now process the restore in background
 	go func() {
 		rctx := GlobalContext
+		// The restore releases the lock it was handed; this covers a path
+		// that does not reach it. Releasing twice is a no-op.
+		defer owner.release()
 		if !rreq.SelectParameters.IsEmpty() {
 			actualSize, err := objInfo.GetActualSize()
 			if err != nil {
@@ -4080,14 +4134,20 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 			rreq.SelectParameters.Close()
 			return
 		}
-		opts := ObjectOptions{
+		// The versioning context and the explicit version POST resolved.
+		restoreOpts := ObjectOptions{
 			Transition: TransitionOptions{
 				RestoreRequest: rreq,
 				RestoreExpiry:  restoreExpiry,
+				attempt:        attempt,
+				owner:          owner,
+				restoreDate:    restoreDate,
 			},
-			VersionID: objInfo.VersionID,
+			VersionID:        attempt.VersionID,
+			Versioned:        opts.Versioned,
+			VersionSuspended: opts.VersionSuspended,
 		}
-		if err := objectAPI.RestoreTransitionedObject(rctx, bucket, object, opts); err != nil {
+		if err := objectAPI.RestoreTransitionedObject(rctx, bucket, object, restoreOpts); err != nil {
 			s3LogIf(ctx, fmt.Errorf("Unable to restore transitioned bucket/object %s/%s: %w", bucket, object, err))
 			return
 		}

cmd/object-lifecycle-read_test.go

@@ -0,0 +1,380 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// An explicit-version read uses ReadVersion. Only the added lifecycle scan
+// uses ReadXL, so its errors/absence can be injected without failing the read.
+type lifecycleReadDisk struct {
+	StorageAPI
+	bucket, object string
+	scans          *atomic.Int64
+	err            error
+}
+
+func (d lifecycleReadDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
+	if bucket == d.bucket && object == d.object {
+		d.scans.Add(1)
+		if d.err != nil {
+			return RawFileInfo{}, d.err
+		}
+	}
+	return d.StorageAPI.ReadXL(ctx, bucket, object, data)
+}
+
+func lifecycleReadDisks(t *testing.T, s *splitILM, object string, failedPool int, err error) *atomic.Int64 {
+	t.Helper()
+	var scans atomic.Int64
+	for p, pool := range s.z.serverPools {
+		pool.erasureDisksMu.Lock()
+		previous := pool.erasureDisks
+		wrapped := make([][]StorageAPI, len(previous))
+		for i, disks := range previous {
+			wrapped[i] = append([]StorageAPI(nil), disks...)
+			for j, disk := range disks {
+				var injected error
+				if p == failedPool {
+					injected = err
+				}
+				wrapped[i][j] = lifecycleReadDisk{StorageAPI: disk, bucket: s.bucket, object: object, scans: &scans, err: injected}
+			}
+		}
+		pool.erasureDisks = wrapped
+		pool.erasureDisksMu.Unlock()
+		t.Cleanup(func() {
+			pool.erasureDisksMu.Lock()
+			pool.erasureDisks = previous
+			pool.erasureDisksMu.Unlock()
+		})
+	}
+	return &scans
+}
+
+// Real signed GET and HEAD requests use the author's two-pool storage fixture.
+// Checking the queue as well as the response catches both false hiding and
+// accidentally authorizing the latest version instead of the version read.
+func TestPoolsLifecycleReadHTTP(t *testing.T) {
+	const bothRules = `<Rule><ID>both</ID><Status>Enabled</Status><Filter><Prefix>both-</Prefix></Filter>` +
+		`<Expiration><Days>1</Days></Expiration><NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration></Rule>`
+	for _, state := range []versioning.State{versioning.Enabled, versioning.Suspended} {
+		t.Run(string(state), func(t *testing.T) {
+			s := newSplitILM(t, state, splitILMRules+bothRules)
+			router := initTestAPIEndPoints(s.z, nil)
+			for _, tc := range []struct {
+				name, prefix string
+				latestAge    float64
+				readLatest   bool
+				implicit     bool
+				action       lifecycle.Action
+				local        lifecycle.Action
+				fault        error
+				wantScan     bool
+			}{
+				{name: "old-version-new-latest", prefix: "exp-", latestAge: 0.01, local: lifecycle.DeleteAction, wantScan: true},
+				{name: "old-version-expired-latest", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, wantScan: true},
+				{name: "old-version-delete-all-latest", prefix: "eav-", latestAge: 3, local: lifecycle.DeleteAllVersionsAction, wantScan: true},
+				{name: "expired-current", prefix: "exp-", latestAge: 3, readLatest: true, local: lifecycle.DeleteAction, action: lifecycle.DeleteAction, wantScan: true},
+				{name: "expired-current-implicit", prefix: "exp-", latestAge: 3, readLatest: true, implicit: true, local: lifecycle.DeleteAction, action: lifecycle.DeleteAction, wantScan: true},
+				{name: "expired-noncurrent", prefix: "both-", latestAge: 3, local: lifecycle.DeleteAction, action: lifecycle.DeleteVersionAction, wantScan: true},
+				{name: "extra-read-quorum-error", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, fault: errDiskNotFound, wantScan: true},
+				{name: "read-version-missing-from-recheck", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, fault: errFileNotFound, wantScan: true},
+				{name: "local-none-global-expired", prefix: "ncv-", latestAge: 3, local: lifecycle.NoneAction},
+			} {
+				for _, null := range []bool{false, true} {
+					for oldPool := range 2 {
+						t.Run(fmt.Sprintf("%s/null=%t/old-pool=%d", tc.name, null, oldPool), func(t *testing.T) {
+							key := fmt.Sprintf("%s%s-%t-%d", tc.prefix, tc.name, null, oldPool)
+							old := splitAt("v1", 5, oldPool)
+							old.null = null
+							s.write(key, old)
+							s.write(key, splitAt("v2", tc.latestAge, 1-oldPool))
+							label := "v1"
+							if tc.readLatest {
+								label = "v2"
+							}
+							vid := s.ids[key+" "+label]
+							if vid == "" {
+								vid = nullVersionID
+							}
+							oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: state == versioning.Enabled, VersionSuspended: state == versioning.Suspended})
+							if err != nil {
+								t.Fatal(err)
+							}
+							lc, err := globalLifecycleSys.Get(s.bucket)
+							if err != nil {
+								t.Fatal(err)
+							}
+							if local := lc.Eval(oi.ToLifecycleOpts()); local.Action != tc.local {
+								t.Fatalf("fixture local event=%v, want %v", local.Action, tc.local)
+							}
+							versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key)
+							if err != nil {
+								t.Fatal(err)
+							}
+							index := 1
+							if tc.readLatest {
+								index = 0
+							}
+							if tc.local == lifecycle.NoneAction && events[index].Action != lifecycle.DeleteVersionAction {
+								t.Fatal("fixture must be globally expired but locally NoneAction")
+							}
+							scans := lifecycleReadDisks(t, s, key, oldPool, tc.fault)
+							for _, method := range []string{http.MethodGet, http.MethodHead} {
+								t.Run(method, func(t *testing.T) {
+									scans.Store(0)
+									requestID := vid
+									if tc.implicit {
+										requestID = ""
+									}
+									rec := consistencyRequest(t, router, method, s.bucket, key, requestID)
+									want := http.StatusOK
+									if tc.action.Delete() {
+										want = http.StatusNotFound
+									}
+									if rec.Code != want {
+										t.Errorf("status=%d want=%d body=%s", rec.Code, want, rec.Body.String())
+									}
+									if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != key+" "+label {
+										t.Errorf("body=%q, want requested version %s", rec.Body.String(), label)
+									}
+									if !tc.implicit && (scans.Load() > 0) != tc.wantScan {
+										t.Errorf("ReadXL calls=%d, want scan=%t", scans.Load(), tc.wantScan)
+									}
+									if tc.action.Delete() {
+										select {
+										case op := <-s.queue:
+											task, ok := op.(expiryTask)
+											src := lcEventSrc_s3GetObject
+											if method == http.MethodHead {
+												src = lcEventSrc_s3HeadObject
+											}
+											global := versions[index]
+											if !ok || task.src != src || task.event != events[index] || task.event.Action != tc.action ||
+												task.objInfo.VersionID != global.VersionID || task.objInfo.IsLatest != global.IsLatest ||
+												task.objInfo.NumVersions != global.NumVersions || !task.objInfo.SuccessorModTime.Equal(global.SuccessorModTime) {
+												t.Errorf("wrong queued expiry: %+v; want %s with global event %+v", task, vid, events[index])
+											}
+										default:
+											t.Error("missing expiry task")
+										}
+									}
+									if n := len(s.queue); n != 0 {
+										t.Errorf("unexpected expiry tasks: %d", n)
+										for len(s.queue) > 0 {
+											<-s.queue
+										}
+									}
+								})
+							}
+						})
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsLifecycleReadAuthorizationAndClose(t *testing.T) {
+	s := newSplitILM(t, versioning.Enabled, splitILMRules)
+	initConfigSubsystem(t.Context(), s.z)
+	globalIAMSys.Init(t.Context(), s.z, globalEtcdClient, 2*time.Second)
+	denied := newObjectAttributesAuthzUser(t, "lifecycle-read", s.bucket, `"s3:PutObject"`)
+	router := initTestAPIEndPoints(s.z, nil)
+	for _, expire := range []bool{false, true} {
+		key := fmt.Sprintf("exp-reader-%t", expire)
+		v := splitAt("v1", 5, 0)
+		v.size = 1 << 20 // a non-inline reader retains the object read lock
+		s.write(key, v)
+		if !expire {
+			s.write(key, splitAt("v2", 0.01, 1))
+		}
+		vid := s.ids[key+" v1"]
+		oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: true})
+		if err != nil || oi.Inlined {
+			t.Fatalf("fixture must retain a reader lock: inlined=%t err=%v", oi.Inlined, err)
+		}
+		scans := lifecycleReadDisks(t, s, key, -1, nil)
+		for _, method := range []string{http.MethodGet, http.MethodHead} {
+			req, err := newTestSignedRequestV4(method, getGetObjectURL("", s.bucket, key)+"?versionId="+vid, 0, nil, denied.AccessKey, denied.SecretKey, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			rec := httptest.NewRecorder()
+			router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusForbidden || scans.Load() != 0 || len(s.queue) != 0 {
+				t.Fatalf("denied %s: status=%d scans=%d queued=%d", method, rec.Code, scans.Load(), len(s.queue))
+			}
+		}
+		rec := consistencyRequest(t, router, http.MethodGet, s.bucket, key, vid)
+		want := http.StatusOK
+		if expire {
+			want = http.StatusNotFound
+		}
+		if rec.Code != want {
+			t.Errorf("authorized GET status=%d want=%d", rec.Code, want)
+		}
+		// The reader owns the multi-pool read lock until Close, including the
+		// early 404 path. A writer must acquire it immediately after the handler.
+		lk := s.z.NewNSLock(s.bucket, key)
+		lkctx, err := lk.GetLock(t.Context(), newDynamicTimeout(time.Second, time.Second))
+		if err != nil {
+			t.Fatalf("GET leaked its reader/lock: %v", err)
+		}
+		lk.Unlock(lkctx)
+		for len(s.queue) > 0 {
+			<-s.queue
+		}
+	}
+}
+
+func TestPoolsLifecycleReadBoundaries(t *testing.T) {
+	const transitionRules = `<Rule><ID>transition</ID><Status>Enabled</Status><Filter><Prefix>transition-</Prefix></Filter>` +
+		`<Transition><Days>1</Days><StorageClass>WARM</StorageClass></Transition></Rule>`
+	for _, tc := range []struct {
+		name       string
+		state      versioning.State
+		null       bool
+		singlePool bool
+		restored   bool
+		transition bool
+		action     lifecycle.Action
+		wantScan   bool
+	}{
+		{name: "single-pool", state: versioning.Enabled, singlePool: true, action: lifecycle.DeleteAction},
+		{name: "single-pool-null", state: versioning.Enabled, singlePool: true, null: true, action: lifecycle.DeleteAction},
+		{name: "unversioned", null: true, action: lifecycle.DeleteAction},
+		{name: "current-null-enabled", state: versioning.Enabled, null: true, action: lifecycle.DeleteAction, wantScan: true},
+		{name: "current-null-suspended", state: versioning.Suspended, null: true, action: lifecycle.DeleteAction, wantScan: true},
+		{name: "transition", state: versioning.Enabled, transition: true, action: lifecycle.TransitionAction},
+		{name: "restored", state: versioning.Enabled, restored: true, action: lifecycle.DeleteRestoredAction},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			s := newSplitILM(t, tc.state, splitILMRules+transitionRules)
+			key := "exp-" + tc.name
+			if tc.restored || tc.transition {
+				key = "transition-" + tc.name
+			}
+			v := splitAt("v1", 5, 0)
+			v.null, v.tiered = tc.null, tc.restored
+			s.write(key, v)
+			vid := s.ids[key+" v1"]
+			if vid == "" {
+				vid = nullVersionID
+			}
+			if tc.restored {
+				globalTierConfigMgr.drivercache["WARM"] = &tierExpiryTestBackend{objects: map[string][]byte{"remote-v1": []byte(key + " v1")}}
+				opts := ObjectOptions{VersionID: vid, Versioned: true}
+				opts.EvalMetadataFn = func(oi *ObjectInfo, _ error) (ReplicateDecision, error) {
+					oi.UserDefined[xhttp.AmzRestore] = completedRestoreObj(UTCNow().Add(-time.Hour)).String()
+					return ReplicateDecision{}, nil
+				}
+				if _, err := s.z.PutObjectMetadata(t.Context(), s.bucket, key, opts); err != nil {
+					t.Fatal(err)
+				}
+			}
+			z := s.z
+			if tc.singlePool {
+				// Exercise the single-pool dispatch with the same storage fixture.
+				z = &erasureServerPools{serverPools: s.z.serverPools[:1]}
+			}
+			router := initTestAPIEndPoints(z, []string{"GetObject", "HeadObject"})
+			scans := lifecycleReadDisks(t, s, key, -1, nil)
+			for _, method := range []string{http.MethodGet, http.MethodHead} {
+				scans.Store(0)
+				rec := consistencyRequest(t, router, method, s.bucket, key, vid)
+				want := http.StatusOK
+				if tc.action.Delete() && !tc.action.DeleteRestored() {
+					want = http.StatusNotFound
+				}
+				if rec.Code != want || (scans.Load() > 0) != tc.wantScan {
+					t.Errorf("%s status=%d want=%d scans=%d wantScan=%t body=%s", method, rec.Code, want, scans.Load(), tc.wantScan, rec.Body.String())
+				}
+				if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != key+" v1" {
+					t.Errorf("%s lost object contents: %q", method, rec.Body.String())
+				}
+				if tc.action.Delete() {
+					select {
+					case op := <-s.queue:
+						task, ok := op.(expiryTask)
+						if !ok || task.event.Action != tc.action || s.key(key, task.objInfo.VersionID) != s.key(key, vid) {
+							t.Errorf("%s wrong expiry: %+v", method, task)
+						}
+					default:
+						t.Errorf("%s missing expiry", method)
+					}
+				}
+				if n := len(s.queue); n != 0 {
+					t.Fatalf("%s queued %d unexpected expiries", method, n)
+				}
+			}
+		})
+	}
+}
+
+func TestPoolsLifecycleReadProxy(t *testing.T) {
+	s := newSplitILM(t, versioning.Enabled, splitILMRules)
+	const key, body = "exp-proxied", "remote object remains readable"
+	vid := mustGetUUID()
+	target := newILMTestTarget(t)
+	var remoteRequests atomic.Int64
+	registerILMTestTarget(t, s.z, s.bucket, target, func(w http.ResponseWriter, r *http.Request) {
+		remoteRequests.Add(1)
+		w.Header().Set(xhttp.ContentLength, fmt.Sprint(len(body)))
+		w.Header().Set(xhttp.LastModified, UTCNow().Add(-5*24*time.Hour).Format(http.TimeFormat))
+		w.Header().Set(xhttp.ETag, `"da39a3ee5e6b4b0d3255bfef95601890"`)
+		w.Header().Set(xhttp.AmzVersionID, vid)
+		if r.Method == http.MethodGet {
+			_, _ = w.Write([]byte(body))
+		}
+	})
+	setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket)
+	// The delete-only target fixture omits credentials; proxy discovery clones
+	// the target descriptor and requires its credentials field to be present.
+	globalBucketTargetSys.Lock()
+	globalBucketTargetSys.targetsMap[s.bucket][0].Credentials = &madmin.Credentials{AccessKey: "proxy-test"}
+	globalBucketTargetSys.Unlock()
+	stats := NewReplicationStats(t.Context(), nil)
+	previous := globalReplicationStats.Swap(stats)
+	t.Cleanup(func() { globalReplicationStats.Store(previous) })
+	scans := lifecycleReadDisks(t, s, key, -1, nil)
+	router := initTestAPIEndPoints(s.z, nil)
+	for _, method := range []string{http.MethodGet, http.MethodHead} {
+		before := remoteRequests.Load()
+		rec := consistencyRequest(t, router, method, s.bucket, key, vid)
+		if rec.Code != http.StatusOK || remoteRequests.Load() == before || scans.Load() != 0 || len(s.queue) != 0 {
+			t.Errorf("proxy %s: status=%d remote=%d scans=%d queued=%d body=%s", method, rec.Code, remoteRequests.Load()-before, scans.Load(), len(s.queue), rec.Body.String())
+		}
+		if method == http.MethodGet && rec.Body.String() != body {
+			t.Errorf("proxy body=%q", rec.Body.String())
+		}
+	}
+}

cmd/object-retention-authz_test.go

@@ -544,6 +544,18 @@ func (p *hookBeforeWrite) CompleteMultipartUpload(ctx context.Context, bucket, k
 	return p.ObjectLayer.CompleteMultipartUpload(ctx, bucket, key, uploadID, parts, opts)
 }
 
+// The COPY handler checks Object Lock under its destination's write lock, so
+// no writer can land between that check and the copy. The latest a writer can
+// land is just before the handler takes its locks.
+func (p *hookBeforeWrite) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) {
+	for _, r := range reqs {
+		if r.write {
+			p.fire(r.object)
+		}
+	}
+	return p.ObjectLayer.lockObjects(ctx, reqs...)
+}
+
 func TestDirectoryObjectLockAfterPreflight(t *testing.T) {
 	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
 		t:                 t,

cmd/peer-rest-server.go

@@ -116,6 +116,7 @@ var (
 	stopRebalanceRPC               = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerStopRebalance, grid.NewMSS, grid.NewNoPayload)
 	updateMetacacheListingRPC      = grid.NewSingleHandler[*metacache, *metacache](grid.HandlerUpdateMetacacheListing, func() *metacache { return &metacache{} }, func() *metacache { return &metacache{} })
 	cleanupUploadIDCacheMetaRPC    = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerClearUploadID, grid.NewMSS, grid.NewNoPayload)
+	restoreCapabilityRPC           = grid.NewSingleHandler[*grid.MSS, *grid.MSS](grid.HandlerRestoreCapability, grid.NewMSS, grid.NewMSS)
 
 	// STREAMS
 	// Set an output capacity of 100 for consoleLog and listenRPC
@@ -1440,6 +1441,7 @@ func registerPeerRESTHandlers(router *mux.Router, gm *grid.Manager) {
 	logger.FatalIf(stopRebalanceRPC.Register(gm, server.StopRebalanceHandler), "unable to register handler")
 	logger.FatalIf(updateMetacacheListingRPC.Register(gm, server.UpdateMetacacheListingHandler), "unable to register handler")
 	logger.FatalIf(cleanupUploadIDCacheMetaRPC.Register(gm, server.HandlerClearUploadID), "unable to register handler")
+	logger.FatalIf(restoreCapabilityRPC.Register(gm, restoreCapabilityHandler), "unable to register handler")
 
 	logger.FatalIf(gm.RegisterStreamingHandler(grid.HandlerTrace, grid.StreamHandler{
 		Handle:      server.TraceHandler,

cmd/restore-attempt.go

@@ -0,0 +1,662 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"maps"
+	"net/http"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// The version that POST marks ongoing names the native restore attempt that
+// owns the marker. Clients cannot set it: requests with reserved metadata are
+// refused, and it is never extracted from headers.
+const restoreAttemptKey = ReservedMetadataPrefixLower + "restore-attempt"
+
+const (
+	// restoreTakeoverWait bounds how long a POST waits for the restore lock
+	// of its version. A running restore holds it for its whole run.
+	restoreTakeoverWait = 2 * time.Second
+	// restoreCleanupTimeout bounds each status reset and convergence pass.
+	restoreCleanupTimeout = time.Minute
+)
+
+// restoreRecoveryDelays are the in-memory retries that clear the ongoing
+// marker of a restore that failed before it could clear the marker itself.
+// A restart loses them; a later POST then takes the marker over.
+var restoreRecoveryDelays = []time.Duration{0, 100 * time.Millisecond, time.Second, 5 * time.Second, 30 * time.Second, 2 * time.Minute}
+
+var (
+	// errRestoreSuperseded: the version no longer belongs to this attempt.
+	// The attempt must not rehydrate, complete or reset anything.
+	errRestoreSuperseded = errors.New("restore attempt superseded")
+	// errRestoreInProgress: a restore holds the restore lock of the version.
+	errRestoreInProgress = errors.New("restore already in progress")
+	// errRestoreUnchanged: a restore status update found nothing to write.
+	errRestoreUnchanged = errors.New("restore status unchanged")
+)
+
+// restoreResetError marks a failed restore that may have left its ongoing
+// marker behind. The pools layer hands it to recoverRestoreStatus.
+type restoreResetError struct{ error }
+
+func (e restoreResetError) Unwrap() error { return e.error }
+
+// restoreVersionID is the explicit version ID of a resolved restore target.
+func restoreVersionID(versionID string) string {
+	if versionID == "" {
+		return nullVersionID
+	}
+	return versionID
+}
+
+// restoreAttempt identifies one native restore of one stored version.
+type restoreAttempt struct {
+	ID        string    `json:"id"`
+	VersionID string    `json:"v"` // "null" for the null version
+	ETag      string    `json:"e"`
+	Size      int64     `json:"s"`
+	ModTime   time.Time `json:"m"`
+}
+
+func newRestoreAttempt(oi ObjectInfo) *restoreAttempt {
+	return &restoreAttempt{
+		ID:        mustGetUUID(),
+		VersionID: restoreVersionID(oi.VersionID),
+		ETag:      oi.ETag,
+		Size:      oi.Size,
+		ModTime:   oi.ModTime.UTC(),
+	}
+}
+
+func (a *restoreAttempt) String() string {
+	b, _ := json.Marshal(a)
+	return string(b)
+}
+
+// targets reports whether oi is still the version the attempt was made for.
+// A replacement under the same version ID differs in ETag, size or ModTime.
+func (a *restoreAttempt) targets(oi ObjectInfo) bool {
+	return a != nil && !oi.DeleteMarker && restoreVersionID(oi.VersionID) == a.VersionID &&
+		oi.ETag == a.ETag && oi.Size == a.Size && oi.ModTime.Equal(a.ModTime) &&
+		oi.TransitionedObject.Status == lifecycle.TransitionComplete
+}
+
+// owns reports whether the version is ongoing under this attempt.
+func (a *restoreAttempt) owns(oi ObjectInfo) bool {
+	return a.targets(oi) && oi.RestoreOngoing && oi.UserDefined[restoreAttemptKey] == a.ID
+}
+
+// ownsGeneration reports whether the attempt owns the authoritative
+// generation of the version. Where the drives disagree only in their restore
+// fields, see restoreGeneration, it owns the version only when its marker is
+// the only ongoing marker there: holding one of several is not enough.
+func (a *restoreAttempt) ownsGeneration(gen restoreGeneration, bucket, object string) bool {
+	oi := gen.fi.ToObjectInfo(bucket, object, false)
+	if !gen.split {
+		return a.owns(oi)
+	}
+	return a.targets(oi) && len(gen.ongoing) == 1 && gen.ongoing[a.ID]
+}
+
+// restoreGeneration is the metadata generation of one version that the drives
+// of a set hold authoritatively: the revision (header and normalized metadata
+// digest, versionRevision) that at least max(n/2+1, its read quorum) drives
+// hold. drives has each drive's own copy of it, with that drive's erasure
+// index and inline data, and an invalid entry where a drive holds another or
+// no revision; disks has nil there.
+//
+// A failed restore status write can leave drives that differ only in their
+// restore fields, with no revision on a quorum. Then the generation is the
+// restore-neutral revision that a quorum holds once those fields are left
+// out: fi has no restore fields, and ongoing names the attempts whose
+// ongoing marker some of its drives hold ("" for one without an attempt).
+// Every other field, protection included, still needs the quorum.
+//
+// revs has, for every drive that holds the version, the full revision of the
+// entry it holds, whether or not that revision is the generation; nil where a
+// drive does not hold the version or cannot be read. A native restore's
+// commit compares the drives against it later, see restorePublication.
+type restoreGeneration struct {
+	fi      FileInfo
+	drives  []FileInfo
+	disks   []StorageAPI
+	split   bool
+	ongoing map[string]bool
+	revs    []*revisionKey
+}
+
+// restoreFieldKeys are the metadata keys a restore status write sets.
+var restoreFieldKeys = []string{xhttp.AmzRestore, xhttp.AmzRestoreExpiryDays, xhttp.AmzRestoreRequestDate, restoreAttemptKey}
+
+// restoreHealMeta returns latest, the metadata heal copies to drives that
+// lack the version, with the restore fields that at least quorum drives of
+// that version hold. When some drive holds restore fields and no set of
+// them has the quorum, healing the version waits.
+func restoreHealMeta(metas []FileInfo, latest FileInfo, quorum int) (FileInfo, error) {
+	counts := make(map[string]int)
+	fields := make(map[string]map[string]string)
+	restoring := false
+	for _, m := range metas {
+		if !m.IsValid() || m.Deleted || !m.ModTime.Equal(latest.ModTime) {
+			continue
+		}
+		f := make(map[string]string)
+		var key strings.Builder
+		for _, k := range restoreFieldKeys {
+			if v, ok := m.Metadata[k]; ok {
+				f[k] = v
+				fmt.Fprintf(&key, "%d:%s=%d:%s;", len(k), k, len(v), v)
+				restoring = true
+			}
+		}
+		counts[key.String()]++
+		fields[key.String()] = f
+	}
+	if !restoring {
+		return latest, nil
+	}
+	best, bestCount := "", 0
+	for key, c := range counts {
+		if c > bestCount {
+			best, bestCount = key, c
+		}
+	}
+	if bestCount < quorum {
+		return latest, errErasureReadQuorum
+	}
+	latest.Metadata = maps.Clone(latest.Metadata)
+	for _, k := range restoreFieldKeys {
+		delete(latest.Metadata, k)
+	}
+	maps.Copy(latest.Metadata, fields[best])
+	return latest, nil
+}
+
+// restoreNeutralRevision is the revision of ver without its restore fields.
+func restoreNeutralRevision(ver xlMetaV2ShallowVersion, metaV uint8) (revisionKey, bool) {
+	var v xlMetaV2Version
+	if _, err := v.unmarshalV(metaV, ver.meta); err != nil || v.Type != ObjectType || v.ObjectV2 == nil {
+		return revisionKey{}, false
+	}
+	o := *v.ObjectV2
+	o.MetaUser, o.MetaSys = maps.Clone(o.MetaUser), maps.Clone(o.MetaSys)
+	for _, k := range restoreFieldKeys {
+		delete(o.MetaUser, k)
+		delete(o.MetaSys, k)
+	}
+	v.ObjectV2 = &o
+	meta, err := v.MarshalMsg(nil)
+	if err != nil {
+		return revisionKey{}, false
+	}
+	return versionRevision(xlMetaV2ShallowVersion{header: v.header(), meta: meta}, xlMetaVersion), true
+}
+
+// readRestoreGeneration selects the authoritative generation of a version,
+// or fails closed: a revision that only some drives hold, such as one a
+// failed metadata update left behind, is never selected. An empty version ID
+// addresses the latest version, as a direct restore may.
+func (er erasureObjects) readRestoreGeneration(ctx context.Context, bucket, object, versionID string) (restoreGeneration, error) {
+	if versionID == "" {
+		fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, ObjectOptions{
+			NoLock:     true,
+			Transition: TransitionOptions{RestoreRequest: &RestoreObjectRequest{}},
+		}, false)
+		if err != nil {
+			return restoreGeneration{}, err
+		}
+		versionID = restoreVersionID(fi.VersionID)
+	}
+	var uv [16]byte
+	if versionID != nullVersionID {
+		id, err := uuid.Parse(versionID)
+		if err != nil {
+			return restoreGeneration{}, errFileVersionNotFound
+		}
+		uv = id
+	}
+	disks := er.getDisks()
+	n := len(disks)
+	majority := n/2 + 1
+	raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, true)
+	type revision struct {
+		ver   xlMetaV2ShallowVersion
+		metaV uint8
+		count int
+	}
+	keys := make([]*revisionKey, n)
+	neutral := make([]*revisionKey, n)
+	revisions := make(map[revisionKey]*revision)
+	neutrals := make(map[revisionKey]*revision)
+	absent := 0
+	for i := range disks {
+		if errs[i] != nil {
+			if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errVolumeNotFound) {
+				absent++
+			}
+			continue
+		}
+		var xl xlMetaV2
+		if xl.LoadOrConvert(raws[i].Buf) != nil {
+			continue
+		}
+		found := false
+		for _, ver := range xl.versions {
+			if ver.header.VersionID != uv || ver.header.FreeVersion() {
+				continue
+			}
+			key := versionRevision(ver, xl.metaV)
+			keys[i], found = &key, true
+			r := revisions[key]
+			if r == nil {
+				r = &revision{ver: ver, metaV: xl.metaV}
+				revisions[key] = r
+			}
+			r.count++
+			if nkey, ok := restoreNeutralRevision(ver, xl.metaV); ok {
+				neutral[i] = &nkey
+				if neutrals[nkey] == nil {
+					neutrals[nkey] = &revision{ver: ver, metaV: xl.metaV}
+				}
+				neutrals[nkey].count++
+			}
+			break
+		}
+		if !found {
+			absent++
+		}
+	}
+	accepted := func(revs map[revisionKey]*revision) (revisionKey, bool) {
+		var best *revision
+		var bestKey revisionKey
+		for key, r := range revs {
+			if best == nil || r.count > best.count {
+				best, bestKey = r, key
+			}
+		}
+		return bestKey, best != nil && best.count >= majority &&
+			best.count >= er.versionReadQuorum(ctx, bucket, object, best.ver, best.metaV, make([]FileInfo, n), make([]error, n))
+	}
+	gen := restoreGeneration{drives: make([]FileInfo, n), disks: make([]StorageAPI, n), revs: slices.Clone(keys)}
+	bestKey, ok := accepted(revisions)
+	if !ok {
+		if bestKey, ok = accepted(neutrals); !ok {
+			if absent >= majority {
+				return restoreGeneration{}, errFileVersionNotFound
+			}
+			return restoreGeneration{}, errErasureReadQuorum
+		}
+		keys, gen.split, gen.ongoing = neutral, true, make(map[string]bool)
+	}
+	for i := range disks {
+		if keys[i] == nil || *keys[i] != bestKey {
+			continue
+		}
+		fi, err := getFileInfo(raws[i].Buf, bucket, object, versionID, fileInfoOpts{Data: true})
+		if err != nil {
+			continue
+		}
+		gen.drives[i], gen.disks[i] = fi, disks[i]
+		if gen.split {
+			if st, err := parseRestoreObjStatus(fi.Metadata[xhttp.AmzRestore]); err == nil && st.Ongoing() {
+				gen.ongoing[fi.Metadata[restoreAttemptKey]] = true
+			}
+		}
+		if !gen.fi.IsValid() {
+			gen.fi = fi
+		}
+	}
+	if !gen.fi.IsValid() && !gen.fi.Deleted {
+		return restoreGeneration{}, errErasureReadQuorum
+	}
+	if gen.split {
+		gen.fi.Metadata = maps.Clone(gen.fi.Metadata)
+		for _, k := range restoreFieldKeys {
+			delete(gen.fi.Metadata, k)
+		}
+	}
+	return gen, nil
+}
+
+// updateRestoreStatus rewrites the restore fields of one stored version. It
+// selects the authoritative generation of the version, see
+// readRestoreGeneration, patches only the restore fields onto exactly that
+// generation, keeping every other field and its ModTime, and writes it back
+// only to the drives that hold it. The caller holds the object lock. update
+// edits the metadata in place, or returns an error, errRestoreUnchanged to
+// write nothing; ongoing names the attempts whose ongoing marker the
+// generation holds. A failed read never removes the version, and a failed
+// write puts the generation it read back.
+func (er erasureObjects) updateRestoreStatus(ctx context.Context, bucket, object, versionID string, update func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error) (ObjectInfo, error) {
+	gen, err := er.readRestoreGeneration(ctx, bucket, object, versionID)
+	if err != nil {
+		return ObjectInfo{}, toObjectErr(err, bucket, object, versionID)
+	}
+	fi := gen.fi
+	oi := fi.ToObjectInfo(bucket, object, false)
+	if fi.Deleted {
+		return oi, toObjectErr(errMethodNotAllowed, bucket, object)
+	}
+	ongoing := gen.ongoing
+	if !gen.split && oi.RestoreOngoing {
+		ongoing = map[string]bool{oi.UserDefined[restoreAttemptKey]: true}
+	}
+	meta := maps.Clone(fi.Metadata)
+	if err := update(oi, meta, ongoing); err != nil {
+		return oi, err
+	}
+	write := func(meta func(FileInfo) map[string]string) error {
+		metaArr := make([]FileInfo, len(gen.drives))
+		for i, d := range gen.drives {
+			if gen.disks[i] == nil {
+				continue
+			}
+			if !d.InlineData() {
+				d.Data = nil
+			}
+			d.Metadata = meta(d)
+			metaArr[i] = d
+		}
+		// The write assigns each drive its slot as erasure index: order the
+		// drives by the version's distribution first.
+		disks, metaArr := shuffleDisksAndPartsMetadata(gen.disks, metaArr, fi)
+		_, err := writeUniqueFileInfo(ctx, disks, "", bucket, object, metaArr, fi.WriteQuorum(er.defaultWQuorum()))
+		return err
+	}
+	if err := write(func(FileInfo) map[string]string { return meta }); err != nil {
+		// Without the read generation back, its drives could hold two
+		// revisions and no quorum for either. A split that stays differs
+		// only in restore fields, which the next write or reset resolves.
+		if rerr := write(func(d FileInfo) map[string]string { return d.Metadata }); rerr != nil {
+			storageLogIf(ctx, fmt.Errorf("Unable to put back the restore status of %s/%s(%s): %w", bucket, object, versionID, rerr))
+		}
+		return oi, toObjectErr(err, bucket, object)
+	}
+	fi.Metadata = meta
+	return fi.ToObjectInfo(bucket, object, false), nil
+}
+
+// clearRestoreStatus makes a version retryable again while attempt a still
+// owns its ongoing marker. It never brings back an earlier completed status:
+// POST only starts a restore when that restored copy is gone.
+func (er erasureObjects) clearRestoreStatus(ctx context.Context, bucket, object, versionID string, a *restoreAttempt) error {
+	_, err := er.updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error {
+		id := ""
+		if a != nil {
+			id = a.ID
+			if !a.targets(oi) {
+				return errRestoreUnchanged
+			}
+		}
+		// Clear only this attempt's marker. Where drives disagree, another
+		// attempt's marker may be the acknowledged one: leave that to it,
+		// or to a later POST, which takes it over under the lock.
+		if !ongoing[id] || len(ongoing) != 1 {
+			return errRestoreUnchanged
+		}
+		delete(meta, xhttp.AmzRestore)
+		delete(meta, xhttp.AmzRestoreExpiryDays)
+		delete(meta, xhttp.AmzRestoreRequestDate)
+		delete(meta, restoreAttemptKey)
+		return nil
+	})
+	if errors.Is(err, errRestoreUnchanged) {
+		return nil
+	}
+	return err
+}
+
+// restoreLock is the object lock that restores and their status updates
+// share: the lock every write of the object takes.
+func (z *erasureServerPools) restoreLock(bucket, object string) RWLocker {
+	return z.NewObjectNSLock(bucket, object)
+}
+
+// restoreRequestStatus is what a POST restore recorded on its version.
+type restoreRequestStatus struct {
+	objInfo ObjectInfo
+	// attempt is the restore that must rehydrate the version; nil when a
+	// restored copy was still there and its expiry was renewed.
+	attempt *restoreAttempt
+	// owner is the restore lock of the version, held for attempt. The
+	// caller hands it to the worker that runs attempt, or releases it.
+	owner *restoreOwner
+	// date is the request date recorded with attempt.
+	date string
+}
+
+// restoreRequest records a POST restore on target, the version that POST
+// resolved, on a fresh read under the object lock. The version must still
+// be target: a replacement under the same version ID is refused. A copy that
+// still holds its restored data is renewed. Otherwise the version is marked
+// ongoing for a new attempt, which also takes over a marker that no running
+// restore holds. Only the copy that is to be restored is written, so no copy
+// claims a restored copy it does not hold.
+//
+// It takes the restore lock R of the version before the object lock O. A
+// running restore holds R for its whole run, so a POST that cannot take R
+// within the takeover wait answers from the drives: in progress when they
+// show the version ongoing, and a retryable timeout otherwise, such as while
+// another POST of the version registers. ctx must outlive the request: a new
+// attempt keeps R in the returned status.
+func (z *erasureServerPools) restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (status restoreRequestStatus, err error) {
+	object = encodeDirObject(object)
+	versionID := restoreVersionID(target.VersionID)
+	owner, err := z.takeRestoreOwner(ctx, bucket, object, versionID, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait))
+	if err != nil {
+		if errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil {
+			running, rerr := z.restoreRunning(ctx, bucket, object, versionID)
+			switch {
+			case rerr != nil:
+				return restoreRequestStatus{}, rerr
+			case running:
+				return restoreRequestStatus{}, errRestoreInProgress
+			}
+		}
+		return restoreRequestStatus{}, err
+	}
+	defer func() {
+		if status.attempt == nil {
+			owner.release()
+		}
+	}()
+	lk := z.restoreLock(bucket, object)
+	lkctx, err := lk.GetLock(owner.ctx, globalOperationTimeout)
+	if err != nil {
+		return restoreRequestStatus{}, err
+	}
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx)
+
+	expected := newRestoreAttempt(target)
+	copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{
+		VersionID:        versionID,
+		Versioned:        opts.Versioned,
+		VersionSuspended: opts.VersionSuspended,
+		Transition:       opts.Transition,
+	})
+	if err != nil {
+		return restoreRequestStatus{}, err
+	}
+	// Restore the copy reads serve first, outside pools being decommissioned,
+	// as RestoreTransitionedObject does.
+	pool := -1
+	for _, c := range copies {
+		if !z.IsSuspended(c.Index) {
+			pool = c.Index
+			break
+		}
+	}
+	if pool < 0 {
+		return restoreRequestStatus{}, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
+	}
+
+	rreq := opts.Transition.RestoreRequest
+	date := UTCNow().Format(http.TimeFormat)
+	var attempt *restoreAttempt
+	oi, err := z.serverPools[pool].getHashedSet(object).updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, _ map[string]bool) error {
+		if !expected.targets(oi) {
+			// Not the version POST resolved, or no longer transitioned.
+			return InvalidObjectState{Bucket: bucket, Object: decodeDirObject(object)}
+		}
+		attempt = nil
+		meta[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days)
+		meta[xhttp.AmzRestoreRequestDate] = date
+		if !oi.RestoreOngoing && !oi.RestoreExpires.IsZero() {
+			// Cleanup removes a restored copy together with its header.
+			meta[xhttp.AmzRestore] = completedRestoreObj(opts.Transition.RestoreExpiry).String()
+			delete(meta, restoreAttemptKey)
+			return nil
+		}
+		attempt = newRestoreAttempt(oi)
+		meta[xhttp.AmzRestore] = ongoingRestoreObj().String()
+		meta[restoreAttemptKey] = attempt.ID
+		return nil
+	})
+	if err != nil {
+		if attempt != nil {
+			// The marker may have reached some drives, or all of them.
+			z.recoverRestoreStatus(bucket, object, ObjectOptions{
+				VersionID:  attempt.VersionID,
+				Transition: TransitionOptions{attempt: attempt},
+			})
+		}
+		return restoreRequestStatus{}, err
+	}
+	if attempt == nil {
+		return restoreRequestStatus{objInfo: oi}, nil
+	}
+	return restoreRequestStatus{objInfo: oi, attempt: attempt, owner: owner, date: date}, nil
+}
+
+// restoreRunning reports, without the object lock and within the takeover
+// wait, whether the drives show a restore of the version ongoing in a pool.
+// An error means that this cannot be told.
+func (z *erasureServerPools) restoreRunning(ctx context.Context, bucket, object, versionID string) (bool, error) {
+	ctx, cancel := context.WithTimeout(ctx, restoreTakeoverWait)
+	defer cancel()
+	for _, pool := range z.serverPools {
+		gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, versionID)
+		if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err == nil && ctx.Err() != nil {
+			err = ctx.Err()
+		}
+		if err != nil {
+			return false, toObjectErr(err, bucket, object, versionID)
+		}
+		if len(gen.ongoing) > 0 || !gen.split && gen.fi.ToObjectInfo(bucket, object, false).RestoreOngoing {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// restorePoolIdx returns the pool whose copy the restore rehydrates: the one
+// POST marked for its attempt, or for a direct caller the existing version.
+func (z *erasureServerPools) restorePoolIdx(ctx context.Context, bucket, object string, opts ObjectOptions) (int, error) {
+	a := opts.Transition.attempt
+	if a == nil {
+		// Restore an existing version, not the latest version's write location.
+		lookupOpts := opts
+		lookupOpts.MetadataChg = true
+		idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts)
+		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return idx, restoreResetError{err}
+		}
+		return idx, err
+	}
+	found := false
+	for idx, pool := range z.serverPools {
+		gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, a.VersionID)
+		if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			return -1, restoreResetError{toObjectErr(err, bucket, object, a.VersionID)}
+		}
+		found = true
+		if !z.IsSuspended(idx) && a.ownsGeneration(gen, bucket, object) {
+			return idx, nil
+		}
+	}
+	if !found {
+		return -1, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: a.VersionID}
+	}
+	return -1, errRestoreSuperseded
+}
+
+// recoverRestoreStatus retries, in memory and for a bounded time, to clear the
+// ongoing marker that a failed restore left behind, so a later POST is not
+// refused. Each try takes the object lock afresh and clears the marker only
+// while it still belongs to this attempt.
+func (z *erasureServerPools) recoverRestoreStatus(bucket, object string, opts ObjectOptions) {
+	go func() {
+		var err error
+		for _, delay := range restoreRecoveryDelays {
+			select {
+			case <-GlobalContext.Done():
+				return
+			case <-time.After(delay):
+			}
+			if err = z.resetRestoreStatus(bucket, object, opts); err == nil {
+				return
+			}
+		}
+		storageLogIf(GlobalContext, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, err))
+	}()
+}
+
+// resetRestoreStatus is one try of recoverRestoreStatus.
+func (z *erasureServerPools) resetRestoreStatus(bucket, object string, opts ObjectOptions) error {
+	ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
+	defer cancel()
+	lk := z.restoreLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return err
+	}
+	defer lk.Unlock(lkctx)
+	ctx = lkctx.Context()
+
+	a := opts.Transition.attempt
+	versionID := opts.VersionID
+	if a != nil {
+		versionID = a.VersionID
+	}
+	for _, pool := range z.serverPools {
+		err := pool.getHashedSet(object).clearRestoreStatus(ctx, bucket, object, versionID, a)
+		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !isErrMethodNotAllowed(err) && !isErrBucketNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}

cmd/restore-capability.go

@@ -0,0 +1,184 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/internal/grid"
+)
+
+// A native restore keeps its multipart upload private and marks the version
+// it rehydrates (restore-attempt.go). A server that predates this ignores
+// the marker: it would list, change, abort or complete the upload. Native
+// restore therefore runs only while every server of the cluster positively
+// reports that it understands the marker. A server that does not answer,
+// or does not know the question, as an older server does not, blocks it.
+
+// nativeRestoreCapability is the native restore protocol this server speaks.
+const nativeRestoreCapability = "2"
+
+const (
+	peerRESTRestoreCapability = "restore"
+	// restoreCapabilityTTL bounds how long a positive answer is reused while
+	// no server reconnected.
+	restoreCapabilityTTL = 10 * time.Second
+	// restoreCapabilityTimeout bounds the question to each server.
+	restoreCapabilityTimeout = 5 * time.Second
+)
+
+var errNativeRestoreUnavailable = errors.New("native restore is unavailable during a rolling upgrade: not every server of the cluster supports it yet")
+
+// restoreCapabilityHandler answers the question on this server.
+func restoreCapabilityHandler(*grid.MSS) (*grid.MSS, *grid.RemoteErr) {
+	return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: nativeRestoreCapability}), nil
+}
+
+// restoreCapabilityPeer is another server of the cluster.
+type restoreCapabilityPeer interface {
+	// restoreConnection identifies the current connection to the server; it
+	// changes when the server reconnects, as a restarted one does. ok is
+	// false when the server is not connected.
+	restoreConnection() (key string, ok bool)
+	restoreCapability(ctx context.Context) (string, error)
+}
+
+type restorePeerClient struct{ *peerRESTClient }
+
+func (c restorePeerClient) restoreConnection() (string, bool) {
+	if c.peerRESTClient == nil || c.gridConn == nil {
+		return "", false
+	}
+	conn := c.gridConn()
+	if conn == nil {
+		return "", false
+	}
+	st := conn.Stats()
+	if st.Connected == 0 {
+		return "", false
+	}
+	// Keys are compared by the position of their server.
+	return fmt.Sprintf("%d/%d", st.LastConnectTime.UnixNano(), st.ReconnectCount), true
+}
+
+func (c restorePeerClient) restoreCapability(ctx context.Context) (string, error) {
+	conn := c.gridConn()
+	if conn == nil {
+		return "", errPeerOffline
+	}
+	resp, err := restoreCapabilityRPC.Call(ctx, conn, grid.NewMSS())
+	if err != nil {
+		return "", err
+	}
+	return resp.Get(peerRESTRestoreCapability), nil
+}
+
+// restoreCapabilityGate decides whether native restore may run, caching a
+// positive answer for restoreCapabilityTTL while the connections to all
+// servers stay the same. A negative answer is not cached.
+type restoreCapabilityGate struct {
+	peers func() []restoreCapabilityPeer
+
+	once    sync.Once
+	turn    chan struct{} // one check at a time
+	checked time.Time
+	keys    []string
+}
+
+var globalRestoreCapability = &restoreCapabilityGate{peers: notificationRestorePeers}
+
+func notificationRestorePeers() []restoreCapabilityPeer {
+	if globalNotificationSys == nil {
+		return nil
+	}
+	peers := make([]restoreCapabilityPeer, 0, len(globalNotificationSys.peerClients))
+	for _, client := range globalNotificationSys.peerClients {
+		peers = append(peers, restorePeerClient{client})
+	}
+	return peers
+}
+
+// nativeRestoreAvailable returns errNativeRestoreUnavailable unless every
+// other server of the cluster supports native restore.
+func nativeRestoreAvailable(ctx context.Context) error {
+	return globalRestoreCapability.check(ctx)
+}
+
+func (g *restoreCapabilityGate) check(ctx context.Context) error {
+	g.once.Do(func() { g.turn = make(chan struct{}, 1) })
+	// One deadline covers the wait for another check and the questions.
+	ctx, cancel := context.WithTimeout(ctx, restoreCapabilityTimeout)
+	defer cancel()
+	select {
+	case g.turn <- struct{}{}:
+	case <-ctx.Done():
+		return errNativeRestoreUnavailable
+	}
+	defer func() { <-g.turn }()
+
+	// Read the connections only now: one may have changed while this check
+	// waited for another.
+	peers := g.peers()
+	connections := func() ([]string, bool) {
+		keys := make([]string, len(peers))
+		for i, p := range peers {
+			key, ok := p.restoreConnection()
+			if !ok {
+				return nil, false
+			}
+			keys[i] = key
+		}
+		return keys, true
+	}
+	keys, ok := connections()
+	if !ok {
+		g.checked, g.keys = time.Time{}, nil
+		return errNativeRestoreUnavailable
+	}
+	if ctx.Err() != nil {
+		return errNativeRestoreUnavailable
+	}
+	if !g.checked.IsZero() && time.Since(g.checked) < restoreCapabilityTTL && slices.Equal(keys, g.keys) {
+		return nil
+	}
+	g.checked, g.keys = time.Time{}, nil
+	answers := make([]bool, len(peers))
+	var wg sync.WaitGroup
+	for i, p := range peers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			version, err := p.restoreCapability(ctx)
+			answers[i] = err == nil && version == nativeRestoreCapability
+		}()
+	}
+	wg.Wait()
+	// An answer counts only for the connection it was asked on: a server
+	// that reconnected meanwhile may be another binary.
+	after, ok := connections()
+	if !ok || !slices.Equal(keys, after) || slices.Contains(answers, false) || ctx.Err() != nil {
+		return errNativeRestoreUnavailable
+	}
+	g.checked, g.keys = time.Now(), keys
+	return nil
+}

cmd/restore-capability_test.go

@@ -0,0 +1,271 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"reflect"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/grid"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// A server answers the capability question over the grid. One that does not
+// know it, as an older server, or answers something else, blocks native
+// restore, and so does one that is not connected.
+func TestNativeRestoreCapabilityGrid(t *testing.T) {
+	tg, err := grid.SetupTestGrid(4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(tg.Cleanup)
+	if err := restoreCapabilityRPC.Register(tg.Managers[1], restoreCapabilityHandler); err != nil {
+		t.Fatal(err)
+	}
+	// Managers[2] never registers the handler.
+	if err := restoreCapabilityRPC.Register(tg.Managers[3], func(*grid.MSS) (*grid.MSS, *grid.RemoteErr) {
+		return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: "0"}), nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	peer := func(i int) restoreCapabilityPeer {
+		return restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[i]) }}}
+	}
+	offline := restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return nil }}}
+	for _, tc := range []struct {
+		name  string
+		peers []restoreCapabilityPeer
+		ok    bool
+	}{
+		{"single-server", nil, true},
+		{"all-capable", []restoreCapabilityPeer{peer(1)}, true},
+		{"older-server", []restoreCapabilityPeer{peer(1), peer(2)}, false},
+		{"other-answer", []restoreCapabilityPeer{peer(1), peer(3)}, false},
+		{"offline", []restoreCapabilityPeer{peer(1), offline}, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return tc.peers }}
+			if err := g.check(t.Context()); (err == nil) != tc.ok {
+				t.Errorf("capability: %v, want available=%v", err, tc.ok)
+			}
+		})
+	}
+}
+
+// restoreCapabilityFake is a server whose connection and answer a test sets.
+type restoreCapabilityFake struct {
+	capable   atomic.Bool
+	connected atomic.Bool
+	conn      atomic.Int64
+	asked     atomic.Int64
+}
+
+func newRestoreCapabilityFake() *restoreCapabilityFake {
+	p := &restoreCapabilityFake{}
+	p.capable.Store(true)
+	p.connected.Store(true)
+	return p
+}
+
+func (p *restoreCapabilityFake) restoreConnection() (string, bool) {
+	return strings.Repeat("c", int(p.conn.Load())), p.connected.Load()
+}
+
+func (p *restoreCapabilityFake) restoreCapability(context.Context) (string, error) {
+	p.asked.Add(1)
+	if !p.capable.Load() {
+		return "", errors.New("Invalid Handler for type: RestoreCapability")
+	}
+	return nativeRestoreCapability, nil
+}
+
+// A positive answer is reused while the connections stay the same; a
+// reconnect asks again, a disconnected server blocks without being asked,
+// and a negative answer is not reused.
+func TestNativeRestoreCapabilityCache(t *testing.T) {
+	p := newRestoreCapabilityFake()
+	g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+	for range 3 {
+		if err := g.check(t.Context()); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if n := p.asked.Load(); n != 1 {
+		t.Errorf("asked %d times while connected and capable, want 1", n)
+	}
+	// The server restarts with an older binary and reconnects.
+	p.capable.Store(false)
+	p.conn.Add(1)
+	if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) {
+		t.Errorf("after a reconnect to an older server: %v", err)
+	}
+	p.capable.Store(true)
+	if err := g.check(t.Context()); err != nil {
+		t.Errorf("a negative answer was reused: %v", err)
+	}
+	asked := p.asked.Load()
+	p.connected.Store(false)
+	if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) || p.asked.Load() != asked {
+		t.Errorf("disconnected server: %v, asked %d -> %d", err, asked, p.asked.Load())
+	}
+}
+
+// During a rolling upgrade POST restore answers 503 and writes nothing, and
+// a restore queued before it creates no upload and leaves the version
+// retryable.
+func TestNativeRestoreRollingUpgradeGate(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	initConfigSubsystem(t.Context(), r.z)
+	oi, _ := restoreIsolationSeed(t, r, 0, false, true)
+	p := newRestoreCapabilityFake()
+	previous := globalRestoreCapability
+	globalRestoreCapability = &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+	t.Cleanup(func() { globalRestoreCapability = previous })
+
+	p.capable.Store(false)
+	before := r.fileInfo(oi.VersionID)
+	layer := newRestoreIsolationLayer(r)
+	rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+	if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "rolling upgrade") {
+		t.Errorf("POST during a rolling upgrade: %d %s", rec.Code, rec.Body.String())
+	}
+	if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before.Metadata, after.Metadata) {
+		t.Error("refused POST changed the version")
+	}
+	select {
+	case err := <-layer.done:
+		t.Fatalf("refused POST started a restore: %v", err)
+	default:
+	}
+
+	p.capable.Store(true)
+	release := make(chan struct{})
+	layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+		<-release
+		return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+	}
+	if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+		close(release)
+		t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+	}
+	// A server is replaced by an older binary while the restore is queued.
+	p.capable.Store(false)
+	p.conn.Add(1)
+	close(release)
+	if err := layer.wait(t); !errors.Is(err, errNativeRestoreUnavailable) {
+		t.Fatalf("queued restore during a rolling upgrade: %v", err)
+	}
+	if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+		t.Errorf("queued restore created an upload during a rolling upgrade: %v", uploads)
+	}
+	info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+	if err != nil || info.RestoreOngoing || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" {
+		t.Errorf("version after a refused queued restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+	}
+}
+
+// restoreCapabilityHeld is a fake server whose answer a test holds.
+type restoreCapabilityHeld struct {
+	restoreCapabilityFake
+	asking  chan struct{}
+	release chan struct{}
+}
+
+func (p *restoreCapabilityHeld) restoreCapability(ctx context.Context) (string, error) {
+	v, err := p.restoreCapabilityFake.restoreCapability(ctx)
+	p.asking <- struct{}{}
+	select {
+	case <-p.release:
+		return v, err
+	case <-ctx.Done():
+		return "", ctx.Err()
+	}
+}
+
+// A check that waits for another reads the connections once its turn
+// comes, an answer counts only if its server did not reconnect meanwhile,
+// and a waiting check ends with its context.
+func TestNativeRestoreCapabilityRevalidates(t *testing.T) {
+	t.Run("reconnect-during-check", func(t *testing.T) {
+		held := func() *restoreCapabilityHeld {
+			p := &restoreCapabilityHeld{asking: make(chan struct{}, 4), release: make(chan struct{})}
+			p.capable.Store(true)
+			p.connected.Store(true)
+			return p
+		}
+		fast, slow := held(), held()
+		g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{fast, slow} }}
+		first := make(chan error, 1)
+		go func() { first <- g.check(t.Context()) }()
+		<-fast.asking
+		<-slow.asking
+		waiting := make(chan error, 1)
+		go func() { waiting <- g.check(t.Context()) }()
+		// Both answered "capable"; then fast reconnects as an older server.
+		fast.capable.Store(false)
+		fast.conn.Add(1)
+		close(fast.release)
+		close(slow.release)
+		if err := <-first; !errors.Is(err, errNativeRestoreUnavailable) {
+			t.Errorf("check whose server reconnected while it asked: %v", err)
+		}
+		if err := <-waiting; !errors.Is(err, errNativeRestoreUnavailable) {
+			t.Errorf("waiting check accepted a reconnected older server: %v", err)
+		}
+	})
+	t.Run("canceled-cached", func(t *testing.T) {
+		p := newRestoreCapabilityFake()
+		g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+		if err := g.check(t.Context()); err != nil {
+			t.Fatal(err)
+		}
+		ctx, cancel := context.WithCancel(t.Context())
+		cancel()
+		if err := g.check(ctx); err == nil {
+			t.Error("a cached answer accepted an ended context")
+		}
+	})
+	t.Run("canceled-wait", func(t *testing.T) {
+		slow := &restoreCapabilityHeld{restoreCapabilityFake: restoreCapabilityFake{}, asking: make(chan struct{}, 4), release: make(chan struct{})}
+		slow.capable.Store(true)
+		slow.connected.Store(true)
+		defer close(slow.release)
+		g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{slow} }}
+		go g.check(t.Context())
+		<-slow.asking
+		ctx, cancel := context.WithCancel(t.Context())
+		waiting := make(chan error, 1)
+		go func() { waiting <- g.check(ctx) }()
+		cancel()
+		select {
+		case err := <-waiting:
+			if err == nil {
+				t.Error("canceled check succeeded")
+			}
+		case <-time.After(time.Second):
+			t.Error("canceled check kept waiting for another check")
+		}
+	})
+}

cmd/restore-native-encoding_test.go

@@ -0,0 +1,248 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/hash"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/pubsub"
+	"github.com/minio/mux"
+)
+
+// Exercise the stored representation produced by real S3 uploads. Native
+// restore copies encoded bytes, but must retain logical sizes and identity.
+func TestNativeRestoreEncodedObjects(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, encoding := range []string{"plain", "compressed", "ssec", "s3-compressed"} {
+			for _, layout := range []string{"single", "multipart", "sparse-one-part"} {
+				t.Run(fmt.Sprintf("pools=%d/%s/%s", pools, encoding, layout), func(t *testing.T) {
+					r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+					initConfigSubsystem(t.Context(), r.z)
+					r.object = "restore-encoded.txt"
+					previousKMS, previousTLS := GlobalKMS, globalIsTLS
+					t.Cleanup(func() { GlobalKMS, globalIsTLS = previousKMS, previousTLS })
+					globalIsTLS = true
+					headers := make(map[string]string)
+					readHeaders := make(map[string]string)
+					if encoding == "compressed" || encoding == "s3-compressed" {
+						enableAttributesPartsCompression(t)
+					}
+					if encoding == "ssec" {
+						headers = ssecTestHeaders(0x53)
+						readHeaders = ssecTestHeaders(0x53)
+					}
+					if encoding == "s3-compressed" {
+						var err error
+						GlobalKMS, err = kms.NewBuiltin("restore-test-key", bytes.Repeat([]byte{0x59}, 32))
+						if err != nil {
+							t.Fatal(err)
+						}
+						headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+					}
+					previousObjectAPI := newObjectLayerFn()
+					t.Cleanup(func() { setObjectLayer(previousObjectAPI) })
+					router := initTestAPIEndPoints(r.z, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "PutObject", "GetObject"})
+					bodies := [][]byte{bytes.Repeat([]byte("encoded-restore-"), compMinIndexSize/16+1)}
+					partNumbers := []int{1}
+					switch layout {
+					case "multipart":
+						bodies = append(bodies, []byte("last encoded part"))
+						partNumbers = []int{1, 2}
+					case "sparse-one-part":
+						partNumbers = []int{5}
+					}
+					want := bytes.Join(bodies, nil)
+					if layout == "single" {
+						headers[xhttp.AmzChecksumCRC32] = mustChecksum(t, hash.ChecksumCRC32, want)
+						putCopyChecksumSource(t, router, globalActiveCred, r.bucket, r.object, want, headers)
+					} else {
+						attributesPartsUpload(t, router, globalActiveCred, r.bucket, r.object, headers, bodies, partNumbers)
+					}
+					before, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{})
+					if err != nil {
+						t.Fatal(err)
+					}
+					if before.IsCompressed() != (encoding == "compressed" || encoding == "s3-compressed") {
+						t.Fatalf("unexpected compression state: %v", before.IsCompressed())
+					}
+					if before.IsCompressed() && len(before.Parts[0].Index) == 0 {
+						t.Fatal("fixture must have a compression index")
+					}
+					if layout == "single" && len(before.Checksum) == 0 {
+						t.Fatal("fixture must have a stored checksum")
+					}
+					objectURL := getGetObjectURL("", r.bucket, r.object) + "?versionId=" + url.QueryEscape(before.VersionID)
+					beforeGet := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, objectURL, nil, readHeaders)
+					if beforeGet.Code != http.StatusOK || !bytes.Equal(beforeGet.Body.Bytes(), want) {
+						t.Fatalf("fixture GET: %d bytes=%d want=%d", beforeGet.Code, beforeGet.Body.Len(), len(want))
+					}
+					beforeETags := beforeGet.Header()[xhttp.ETag]
+					if len(beforeETags) != 1 || beforeETags[0] == "" {
+						t.Fatal("fixture GET must expose a nonempty ETag")
+					}
+					beforeRaw := nativeRestoreSnapshot(t, r)
+					topts := r.opts(before.VersionID)
+					topts.MTime = before.ModTime
+					topts.Transition = TransitionOptions{Tier: "WARM", ETag: before.ETag}
+					if err := r.z.TransitionObject(t.Context(), r.bucket, r.object, topts); err != nil {
+						t.Fatal(err)
+					}
+					observed := nativeRestoreHTTPObserver{r.z, make(chan nativeRestoreHTTPResult, 1)}
+					api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return observed }}
+					events, done := make(chan event.Event, 1), make(chan struct{})
+					defer close(done)
+					if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRestoreCompleted), events, done, nil); err != nil {
+						t.Fatal(err)
+					}
+					restoreXML := []byte(`<RestoreRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Days>1</Days></RestoreRequest>`)
+					req, err := newTestSignedRequestV4(http.MethodPost, objectURL+"&restore", int64(len(restoreXML)), bytes.NewReader(restoreXML), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+					if err != nil {
+						t.Fatal(err)
+					}
+					if err := req.ParseForm(); err != nil {
+						t.Fatal(err)
+					}
+					req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object})
+					rec := httptest.NewRecorder()
+					api.PostRestoreObjectHandler(rec, req)
+					if rec.Code != http.StatusOK {
+						t.Fatalf("restore POST: %d %s", rec.Code, rec.Body.String())
+					}
+					select {
+					case result := <-observed.done:
+						if result.err != nil {
+							t.Fatalf("background restore: %v", result.err)
+						}
+					case <-time.After(5 * time.Second):
+						t.Fatal("restore did not finish")
+					}
+					select {
+					case <-events:
+					case <-time.After(5 * time.Second):
+						t.Fatal("restore completion event missing")
+					}
+					after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(before.VersionID))
+					if err != nil {
+						t.Fatal(err)
+					}
+					if after.ETag != before.ETag || after.Size != before.Size || !after.ModTime.Equal(before.ModTime) {
+						t.Error("restore changed content identity")
+					}
+					afterRaw := nativeRestoreSnapshot(t, r)
+					for pool := range beforeRaw {
+						if len(afterRaw[pool]) != len(beforeRaw[pool]) {
+							t.Fatalf("pool %d version count changed", pool)
+						}
+						for i, fi := range beforeRaw[pool] {
+							if !bytes.Equal(fi.Checksum, afterRaw[pool][i].Checksum) {
+								t.Error("restore lost stored checksum")
+							}
+						}
+					}
+					if len(after.Parts) != len(before.Parts) {
+						t.Fatal("restore changed part count")
+					}
+					for i, part := range after.Parts {
+						old := before.Parts[i]
+						if part.Number != old.Number || part.Size != old.Size || part.ActualSize != old.ActualSize || !bytes.Equal(part.Index, old.Index) {
+							t.Errorf("part %d layout or compression index changed", i)
+						}
+					}
+					tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+					reads := tier.gets.Load()
+					got := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, objectURL, nil, readHeaders)
+					if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), want) {
+						t.Errorf("restored GET: %d bytes=%d want=%d", got.Code, got.Body.Len(), len(want))
+					}
+					afterETags := got.Header()[xhttp.ETag]
+					if len(afterETags) != 1 || afterETags[0] == "" || afterETags[0] != beforeETags[0] {
+						t.Error("restore changed client ETag")
+					}
+					for i, number := range partNumbers {
+						part := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, objectURL+fmt.Sprintf("&partNumber=%d", number), nil, readHeaders)
+						if part.Code != http.StatusPartialContent || !bytes.Equal(part.Body.Bytes(), bodies[i]) {
+							t.Errorf("restored part %d: %d bytes=%d want=%d", number, part.Code, part.Body.Len(), len(bodies[i]))
+						}
+					}
+					if tier.gets.Load() != reads {
+						t.Error("restored GET still used remote tier")
+					}
+				})
+			}
+		}
+	}
+}
+
+// XL metadata predating compression has no ActualSize. Conversion and healing
+// preserve those zero values; a plain multipart restore must derive the size.
+func TestNativeRestoreLegacyPartSizes(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, "")
+	oi, body := nativeRestoreSeed(t, r, 0, true, true)
+	for _, disk := range r.z.serverPools[0].getHashedSet(r.object).getDisks() {
+		fi, err := disk.ReadVersion(t.Context(), r.bucket, r.bucket, r.object, oi.VersionID, ReadOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		for i := range fi.Parts {
+			fi.Parts[i].ActualSize = 0
+		}
+		delete(fi.Metadata, ReservedMetadataPrefix+"actual-size")
+		if err := disk.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+			t.Fatal(err)
+		}
+	}
+	before, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, part := range before.Parts {
+		if part.ActualSize != 0 {
+			t.Fatal("legacy fixture was normalized")
+		}
+	}
+	_, got, err := r.read(oi.VersionID)
+	if err != nil || !bytes.Equal(got, body) {
+		t.Fatalf("legacy fixture read: %v", err)
+	}
+	ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+	defer cancel()
+	if err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+		t.Fatal(err)
+	}
+	tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+	reads := tier.gets.Load()
+	_, got, err = r.read(oi.VersionID)
+	if err != nil || !bytes.Equal(got, body) {
+		t.Fatalf("legacy restored read: %v", err)
+	}
+	if tier.gets.Load() != reads {
+		t.Error("legacy restored read used the remote tier")
+	}
+}

cmd/restore-native-generation_test.go

@@ -0,0 +1,724 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"maps"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"runtime"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// restoreGenerationDisk fails metadata writes or updates of the object, or
+// metadata reads of uploads, while its toggles are set.
+type restoreGenerationDisk struct {
+	StorageAPI
+	bucket, object            string
+	write, update, uploadRead *atomic.Bool
+}
+
+func (d restoreGenerationDisk) WriteMetadata(ctx context.Context, orig, vol, path string, fi FileInfo) error {
+	if d.write != nil && d.write.Load() && vol == d.bucket && path == d.object {
+		return errFaultyDisk
+	}
+	return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi)
+}
+
+func (d restoreGenerationDisk) UpdateMetadata(ctx context.Context, vol, path string, fi FileInfo, opts UpdateMetadataOpts) error {
+	if d.update != nil && d.update.Load() && vol == d.bucket && path == d.object {
+		return errFaultyDisk
+	}
+	return d.StorageAPI.UpdateMetadata(ctx, vol, path, fi, opts)
+}
+
+func (d restoreGenerationDisk) ReadVersion(ctx context.Context, orig, vol, path, version string, opts ReadOptions) (FileInfo, error) {
+	if d.uploadRead != nil && d.uploadRead.Load() && vol == minioMetaMultipartBucket {
+		return FileInfo{}, errFaultyDisk
+	}
+	return d.StorageAPI.ReadVersion(ctx, orig, vol, path, version, opts)
+}
+
+// restoreGenerationOffline makes a drive's object metadata unreadable and
+// unwritable while set, its uploads still readable.
+type restoreGenerationOffline struct {
+	StorageAPI
+	bucket, object string
+	fail           *atomic.Bool
+}
+
+func (d restoreGenerationOffline) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
+	if d.fail.Load() && bucket == d.bucket && object == d.object {
+		return RawFileInfo{}, errFaultyDisk
+	}
+	return d.StorageAPI.ReadXL(ctx, bucket, object, data)
+}
+
+func (d restoreGenerationOffline) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
+	if d.fail.Load() && bucket == d.bucket && object == d.object {
+		return FileInfo{}, errFaultyDisk
+	}
+	return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
+}
+
+func (d restoreGenerationOffline) WriteMetadata(ctx context.Context, orig, bucket, object string, fi FileInfo) error {
+	if d.fail.Load() && bucket == d.bucket && object == d.object {
+		return errFaultyDisk
+	}
+	return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi)
+}
+
+// restoreGenerationMetadata returns the version's metadata on each drive.
+func restoreGenerationMetadata(t *testing.T, disks []StorageAPI, r *restoreTest, versionID string) []map[string]string {
+	t.Helper()
+	metas := make([]map[string]string, len(disks))
+	for i, d := range disks {
+		fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, versionID, ReadOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		metas[i] = fi.Metadata
+	}
+	return metas
+}
+
+// An Object Lock update acknowledged on a quorum, which one drive missed,
+// survives POST restore: POST patches the restore fields onto the quorum's
+// generation and never writes the stale drive's metadata anywhere.
+func TestRestoreStatusUsesQuorumGeneration(t *testing.T) {
+	for _, kind := range []string{"legal-hold", "retention"} {
+		t.Run(kind, func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			initConfigSubsystem(t.Context(), r.z)
+			enableBucketObjectLock(t, r.bucket)
+			oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+			if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+				t.Fatal(err)
+			}
+			initial := []byte(`<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>OFF</Status></LegalHold>`)
+			updated := []byte(`<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>ON</Status></LegalHold>`)
+			key := strings.ToLower(xhttp.AmzObjectLockLegalHold)
+			if kind == "retention" {
+				initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour))
+				updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))
+				key = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
+			}
+			if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK {
+				t.Fatal(rec.Body.String())
+			}
+			er := r.z.serverPools[0].getHashedSet(r.object)
+			disks := er.getDisks()
+			var fail atomic.Bool
+			wrapped := append([]StorageAPI(nil), disks...)
+			wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, update: &fail}
+			er.getDisks = func() []StorageAPI { return wrapped }
+			fail.Store(true)
+			if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK {
+				t.Fatalf("protection update: %d %s", rec.Code, rec.Body.String())
+			}
+			fail.Store(false)
+			before := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+			acknowledged := before[1][key]
+			if before[0][key] == acknowledged {
+				t.Fatal("fixture: drive 0 did not miss the update")
+			}
+			for i := 1; i < len(disks); i++ {
+				if before[i][key] != acknowledged {
+					t.Fatal("fixture: the update did not reach the other drives")
+				}
+			}
+			if rec := restoreIsolationPOST(t, r, r.z, oi.VersionID); rec.Code != http.StatusOK {
+				t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+			}
+			after := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+			for i := 1; i < len(disks); i++ {
+				if after[i][key] != acknowledged {
+					t.Errorf("drive %d: POST wrote %q over the acknowledged %q", i, after[i][key], acknowledged)
+				}
+				if after[i][xhttp.AmzRestoreRequestDate] == before[i][xhttp.AmzRestoreRequestDate] {
+					t.Errorf("drive %d: POST did not record its request", i)
+				}
+			}
+			// The stale drive is left to heal; reads that pick it are not
+			// changed here.
+			if after[0][key] != before[0][key] || after[0][xhttp.AmzRestoreRequestDate] != before[0][xhttp.AmzRestoreRequestDate] {
+				t.Error("POST wrote the stale drive instead of leaving it to heal")
+			}
+		})
+	}
+}
+
+// The attempt is part of a version's metadata generation. A POST whose
+// marker reached all drives but one is acknowledged: its restore owns the
+// version, and the reset of an older attempt leaves the newer marker alone.
+func TestRestoreAttemptGenerationFencing(t *testing.T) {
+	for _, mode := range []string{"worker", "old-reset", "degraded-old-reset"} {
+		t.Run(mode, func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			initConfigSubsystem(t.Context(), r.z)
+			oi, body := restoreIsolationSeed(t, r, 0, false, false)
+			er := r.z.serverPools[0].getHashedSet(r.object)
+			disks := er.getDisks()
+			var fail, offline atomic.Bool
+			wrapped := append([]StorageAPI(nil), disks...)
+			wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, write: &fail}
+			wrapped[1] = restoreGenerationOffline{StorageAPI: disks[1], bucket: r.bucket, object: r.object, fail: &offline}
+			er.getDisks = func() []StorageAPI { return wrapped }
+			captured := make(chan ObjectOptions, 2)
+			layer := newRestoreIsolationLayer(r)
+			layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error {
+				captured <- opts
+				return nil
+			}
+			var older ObjectOptions
+			if mode != "worker" {
+				if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+					t.Fatal(rec.Body.String())
+				}
+				older = <-captured
+				layer.wait(t)
+			}
+			fail.Store(true)
+			rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+			fail.Store(false)
+			if rec.Code != http.StatusOK {
+				t.Fatalf("POST on all drives but one: %d %s", rec.Code, rec.Body.String())
+			}
+			newer := <-captured
+			layer.wait(t)
+			id := newer.Transition.attempt.ID
+			if mode == "worker" {
+				if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, newer); err != nil {
+					t.Fatalf("restore of the acknowledged attempt: %v", err)
+				}
+				if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+					t.Fatalf("restored contents: %v", err)
+				}
+				return
+			}
+			if mode == "degraded-old-reset" {
+				// One drive of the newer attempt is unreadable: the drives
+				// that answer no longer show which attempt is acknowledged.
+				offline.Store(true)
+			}
+			if err := r.z.resetRestoreStatus(r.bucket, r.object, older); err != nil {
+				t.Fatal(err)
+			}
+			offline.Store(false)
+			after := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+			for i := 1; i < len(disks); i++ {
+				if after[i][restoreAttemptKey] != id || !strings.Contains(after[i][xhttp.AmzRestore], "true") {
+					t.Errorf("drive %d: the older attempt's reset cleared the newer marker: %q %q", i, after[i][restoreAttemptKey], after[i][xhttp.AmzRestore])
+				}
+			}
+			if after[0][restoreAttemptKey] != older.Transition.attempt.ID {
+				t.Error("the reset wrote the stale drive")
+			}
+			// The newer attempt still owns the version and restores it.
+			if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, newer); err != nil {
+				t.Fatalf("restore of the newer attempt after the older reset: %v", err)
+			}
+			if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("restored contents: %v", err)
+			}
+		})
+	}
+}
+
+// POST restores the version it resolved with that version's identity. A
+// replacement under the same version ID between its read and its status
+// write is refused, and nothing restores it.
+func TestRestorePOSTRejectsReplacedTarget(t *testing.T) {
+	for _, state := range []versioning.State{"", versioning.Suspended} {
+		t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, state)
+			initConfigSubsystem(t.Context(), r.z)
+			oi, _ := restoreIsolationSeed(t, r, 0, true, false)
+			layer := newRestoreIsolationLayer(r)
+			var replacement ObjectInfo
+			layer.afterRead = func() {
+				data := []byte("replacement after POST resolved its target")
+				var err error
+				replacement, err = r.z.PutObject(t.Context(), r.bucket, r.object,
+					mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: state == versioning.Suspended})
+				if err != nil {
+					t.Fatal(err)
+				}
+				opts := r.opts(nullVersionID)
+				opts.MTime = replacement.ModTime
+				opts.Transition = TransitionOptions{Tier: "WARM", ETag: replacement.ETag}
+				if err := r.z.serverPools[0].TransitionObject(t.Context(), r.bucket, r.object, opts); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code < 400 {
+				layer.wait(t)
+				t.Errorf("POST accepted a replacement of its target: %d", rec.Code)
+			}
+			after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(nullVersionID))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if after.ETag != replacement.ETag || after.UserDefined[xhttp.AmzRestore] != "" || after.ETag == oi.ETag {
+				t.Errorf("replacement %q changed by POST: etag %q restore %q", replacement.ETag, after.ETag, after.UserDefined[xhttp.AmzRestore])
+			}
+		})
+	}
+}
+
+// A POST on a version whose restore holds the object lock answers within
+// the two-second takeover bound: its first read does not wait for the lock.
+func TestRestorePOSTOngoingBound(t *testing.T) {
+	for _, mode := range []string{"pools=1", "pools=2", "stale-minority", "failed-activity-read"} {
+		t.Run(mode, func(t *testing.T) {
+			pools := 1
+			if mode == "pools=2" {
+				pools = 2
+			}
+			r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+			initConfigSubsystem(t.Context(), r.z)
+			oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+			tier := newRestoreIsolationGateTier(0, nil)
+			globalTierConfigMgr.drivercache["WARM"] = tier
+			defer tier.unblock()
+			var fail atomic.Bool
+			if mode == "stale-minority" {
+				// The first read may pick a drive that missed the marker.
+				er := r.z.serverPools[0].getHashedSet(r.object)
+				disks := er.getDisks()
+				wrapped := append([]StorageAPI(nil), disks...)
+				wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, write: &fail}
+				er.getDisks = func() []StorageAPI { return wrapped }
+				fail.Store(true)
+			}
+			layer := newRestoreIsolationLayer(r)
+			rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+			fail.Store(false)
+			if rec.Code != http.StatusOK {
+				t.Fatal(rec.Body.String())
+			}
+			restoreIsolationEntered(t, tier)
+			var unreadable atomic.Bool
+			second := newRestoreIsolationLayer(r)
+			if mode == "failed-activity-read" {
+				// Once POST resolved its target, nothing tells whether it
+				// is restoring.
+				er := r.z.serverPools[0].getHashedSet(r.object)
+				disks := er.getDisks()
+				wrapped := make([]StorageAPI, len(disks))
+				for i, d := range disks {
+					wrapped[i] = restoreGenerationOffline{StorageAPI: d, bucket: r.bucket, object: r.object, fail: &unreadable}
+				}
+				er.getDisks = func() []StorageAPI { return wrapped }
+				second.afterRead = func() { unreadable.Store(true) }
+			}
+			done := make(chan *httptest.ResponseRecorder, 1)
+			start := time.Now()
+			go func() { done <- restoreIsolationPOST(t, r, second, oi.VersionID) }()
+			select {
+			case rec := <-done:
+				want := rec.Code == http.StatusConflict
+				if mode == "failed-activity-read" {
+					want = rec.Code >= 500
+				}
+				if !want || time.Since(start) > restoreTakeoverWait+time.Second {
+					t.Errorf("POST during a running restore: %d after %s, want a bounded refusal", rec.Code, time.Since(start))
+				}
+			case <-time.After(restoreTakeoverWait + 2*time.Second):
+				t.Error("POST during a running restore waited for the object lock")
+				unreadable.Store(false)
+				tier.unblock()
+				<-done
+			}
+			unreadable.Store(false)
+			tier.unblock()
+			if err := layer.wait(t); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+// A POST whose status write fails on drives that stay unwritten puts the
+// generation it read back, and a split that a failed put-back leaves
+// differs only in restore fields: the attempt's reset and a later POST both
+// resolve it, while every other field still needs the quorum.
+func TestRestorePOSTStatusWriteSplit(t *testing.T) {
+	for _, mode := range []string{"put-back", "split-reset", "split-post"} {
+		t.Run(mode, func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			initConfigSubsystem(t.Context(), r.z)
+			oi, body := restoreIsolationSeed(t, r, 0, false, false)
+			er := r.z.serverPools[0].getHashedSet(r.object)
+			disks := er.getDisks()
+			var fail atomic.Bool
+			wrapped := append([]StorageAPI(nil), disks...)
+			for i := range 2 {
+				wrapped[i] = restoreGenerationDisk{StorageAPI: disks[i], bucket: r.bucket, object: r.object, write: &fail}
+			}
+			er.getDisks = func() []StorageAPI { return wrapped }
+			before := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+			var attempt *restoreAttempt
+			if mode == "put-back" {
+				fail.Store(true)
+				rec := restoreIsolationPOST(t, r, r.z, oi.VersionID)
+				fail.Store(false)
+				if rec.Code < 400 {
+					t.Fatalf("POST acknowledged a status write on two of four drives: %d", rec.Code)
+				}
+				if after := restoreGenerationMetadata(t, disks, r, oi.VersionID); !reflect.DeepEqual(before, after) {
+					t.Error("failed status write was not put back")
+				}
+			} else {
+				// A status write and its put-back both failed on two drives.
+				attempt = newRestoreAttempt(oi)
+				for _, d := range disks[2:] {
+					fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true})
+					if err != nil {
+						t.Fatal(err)
+					}
+					fi.Metadata[xhttp.AmzRestore] = ongoingRestoreObj().String()
+					fi.Metadata[restoreAttemptKey] = attempt.ID
+					if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+						t.Fatal(err)
+					}
+				}
+				if _, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); err != nil {
+					t.Fatalf("a split in restore fields only has no generation: %v", err)
+				}
+			}
+			if mode == "split-reset" {
+				opts := nativeRestoreOptions(r, oi.VersionID)
+				opts.Transition.attempt = attempt
+				if err := r.z.resetRestoreStatus(r.bucket, r.object, opts); err != nil {
+					t.Fatalf("reset of the split attempt: %v", err)
+				}
+				for i, meta := range restoreGenerationMetadata(t, disks, r, oi.VersionID) {
+					if meta[xhttp.AmzRestore] != "" || meta[restoreAttemptKey] != "" {
+						t.Errorf("drive %d keeps the split attempt: %q %q", i, meta[xhttp.AmzRestore], meta[restoreAttemptKey])
+					}
+				}
+			}
+			layer := newRestoreIsolationLayer(r)
+			if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+				t.Fatalf("POST after the failed status write: %d %s", rec.Code, rec.Body.String())
+			}
+			if err := layer.wait(t); err != nil {
+				t.Fatal(err)
+			}
+			if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("restore contents: %v", err)
+			}
+		})
+	}
+}
+
+// A split in any other field is not resolved: the generation fails closed.
+func TestRestoreGenerationSplitNeedsQuorum(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+	er := r.z.serverPools[0].getHashedSet(r.object)
+	for i, d := range er.getDisks()[2:] {
+		fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true})
+		if err != nil {
+			t.Fatal(err)
+		}
+		fi.Metadata[xhttp.AmzRestore] = ongoingRestoreObj().String()
+		fi.Metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = fmt.Sprint("ON", i)
+		if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if _, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); !errors.Is(err, errErasureReadQuorum) {
+		t.Fatalf("split outside the restore fields: %v, want read quorum", err)
+	}
+	if rec := restoreIsolationPOST(t, r, r.z, oi.VersionID); rec.Code < 400 {
+		t.Errorf("POST over a split outside the restore fields: %d", rec.Code)
+	}
+}
+
+// restoreGenerationCleanupDisk misses restored-copy cleanups and metadata
+// writes of the object while its toggles are set.
+type restoreGenerationCleanupDisk struct {
+	StorageAPI
+	bucket, object      string
+	cleanup, metaWrites *atomic.Bool
+}
+
+func (d restoreGenerationCleanupDisk) DeleteVersion(ctx context.Context, bucket, object string, fi FileInfo, force bool, opts DeleteOptions) error {
+	if bucket == d.bucket && object == d.object && fi.ExpireRestored && d.cleanup.Load() {
+		return errFaultyDisk
+	}
+	return d.StorageAPI.DeleteVersion(ctx, bucket, object, fi, force, opts)
+}
+
+func (d restoreGenerationCleanupDisk) WriteMetadata(ctx context.Context, orig, bucket, object string, fi FileInfo) error {
+	if bucket == d.bucket && object == d.object && d.metaWrites.Load() {
+		return errFaultyDisk
+	}
+	return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi)
+}
+
+// Restored-copy expiry decides on the version's authoritative generation. A
+// drive that missed a cleanup and the POST after it keeps an expired header,
+// which must not clear the quorum's ongoing restore; drives split between
+// an expired and a renewed header authorize nothing.
+func TestExpireRestoredUsesQuorumGeneration(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, mode := range []string{"after-partial-cleanup", "split-renewal"} {
+			t.Run(fmt.Sprintf("pools=%d/%s", pools, mode), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+				initConfigSubsystem(t.Context(), r.z)
+				oi := r.put("cache", false, true)
+				r.restore(oi)
+				stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+				if err != nil {
+					t.Fatal(err)
+				}
+				er := r.z.serverPools[0].getHashedSet(r.object)
+				disks := er.getDisks()
+				expire := func() error {
+					return expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+				}
+				if mode == "split-renewal" {
+					// A renewal reached two drives; the other two keep the
+					// expired header.
+					dataDir := r.fileInfo(oi.VersionID).DataDir
+					for _, d := range disks[2:] {
+						fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true})
+						if err != nil {
+							t.Fatal(err)
+						}
+						fi.Metadata[xhttp.AmzRestore] = completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()
+						if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+							t.Fatal(err)
+						}
+					}
+					if err := expire(); err == nil || r.dataDirs(dataDir) != len(disks) {
+						t.Errorf("expiry on drives split between expired and renewed: %v, %d/%d cache dirs", err, r.dataDirs(dataDir), len(disks))
+					}
+					return
+				}
+				var cleanup, metaWrites atomic.Bool
+				wrapped := append([]StorageAPI(nil), disks...)
+				wrapped[0] = restoreGenerationCleanupDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, cleanup: &cleanup, metaWrites: &metaWrites}
+				er.getDisks = func() []StorageAPI { return wrapped }
+				cleanup.Store(true)
+				err = expire()
+				cleanup.Store(false)
+				if err != nil {
+					t.Fatalf("cleanup on all drives but one: %v", err)
+				}
+				captured := make(chan ObjectOptions, 1)
+				layer := newRestoreIsolationLayer(r)
+				layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error {
+					captured <- opts
+					return nil
+				}
+				metaWrites.Store(true)
+				rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+				metaWrites.Store(false)
+				if rec.Code != http.StatusOK {
+					t.Fatalf("POST after the cleanup: %d %s", rec.Code, rec.Body.String())
+				}
+				opts := <-captured
+				layer.wait(t)
+				// The same cleanup, queued twice, runs again.
+				if err := expire(); err == nil {
+					t.Error("a drive's expired header cleared the quorum's ongoing restore")
+				}
+				if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, opts); err != nil {
+					t.Fatalf("restore of the acknowledged attempt: %v", err)
+				}
+				if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, r.bodies["cache"]) {
+					t.Fatalf("restored contents: %v", err)
+				}
+			})
+		}
+	}
+}
+
+// restoreGenerationHidden hides a drive's copy of the object version while
+// set, as a drive that lost it and that heal writes again.
+type restoreGenerationHidden struct {
+	StorageAPI
+	bucket, object string
+	hidden         *atomic.Bool
+}
+
+func (d restoreGenerationHidden) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
+	if d.hidden.Load() && bucket == d.bucket && object == d.object {
+		return FileInfo{}, errFileNotFound
+	}
+	return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
+}
+
+// Heal copies the restore fields to a drive only as a quorum of the drives
+// holds them. While they disagree, healing the version waits: it must not
+// create a quorum for an expired header over a renewal, nor for a
+// superseded attempt. Fields a quorum agrees on are copied.
+func TestRestoreHealNeedsRestoreQuorum(t *testing.T) {
+	for _, mode := range []string{"expired-over-renewal", "superseded-attempt", "agreed"} {
+		t.Run(mode, func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			initConfigSubsystem(t.Context(), r.z)
+			er := r.z.serverPools[0].getHashedSet(r.object)
+			disks := er.getDisks()
+			setRestore := func(versionID string, drives []StorageAPI, fields map[string]string) {
+				for _, d := range drives {
+					fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, versionID, ReadOptions{ReadData: true})
+					if err != nil {
+						t.Fatal(err)
+					}
+					maps.Copy(fi.Metadata, fields)
+					if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+			var versionID string
+			var stale ObjectInfo
+			var older ObjectOptions
+			hide := 3
+			switch mode {
+			case "expired-over-renewal", "agreed":
+				oi := r.put("cache", false, true)
+				r.restore(oi)
+				versionID = oi.VersionID
+				var err error
+				if stale, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(versionID)); err != nil {
+					t.Fatal(err)
+				}
+				renewed := map[string]string{xhttp.AmzRestore: completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()}
+				if mode == "agreed" {
+					// Three drives renewed, the hidden one expired.
+					setRestore(versionID, disks[:3], renewed)
+				} else {
+					// No quorum: two drives expired an hour ago, one renewed,
+					// and the hidden one expired a day ago.
+					setRestore(versionID, disks[:2], map[string]string{xhttp.AmzRestore: completedRestoreObj(UTCNow().Add(-time.Hour)).String()})
+					setRestore(versionID, disks[2:3], renewed)
+				}
+			case "superseded-attempt":
+				oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+				versionID = oi.VersionID
+				captured := make(chan ObjectOptions, 1)
+				layer := newRestoreIsolationLayer(r)
+				layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error {
+					captured <- opts
+					return nil
+				}
+				if rec := restoreIsolationPOST(t, r, layer, versionID); rec.Code != http.StatusOK {
+					t.Fatal(rec.Body.String())
+				}
+				older = <-captured
+				layer.wait(t)
+				// A newer attempt reached two drives.
+				setRestore(versionID, disks[2:], map[string]string{xhttp.AmzRestore: ongoingRestoreObj().String(), restoreAttemptKey: newRestoreAttempt(oi).ID})
+				hide = 2
+			}
+			before := restoreGenerationMetadata(t, disks, r, versionID)
+			var hidden atomic.Bool
+			wrapped := append([]StorageAPI(nil), disks...)
+			wrapped[hide] = restoreGenerationHidden{StorageAPI: disks[hide], bucket: r.bucket, object: r.object, hidden: &hidden}
+			er.getDisks = func() []StorageAPI { return wrapped }
+			hidden.Store(true)
+			_, healErr := er.HealObject(t.Context(), r.bucket, r.object, versionID, madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+			hidden.Store(false)
+			er.getDisks = func() []StorageAPI { return disks }
+			after := restoreGenerationMetadata(t, disks, r, versionID)
+			if mode == "agreed" {
+				if healErr != nil || after[hide][xhttp.AmzRestore] != after[0][xhttp.AmzRestore] {
+					t.Errorf("heal did not copy the restore fields a quorum holds: %v, %q", healErr, after[hide][xhttp.AmzRestore])
+				}
+				return
+			}
+			if healErr == nil {
+				t.Error("heal proceeded on drives that disagree on the restore fields")
+			}
+			if after[hide][xhttp.AmzRestore] != before[hide][xhttp.AmzRestore] || after[hide][restoreAttemptKey] != before[hide][restoreAttemptKey] {
+				t.Errorf("heal wrote a minority restore state: %q -> %q", before[hide][xhttp.AmzRestore], after[hide][xhttp.AmzRestore])
+			}
+			if mode == "expired-over-renewal" {
+				dataDir := r.fileInfo(versionID).DataDir
+				if err := expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err == nil || r.dataDirs(dataDir) != len(disks) {
+					t.Errorf("expiry after heal: %v, %d/%d cache dirs", err, r.dataDirs(dataDir), len(disks))
+				}
+				return
+			}
+			if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, older); !errors.Is(err, errRestoreSuperseded) {
+				t.Errorf("superseded attempt after heal: %v", err)
+			}
+		})
+	}
+}
+
+// restoreGenerationRand draws UUIDs from crypto/rand for the whole test
+// binary, installed before any goroutine runs. While a pause is set, it holds
+// the UUID that stale cleanup draws in its deadline-bound worker, for the
+// trash path right before its rename, until the pause is released.
+var restoreGenerationRand = &restoreGenerationRandReader{}
+
+func init() { uuid.SetRand(restoreGenerationRand) }
+
+type restoreGenerationRandReader struct {
+	pause atomic.Pointer[restoreGenerationPause]
+}
+
+type restoreGenerationPause struct {
+	entered, release chan struct{}
+}
+
+func (r *restoreGenerationRandReader) Read(b []byte) (int, error) {
+	if p := r.pause.Load(); p != nil && restoreGenerationInCleanupRename() && r.pause.CompareAndSwap(p, nil) {
+		close(p.entered)
+		<-p.release
+	}
+	return rand.Read(b)
+}
+
+// restoreGenerationInCleanupRename reports whether the caller runs in stale
+// cleanup's deadline-bound worker.
+func restoreGenerationInCleanupRename() bool {
+	pcs := make([]uintptr, 32)
+	frames := runtime.CallersFrames(pcs[:runtime.Callers(3, pcs)])
+	cleanup, worker := false, false
+	for {
+		f, more := frames.Next()
+		cleanup = cleanup || strings.Contains(f.Function, "cleanupStaleUploadsOnDisk")
+		worker = worker || strings.Contains(f.Function, "ioutil.WithDeadline")
+		if !more {
+			return cleanup && worker
+		}
+	}
+}

cmd/restore-native-isolation_test.go

@@ -0,0 +1,930 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"reflect"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/mux"
+)
+
+// The private attempt metadata of a native restore, as stored. A literal name
+// keeps these regressions meaningful on a tree without the key.
+const restoreIsolationAttemptKey = "x-minio-internal-restore-attempt"
+
+// restoreIsolationLayer hooks the object layer a POST restore handler uses.
+// afterRead runs once POST has read the version, before POST records its
+// status. restore, when set, replaces the background restore; every
+// background restore result is sent to done.
+type restoreIsolationLayer struct {
+	ObjectLayer
+	afterRead func()
+	once      sync.Once
+	restore   func(ctx context.Context, bucket, object string, opts ObjectOptions) error
+	done      chan error
+}
+
+func newRestoreIsolationLayer(r *restoreTest) *restoreIsolationLayer {
+	return &restoreIsolationLayer{ObjectLayer: r.z, done: make(chan error, 4)}
+}
+
+func (l *restoreIsolationLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	oi, err := l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+	if err == nil && l.afterRead != nil {
+		l.once.Do(l.afterRead)
+	}
+	return oi, err
+}
+
+func (l *restoreIsolationLayer) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+	var err error
+	if l.restore != nil {
+		err = l.restore(ctx, bucket, object, opts)
+	} else {
+		err = l.ObjectLayer.RestoreTransitionedObject(ctx, bucket, object, opts)
+	}
+	l.done <- err
+	return err
+}
+
+// restoreTestWait bounds how long a restore test waits for a step that does
+// real drive work: metadata writes with synchronous persistence and renames
+// on every drive. It is a liveness bound, not a latency contract; a loaded
+// machine has stalled such a step past ten seconds.
+const restoreTestWait = time.Minute
+
+// wait returns the result of the next background restore.
+func (l *restoreIsolationLayer) wait(t *testing.T) error {
+	t.Helper()
+	select {
+	case err := <-l.done:
+		return err
+	case <-time.After(restoreTestWait):
+		buf := make([]byte, 1<<20)
+		t.Fatalf("background restore did not finish in %v; goroutines:\n%s", restoreTestWait, buf[:runtime.Stack(buf, true)])
+		return nil
+	}
+}
+
+func restoreIsolationPOST(t *testing.T, r *restoreTest, layer ObjectLayer, versionID string) *httptest.ResponseRecorder {
+	t.Helper()
+	body := []byte(`<RestoreRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Days>1</Days></RestoreRequest>`)
+	path := "/" + r.bucket + "/" + r.object + "?restore"
+	if versionID != "" {
+		path += "&versionId=" + url.QueryEscape(versionID)
+	}
+	req, err := newTestSignedRequestV4(http.MethodPost, path, int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := req.ParseForm(); err != nil {
+		t.Fatal(err)
+	}
+	req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object})
+	rec := httptest.NewRecorder()
+	api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return layer }}
+	api.PostRestoreObjectHandler(rec, req)
+	return rec
+}
+
+// restoreIsolationSeed is a transitioned version without the ongoing marker
+// that nativeRestoreSeed leaves for direct restores: POST sets its own.
+func restoreIsolationSeed(t *testing.T, r *restoreTest, pool int, null, multipart bool) (ObjectInfo, []byte) {
+	t.Helper()
+	oi, body := nativeRestoreSeed(t, r, pool, null, multipart)
+	opts := r.opts(oi.VersionID)
+	opts.MTime = oi.ModTime
+	opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+		o.UserDefined[xhttp.AmzRestore] = ""
+		return ReplicateDecision{}, nil
+	}
+	if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil {
+		t.Fatal(err)
+	}
+	return oi, body
+}
+
+// restoreIsolationSettled waits until the version is no longer ongoing.
+func restoreIsolationSettled(t *testing.T, r *restoreTest, versionID string) ObjectInfo {
+	t.Helper()
+	deadline := time.Now().Add(10 * time.Second)
+	for {
+		oi, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(versionID))
+		if err == nil && !oi.RestoreOngoing {
+			return oi
+		}
+		if time.Now().After(deadline) {
+			t.Fatalf("version stays ongoing, refusing every later POST: %q, %v", oi.UserDefined[xhttp.AmzRestore], err)
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+}
+
+// restoreIsolationUploads returns the upload directories of the object on
+// each drive of a pool, as stored, whatever listings show.
+func restoreIsolationUploads(t *testing.T, r *restoreTest, pool int) map[string]int {
+	t.Helper()
+	er := r.z.serverPools[pool].getHashedSet(r.object)
+	found := make(map[string]int)
+	for _, disk := range er.getDisks() {
+		entries, err := disk.ListDir(t.Context(), r.bucket, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), -1)
+		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		for _, entry := range entries {
+			found[strings.TrimSuffix(entry, SlashSeparator)]++
+		}
+	}
+	return found
+}
+
+// restoreIsolationGateTier holds a download after its first bytes and can
+// fail it once released.
+type restoreIsolationGateTier struct {
+	WarmBackend
+	first   int
+	entered chan struct{}
+	release chan struct{}
+	failure error
+	enter   sync.Once
+	open    sync.Once
+}
+
+func newRestoreIsolationGateTier(first int, failure error) *restoreIsolationGateTier {
+	return &restoreIsolationGateTier{
+		WarmBackend: globalTierConfigMgr.drivercache["WARM"],
+		first:       first, failure: failure,
+		entered: make(chan struct{}), release: make(chan struct{}),
+	}
+}
+
+func (b *restoreIsolationGateTier) unblock() { b.open.Do(func() { close(b.release) }) }
+
+func (b *restoreIsolationGateTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+	rd, err := b.WarmBackend.Get(ctx, object, version, opts)
+	if err != nil {
+		return nil, err
+	}
+	return &restoreIsolationGateReader{ReadCloser: rd, tier: b, ctx: ctx, remaining: b.first}, nil
+}
+
+type restoreIsolationGateReader struct {
+	io.ReadCloser
+	tier      *restoreIsolationGateTier
+	ctx       context.Context
+	remaining int
+}
+
+func (r *restoreIsolationGateReader) Read(p []byte) (int, error) {
+	if r.remaining > 0 {
+		if len(p) > r.remaining {
+			p = p[:r.remaining]
+		}
+		n, err := r.ReadCloser.Read(p)
+		r.remaining -= n
+		return n, err
+	}
+	r.tier.enter.Do(func() { close(r.tier.entered) })
+	select {
+	case <-r.tier.release:
+	case <-r.ctx.Done():
+		return 0, r.ctx.Err()
+	}
+	if r.tier.failure != nil {
+		return 0, r.tier.failure
+	}
+	return r.ReadCloser.Read(p)
+}
+
+func restoreIsolationEntered(t *testing.T, tier *restoreIsolationGateTier) {
+	t.Helper()
+	select {
+	case <-tier.entered:
+	case <-time.After(restoreTestWait):
+		t.Fatal("restore never reached its download gate")
+	}
+}
+
+// restoreIsolationFaultDisk fails metadata reads of the object while
+// failReads is set, and reports writes of its metadata as failed, after
+// they were made, while failWrites is set. Toggles keep the wrapper in
+// place, so no restore retry races a swap of the drives.
+type restoreIsolationFaultDisk struct {
+	StorageAPI
+	bucket, object string
+	failReads      *atomic.Bool
+	failWrites     *atomic.Bool
+}
+
+func (d restoreIsolationFaultDisk) ReadVersion(ctx context.Context, origvolume, volume, path, version string, opts ReadOptions) (FileInfo, error) {
+	if d.failReads.Load() && volume == d.bucket && path == d.object {
+		return FileInfo{}, errDiskNotFound
+	}
+	return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, version, opts)
+}
+
+func (d restoreIsolationFaultDisk) ReadXL(ctx context.Context, volume, path string, readData bool) (RawFileInfo, error) {
+	if d.failReads.Load() && volume == d.bucket && path == d.object {
+		return RawFileInfo{}, errDiskNotFound
+	}
+	return d.StorageAPI.ReadXL(ctx, volume, path, readData)
+}
+
+func (d restoreIsolationFaultDisk) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error {
+	err := d.StorageAPI.WriteMetadata(ctx, origvolume, volume, path, fi)
+	if err == nil && d.failWrites != nil && d.failWrites.Load() && volume == d.bucket && path == d.object {
+		return errFaultyDisk
+	}
+	return err
+}
+
+// installRestoreIsolationFaults wraps the drives of the object's set in a
+// pool for the rest of the test; failing marks the drives whose writes fail.
+func installRestoreIsolationFaults(r *restoreTest, pool int, failing int) (reads, writes *atomic.Bool) {
+	reads, writes = new(atomic.Bool), new(atomic.Bool)
+	er := r.z.serverPools[pool].getHashedSet(r.object)
+	disks := er.getDisks()
+	wrapped := make([]StorageAPI, len(disks))
+	for i, disk := range disks {
+		d := restoreIsolationFaultDisk{StorageAPI: disk, bucket: r.bucket, object: r.object, failReads: reads}
+		if i < failing {
+			d.failWrites = writes
+		}
+		wrapped[i] = d
+	}
+	er.getDisks = func() []StorageAPI { return wrapped }
+	return reads, writes
+}
+
+// POST records its status on a fresh read under the object lock, so an
+// Object Lock update it did not see survives.
+func TestRestorePOSTKeepsConcurrentProtection(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, kind := range []string{"legal-hold", "retention"} {
+			t.Run(fmt.Sprintf("pools=%d/%s", pools, kind), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+				initConfigSubsystem(t.Context(), r.z)
+				enableBucketObjectLock(t, r.bucket)
+				oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+				if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+					t.Fatal(err)
+				}
+				initial := []byte(`<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>OFF</Status></LegalHold>`)
+				updated := []byte(`<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>ON</Status></LegalHold>`)
+				key := strings.ToLower(xhttp.AmzObjectLockLegalHold)
+				if kind == "retention" {
+					initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour))
+					updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))
+					key = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
+				}
+				if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK {
+					t.Fatalf("initial protection: %d %s", rec.Code, rec.Body.String())
+				}
+				var accepted ObjectInfo
+				layer := newRestoreIsolationLayer(r)
+				layer.afterRead = func() {
+					if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK {
+						t.Fatalf("update: %d %s", rec.Code, rec.Body.String())
+					}
+					var err error
+					if accepted, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err != nil {
+						t.Fatal(err)
+					}
+				}
+				if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code >= 400 {
+					t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+				}
+				after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+				if err != nil {
+					t.Fatal(err)
+				}
+				if accepted.UserDefined[key] == "" || after.UserDefined[key] != accepted.UserDefined[key] {
+					t.Errorf("POST restore overwrote acknowledged Object Lock metadata: %q -> %q", accepted.UserDefined[key], after.UserDefined[key])
+				}
+				if !after.ModTime.Equal(oi.ModTime) || after.ETag != oi.ETag {
+					t.Error("POST restore changed the version identity")
+				}
+			})
+		}
+	}
+}
+
+// A restored copy removed between POST's read and its status write leads to
+// a new restore, never to a completed header without the restored data.
+func TestRestorePOSTRacesCacheExpiry(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+			initConfigSubsystem(t.Context(), r.z)
+			oi := r.put("restored", false, true)
+			r.restore(oi)
+			layer := newRestoreIsolationLayer(r)
+			layer.afterRead = func() {
+				if err := expireTransitionedObject(t.Context(), r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err != nil {
+					t.Fatal(err)
+				}
+			}
+			if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code >= 400 {
+				t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+			}
+			select {
+			case err := <-layer.done:
+				if err != nil {
+					t.Fatalf("rehydration: %v", err)
+				}
+			case <-time.After(5 * time.Second):
+				t.Error("POST renewed a removed restored copy instead of restoring it")
+			}
+			_, body, err := r.read(oi.VersionID)
+			if err != nil || !bytes.Equal(body, r.bodies["restored"]) {
+				t.Errorf("version unreadable after POST raced cache expiry: %v", err)
+			}
+		})
+	}
+}
+
+// A restore queued with the versioning POST captured writes back the null
+// version it resolved, even after versioning was enabled in between.
+func TestRestoreKeepsCapturedNullIdentity(t *testing.T) {
+	enable := func(t *testing.T, r *restoreTest) {
+		meta, err := globalBucketMetadataSys.Get(r.bucket)
+		if err != nil {
+			t.Fatal(err)
+		}
+		meta.VersioningConfigXML = []byte(`<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>`)
+		if err := meta.Save(t.Context(), r.z); err != nil {
+			t.Fatal(err)
+		}
+		globalBucketMetadataSys.Set(r.bucket, meta)
+	}
+	for _, mode := range []string{"direct", "post"} {
+		t.Run(mode, func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, "")
+			initConfigSubsystem(t.Context(), r.z)
+			var oi ObjectInfo
+			var body []byte
+			if mode == "direct" {
+				oi, body = nativeRestoreSeed(t, r, 0, true, false)
+				// These are the options that POST captured while versioning was off.
+				queued := nativeRestoreOptions(r, "")
+				enable(t, r)
+				if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, queued); err != nil {
+					t.Fatal(err)
+				}
+			} else {
+				oi, body = restoreIsolationSeed(t, r, 0, true, false)
+				release := make(chan struct{})
+				layer := newRestoreIsolationLayer(r)
+				layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+					<-release
+					return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+				}
+				if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code != http.StatusOK {
+					t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+				}
+				enable(t, r)
+				close(release)
+				if err := layer.wait(t); err != nil {
+					t.Fatal(err)
+				}
+			}
+			versions := nativeRestoreSnapshot(t, r)[0]
+			if len(versions) != 1 || versions[0].VersionID != "" {
+				t.Errorf("restore created a UUID alias of the queued null version: %d versions", len(versions))
+			}
+			_, got, err := r.read(oi.VersionID)
+			if err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("null original: %v", err)
+			}
+		})
+	}
+}
+
+// A restore that fails before it reads the version, on its lock wait or on
+// its first read, must not leave POST's ongoing marker for every later POST
+// to refuse: bounded retries clear it once the version is readable again.
+func TestRestoreRecoversStrandedMarker(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, stage := range []string{"lock-wait", "initial-read"} {
+			t.Run(fmt.Sprintf("pools=%d/%s", pools, stage), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+				oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+				var err error
+				if stage == "lock-wait" {
+					lk := r.z.NewNSLock(r.bucket, r.object)
+					if pools == 1 {
+						lk = r.z.serverPools[0].getHashedSet(r.object).NewNSLock(r.bucket, r.object)
+					}
+					lkctx, lerr := lk.GetLock(t.Context(), globalOperationTimeout)
+					if lerr != nil {
+						t.Fatal(lerr)
+					}
+					ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+					err = r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+					cancel()
+					lk.Unlock(lkctx)
+				} else {
+					reads, _ := installRestoreIsolationFaults(r, 0, 0)
+					reads.Store(true)
+					err = r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+					reads.Store(false)
+				}
+				if err == nil {
+					t.Fatal("restore succeeded without its lock or its version")
+				}
+				after := restoreIsolationSettled(t, r, oi.VersionID)
+				if !after.IsRemote() || !after.ModTime.Equal(oi.ModTime) {
+					t.Error("recovery changed more than the restore status")
+				}
+			})
+		}
+	}
+}
+
+// A later POST takes over a marker that no running restore holds, and the
+// superseded attempt then neither downloads nor clears the newer status,
+// also when both POSTs share a second, the precision of the request date.
+func TestRestoreSupersededAttempt(t *testing.T) {
+	for _, mode := range []string{"newer-completes", "newer-fails"} {
+		t.Run(mode, func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			initConfigSubsystem(t.Context(), r.z)
+			oi, body := restoreIsolationSeed(t, r, 0, false, false)
+			tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+			release := make(chan struct{})
+			var releaseOnce sync.Once
+			unblock := func() { releaseOnce.Do(func() { close(release) }) }
+			t.Cleanup(unblock)
+			older := newRestoreIsolationLayer(r)
+			older.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+				// The queued worker loses the restore lock POST handed it,
+				// as when its refresh found it gone: the marker then belongs
+				// to no running restore.
+				opts.Transition.owner.release()
+				<-release
+				return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+			}
+			newer := newRestoreIsolationLayer(r)
+			time.Sleep(time.Until(time.Now().Truncate(time.Second).Add(time.Second + 10*time.Millisecond)))
+			if rec := restoreIsolationPOST(t, r, older, oi.VersionID); rec.Code != http.StatusOK {
+				t.Fatalf("first POST: %d %s", rec.Code, rec.Body.String())
+			}
+			first := r.fileInfo(oi.VersionID).Metadata[xhttp.AmzRestoreRequestDate]
+			if mode == "newer-fails" {
+				globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, errors.New("injected remote failure")}
+			}
+			if rec := restoreIsolationPOST(t, r, newer, oi.VersionID); rec.Code != http.StatusOK {
+				t.Fatalf("POST over a marker no restore holds: %d %s", rec.Code, rec.Body.String())
+			}
+			second := r.fileInfo(oi.VersionID).Metadata[xhttp.AmzRestoreRequestDate]
+			t.Logf("request dates %q %q, same second: %v", first, second, first == second)
+			newerErr := newer.wait(t)
+			globalTierConfigMgr.drivercache["WARM"] = tier
+			before := r.fileInfo(oi.VersionID)
+			gets := tier.gets.Load()
+			unblock()
+			if err := older.wait(t); err == nil {
+				t.Error("superseded restore reported success")
+			}
+			if tier.gets.Load() != gets {
+				t.Error("superseded restore downloaded the version again")
+			}
+			after := r.fileInfo(oi.VersionID)
+			if !reflect.DeepEqual(before.Metadata, after.Metadata) || before.DataDir != after.DataDir {
+				t.Errorf("superseded restore changed the version: %q -> %q", before.Metadata[xhttp.AmzRestore], after.Metadata[xhttp.AmzRestore])
+			}
+			info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+			if err != nil || info.RestoreOngoing {
+				t.Fatalf("version left ongoing: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+			}
+			if mode == "newer-completes" {
+				if newerErr != nil || info.IsRemote() {
+					t.Fatalf("newer restore: %v remote=%v", newerErr, info.IsRemote())
+				}
+				if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) || tier.gets.Load() != gets {
+					t.Errorf("restored contents: %v", err)
+				}
+				return
+			}
+			if newerErr == nil || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" {
+				t.Fatalf("failed newer restore is not retryable: %v %q", newerErr, info.UserDefined[xhttp.AmzRestore])
+			}
+			if rec := restoreIsolationPOST(t, r, newer, oi.VersionID); rec.Code != http.StatusOK {
+				t.Fatalf("retry POST: %d %s", rec.Code, rec.Body.String())
+			}
+			if err := newer.wait(t); err != nil {
+				t.Fatal(err)
+			}
+			if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+				t.Errorf("retried restore contents: %v", err)
+			}
+		})
+	}
+}
+
+// A restart loses the restore that POST started; a POST to the restarted
+// server takes the stranded marker over and restores the version.
+func TestRestoreRestartTakeover(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	initConfigSubsystem(t.Context(), r.z)
+	oi, body := restoreIsolationSeed(t, r, 0, false, true)
+	lost := newRestoreIsolationLayer(r)
+	lost.restore = func(context.Context, string, string, ObjectOptions) error { return nil }
+	if rec := restoreIsolationPOST(t, r, lost, oi.VersionID); rec.Code != http.StatusOK {
+		t.Fatalf("POST before restart: %d %s", rec.Code, rec.Body.String())
+	}
+	lost.wait(t)
+	if info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err != nil || !info.RestoreOngoing {
+		t.Fatalf("fixture: %q %v", info.UserDefined[xhttp.AmzRestore], err)
+	}
+
+	endpoints := make(EndpointServerPools, len(r.z.serverPools))
+	for i, pool := range r.z.serverPools {
+		endpoints[i] = pool.endpoints
+	}
+	obj, err := newErasureServerPools(t.Context(), endpoints)
+	if err != nil {
+		t.Fatal(err)
+	}
+	restarted := obj.(*erasureServerPools)
+	t.Cleanup(func() { restarted.Shutdown(context.Background()) })
+	r.z = restarted
+	layer := newRestoreIsolationLayer(r)
+	if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+		t.Fatalf("POST after restart: %d %s", rec.Code, rec.Body.String())
+	}
+	if err := layer.wait(t); err != nil {
+		t.Fatal(err)
+	}
+	info := restoreIsolationSettled(t, r, oi.VersionID)
+	if info.IsRemote() || info.UserDefined[restoreIsolationAttemptKey] != "" {
+		t.Errorf("restore after restart incomplete: remote=%v attempt=%q", info.IsRemote(), info.UserDefined[restoreIsolationAttemptKey])
+	}
+	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+		t.Errorf("contents after restart: %v", err)
+	}
+}
+
+// A null version replaced while its restore was queued is another version:
+// the restore must not rehydrate it, even if it is transitioned as well.
+func TestRestoreReplacementNullObject(t *testing.T) {
+	for _, state := range []versioning.State{"", versioning.Suspended} {
+		t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, state)
+			initConfigSubsystem(t.Context(), r.z)
+			oi, _ := restoreIsolationSeed(t, r, 0, true, false)
+			release := make(chan struct{})
+			layer := newRestoreIsolationLayer(r)
+			layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+				<-release
+				return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+			}
+			if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code != http.StatusOK {
+				close(release)
+				t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+			}
+			replacement := []byte("replacement null object")
+			repl, err := r.z.serverPools[0].PutObject(t.Context(), r.bucket, r.object,
+				mustGetPutObjReader(t, bytes.NewReader(replacement), int64(len(replacement)), "", ""),
+				ObjectOptions{VersionSuspended: state == versioning.Suspended})
+			if err != nil {
+				close(release)
+				t.Fatal(err)
+			}
+			topts := r.opts(nullVersionID)
+			topts.MTime = repl.ModTime
+			topts.Transition = TransitionOptions{Tier: "WARM", ETag: repl.ETag}
+			if err := r.z.serverPools[0].TransitionObject(t.Context(), r.bucket, r.object, topts); err != nil {
+				close(release)
+				t.Fatal(err)
+			}
+			tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+			before := r.fileInfo(nullVersionID)
+			gets := tier.gets.Load()
+			close(release)
+			layer.wait(t)
+			after := r.fileInfo(nullVersionID)
+			if tier.gets.Load() != gets || after.Metadata[xhttp.AmzRestore] != "" || !reflect.DeepEqual(before.Metadata, after.Metadata) {
+				t.Errorf("queued restore rehydrated the replacement null object: %q", after.Metadata[xhttp.AmzRestore])
+			}
+			if after.ModTime.Equal(oi.ModTime) || !after.ModTime.Equal(repl.ModTime) {
+				t.Fatal("fixture did not replace the null version")
+			}
+			if _, got, err := r.read(nullVersionID); err != nil || !bytes.Equal(got, replacement) {
+				t.Errorf("replacement contents: %v", err)
+			}
+		})
+	}
+}
+
+// POST's own status write can fail after its ongoing marker reached the
+// drives; the marker must not stay for every later POST to refuse.
+func TestRestorePOSTStatusWritePartialFailure(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	initConfigSubsystem(t.Context(), r.z)
+	oi, body := restoreIsolationSeed(t, r, 0, false, false)
+	_, writes := installRestoreIsolationFaults(r, 0, 2)
+	layer := newRestoreIsolationLayer(r)
+	writes.Store(true)
+	rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+	writes.Store(false)
+	if rec.Code < 400 {
+		t.Fatalf("POST acknowledged a failed status write: %d %s", rec.Code, rec.Body.String())
+	}
+	if info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err == nil && !info.RestoreOngoing {
+		t.Log("the failed status write left no marker in quorum")
+	}
+	restoreIsolationSettled(t, r, oi.VersionID)
+	if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+		t.Fatalf("POST after a failed status write: %d %s", rec.Code, rec.Body.String())
+	}
+	if err := layer.wait(t); err != nil {
+		t.Fatal(err)
+	}
+	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+		t.Errorf("restore after a failed status write: %v", err)
+	}
+}
+
+// Duplicate copies of a version in two pools can differ in whether they
+// hold a restored copy. A completed restore header must never reach a copy
+// without one; POST renews or restores only the copy that restores serve.
+func TestRestoreDuplicatePoolCacheStates(t *testing.T) {
+	for _, mode := range []string{"suspended-owner", "unrestored-first", "restored-first"} {
+		t.Run(mode, func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+			initConfigSubsystem(t.Context(), r.z)
+			oi, body := restoreIsolationSeed(t, r, 0, false, false)
+			// The duplicate is the same stored version, metadata only.
+			src := r.z.serverPools[0].getHashedSet(r.object).getDisks()
+			dst := r.z.serverPools[1].getHashedSet(r.object).getDisks()
+			for i := range src {
+				raw, err := src[i].ReadAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile))
+				if err != nil {
+					t.Fatal(err)
+				}
+				if err := dst[i].WriteAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile), raw); err != nil {
+					t.Fatal(err)
+				}
+			}
+			owner := 0
+			if mode == "unrestored-first" {
+				owner = 1
+			}
+			if err := r.z.serverPools[owner].RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+				t.Fatal(err)
+			}
+			if mode == "suspended-owner" {
+				r.z.poolMetaMutex.Lock()
+				r.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+				r.z.poolMetaMutex.Unlock()
+				t.Cleanup(func() {
+					r.z.poolMetaMutex.Lock()
+					r.z.poolMeta.Pools[0].Decommission = nil
+					r.z.poolMetaMutex.Unlock()
+				})
+			}
+			before := nativeRestoreSnapshot(t, r)
+			layer := newRestoreIsolationLayer(r)
+			if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+				t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+			}
+			if mode != "restored-first" {
+				select {
+				case err := <-layer.done:
+					if err != nil {
+						t.Fatal(err)
+					}
+				case <-time.After(5 * time.Second):
+					t.Error("POST did not restore the copy without restored data")
+				}
+			}
+			after := nativeRestoreSnapshot(t, r)
+			for pool := range r.z.serverPools {
+				gr, err := r.z.serverPools[pool].GetObjectNInfo(t.Context(), r.bucket, r.object, nil, nil, ObjectOptions{VersionID: oi.VersionID})
+				if err != nil {
+					t.Errorf("pool %d copy unreadable, header %q: %v", pool, after[pool][0].Metadata[xhttp.AmzRestore], err)
+					continue
+				}
+				got, err := io.ReadAll(gr)
+				gr.Close()
+				if err != nil || !bytes.Equal(got, body) {
+					t.Errorf("pool %d copy contents: %v", pool, err)
+				}
+			}
+			// The copy that is neither renewed nor restored stays as it was.
+			untouched := 1
+			if mode == "suspended-owner" {
+				untouched = 0
+			}
+			if !reflect.DeepEqual(before[untouched], after[untouched]) {
+				t.Errorf("POST changed the pool %d copy: %q -> %q", untouched, before[untouched][0].Metadata[xhttp.AmzRestore], after[untouched][0].Metadata[xhttp.AmzRestore])
+			}
+		})
+	}
+}
+
+// A restored copy expires only by its own reason, on a fresh read: its
+// restore deadline passed and nothing else. A renewed deadline survives
+// another rule falling due and an expiry checked across all pools.
+func TestExpireRestoredNeedsOwnReason(t *testing.T) {
+	for _, mode := range []string{"other-expiry-due", "checked-all-pools", "ongoing", "renewed", "expired"} {
+		t.Run(mode, func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			oi := r.put("restored", false, true)
+			r.restore(oi)
+			before := r.fileInfo(oi.VersionID)
+			if mode == "other-expiry-due" {
+				meta, err := globalBucketMetadataSys.Get(r.bucket)
+				if err != nil {
+					t.Fatal(err)
+				}
+				meta.LifecycleConfigXML = []byte(`<LifecycleConfiguration><Rule><ID>delete</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter><Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration></Rule></LifecycleConfiguration>`)
+				if err := meta.Save(t.Context(), r.z); err != nil {
+					t.Fatal(err)
+				}
+				globalBucketMetadataSys.Set(r.bucket, meta)
+			}
+			header := completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()
+			switch mode {
+			case "ongoing":
+				header = ongoingRestoreObj().String()
+			case "expired":
+				header = ""
+			}
+			if header != "" {
+				opts := r.opts(oi.VersionID)
+				opts.MTime = oi.ModTime
+				opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+					o.UserDefined[xhttp.AmzRestore] = header
+					return ReplicateDecision{}, nil
+				}
+				if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil {
+					t.Fatal(err)
+				}
+			}
+			var err error
+			if mode == "checked-all-pools" {
+				opts := r.opts(oi.VersionID)
+				opts.Expiration = ExpirationOptions{Expire: true, CheckedAllPools: true}
+				opts.Transition.ExpireRestored = true
+				_, err = r.z.serverPools[0].getHashedSet(r.object).DeleteObject(t.Context(), r.bucket, r.object, opts)
+			} else {
+				err = expireTransitionedObject(t.Context(), r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+			}
+			after := r.fileInfo(oi.VersionID)
+			if mode == "expired" {
+				// Control: an elapsed deadline is the reason itself.
+				if err != nil || after.Metadata[xhttp.AmzRestore] != "" || r.dataDirs(before.DataDir) != 0 {
+					t.Errorf("expired restored copy not removed: %v", err)
+				}
+				return
+			}
+			t.Logf("cleanup=%v header=%q", err, after.Metadata[xhttp.AmzRestore])
+			if err == nil || after.Metadata[xhttp.AmzRestore] != header || r.dataDirs(before.DataDir) != len(r.z.serverPools[0].getHashedSet(r.object).getDisks()) {
+				t.Error("cache cleanup removed a restored copy whose own deadline has not passed")
+			}
+		})
+	}
+}
+
+// Controls: an ordinary POST restore, a renewal, and a client's multipart
+// upload of the same key while a restore runs.
+func TestRestorePOSTControls(t *testing.T) {
+	t.Run("restore", func(t *testing.T) {
+		r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+		initConfigSubsystem(t.Context(), r.z)
+		oi, body := restoreIsolationSeed(t, r, 0, false, true)
+		layer := newRestoreIsolationLayer(r)
+		if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+			t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+		}
+		if err := layer.wait(t); err != nil {
+			t.Fatal(err)
+		}
+		info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+		if err != nil || info.IsRemote() || info.RestoreOngoing || info.UserDefined[restoreIsolationAttemptKey] != "" {
+			t.Fatalf("restored status: %q %q, %v", info.UserDefined[xhttp.AmzRestore], info.UserDefined[restoreIsolationAttemptKey], err)
+		}
+		if !info.ModTime.Equal(oi.ModTime) || info.ETag != oi.ETag || info.UserDefined["x-amz-meta-preserve"] != "keep-me" {
+			t.Error("restore changed the version identity or metadata")
+		}
+		if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+			t.Errorf("restored contents: %v", err)
+		}
+		if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+			t.Errorf("completed restore left its upload: %v", uploads)
+		}
+	})
+	t.Run("renewal", func(t *testing.T) {
+		r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+		initConfigSubsystem(t.Context(), r.z)
+		oi, body := nativeRestoreSeed(t, r, 0, false, false)
+		if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+			t.Fatal(err)
+		}
+		tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+		before := r.fileInfo(oi.VersionID)
+		gets := tier.gets.Load()
+		layer := newRestoreIsolationLayer(r)
+		if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+			t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+		}
+		select {
+		case err := <-layer.done:
+			t.Fatalf("renewal restored again: %v", err)
+		case <-time.After(200 * time.Millisecond):
+		}
+		after := r.fileInfo(oi.VersionID)
+		if after.DataDir != before.DataDir || !after.ModTime.Equal(before.ModTime) || after.Metadata[xhttp.AmzRestore] == "" {
+			t.Error("renewal changed the restored copy")
+		}
+		if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) || tier.gets.Load() != gets {
+			t.Errorf("renewed copy not served locally: %v", err)
+		}
+	})
+	t.Run("public-multipart-same-key", func(t *testing.T) {
+		r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+		initConfigSubsystem(t.Context(), r.z)
+		oi, body := restoreIsolationSeed(t, r, 0, false, true)
+		tier := newRestoreIsolationGateTier(5<<20, nil)
+		globalTierConfigMgr.drivercache["WARM"] = tier
+		defer tier.unblock()
+		layer := newRestoreIsolationLayer(r)
+		if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+			t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+		}
+		restoreIsolationEntered(t, tier)
+		upload, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true})
+		if err != nil {
+			t.Fatal(err)
+		}
+		uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 100)
+		listed := false
+		for _, u := range uploads.Uploads {
+			listed = listed || u.UploadID == upload.UploadID
+		}
+		if err != nil || !listed {
+			t.Fatalf("client upload not listed: %+v, %v", uploads.Uploads, err)
+		}
+		tier.unblock()
+		if err := layer.wait(t); err != nil {
+			t.Fatal(err)
+		}
+		// Single-pool part writes wait for the object lock of a running restore.
+		data := []byte("client data")
+		part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		parts, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, upload.UploadID, 0, 10, ObjectOptions{})
+		if err != nil || len(parts.Parts) != 1 {
+			t.Fatalf("client parts: %+v, %v", parts, err)
+		}
+		newer, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, got, err := r.read(newer.VersionID); err != nil || !bytes.Equal(got, data) {
+			t.Errorf("client version: %v", err)
+		}
+		if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+			t.Errorf("restored version: %v", err)
+		}
+	})
+}

cmd/restore-native_test.go

@@ -0,0 +1,935 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/event"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/pubsub"
+	"github.com/minio/mux"
+)
+
+func nativeRestoreSeed(t *testing.T, r *restoreTest, pool int, null, multipart bool) (ObjectInfo, []byte) {
+	t.Helper()
+	ctx := t.Context()
+	opts := ObjectOptions{Versioned: !null, MTime: UTCNow().Add(-2 * time.Hour), UserDefined: map[string]string{
+		"content-type": "application/x-restore-test", "x-amz-meta-preserve": "keep-me",
+	}}
+	body := bytes.Repeat([]byte("restore-payload"), 100000)
+	var oi ObjectInfo
+	var err error
+	if multipart {
+		parts := [][]byte{bytes.Repeat([]byte("a"), 5<<20), []byte("last-part")}
+		body = bytes.Join(parts, nil)
+		upload, err := r.z.serverPools[pool].NewMultipartUpload(ctx, r.bucket, r.object, opts)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var complete []CompletePart
+		for i, data := range parts {
+			p, err := r.z.serverPools[pool].PutObjectPart(ctx, r.bucket, r.object, upload.UploadID, i+1,
+				mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			complete = append(complete, CompletePart{PartNumber: i + 1, ETag: p.ETag})
+		}
+		oi, err = r.z.serverPools[pool].CompleteMultipartUpload(ctx, r.bucket, r.object, upload.UploadID, complete, opts)
+		if err == nil && len(oi.Parts) != 2 {
+			t.Fatalf("fixture has %d parts, want 2", len(oi.Parts))
+		}
+	} else {
+		oi, err = r.z.serverPools[pool].PutObject(ctx, r.bucket, r.object,
+			mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), opts)
+	}
+	if err != nil {
+		t.Fatal(err)
+	}
+	oi.VersionID = restoreTestVersionKey(oi.VersionID)
+	topts := r.opts(oi.VersionID)
+	topts.MTime = oi.ModTime
+	topts.Transition = TransitionOptions{Tier: "WARM", ETag: oi.ETag}
+	if err := r.z.serverPools[pool].TransitionObject(ctx, r.bucket, r.object, topts); err != nil {
+		t.Fatal(err)
+	}
+	// Use the production metadata writer to mark the request ongoing.
+	topts.Transition = TransitionOptions{}
+	topts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+		o.UserDefined[xhttp.AmzRestore] = ongoingRestoreObj().String()
+		return ReplicateDecision{}, nil
+	}
+	if _, err := r.z.PutObjectMetadata(ctx, r.bucket, r.object, topts); err != nil {
+		t.Fatal(err)
+	}
+	return oi, body
+}
+
+func nativeRestoreSnapshot(t *testing.T, r *restoreTest) [][]FileInfo {
+	t.Helper()
+	result := make([][]FileInfo, len(r.z.serverPools))
+	for p, pool := range r.z.serverPools {
+		raw, err := pool.getHashedSet(r.object).getDisks()[0].ReadXL(t.Context(), r.bucket, r.object, false)
+		if errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		infos, err := getFileInfoVersions(raw.Buf, r.bucket, r.object, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		result[p] = append(infos.Versions, infos.FreeVersions...)
+	}
+	return result
+}
+
+func nativeRestoreOptions(r *restoreTest, id string) ObjectOptions {
+	opts := r.opts(id)
+	opts.Transition = TransitionOptions{RestoreRequest: &RestoreObjectRequest{Days: 1}}
+	return opts
+}
+
+type nativeRestoreFailTier struct {
+	WarmBackend
+	failure error
+}
+
+func (b nativeRestoreFailTier) Get(context.Context, string, remoteVersionID, WarmBackendGetOpts) (io.ReadCloser, error) {
+	return nil, b.failure
+}
+
+func TestNativeRestorePools(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, multipart := range []bool{false, true} {
+			for _, fail := range []bool{false, true} {
+				t.Run(fmt.Sprintf("pools=%d/multipart=%v/failure=%v", pools, multipart, fail), func(t *testing.T) {
+					r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+					oi, body := nativeRestoreSeed(t, r, 0, false, multipart)
+					before := nativeRestoreSnapshot(t, r)
+					tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+					failure := errors.New("injected remote download failure")
+					if fail {
+						globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, failure}
+					}
+					ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+					defer cancel()
+					err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+					if fail {
+						if !errors.Is(err, failure) {
+							t.Fatalf("restore error=%v, want remote failure", err)
+						}
+						got, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+						if err != nil || got.UserDefined[xhttp.AmzRestore] != "" {
+							t.Fatalf("failed restore did not clear ongoing status: %v, %q", err, got.UserDefined[xhttp.AmzRestore])
+						}
+					} else {
+						if err != nil {
+							t.Fatal(err)
+						}
+						reads := tier.gets.Load()
+						_, got, err := r.read(oi.VersionID)
+						if err != nil || !bytes.Equal(got, body) {
+							t.Fatalf("restored contents differ: %v", err)
+						}
+						if tier.gets.Load() != reads {
+							t.Error("restored read still fetched the remote tier")
+						}
+					}
+					after := nativeRestoreSnapshot(t, r)
+					if len(after[0]) != len(before[0]) || after[0][0].VersionID != before[0][0].VersionID || after[0][0].TransitionedObjName != before[0][0].TransitionedObjName {
+						t.Fatal("restore changed the version stack or remote object identity")
+					}
+					if after[0][0].Metadata["content-type"] != "application/x-restore-test" || after[0][0].Metadata["x-amz-meta-preserve"] != "keep-me" {
+						t.Error("restore lost original metadata")
+					}
+				})
+			}
+		}
+	}
+}
+
+func TestNativeRestoreVersionRouting(t *testing.T) {
+	for _, pool := range []int{0, 1} {
+		for _, null := range []bool{false, true} {
+			t.Run(fmt.Sprintf("target-pool=%d/null=%v", pool, null), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+				oi, body := nativeRestoreSeed(t, r, pool, null, false)
+				latest := putConsistencyObject(t, r.z, r.bucket, r.object, 1-pool, "latest-stays", ObjectOptions{Versioned: true, MTime: oi.ModTime.Add(time.Hour)})
+				before := nativeRestoreSnapshot(t, r)
+				ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+				defer cancel()
+				if err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+					t.Fatal(err)
+				}
+				_, got, err := r.read(oi.VersionID)
+				if err != nil || !bytes.Equal(got, body) {
+					t.Fatalf("requested version not restored: %v", err)
+				}
+				after := nativeRestoreSnapshot(t, r)
+				if !reflect.DeepEqual(before[1-pool], after[1-pool]) {
+					t.Error("restore changed the latest version in another pool")
+				}
+				if len(after[pool]) != 1 || after[pool][0].VersionID != before[pool][0].VersionID {
+					t.Error("restore created a version alias")
+				}
+				gotInfo, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{})
+				if err != nil || gotInfo.VersionID != latest.VersionID {
+					t.Errorf("latest changed: %v", err)
+				}
+				before = nativeRestoreSnapshot(t, r)
+				if err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, mustGetUUID())); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+					t.Errorf("missing version: %v", err)
+				}
+				if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
+					t.Error("missing version restore mutated metadata")
+				}
+			})
+		}
+	}
+}
+
+func TestNativeRestoreInitialFailurePreservesMetadata(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	oi := putConsistencyObject(t, r.z, r.bucket, r.object, 0, "original-latest", ObjectOptions{Versioned: true, UserDefined: map[string]string{"content-type": "application/x-original", "x-amz-meta-preserve": "keep"}})
+	before := nativeRestoreSnapshot(t, r)
+	for _, missing := range []string{mustGetUUID(), nullVersionID} {
+		err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, missing))
+		if !isErrVersionNotFound(err) {
+			t.Errorf("missing restore %s: %v", missing, err)
+		}
+		if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
+			t.Error("initial restore failure changed stored metadata or version stack")
+		}
+	}
+	_, body, err := r.read(oi.VersionID)
+	if err != nil || string(body) != "original-latest" {
+		t.Errorf("initial restore failure changed content: %v", err)
+	}
+}
+
+func TestNativeRestoreWithoutOuterLock(t *testing.T) {
+	for _, mode := range []string{"single-part", "multipart", "failure-metadata"} {
+		t.Run(mode, func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			oi, _ := nativeRestoreSeed(t, r, 0, false, mode == "multipart")
+			tier := globalTierConfigMgr.drivercache["WARM"]
+			failure := errors.New("injected remote failure")
+			if mode == "failure-metadata" {
+				globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, failure}
+			}
+			er := r.z.serverPools[0].getHashedSet(r.object)
+			lock := er.NewNSLock(r.bucket, r.object)
+			lkctx, err := lock.GetLock(t.Context(), globalOperationTimeout)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer lock.Unlock(lkctx)
+			before := nativeRestoreSnapshot(t, r)
+			ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+			defer cancel()
+			err = er.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+			if err == nil {
+				t.Error("restore wrote through another caller's lock")
+			}
+			if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
+				t.Error("NoLock=false failed to protect the stored version")
+			}
+		})
+	}
+}
+
+type nativeRestoreHTTPResult struct {
+	opts ObjectOptions
+	err  error
+}
+type nativeRestoreHTTPObserver struct {
+	ObjectLayer
+	done chan nativeRestoreHTTPResult
+}
+
+func (o nativeRestoreHTTPObserver) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+	ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+	err := o.ObjectLayer.RestoreTransitionedObject(ctx, bucket, object, opts)
+	o.done <- nativeRestoreHTTPResult{opts, err}
+	return err
+}
+
+func TestNativeRestorePOSTVersionIdentity(t *testing.T) {
+	for _, state := range []versioning.State{versioning.Enabled, versioning.Suspended} {
+		for _, target := range []int{0, 1} {
+			for _, split := range []bool{false, true} {
+				for _, multipart := range []bool{false, true} {
+					t.Run(fmt.Sprintf("%s/target=%d/split=%v/multipart=%v", state, target, split, multipart), func(t *testing.T) {
+						r := newRestoreSafetyTest(t, 2, state)
+						initConfigSubsystem(t.Context(), r.z)
+						oi, body := nativeRestoreSeed(t, r, target, !multipart, multipart)
+						metaOpts := r.opts(oi.VersionID)
+						metaOpts.MTime = oi.ModTime
+						metaOpts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+							o.UserDefined[xhttp.AmzRestore] = ""
+							return ReplicateDecision{}, nil
+						}
+						if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, metaOpts); err != nil {
+							t.Fatal(err)
+						}
+						if split {
+							putConsistencyObject(t, r.z, r.bucket, r.object, 1-target, "newer-version", ObjectOptions{Versioned: true, MTime: oi.ModTime.Add(time.Hour)})
+						}
+						before := nativeRestoreSnapshot(t, r)
+						observed := nativeRestoreHTTPObserver{r.z, make(chan nativeRestoreHTTPResult, 1)}
+						api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return observed }}
+						events := make(chan event.Event, 1)
+						done := make(chan struct{})
+						defer close(done)
+						if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRestoreCompleted), events, done, nil); err != nil {
+							t.Fatal(err)
+						}
+						bodyXML := []byte(`<RestoreRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Days>1</Days></RestoreRequest>`)
+						path := "/" + r.bucket + "/" + r.object + "?restore"
+						if split {
+							path += "&versionId=" + url.QueryEscape(oi.VersionID)
+						}
+						req, err := newTestSignedRequestV4(http.MethodPost, path, int64(len(bodyXML)), bytes.NewReader(bodyXML), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+						if err != nil {
+							t.Fatal(err)
+						}
+						if err = req.ParseForm(); err != nil {
+							t.Fatal(err)
+						}
+						req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object})
+						rec := httptest.NewRecorder()
+						api.PostRestoreObjectHandler(rec, req)
+						if rec.Code != http.StatusOK {
+							t.Fatalf("POST restore: %d %s", rec.Code, rec.Body.String())
+						}
+						select {
+						case result := <-observed.done:
+							if result.err != nil {
+								t.Fatal(result.err)
+							}
+							if result.opts.VersionID != oi.VersionID || result.opts.Versioned != (state == versioning.Enabled) || result.opts.VersionSuspended != (state == versioning.Suspended) {
+								t.Errorf("background options lost version context: %+v", result.opts)
+							}
+						case <-time.After(5 * time.Second):
+							t.Fatal("background restore did not return")
+						}
+						select {
+						case <-events:
+						case <-time.After(5 * time.Second):
+							t.Fatal("no restore completion event")
+						}
+						after := nativeRestoreSnapshot(t, r)
+						if len(after[target]) != 1 || after[target][0].VersionID != before[target][0].VersionID || after[target][0].TransitionedObjName != before[target][0].TransitionedObjName {
+							t.Fatal("POST restore added a version alias")
+						}
+						if !reflect.DeepEqual(before[1-target], after[1-target]) {
+							t.Error("POST changed another pool's version")
+						}
+						_, got, err := r.read(oi.VersionID)
+						if err != nil || !bytes.Equal(got, body) {
+							t.Fatalf("POST-restored contents differ: %v", err)
+						}
+					})
+				}
+			}
+		}
+	}
+}
+
+// A restore request is internal context, not an exception for multipart
+// overwrites: a native restore never completes an upload, so no completion,
+// whatever restore context it carries, writes an existing version back.
+func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
+	for _, mode := range []string{"matching-restore", "ordinary-complete", "restore-without-attempt", "other-attempt", "not-tiered", "changed-time", "changed-size", "changed-etag"} {
+		t.Run(mode, func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			oi, body := nativeRestoreSeed(t, r, 0, false, true)
+			pool := r.z.serverPools[0]
+			if mode == "not-tiered" {
+				er := pool.getHashedSet(r.object)
+				fi, _, _, err := er.getObjectFileInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID), false)
+				if err != nil {
+					t.Fatal(err)
+				}
+				fi.TransitionStatus = lifecycle.TransitionPending
+				for _, disk := range er.getDisks() {
+					if err := disk.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+						t.Fatal(err)
+					}
+				}
+			}
+			current, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+			if err != nil {
+				t.Fatal(err)
+			}
+			opts := putRestoreOpts(&RestoreObjectRequest{Days: 1}, current, r.opts(oi.VersionID))
+			opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+			opts.PreserveETag = current.ETag
+			if mode == "ordinary-complete" {
+				opts.Transition.RestoreRequest = nil
+			}
+			if mode == "changed-time" {
+				opts.MTime = opts.MTime.Add(time.Second)
+			}
+			// Hold the ETag constant for the size case, so only size can reject it.
+			if mode == "changed-size" {
+				body = append(body, 'x')
+			}
+			if mode == "changed-etag" {
+				opts.PreserveETag = "different-etag"
+			}
+			// The attempt claims what its upload claims; only the matching
+			// mode claims exactly the stored version.
+			if mode != "ordinary-complete" && mode != "restore-without-attempt" {
+				opts.Transition.attempt = &restoreAttempt{ID: mustGetUUID(), VersionID: oi.VersionID, ETag: opts.PreserveETag, Size: current.Size, ModTime: opts.MTime}
+			}
+			uopts := opts
+			uopts.MTime = time.Time{}
+			upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, uopts)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var complete []CompletePart
+			for i, data := range [][]byte{body[:5<<20], body[5<<20:]} {
+				part, err := pool.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, i+1,
+					mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Transition: opts.Transition})
+				if err != nil {
+					t.Fatal(err)
+				}
+				complete = append(complete, CompletePart{PartNumber: i + 1, ETag: part.ETag})
+			}
+			if mode != "changed-size" && getCompleteMultipartMD5(complete) != current.ETag {
+				t.Fatal("fixture multipart ETag does not match original")
+			}
+			if mode == "other-attempt" {
+				other := *opts.Transition.attempt
+				other.ID = mustGetUUID()
+				opts.Transition.attempt = &other
+			}
+			before := nativeRestoreSnapshot(t, r)
+			_, err = pool.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, complete, opts)
+			if !errors.Is(err, errAuthentication) {
+				t.Fatalf("existing version overwrite: %v", err)
+			}
+			if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
+				t.Error("rejected completion changed the target")
+			}
+		})
+	}
+}
+
+// Hold the download after the restore snapshot has been read. Object Lock
+// updates must serialize with both successful rehydration and failure cleanup.
+type nativeRestoreBlockedTier struct {
+	WarmBackend
+	entered        chan struct{}
+	release        chan struct{}
+	once           sync.Once
+	failure        error
+	afterFirstPart bool
+}
+
+func (b *nativeRestoreBlockedTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+	if b.afterFirstPart {
+		r, err := b.WarmBackend.Get(ctx, object, version, opts)
+		if err != nil {
+			return nil, err
+		}
+		return &nativeRestorePartGateReader{ReadCloser: r, gate: b, ctx: ctx, remaining: 5 << 20}, nil
+	}
+	b.once.Do(func() { close(b.entered) })
+	select {
+	case <-b.release:
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+	if b.failure != nil {
+		return nil, b.failure
+	}
+	return b.WarmBackend.Get(ctx, object, version, opts)
+}
+
+func nativeRestoreLockRequest(ctx context.Context, t *testing.T, r *restoreTest, id, kind string, body []byte) *httptest.ResponseRecorder {
+	t.Helper()
+	sum := md5.Sum(body)
+	req, err := newTestSignedRequestV4(http.MethodPut, "/"+r.bucket+"/"+r.object+"?"+kind+"&versionId="+url.QueryEscape(id), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := req.ParseForm(); err != nil {
+		t.Fatal(err)
+	}
+	req = mux.SetURLVars(req.WithContext(ctx), map[string]string{"bucket": r.bucket, "object": r.object})
+	rec := httptest.NewRecorder()
+	api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return r.z }}
+	if kind == "legal-hold" {
+		api.PutObjectLegalHoldHandler(rec, req)
+	} else {
+		api.PutObjectRetentionHandler(rec, req)
+	}
+	return rec
+}
+
+// A native restore downloads without the object lock. An Object Lock
+// update made meanwhile commits at once, and the restore, whether it
+// completes or fails, keeps it.
+func TestNativeRestoreKeepsObjectLockUpdate(t *testing.T) {
+	for _, kind := range []string{"legal-hold", "retention"} {
+		for _, fail := range []bool{false, true} {
+			t.Run(fmt.Sprintf("%s/failure=%v", kind, fail), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+				initConfigSubsystem(t.Context(), r.z)
+				enableBucketObjectLock(t, r.bucket)
+				oi, _ := nativeRestoreSeed(t, r, 0, false, true)
+				initial := []byte(`<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>OFF</Status></LegalHold>`)
+				updated := []byte(`<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>ON</Status></LegalHold>`)
+				keys := []string{strings.ToLower(xhttp.AmzObjectLockLegalHold), ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp}
+				if kind == "retention" {
+					initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour))
+					updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))
+					keys = []string{strings.ToLower(xhttp.AmzObjectLockMode), strings.ToLower(xhttp.AmzObjectLockRetainUntilDate), ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp}
+				}
+				if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK {
+					t.Fatalf("initial protection: %d %s", rec.Code, rec.Body.String())
+				}
+				before, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+				if err != nil {
+					t.Fatal(err)
+				}
+				blocked := &nativeRestoreBlockedTier{WarmBackend: globalTierConfigMgr.drivercache["WARM"], entered: make(chan struct{}), release: make(chan struct{})}
+				if fail {
+					blocked.failure = errors.New("injected download failure after snapshot")
+				}
+				globalTierConfigMgr.drivercache["WARM"] = blocked
+				var release sync.Once
+				unblock := func() { release.Do(func() { close(blocked.release) }) }
+				defer unblock()
+				ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+				defer cancel()
+				result := make(chan error, 1)
+				go func() {
+					result <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+				}()
+				select {
+				case <-blocked.entered:
+				case <-ctx.Done():
+					t.Fatal("restore never started download")
+				}
+				// The download holds no object lock: an Object Lock update
+				// commits at once, and the restore's commit keeps it.
+				updateCtx, updateCancel := context.WithTimeout(t.Context(), 5*time.Second)
+				rec := nativeRestoreLockRequest(updateCtx, t, r, oi.VersionID, kind, updated)
+				updateCancel()
+				if rec.Code != http.StatusOK {
+					t.Fatalf("Object Lock update during the restore download: %d %s", rec.Code, rec.Body.String())
+				}
+				readOpts := r.opts(oi.VersionID)
+				readOpts.NoLock = true
+				during, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, readOpts)
+				if err != nil {
+					t.Fatal(err)
+				}
+				unblock()
+				select {
+				case err := <-result:
+					if !errors.Is(err, blocked.failure) {
+						t.Fatalf("restore returned %v, want %v", err, blocked.failure)
+					}
+				case <-ctx.Done():
+					t.Fatal("restore failed to finish")
+				}
+				after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+				if err != nil {
+					t.Fatal(err)
+				}
+				for _, key := range keys {
+					if during.UserDefined[key] == "" || after.UserDefined[key] != during.UserDefined[key] {
+						t.Errorf("restore changed protection %s: %q -> %q", key, during.UserDefined[key], after.UserDefined[key])
+					}
+				}
+				changed := keys[0]
+				if kind == "retention" {
+					changed = keys[1]
+				}
+				if after.UserDefined[changed] == before.UserDefined[changed] {
+					t.Error("the update made during the download was not kept")
+				}
+				if !after.ModTime.Equal(before.ModTime) || after.ETag != before.ETag || after.Size != before.Size {
+					t.Error("Object Lock update changed content identity")
+				}
+			})
+		}
+	}
+}
+
+// The POST handler marks a version ongoing before its background restore
+// takes the object lock, and only that restore clears the marker. A restore
+// queued behind another version's download must keep waiting: one that gave up
+// after a lock timeout would leave the marker for every later POST to refuse.
+// The wait still ends with its context.
+func TestNativeRestoreQueuedBehindDownload(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+			first, _ := nativeRestoreSeed(t, r, 0, false, false)
+			queued, body := nativeRestoreSeed(t, r, pools-1, false, false)
+			prev := globalRestoreLockTimeout
+			globalRestoreLockTimeout = newDynamicTimeout(50*time.Millisecond, 50*time.Millisecond)
+			t.Cleanup(func() { globalRestoreLockTimeout = prev })
+			tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+			blocked := &nativeRestoreBlockedTier{WarmBackend: tier, entered: make(chan struct{}), release: make(chan struct{})}
+			globalTierConfigMgr.drivercache["WARM"] = blocked
+			var once sync.Once
+			unblock := func() { once.Do(func() { close(blocked.release) }) }
+			defer unblock()
+			ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
+			defer cancel()
+			firstDone := make(chan error, 1)
+			go func() {
+				firstDone <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, first.VersionID))
+			}()
+			select {
+			case <-blocked.entered:
+			case <-ctx.Done():
+				t.Fatal("first restore never reached the tier")
+			}
+			short, shortCancel := context.WithTimeout(ctx, 200*time.Millisecond)
+			defer shortCancel()
+			ended := make(chan error, 1)
+			go func() {
+				ended <- r.z.RestoreTransitionedObject(short, r.bucket, r.object, nativeRestoreOptions(r, queued.VersionID))
+			}()
+			select {
+			case err := <-ended:
+				if err == nil {
+					t.Fatal("restore finished without the object lock")
+				}
+			case <-time.After(5 * time.Second):
+				t.Fatal("restore kept waiting for the lock after its context ended")
+			}
+			queuedDone := make(chan error, 1)
+			go func() {
+				queuedDone <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, queued.VersionID))
+			}()
+			select {
+			case err := <-queuedDone:
+				t.Fatalf("queued restore gave up behind the download: %v", err)
+			case <-time.After(10 * 50 * time.Millisecond):
+			}
+			unblock()
+			for _, done := range []chan error{firstDone, queuedDone} {
+				select {
+				case err := <-done:
+					if err != nil {
+						t.Fatalf("restore: %v", err)
+					}
+				case <-ctx.Done():
+					t.Fatal("restore did not finish")
+				}
+			}
+			info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(queued.VersionID))
+			if err != nil || info.RestoreOngoing {
+				t.Fatalf("queued version still ongoing after restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+			}
+			reads := tier.gets.Load()
+			if _, got, err := r.read(queued.VersionID); err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("queued restore contents: %v", err)
+			}
+			if tier.gets.Load() != reads {
+				t.Error("queued version was not restored locally")
+			}
+		})
+	}
+}
+
+type nativeRestorePartGateReader struct {
+	io.ReadCloser
+	gate      *nativeRestoreBlockedTier
+	ctx       context.Context
+	remaining int
+}
+
+func (r *nativeRestorePartGateReader) Read(p []byte) (int, error) {
+	if r.remaining > 0 {
+		if len(p) > r.remaining {
+			p = p[:r.remaining]
+		}
+		n, err := r.ReadCloser.Read(p)
+		r.remaining -= n
+		return n, err
+	}
+	r.gate.once.Do(func() { close(r.gate.entered) })
+	select {
+	case <-r.gate.release:
+	case <-r.ctx.Done():
+		return 0, r.ctx.Err()
+	}
+	return r.ReadCloser.Read(p)
+}
+
+// Invalid upload paths must retain the backend's normal error precedence,
+// rather than reacquiring the object lock as the upload lock.
+func TestMultipartCompletionLockAlias(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, id := range []string{"", ".", "./"} {
+			t.Run(fmt.Sprintf("pools=%d/id=%q", pools, id), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+				for _, missingBucket := range []bool{false, true} {
+					bucket := r.bucket
+					if missingBucket {
+						bucket = "missing-bucket"
+					}
+					ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+					_, err := r.z.CompleteMultipartUpload(ctx, bucket, r.object, id, nil, ObjectOptions{})
+					cancel()
+					switch {
+					case id != "":
+						if !errors.As(err, &MalformedUploadID{}) {
+							t.Errorf("malformed upload: %v", err)
+						}
+					case missingBucket:
+						if !isErrBucketNotFound(err) {
+							t.Errorf("missing bucket: %v", err)
+						}
+					default:
+						if !errors.As(err, &InvalidUploadID{}) {
+							t.Errorf("invalid upload: %v", err)
+						}
+					}
+				}
+			})
+		}
+	}
+}
+
+type nativeRestoreReadHook struct {
+	io.ReadCloser
+	remaining int
+	mutate    func()
+}
+
+func (r *nativeRestoreReadHook) Read(p []byte) (int, error) {
+	n, err := r.ReadCloser.Read(p)
+	r.remaining -= n
+	if r.remaining == 0 && r.mutate != nil {
+		f := r.mutate
+		r.mutate = nil
+		f()
+	}
+	return n, err
+}
+
+type nativeRestoreReadHookTier struct {
+	WarmBackend
+	size   int
+	mutate func()
+}
+
+func (b nativeRestoreReadHookTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+	rd, err := b.WarmBackend.Get(ctx, object, version, opts)
+	if err != nil {
+		return nil, err
+	}
+	return &nativeRestoreReadHook{rd, b.size, b.mutate}, nil
+}
+
+// Remove only the addressed entry from a majority of disks, keeping shards
+// and the healthy newer version. Return errors to the synchronous caller.
+func nativeRestoreLoseMetadata(ctx context.Context, disks []StorageAPI, bucket, object, id string) error {
+	for _, disk := range disks[1:] {
+		raw, err := disk.ReadXL(ctx, bucket, object, true)
+		if err != nil {
+			return err
+		}
+		var meta xlMetaV2
+		if err := meta.Load(raw.Buf); err != nil {
+			return err
+		}
+		del := FileInfo{VersionID: id}
+		del.SetSkipTierFreeVersion()
+		if _, err := meta.DeleteVersion(del); err != nil {
+			return err
+		}
+		data, err := meta.AppendTo(nil)
+		if err != nil {
+			return err
+		}
+		if err := disk.WriteAll(ctx, bucket, filepath.Join(object, xlStorageFormatFile), data); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// A target can lose metadata quorum after the restore snapshot. Nested
+// completion/retention reads must not turn rehydration into remote GC.
+func TestNativeRestoreCompletionReadSafety(t *testing.T) {
+	for _, null := range []bool{false, true} {
+		for _, multipart := range []bool{false, true} {
+			t.Run(fmt.Sprintf("null=%v/multipart=%v", null, multipart), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+				oi, body := nativeRestoreSeed(t, r, 0, null, multipart)
+				latest := r.put("newer", false, false)
+				enableBucketObjectLock(t, r.bucket)
+				disks := r.z.serverPools[0].getHashedSet(r.object).getDisks()
+				var mutationErr error
+				mutated := false
+				mutate := func() {
+					mutated = true
+					mutationErr = nativeRestoreLoseMetadata(t.Context(), disks, r.bucket, r.object, oi.VersionID)
+				}
+				tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+				globalTierConfigMgr.drivercache["WARM"] = nativeRestoreReadHookTier{tier, len(body), mutate}
+				ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+				defer cancel()
+				err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+				if ctx.Err() != nil {
+					t.Fatalf("restore timed out: %v", err)
+				}
+				if !mutated || mutationErr != nil {
+					t.Fatalf("invalid injection: mutated=%v err=%v", mutated, mutationErr)
+				}
+				t.Logf("restore after addressed metadata lost: %v", err)
+				for i, disk := range disks {
+					raw, e := disk.ReadXL(t.Context(), r.bucket, r.object, true)
+					if e != nil {
+						t.Fatal(e)
+					}
+					versions, e := getFileInfoVersions(raw.Buf, r.bucket, r.object, false)
+					if e != nil {
+						t.Fatal(e)
+					}
+					if len(versions.FreeVersions) > 0 {
+						t.Errorf("disk %d restore manufactured %d remote GC entries", i, len(versions.FreeVersions))
+					}
+				}
+				if err == nil {
+					reads := tier.gets.Load()
+					_, got, readErr := r.read(oi.VersionID)
+					if readErr != nil || !bytes.Equal(got, body) || tier.gets.Load() != reads {
+						t.Errorf("successful restore did not rehydrate its own content: %v", readErr)
+					}
+				}
+				_, got, readErr := r.read(latest.VersionID)
+				if readErr != nil || !bytes.Equal(got, r.bodies["newer"]) {
+					t.Errorf("restore changed the other version: %v", readErr)
+				}
+			})
+		}
+	}
+}
+
+// nativeRestoreStatusHook runs afterRead once POST has read the version and
+// before POST records its status on it.
+type nativeRestoreStatusHook struct {
+	ObjectLayer
+	afterRead        func()
+	readCalled       bool
+	backgroundCalled atomic.Bool
+}
+
+func (h *nativeRestoreStatusHook) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	oi, err := h.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+	if err == nil && !h.readCalled {
+		h.readCalled = true
+		h.afterRead()
+	}
+	return oi, err
+}
+
+func (h *nativeRestoreStatusHook) RestoreTransitionedObject(context.Context, string, string, ObjectOptions) error {
+	h.backgroundCalled.Store(true)
+	return errors.New("unexpected background restore after failed status update")
+}
+
+func TestNativeRestorePOSTReadSafety(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, stage := range []string{"initial", "status-copy"} {
+			t.Run(fmt.Sprintf("pools=%d/%s", pools, stage), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+				initConfigSubsystem(t.Context(), r.z)
+				oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+				metaOpts := r.opts(oi.VersionID)
+				metaOpts.MTime = oi.ModTime
+				metaOpts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+					o.UserDefined[xhttp.AmzRestore] = ""
+					return ReplicateDecision{}, nil
+				}
+				if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, metaOpts); err != nil {
+					t.Fatal(err)
+				}
+				r.put("newer", false, false)
+				disks := r.z.serverPools[0].getHashedSet(r.object).getDisks()
+				var before [][]byte
+				mutate := func() {
+					if err := nativeRestoreLoseMetadata(t.Context(), disks, r.bucket, r.object, oi.VersionID); err != nil {
+						t.Fatal(err)
+					}
+					before = restoreBoundarySnapshot(t, r)
+				}
+				hook := &nativeRestoreStatusHook{ObjectLayer: r.z, afterRead: mutate}
+				if stage == "initial" {
+					mutate()
+				}
+				api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return hook }}
+				body := []byte(`<RestoreRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Days>1</Days></RestoreRequest>`)
+				req, err := newTestSignedRequestV4(http.MethodPost, "/"+r.bucket+"/"+r.object+"?restore&versionId="+url.QueryEscape(oi.VersionID), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if err := req.ParseForm(); err != nil {
+					t.Fatal(err)
+				}
+				req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object})
+				rec := httptest.NewRecorder()
+				api.PostRestoreObjectHandler(rec, req)
+				if rec.Code < 400 {
+					t.Fatalf("POST acknowledged failed read: %d %s", rec.Code, rec.Body.String())
+				}
+				if hook.readCalled != (stage == "status-copy") || hook.backgroundCalled.Load() {
+					t.Fatal("request followed the wrong failure path")
+				}
+				if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+					t.Error("failed POST read changed stored metadata or free versions")
+				}
+			})
+		}
+	}
+}

cmd/restore-stage.go

@@ -0,0 +1,900 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"maps"
+	"net/http"
+	"slices"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/pgsty/silo-pkg/v3/sync/errgroup"
+)
+
+// A native restore runs in two phases. It downloads the tier copy of a
+// version and erasure-codes it into the tmp volume of the version's set
+// without the object lock O, then takes O only to check the version again
+// and publish the staged copy drive by drive, as heal does. Hot versions of
+// the key stay readable and writable during the download.
+//
+// The restore lock R of the version tells whether a restore of it runs: a
+// POST takes R before O, and the worker that rehydrates the version holds R
+// for its whole run. Nothing else takes R, so nothing holds O while it waits
+// for R. R is a liveness hint, not an exclusion: a lost refresh can let two
+// workers overlap. Only the checks under O decide what is written.
+
+// restoreOwnerPrefix names the restore locks under minioMetaBucket.
+const restoreOwnerPrefix = "restore"
+
+// restorePublicationDelays are the in-memory retries of a convergence after
+// the one under the commit's own lock hold, see recoverRestorePublication.
+var restorePublicationDelays = []time.Duration{100 * time.Millisecond, time.Second, 5 * time.Second, 30 * time.Second, 2 * time.Minute}
+
+var (
+	// errRestoreMoved: the version moved to another pool after the restore
+	// staged its copy in this one.
+	errRestoreMoved = errors.New("restored version moved to another pool")
+	// errRestoreNotConverged: a publication that reached some drives but not
+	// a write quorum could not be brought back to the unrestored version yet.
+	errRestoreNotConverged = errors.New("restore publication did not converge")
+)
+
+// restoreOwner is the restore lock R of one version, held by a POST that
+// registers an attempt and then by the worker that runs it. ctx ends when the
+// lock is released or when its holder learns that it lost it.
+type restoreOwner struct {
+	lk     RWLocker
+	lkctx  LockContext
+	ctx    context.Context
+	cancel context.CancelFunc
+	once   sync.Once
+}
+
+// release unlocks R once. It is a no-op on a nil owner.
+func (o *restoreOwner) release() {
+	if o == nil {
+		return
+	}
+	o.once.Do(func() {
+		o.cancel()
+		o.lk.Unlock(o.lkctx)
+	})
+}
+
+// restoreOwnerResource is the name of the restore lock of a version. Bucket
+// names hold no slash, object names no ".." element, and the version ID has
+// a fixed form, so the name is unambiguous.
+func restoreOwnerResource(bucket, object, versionID string) string {
+	return pathJoin(restoreOwnerPrefix, bucket, object, restoreVersionID(versionID))
+}
+
+// takeRestoreOwner takes the restore lock of a version. With a nil timeout it
+// waits for as long as ctx lives, as a direct caller does; otherwise it gives
+// up after timeout. ctx must outlive the restore that the lock is held for.
+func (z *erasureServerPools) takeRestoreOwner(ctx context.Context, bucket, object, versionID string, timeout *dynamicTimeout) (*restoreOwner, error) {
+	for {
+		// A new instance for each try: a distributed unlock retries in the
+		// background and could release a later try of the same instance.
+		lk := z.NewNSLock(minioMetaBucket, restoreOwnerResource(bucket, object, versionID))
+		wait := timeout
+		if wait == nil {
+			wait = globalRestoreLockTimeout
+		}
+		lkctx, err := lk.GetLock(ctx, wait)
+		if err != nil {
+			if timeout == nil && errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil {
+				continue
+			}
+			return nil, err
+		}
+		octx, cancel := context.WithCancel(lkctx.Context())
+		return &restoreOwner{lk: lk, lkctx: lkctx, ctx: octx, cancel: cancel}, nil
+	}
+}
+
+// restoreStage is the copy of one version that a native restore encoded into
+// the tmp volume of the version's set, under tmpID/dataDir, before it takes
+// the object lock.
+type restoreStage struct {
+	versionID string
+	// fi is the version as the download read it: the commit only publishes
+	// the copy for a version that still has this content, see
+	// restoreSameContent.
+	fi      FileInfo
+	tmpID   string
+	dataDir string
+	// staged tells, in the order of the version's erasure distribution,
+	// which drives hold every part of the copy.
+	staged []bool
+	expiry time.Time
+	// request is set for an attempt that POST registered: days and date
+	// are what it recorded. A direct caller keeps the version's own.
+	request bool
+	days    int
+	date    string
+}
+
+// restoreSameContent reports whether two reads of a version name the same
+// encoded bytes, laid out the same way: the version, its tier copy, its parts
+// and its erasure layout. The tier copy is never rewritten under the same
+// name and remote version, so these decide the bytes. Every other field, such
+// as tags, Object Lock, replication state or a rewrapped object key, is taken
+// from the version as the commit reads it.
+func restoreSameContent(a, b FileInfo) bool {
+	if a.VersionID != b.VersionID || a.Deleted || b.Deleted || !a.ModTime.Equal(b.ModTime) ||
+		a.Metadata["etag"] != b.Metadata["etag"] || a.Size != b.Size ||
+		a.TransitionStatus != lifecycle.TransitionComplete || b.TransitionStatus != lifecycle.TransitionComplete ||
+		a.TransitionTier != b.TransitionTier || a.TransitionedObjName != b.TransitionedObjName ||
+		a.TransitionVersionID != b.TransitionVersionID {
+		return false
+	}
+	if len(a.Parts) != len(b.Parts) {
+		return false
+	}
+	for i := range a.Parts {
+		if a.Parts[i].Number != b.Parts[i].Number || a.Parts[i].Size != b.Parts[i].Size {
+			return false
+		}
+	}
+	ea, eb := a.Erasure, b.Erasure
+	return ea.Algorithm == eb.Algorithm && ea.DataBlocks == eb.DataBlocks && ea.ParityBlocks == eb.ParityBlocks &&
+		ea.BlockSize == eb.BlockSize && slices.Equal(ea.Distribution, eb.Distribution)
+}
+
+// restoreInlineKey marks a version whose data lives in its xl.meta.
+const restoreInlineKey = ReservedMetadataPrefixLower + "inline-data"
+
+// stageRestore downloads the tier copy of the version opts address and
+// erasure-codes it, part by part, into the tmp volume of the drives of this
+// set, as heal writes the parts it rebuilds. It holds no object lock. A drive
+// that fails to write any part, or that misses a part afterwards, for
+// example because its tmp volume was moved away when its server restarted,
+// takes no further part and no part in the commit.
+func (er erasureObjects) stageRestore(ctx context.Context, bucket, object string, opts ObjectOptions) (*restoreStage, error) {
+	rreq := opts.Transition.RestoreRequest
+	if rreq == nil {
+		return nil, errInvalidArgument
+	}
+	gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
+	if err != nil {
+		return nil, toObjectErr(err, bucket, object, opts.VersionID)
+	}
+	fi := gen.fi
+	if fi.Deleted {
+		return nil, toObjectErr(errMethodNotAllowed, bucket, object)
+	}
+	if a := opts.Transition.attempt; a != nil && !a.ownsGeneration(gen, bucket, object) {
+		// A later POST took the version over, or it was replaced.
+		return nil, errRestoreSuperseded
+	}
+	disks := er.getDisks()
+	var total int64
+	for _, p := range fi.Parts {
+		if p.Size < 0 {
+			return nil, toObjectErr(errFileCorrupt, bucket, object)
+		}
+		total += p.Size
+	}
+	if fi.TransitionStatus != lifecycle.TransitionComplete || len(fi.Parts) == 0 || total != fi.Size ||
+		len(fi.Erasure.Distribution) != len(disks) {
+		return nil, InvalidObjectState{Bucket: bucket, Object: object}
+	}
+	erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
+	if err != nil {
+		return nil, toObjectErr(err, bucket, object)
+	}
+	writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
+	st := &restoreStage{
+		versionID: restoreVersionID(fi.VersionID),
+		fi:        fi,
+		tmpID:     mustGetUUID(),
+		dataDir:   mustGetUUID(),
+		expiry:    lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days),
+	}
+	if opts.Transition.restoreDate != "" {
+		st.request, st.days, st.date = true, rreq.Days, opts.Transition.restoreDate
+	}
+	staged := false
+	defer func() {
+		if !staged {
+			er.deleteAll(context.Background(), minioMetaTmpBucket, st.tmpID)
+		}
+	}()
+
+	// The tier copy is the encoded stream of the parts, encrypted or
+	// compressed as stored: the parts get exactly these bytes back.
+	oi := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
+	gr, err := getTransitionedObjectReader(ctx, bucket, object, nil, http.Header{}, oi, opts)
+	if err != nil {
+		return nil, toObjectErr(err, bucket, object)
+	}
+	defer gr.Close()
+
+	var buffer []byte
+	pool := globalBytePoolCap.Load()
+	if int64(pool.Width()) < fi.Erasure.BlockSize {
+		buffer = make([]byte, fi.Erasure.BlockSize)
+	} else {
+		buffer = pool.Get()
+		defer pool.Put(buffer)
+	}
+	buffer = buffer[:fi.Erasure.BlockSize]
+
+	// Slots follow the erasure distribution: the drive in slot s gets shard
+	// s+1 of every part, and the commit gives it erasure index s+1.
+	slots := shuffleDisks(disks, fi.Erasure.Distribution)
+	for _, part := range fi.Parts {
+		partPath := pathJoin(st.tmpID, st.dataDir, fmt.Sprintf("part.%d", part.Number))
+		writers := make([]io.Writer, len(slots))
+		for s, disk := range slots {
+			if disk != nil {
+				writers[s] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, partPath,
+					erasure.ShardFileSize(part.Size), DefaultBitrotAlgorithm, erasure.ShardSize())
+			}
+		}
+		n, err := erasure.Encode(ctx, io.LimitReader(gr, part.Size), writers, buffer, writeQuorum)
+		closeErrs := closeBitrotWriters(writers)
+		if err != nil {
+			return nil, toObjectErr(err, bucket, object)
+		}
+		if n != part.Size {
+			return nil, IncompleteBody{Bucket: bucket, Object: object}
+		}
+		for s := range slots {
+			if writers[s] == nil || closeErrs[s] != nil {
+				slots[s] = nil
+			}
+		}
+		if countOnlineDisks(slots) < writeQuorum {
+			return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
+		}
+	}
+	// The tier copy must end with the last part.
+	var extra [1]byte
+	if n, _ := io.ReadFull(gr, extra[:]); n != 0 {
+		return nil, InvalidObjectState{Bucket: bucket, Object: object}
+	}
+
+	// A tmp volume that was moved away and recreated by a later part lacks the
+	// earlier parts: find them before anything is published. CheckParts sees a
+	// part that is present and not shorter than its shards; a staged drive
+	// that wrote every part without an error has exactly that.
+	check := fi
+	check.DataDir = st.dataDir
+	g := errgroup.WithNErrs(len(slots))
+	for s, disk := range slots {
+		if disk == nil {
+			continue
+		}
+		g.Go(func() error {
+			resp, err := disk.CheckParts(ctx, minioMetaTmpBucket, st.tmpID, check)
+			if err != nil {
+				return err
+			}
+			for _, result := range resp.Results {
+				if result != checkPartSuccess {
+					return errFileCorrupt
+				}
+			}
+			return nil
+		}, s)
+	}
+	st.staged = make([]bool, len(slots))
+	count := 0
+	for s, err := range g.Wait() {
+		if slots[s] != nil && err == nil {
+			st.staged[s] = true
+			count++
+		}
+	}
+	if count < writeQuorum {
+		return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
+	}
+	staged = true
+	return st, nil
+}
+
+// fileInfoRevisions are the revisions the entry that writing fi creates can
+// have, as readRestoreGeneration and the convergence identify the entries
+// they read. The header flag that tells whether a transitioned version uses
+// its data directory follows whether its restored copy has expired at the
+// moment a drive writes the entry, so a restore whose copy expires while it
+// publishes can find either value: both are this entry. Every other field of
+// the header and the full metadata digest still have to match.
+func fileInfoRevisions(fi FileInfo) ([]revisionKey, error) {
+	var x xlMetaV2
+	if err := x.AddVersion(fi); err != nil {
+		return nil, err
+	}
+	buf, err := x.AppendTo(nil)
+	if err != nil {
+		return nil, err
+	}
+	var y xlMetaV2
+	if err := y.Load(buf); err != nil {
+		return nil, err
+	}
+	if len(y.versions) != 1 {
+		return nil, errFileCorrupt
+	}
+	ver := y.versions[0]
+	other := ver
+	other.header.Flags ^= xlFlagUsesDataDir
+	return []revisionKey{versionRevision(ver, y.metaV), versionRevision(other, y.metaV)}, nil
+}
+
+// restorePublication remembers, for a commit that published the staged copy
+// on some drives but not on a write quorum, what each drive held before the
+// commit (Sᵢ), what the commit wrote there (Pᵢ) and what a convergence writes
+// there (Vᵢ), in the order of the version's erasure distribution. Only the
+// worker that published knows them; a convergence touches a version only
+// while every drive holds one of them.
+type restorePublication struct {
+	bucket, object string
+	versionID      string
+	uv             [16]byte
+	dataDir        string
+	attempt        *restoreAttempt
+	quorum         int
+	dist           []int
+	before         []*revisionKey
+	anyBefore      map[revisionKey]bool
+	published      map[revisionKey]bool
+	conv           []FileInfo
+	convKeys       map[revisionKey]bool
+}
+
+// restorePublished is the entry a commit publishes on the drive in slot s:
+// the version as the commit read it on that drive, with the staged data
+// directory and a completed restore. Restore fields are written alike on
+// every drive: the request POST recorded, or for a direct caller the one
+// the authoritative generation auth holds.
+func restorePublished(base, auth FileInfo, st *restoreStage, slot int) FileInfo {
+	fi := base
+	fi.Metadata = maps.Clone(base.Metadata)
+	if fi.Metadata == nil {
+		fi.Metadata = make(map[string]string)
+	}
+	for _, k := range restoreFieldKeys {
+		delete(fi.Metadata, k)
+	}
+	// An earlier restore may have inlined the data, and expiring that copy
+	// leaves the inline marker behind; this copy lives in its data directory.
+	delete(fi.Metadata, restoreInlineKey)
+	fi.Metadata[xhttp.AmzRestore] = completedRestoreObj(st.expiry).String()
+	if st.request {
+		fi.Metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(st.days)
+		fi.Metadata[xhttp.AmzRestoreRequestDate] = st.date
+	} else {
+		for _, k := range []string{xhttp.AmzRestoreExpiryDays, xhttp.AmzRestoreRequestDate} {
+			if v, ok := auth.Metadata[k]; ok {
+				fi.Metadata[k] = v
+			}
+		}
+	}
+	fi.DataDir = st.dataDir
+	fi.Data = nil
+	fi.Parts = slices.Clone(base.Parts)
+	fi.Erasure.Index = slot + 1
+	// The staged parts use the default streaming bitrot algorithm.
+	fi.Erasure.Checksums = nil
+	return fi
+}
+
+// restoreUnrestored is the entry a convergence writes on the drive in slot s:
+// the version as the commit read it, with no restore fields, so that its tier
+// copy serves reads again and a later POST can restore it.
+func restoreUnrestored(base FileInfo, slot int) FileInfo {
+	fi := base
+	fi.Metadata = maps.Clone(base.Metadata)
+	for _, k := range restoreFieldKeys {
+		delete(fi.Metadata, k)
+	}
+	delete(fi.Metadata, restoreInlineKey)
+	fi.Data = nil
+	fi.Erasure.Index = slot + 1
+	return fi
+}
+
+// commitRestore publishes a staged copy while the caller holds the object
+// lock. It reads the version again, checks that the attempt still owns it and
+// that it still has the content the copy was staged for, then publishes the
+// copy on every staged drive that holds the version, with RenameData on each
+// drive, as heal does: never through erasure renameData, whose undo of a
+// replaced version could delete the new data and keep metadata that points
+// at it.
+//
+// It returns nil when the copy reached a write quorum. Otherwise it returns
+// an error, and a publication when some drive may already have the copy; the
+// caller must then converge, see convergeRestore.
+func (er erasureObjects) commitRestore(ctx context.Context, bucket, object string, st *restoreStage, opts ObjectOptions, placed func(context.Context) error) (*restorePublication, error) {
+	if placed != nil {
+		if err := placed(ctx); err != nil {
+			return nil, err
+		}
+	}
+	gen, err := er.readRestoreGeneration(ctx, bucket, object, st.versionID)
+	if err != nil {
+		return nil, toObjectErr(err, bucket, object, st.versionID)
+	}
+	if gen.fi.Deleted {
+		return nil, toObjectErr(errMethodNotAllowed, bucket, object)
+	}
+	a := opts.Transition.attempt
+	if a != nil && !a.ownsGeneration(gen, bucket, object) {
+		return nil, errRestoreSuperseded
+	}
+	if !restoreSameContent(st.fi, gen.fi) {
+		// Replaced, or its tier copy or layout changed since the download.
+		return nil, InvalidObjectState{Bucket: bucket, Object: object}
+	}
+	disks := er.getDisks()
+	dist := gen.fi.Erasure.Distribution
+	if len(dist) != len(disks) || len(gen.revs) != len(disks) {
+		return nil, InvalidObjectState{Bucket: bucket, Object: object}
+	}
+	n := len(disks)
+	writeQuorum := gen.fi.WriteQuorum(er.defaultWQuorum())
+
+	pub := &restorePublication{
+		bucket: bucket, object: object, versionID: st.versionID, dataDir: st.dataDir,
+		attempt: a, quorum: writeQuorum, dist: slices.Clone(dist),
+		before:    make([]*revisionKey, n),
+		anyBefore: make(map[revisionKey]bool),
+		published: make(map[revisionKey]bool),
+		conv:      make([]FileInfo, n),
+		convKeys:  make(map[revisionKey]bool),
+	}
+	if st.versionID != nullVersionID {
+		id, err := uuid.Parse(st.versionID)
+		if err != nil {
+			return nil, toObjectErr(errFileVersionNotFound, bucket, object, st.versionID)
+		}
+		pub.uv = id
+	}
+	targets := make([]StorageAPI, n)
+	files := make([]FileInfo, n)
+	for i := range disks {
+		s := dist[i] - 1
+		if s < 0 || s >= n {
+			return nil, InvalidObjectState{Bucket: bucket, Object: object}
+		}
+		if gen.revs[i] != nil {
+			pub.before[s] = gen.revs[i]
+			pub.anyBefore[*gen.revs[i]] = true
+		}
+		v := restoreUnrestored(gen.fi, s)
+		keys, err := fileInfoRevisions(v)
+		if err != nil {
+			return nil, toObjectErr(err, bucket, object)
+		}
+		pub.conv[s] = v
+		// The unrestored entry has no restore header: its flag never varies.
+		pub.convKeys[keys[0]] = true
+		if gen.disks[i] == nil || !st.staged[s] {
+			continue
+		}
+		fi := restorePublished(gen.drives[i], gen.fi, st, s)
+		if keys, err = fileInfoRevisions(fi); err != nil {
+			return nil, toObjectErr(err, bucket, object)
+		}
+		for _, key := range keys {
+			pub.published[key] = true
+		}
+		targets[s], files[s] = gen.disks[i], fi
+	}
+	if countOnlineDisks(targets) < writeQuorum {
+		// Nothing is published.
+		return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
+	}
+
+	errs := make([]error, n)
+	olds := make([]string, n)
+	var wg sync.WaitGroup
+	for s, disk := range targets {
+		if disk == nil {
+			errs[s] = errDiskNotFound
+			continue
+		}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			resp, err := disk.RenameData(ctx, minioMetaTmpBucket, st.tmpID, files[s], bucket, object, RenameOptions{})
+			olds[s], errs[s] = resp.OldDataDir, err
+		}()
+	}
+	wg.Wait()
+
+	if countErrs(errs, nil) < writeQuorum {
+		return pub, toObjectErr(errErasureWriteQuorum, bucket, object)
+	}
+	// The data directory of the replaced entry, which held a former restored
+	// copy, goes only when every drive that published reports the same one:
+	// RenameData reports none where another version shares it, and then the
+	// directory stays everywhere. It goes only from the drives that reported
+	// it.
+	old := ""
+	cleanup := make([]StorageAPI, n)
+	for s := range targets {
+		if errs[s] != nil {
+			continue
+		}
+		if olds[s] == "" || (old != "" && olds[s] != old) {
+			old = ""
+			break
+		}
+		old = olds[s]
+		cleanup[s] = targets[s]
+	}
+	if old != "" && old != st.dataDir {
+		er.commitRenameDataDir(ctx, bucket, object, old, cleanup, 1)
+	}
+	if countErrs(errs, nil) < n {
+		er.addPartial(bucket, object, gen.fi.VersionID)
+	}
+	return nil, nil
+}
+
+// restoreConvergence is the outcome of one convergence pass.
+type restoreConvergence int
+
+const (
+	// restoreIncomplete: the drives are not settled yet; try again.
+	restoreIncomplete restoreConvergence = iota
+	// restoreConverged: every drive that holds the version holds it
+	// unrestored again.
+	restoreConverged
+	// restoreCommitted: the copy reached a write quorum after all, such as
+	// when a drive applied a publication whose answer was lost.
+	restoreCommitted
+	// restoreStopped: some drive holds an entry that neither this attempt
+	// nor the version before it wrote: someone else changed the version.
+	restoreStopped
+)
+
+// classify reads the entry of the version on every drive and sorts it into
+// the slot of its drive: held tells which slots hold an entry this
+// publication knows, published and converged count the drives that hold
+// what the commit and the convergence write. A drive that cannot be read is
+// left out. An entry the publication does not know stops it.
+func (pub *restorePublication) classify(ctx context.Context, disks []StorageAPI) (held []bool, published, converged int, err error) {
+	raws, errs := readAllRawFileInfo(ctx, disks, pub.bucket, pub.object, false)
+	held = make([]bool, len(disks))
+	for i := range disks {
+		if errs[i] != nil || i >= len(pub.dist) {
+			continue
+		}
+		var xl xlMetaV2
+		if xl.LoadOrConvert(raws[i].Buf) != nil {
+			continue
+		}
+		s := pub.dist[i] - 1
+		for _, ver := range xl.versions {
+			if ver.header.VersionID != pub.uv || ver.header.FreeVersion() {
+				continue
+			}
+			key := versionRevision(ver, xl.metaV)
+			switch {
+			case pub.published[key]:
+				published++
+			case pub.convKeys[key]:
+				converged++
+			case pub.before[s] != nil && *pub.before[s] == key:
+			case pub.before[s] == nil && pub.anyBefore[key]:
+			default:
+				return nil, 0, 0, errRestoreSuperseded
+			}
+			held[s] = true
+			break
+		}
+	}
+	return held, published, converged, nil
+}
+
+// convergeRestore settles a publication while the caller holds the object
+// lock. When every drive still holds what the version held before the commit,
+// what the commit wrote, or what an earlier pass wrote, it either finds the
+// copy on a write quorum, or writes the unrestored version to every drive
+// that holds the version. It deletes no directory: an unreferenced staged
+// directory is left to abandoned-data cleanup. Any other entry stops it, so
+// that it never overwrites what someone else wrote between two passes.
+func (er erasureObjects) convergeRestore(ctx context.Context, pub *restorePublication) (restoreConvergence, error) {
+	disks := er.getDisks()
+	held, published, _, err := pub.classify(ctx, disks)
+	if err != nil {
+		return restoreStopped, err
+	}
+	if published >= pub.quorum {
+		er.addPartial(pub.bucket, pub.object, pub.versionID)
+		return restoreCommitted, nil
+	}
+	slots := shuffleDisks(disks, pub.dist)
+	files := make([]FileInfo, len(slots))
+	for s := range slots {
+		if !held[s] {
+			slots[s] = nil
+			continue
+		}
+		files[s] = pub.conv[s]
+	}
+	_, werr := writeUniqueFileInfo(ctx, slots, "", pub.bucket, pub.object, files, pub.quorum)
+
+	held, published, converged, err := pub.classify(ctx, disks)
+	if err != nil {
+		return restoreStopped, err
+	}
+	holding := 0
+	for _, h := range held {
+		if h {
+			holding++
+		}
+	}
+	if published == 0 && converged == holding && converged >= pub.quorum {
+		return restoreConverged, nil
+	}
+	if werr == nil {
+		werr = errErasureWriteQuorum
+	}
+	return restoreIncomplete, werr
+}
+
+// recoverRestorePublication retries a convergence in memory, for a bounded
+// time, each pass under the object lock taken afresh. A restart loses the
+// retries; the version then stays as the drives hold it, see the design's
+// excluded residuals.
+func (er erasureObjects) recoverRestorePublication(pub *restorePublication, lock func(context.Context) (LockContext, func(), error)) {
+	go func() {
+		err := errRestoreNotConverged
+		for _, delay := range restorePublicationDelays {
+			select {
+			case <-GlobalContext.Done():
+				return
+			case <-time.After(delay):
+			}
+			ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
+			lkctx, unlock, lerr := lock(ctx)
+			if lerr != nil {
+				cancel()
+				err = lerr
+				continue
+			}
+			var res restoreConvergence
+			res, err = er.convergeRestore(lkctx.Context(), pub)
+			unlock()
+			cancel()
+			if res == restoreConverged || res == restoreCommitted {
+				return
+			}
+			if res == restoreStopped {
+				break
+			}
+		}
+		er.logRestoreNotConverged(pub, err)
+	}()
+}
+
+func (er erasureObjects) logRestoreNotConverged(pub *restorePublication, err error) {
+	attempt := ""
+	if pub.attempt != nil {
+		attempt = pub.attempt.ID
+	}
+	storageLogIf(GlobalContext, fmt.Errorf("%w: %s/%s(%s), attempt %q, data dir %s: %v",
+		errRestoreNotConverged, pub.bucket, pub.object, pub.versionID, attempt, pub.dataDir, err))
+}
+
+// restoreHooks tie a restore of one set to its caller.
+type restoreHooks struct {
+	// lock takes the object lock O of the version, in the lock domain of the
+	// object's writes, and returns its release.
+	lock func(ctx context.Context) (LockContext, func(), error)
+	// placed tells, under O, whether the version still lives in this set.
+	placed func(ctx context.Context) error
+	// clear clears the attempt's ongoing marker after a failure that
+	// published nothing, taking O itself, and retries in the background.
+	clear func()
+}
+
+// restoreVersion rehydrates the transitioned version opts address in this
+// set: it stages the copy without the object lock, then commits it under the
+// lock. A failure that published nothing clears the attempt's marker. A
+// publication that reached some drives but not a write quorum converges
+// under the same lock hold, and in bounded background retries after it.
+func (er erasureObjects) restoreVersion(ctx context.Context, bucket, object string, opts ObjectOptions, h restoreHooks) error {
+	st, err := er.stageRestore(ctx, bucket, object, opts)
+	if err != nil {
+		return restoreAbandon(err, h)
+	}
+	defer er.deleteAll(context.Background(), minioMetaTmpBucket, st.tmpID)
+
+	lkctx, unlock, err := h.lock(ctx)
+	if err != nil {
+		return restoreAbandon(err, h)
+	}
+	pub, err := er.commitRestore(lkctx.Context(), bucket, object, st, opts, h.placed)
+	if err == nil {
+		unlock()
+		return nil
+	}
+	if pub == nil {
+		unlock()
+		return restoreAbandon(err, h)
+	}
+	res, cerr := er.convergeRestore(lkctx.Context(), pub)
+	unlock()
+	switch res {
+	case restoreCommitted:
+		return nil
+	case restoreConverged:
+		return err
+	case restoreStopped:
+		er.logRestoreNotConverged(pub, cerr)
+		return err
+	default:
+		er.recoverRestorePublication(pub, h.lock)
+		return err
+	}
+}
+
+// restoreAbandon clears the marker of an attempt that published nothing,
+// unless another attempt owns the version. A not-found error may come from the
+// tier rather than from the version, and the clear checks the marker's owner
+// and finds nothing to do for a version that is gone, so it runs then too.
+func restoreAbandon(err error, h restoreHooks) error {
+	if errors.Is(err, errRestoreSuperseded) {
+		return err
+	}
+	if h.clear != nil {
+		h.clear()
+	}
+	return err
+}
+
+// RestoreTransitionedObject - restore transitioned object content locally on
+// this cluster. This is similar to PostObjectRestore from AWS GLACIER storage
+// class. When PostObjectRestore API is called, a temporary copy of the object
+// is restored locally to the bucket on source cluster until the restore
+// expiry date. The copy that was transitioned continues to reside in the
+// transitioned tier.
+//
+// The set entry serves direct callers of one set. It takes the object lock in
+// the set's own domain and no restore lock; native restores of a cluster go
+// through erasureServerPools.
+func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+	lock := func(ctx context.Context) (LockContext, func(), error) {
+		lk := er.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return LockContext{}, nil, err
+		}
+		return lkctx, func() { lk.Unlock(lkctx) }, nil
+	}
+	clearMarker := func() {
+		ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
+		defer cancel()
+		lkctx, unlock, err := lock(ctx)
+		if err != nil {
+			storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, err))
+			return
+		}
+		defer unlock()
+		versionID := opts.VersionID
+		if a := opts.Transition.attempt; a != nil {
+			versionID = a.VersionID
+		}
+		if err := er.clearRestoreStatus(lkctx.Context(), bucket, object, versionID, opts.Transition.attempt); err != nil &&
+			!isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !isErrMethodNotAllowed(err) {
+			storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, versionID, err))
+		}
+	}
+	return er.restoreVersion(ctx, bucket, object, opts, restoreHooks{lock: lock, clear: clearMarker})
+}
+
+// RestoreTransitionedObject - restore transitioned object content locally on
+// this cluster, see erasureObjects.RestoreTransitionedObject.
+//
+// A POST hands over the restore lock R of the version it registered in
+// opts.Transition.owner; a direct caller takes R here, waiting while ctx
+// lives. The restore runs while R is held, and releases it when it returns.
+func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+	object = encodeDirObject(object)
+	owner := opts.Transition.owner
+	opts.Transition.owner = nil
+	if owner != nil && owner.ctx.Err() != nil {
+		// The lock POST handed over ended before the restore ran, such as
+		// when its holder lost it while the restore was queued: take it again
+		// as a direct caller does. The attempt still has to own the version.
+		owner = nil
+	}
+	defer func() { owner.release() }()
+
+	// Avoid restoring object from a pool being decommissioned.
+	opts.SkipDecommissioned = true
+	opts.NoLock = true
+	clearMarker := func() {
+		if err := z.resetRestoreStatus(bucket, object, opts); err != nil {
+			z.recoverRestoreStatus(bucket, object, opts)
+		}
+	}
+	idx, err := z.restorePoolIdx(ctx, bucket, object, opts)
+	if err != nil {
+		var reset restoreResetError
+		if errors.As(err, &reset) {
+			clearMarker()
+			return reset.error
+		}
+		if isErrObjectNotFound(err) && opts.VersionID != "" {
+			err = VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID}
+		}
+		return err
+	}
+	// A server may have been replaced by an older binary while the restore
+	// was queued: such a server knows neither the restore lock nor the
+	// staged commit.
+	if err := nativeRestoreAvailable(ctx); err != nil {
+		clearMarker()
+		return err
+	}
+	er := z.serverPools[idx].getHashedSet(object)
+	if owner == nil {
+		if opts.VersionID == "" {
+			// Restore the version that is latest now, by its own ID.
+			gen, err := er.readRestoreGeneration(ctx, bucket, object, "")
+			if err != nil {
+				return toObjectErr(err, bucket, object)
+			}
+			opts.VersionID = restoreVersionID(gen.fi.VersionID)
+		}
+		if owner, err = z.takeRestoreOwner(ctx, bucket, object, opts.VersionID, nil); err != nil {
+			return err
+		}
+	}
+	return er.restoreVersion(owner.ctx, bucket, object, opts, restoreHooks{
+		lock: func(ctx context.Context) (LockContext, func(), error) {
+			lk := z.restoreLock(bucket, object)
+			lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+			if err != nil {
+				return LockContext{}, nil, err
+			}
+			return lkctx, func() { lk.Unlock(lkctx) }, nil
+		},
+		placed: func(ctx context.Context) error {
+			got, err := z.restorePoolIdx(ctx, bucket, object, opts)
+			var reset restoreResetError
+			if errors.As(err, &reset) {
+				return reset.error
+			}
+			if err != nil {
+				return err
+			}
+			if got != idx {
+				return errRestoreMoved
+			}
+			return nil
+		},
+		clear: clearMarker,
+	})
+}

cmd/restore-two-phase_test.go

@@ -0,0 +1,858 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/config/storageclass"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// A native restore downloads its tier copy without the object lock: while
+// the download of an old version is paused, the hot versions of the same key
+// stay readable and writable. Only the short commit takes the object lock.
+func TestRestoreDownloadKeepsKeyAvailable(t *testing.T) {
+	for _, pools := range []int{1, 2} {
+		for _, multipart := range []bool{false, true} {
+			t.Run(fmt.Sprintf("pools=%d/multipart=%v", pools, multipart), func(t *testing.T) {
+				r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+				initConfigSubsystem(t.Context(), r.z)
+				oi, body := nativeRestoreSeed(t, r, 0, false, multipart)
+				hot := putConsistencyObject(t, r.z, r.bucket, r.object, 0, "hot latest", ObjectOptions{Versioned: true, MTime: oi.ModTime.Add(time.Hour)})
+				putConsistencyObject(t, r.z, r.bucket, "unrelated", 0, "unrelated hot object", ObjectOptions{Versioned: true})
+				gate := newRestoreIsolationGateTier(1, nil)
+				globalTierConfigMgr.drivercache["WARM"] = gate
+				defer gate.unblock()
+				done := make(chan error, 1)
+				go func() {
+					done <- r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+				}()
+				restoreIsolationEntered(t, gate)
+
+				within := func(name string, fn func(ctx context.Context) error) {
+					t.Helper()
+					ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+					defer cancel()
+					start := time.Now()
+					if err := fn(ctx); err != nil {
+						t.Fatalf("%s during a paused restore download: %v after %s", name, err, time.Since(start))
+					}
+				}
+				for _, key := range []string{"unrelated", r.object} {
+					within("GET "+key, func(ctx context.Context) error {
+						gr, err := r.z.GetObjectNInfo(ctx, r.bucket, key, nil, http.Header{}, ObjectOptions{})
+						if err != nil {
+							return err
+						}
+						defer gr.Close()
+						_, err = io.ReadAll(gr)
+						return err
+					})
+				}
+				within("HEAD hot latest", func(ctx context.Context) error {
+					got, err := r.z.GetObjectInfo(ctx, r.bucket, r.object, ObjectOptions{})
+					if err == nil && got.VersionID != hot.VersionID {
+						err = fmt.Errorf("latest is %s, want %s", got.VersionID, hot.VersionID)
+					}
+					return err
+				})
+				within("PutObjectTags hot latest", func(ctx context.Context) error {
+					_, err := r.z.PutObjectTags(ctx, r.bucket, r.object, "during=restore", ObjectOptions{VersionID: hot.VersionID, Versioned: true})
+					return err
+				})
+				// The download publishes nothing: no upload, public or not.
+				if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+					t.Errorf("restore created uploads: %v", uploads)
+				}
+				within("PUT new version", func(ctx context.Context) error {
+					data := []byte("written during restore")
+					_, err := r.z.PutObject(ctx, r.bucket, r.object,
+						mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true})
+					return err
+				})
+
+				gate.unblock()
+				select {
+				case err := <-done:
+					if err != nil {
+						t.Fatal(err)
+					}
+				case <-time.After(time.Minute):
+					t.Fatal("restore did not complete after releasing the tier")
+				}
+				if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+					t.Fatalf("restored version mismatch: %v", err)
+				}
+			})
+		}
+	}
+}
+
+// restoreTwoPhaseStagedCommitFailure makes a multipart completion stage its
+// object metadata in the upload and then fail before moving any data.
+type restoreTwoPhaseStagedCommitFailure struct {
+	StorageAPI
+	bucket, object string
+}
+
+func (d restoreTwoPhaseStagedCommitFailure) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
+	if sv == minioMetaMultipartBucket && dv == d.bucket && dp == d.object {
+		if err := d.WriteMetadata(ctx, "", sv, sp, fi); err != nil {
+			return RenameDataResp{}, err
+		}
+		return RenameDataResp{}, errFaultyDisk
+	}
+	return d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
+}
+
+// An ordinary upload whose completion failed after staging stays usable as
+// on main: the client re-uploads its parts and completes, or aborts.
+func TestMultipartFailedCommitRetriesLikeMain(t *testing.T) {
+	for _, abort := range []bool{false, true} {
+		t.Run(fmt.Sprintf("abort=%v", abort), func(t *testing.T) {
+			z, bucket := consistencyPools(t)
+			initConfigSubsystem(t.Context(), z)
+			const object = "public-upload"
+			er := z.serverPools[0].getHashedSet(object)
+			ctx := t.Context()
+			mp, err := er.NewMultipartUpload(ctx, bucket, object, ObjectOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			body := []byte("ordinary multipart data")
+			putPart := func() (PartInfo, error) {
+				return er.PutObjectPart(ctx, bucket, object, mp.UploadID, 1,
+					mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
+			}
+			p, err := putPart()
+			if err != nil {
+				t.Fatal(err)
+			}
+			disks := er.getDisks()
+			wrapped := make([]StorageAPI, len(disks))
+			for i, d := range disks {
+				wrapped[i] = restoreTwoPhaseStagedCommitFailure{d, bucket, object}
+			}
+			previous := er.getDisks
+			er.getDisks = func() []StorageAPI { return wrapped }
+			_, err = er.CompleteMultipartUpload(ctx, bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: p.ETag}}, ObjectOptions{})
+			er.getDisks = previous
+			if err == nil {
+				t.Fatal("fault injection failed")
+			}
+			if _, err := er.GetMultipartInfo(ctx, bucket, object, mp.UploadID, ObjectOptions{}); err != nil {
+				t.Fatalf("upload after a failed commit: %v", err)
+			}
+			if abort {
+				if err := er.AbortMultipartUpload(ctx, bucket, object, mp.UploadID, ObjectOptions{}); err != nil {
+					t.Fatalf("abort after a failed commit: %v", err)
+				}
+				if _, err := er.GetMultipartInfo(ctx, bucket, object, mp.UploadID, ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) {
+					t.Fatalf("aborted upload still answers: %v", err)
+				}
+				return
+			}
+			if p, err = putPart(); err != nil {
+				t.Fatalf("re-upload a part after a failed commit: %v", err)
+			}
+			if _, err := er.CompleteMultipartUpload(ctx, bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: p.ETag}}, ObjectOptions{}); err != nil {
+				t.Fatalf("complete after re-uploading: %v", err)
+			}
+			gr, err := er.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{})
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer gr.Close()
+			if got, err := io.ReadAll(gr); err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("completed object = %q, %v", got, err)
+			}
+		})
+	}
+}
+
+// restoreTwoPhaseFaults are the faults restoreTwoPhaseDisk injects, shared by
+// a test and its drives so that the test can turn them off.
+type restoreTwoPhaseFaults struct {
+	// rename: "" passes the publication through, "fail" fails it without
+	// applying it, "apply-fail" applies it and then reports a failure.
+	rename atomic.Value
+	// oldDataDir, when set, replaces the old data directory a publication
+	// reports.
+	oldDataDir atomic.Pointer[string]
+	// writeFails fails this many metadata writes of the object.
+	writeFails atomic.Int32
+	// writes counts the metadata writes of the object.
+	writes atomic.Int32
+	// recreateTmp drops the staging directory before the second part is
+	// written, as a server restart moves the tmp volume away.
+	recreateTmp atomic.Bool
+}
+
+func (f *restoreTwoPhaseFaults) renameMode() string {
+	m, _ := f.rename.Load().(string)
+	return m
+}
+
+type restoreTwoPhaseDisk struct {
+	StorageAPI
+	bucket, object string
+	f              *restoreTwoPhaseFaults
+}
+
+func (d restoreTwoPhaseDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
+	if sv != minioMetaTmpBucket || dv != d.bucket || dp != d.object {
+		return d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
+	}
+	switch d.f.renameMode() {
+	case "fail":
+		return RenameDataResp{}, errFaultyDisk
+	case "apply-fail":
+		if _, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts); err != nil {
+			return RenameDataResp{}, err
+		}
+		return RenameDataResp{}, errFaultyDisk
+	}
+	resp, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
+	if old := d.f.oldDataDir.Load(); old != nil {
+		resp.OldDataDir = *old
+	}
+	return resp, err
+}
+
+func (d restoreTwoPhaseDisk) WriteMetadata(ctx context.Context, orig, vol, path string, fi FileInfo) error {
+	if vol == d.bucket && path == d.object {
+		d.f.writes.Add(1)
+		if d.f.writeFails.Load() > 0 {
+			d.f.writeFails.Add(-1)
+			return errFaultyDisk
+		}
+	}
+	return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi)
+}
+
+func (d restoreTwoPhaseDisk) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) error {
+	if volume == minioMetaTmpBucket && strings.HasSuffix(path, "/part.2") && d.f.recreateTmp.Load() {
+		tmpID, _, _ := strings.Cut(path, "/")
+		if err := os.RemoveAll(filepath.Join(d.Endpoint().Path, minioMetaTmpBucket, tmpID)); err != nil {
+			return err
+		}
+	}
+	return d.StorageAPI.CreateFile(ctx, origvolume, volume, path, size, reader)
+}
+
+// restoreTwoPhaseFaulty wraps the drives of the object's set in the first
+// pool; faults[i] is nil for a drive left as it is. It returns the plain
+// drives, for reads that see what is stored.
+func restoreTwoPhaseFaulty(t *testing.T, r *restoreTest, faults ...*restoreTwoPhaseFaults) []StorageAPI {
+	t.Helper()
+	er := r.z.serverPools[0].getHashedSet(r.object)
+	disks := er.getDisks()
+	previous := er.getDisks
+	t.Cleanup(func() { er.getDisks = previous })
+	wrapped := append([]StorageAPI(nil), disks...)
+	for i, f := range faults {
+		if f != nil {
+			wrapped[i] = restoreTwoPhaseDisk{StorageAPI: disks[i], bucket: r.bucket, object: r.object, f: f}
+		}
+	}
+	er.getDisks = func() []StorageAPI { return wrapped }
+	return disks
+}
+
+// restoreTwoPhaseEntries reads the version from each drive.
+func restoreTwoPhaseEntries(t *testing.T, r *restoreTest, disks []StorageAPI, versionID string) []FileInfo {
+	t.Helper()
+	infos, errs := readAllFileInfo(t.Context(), disks, "", r.bucket, r.object, restoreTestVersionKey(versionID), false, false)
+	for i, err := range errs {
+		if err != nil {
+			t.Fatalf("drive %d: %v", i, err)
+		}
+	}
+	return infos
+}
+
+func restoreTwoPhaseUnrestored(fi FileInfo) bool {
+	return fi.Metadata[xhttp.AmzRestore] == "" && fi.Metadata[restoreAttemptKey] == "" && fi.IsRemote()
+}
+
+func restoreTwoPhaseRestored(fi FileInfo) bool {
+	st, err := parseRestoreObjStatus(fi.Metadata[xhttp.AmzRestore])
+	return err == nil && !st.Ongoing() && !fi.IsRemote() && fi.Metadata[restoreAttemptKey] == ""
+}
+
+// restoreTwoPhaseDelays shortens the convergence retries for a test.
+func restoreTwoPhaseDelays(t *testing.T, delays ...time.Duration) {
+	previous := restorePublicationDelays
+	restorePublicationDelays = delays
+	t.Cleanup(func() { restorePublicationDelays = previous })
+}
+
+// A publication that reaches fewer drives than the write quorum is brought
+// back to the unrestored version on every drive, under the same lock hold.
+// No directory is deleted: the staged one is left to abandoned-data cleanup.
+// The version restores again once the drives work.
+func TestRestorePublicationBelowQuorumReverts(t *testing.T) {
+	for _, multipart := range []bool{false, true} {
+		t.Run(fmt.Sprintf("multipart=%v", multipart), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			oi, body := nativeRestoreSeed(t, r, 0, false, multipart)
+			faults := &restoreTwoPhaseFaults{}
+			faults.rename.Store("fail")
+			disks := restoreTwoPhaseFaulty(t, r, faults, faults)
+			before := restoreTwoPhaseEntries(t, r, disks, oi.VersionID)
+			err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+			if err == nil {
+				t.Fatal("a publication on two of four drives succeeded")
+			}
+			for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+				if !restoreTwoPhaseUnrestored(fi) || fi.DataDir != before[i].DataDir {
+					t.Errorf("drive %d after a failed publication: restore %q, remote %v, data dir %s (was %s)",
+						i, fi.Metadata[xhttp.AmzRestore], fi.IsRemote(), fi.DataDir, before[i].DataDir)
+				}
+			}
+			// The drives that published keep the staged directory.
+			for i := 2; i < len(disks); i++ {
+				entries, err := os.ReadDir(filepath.Join(disks[i].Endpoint().Path, r.bucket, r.object))
+				if err != nil {
+					t.Fatal(err)
+				}
+				dirs := 0
+				for _, e := range entries {
+					if e.IsDir() && e.Name() != before[i].DataDir {
+						dirs++
+					}
+				}
+				if dirs == 0 {
+					t.Errorf("drive %d: the published data directory was deleted", i)
+				}
+			}
+			faults.rename.Store("")
+			if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+				t.Fatalf("restore after the drives recovered: %v", err)
+			}
+			if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("restored contents: %v", err)
+			}
+		})
+	}
+}
+
+// A publication whose drives applied it but did not answer is found on a
+// write quorum when the worker reads the drives again: the restore stands.
+func TestRestorePublicationUncertainCommits(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	oi, body := nativeRestoreSeed(t, r, 0, false, true)
+	faults := &restoreTwoPhaseFaults{}
+	faults.rename.Store("apply-fail")
+	disks := restoreTwoPhaseFaulty(t, r, faults, faults)
+	if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+		t.Fatalf("restore whose answers were lost: %v", err)
+	}
+	for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+		if !restoreTwoPhaseRestored(fi) {
+			t.Errorf("drive %d: not restored: %q", i, fi.Metadata[xhttp.AmzRestore])
+		}
+	}
+	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+		t.Fatalf("restored contents: %v", err)
+	}
+}
+
+// A convergence that cannot write every drive under the commit's lock hold
+// finishes in a background retry.
+func TestRestoreConvergenceRetries(t *testing.T) {
+	restoreTwoPhaseDelays(t, 50*time.Millisecond, 50*time.Millisecond, 50*time.Millisecond)
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+	failing, flaky := &restoreTwoPhaseFaults{}, &restoreTwoPhaseFaults{}
+	failing.rename.Store("fail")
+	flaky.writeFails.Store(1)
+	disks := restoreTwoPhaseFaulty(t, r, failing, failing, flaky)
+	if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err == nil {
+		t.Fatal("a publication on two of four drives succeeded")
+	}
+	deadline := time.Now().Add(restoreTestWait)
+	for {
+		settled := true
+		for _, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+			settled = settled && restoreTwoPhaseUnrestored(fi)
+		}
+		if settled {
+			return
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("background convergence did not finish")
+		}
+		time.Sleep(20 * time.Millisecond)
+	}
+}
+
+// A convergence retry never overwrites what someone else wrote between two
+// lock holds: a metadata update or a newer attempt's marker stops it.
+func TestRestoreConvergenceStopsOnOtherWrites(t *testing.T) {
+	for _, other := range []string{"tags", "successor"} {
+		t.Run(other, func(t *testing.T) {
+			restoreTwoPhaseDelays(t, 300*time.Millisecond, 300*time.Millisecond, 300*time.Millisecond)
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			initConfigSubsystem(t.Context(), r.z)
+			oi, body := restoreIsolationSeed(t, r, 0, false, false)
+			failing, stuck := &restoreTwoPhaseFaults{}, &restoreTwoPhaseFaults{}
+			failing.rename.Store("fail")
+			stuck.writeFails.Store(1 << 20)
+			disks := restoreTwoPhaseFaulty(t, r, failing, failing, stuck)
+			if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err == nil {
+				t.Fatal("a publication on two of four drives succeeded")
+			}
+			var successor string
+			release := make(chan struct{})
+			layer := newRestoreIsolationLayer(r)
+			switch other {
+			case "tags":
+				if _, err := r.z.PutObjectTags(t.Context(), r.bucket, r.object, "kept=yes", r.opts(oi.VersionID)); err != nil {
+					t.Fatal(err)
+				}
+			case "successor":
+				layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+					<-release
+					return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+				}
+				if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+					close(release)
+					t.Fatalf("successor POST: %d %s", rec.Code, rec.Body.String())
+				}
+				successor = restoreTwoPhaseEntries(t, r, disks, oi.VersionID)[0].Metadata[restoreAttemptKey]
+				if successor == "" {
+					close(release)
+					t.Fatal("successor POST recorded no attempt")
+				}
+			}
+			writes := stuck.writes.Load()
+			time.Sleep(time.Second)
+			if got := stuck.writes.Load(); got != writes {
+				t.Errorf("a stopped convergence kept writing: %d -> %d writes", writes, got)
+			}
+			for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+				if i == 2 {
+					continue // the drive the convergence could not write
+				}
+				switch other {
+				case "tags":
+					if fi.Metadata[xhttp.AmzObjectTagging] != "kept=yes" {
+						t.Errorf("drive %d: the tags written between retries were overwritten: %q", i, fi.Metadata[xhttp.AmzObjectTagging])
+					}
+				case "successor":
+					if fi.Metadata[restoreAttemptKey] != successor {
+						t.Errorf("drive %d: the successor's marker was overwritten: %q", i, fi.Metadata[restoreAttemptKey])
+					}
+				}
+			}
+			if other == "successor" {
+				failing.rename.Store("")
+				stuck.writeFails.Store(0)
+				close(release)
+				if err := layer.wait(t); err != nil {
+					t.Fatalf("successor restore: %v", err)
+				}
+				if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+					t.Fatalf("restored contents: %v", err)
+				}
+			}
+		})
+	}
+}
+
+// The data directory of the replaced entry goes only where every drive that
+// reports it agrees on it; a drive that does not report it, as when another
+// version shares the directory, keeps the directory everywhere.
+func TestRestoreRemovesOldDataDirOnlyWhenUnanimous(t *testing.T) {
+	for _, mixed := range []bool{false, true} {
+		t.Run(fmt.Sprintf("mixed=%v", mixed), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			oi, body := nativeRestoreSeed(t, r, 0, false, true)
+			old := r.fileInfo(oi.VersionID).DataDir
+			if old == "" {
+				t.Fatal("fixture: the transitioned version has no data directory")
+			}
+			faults := &restoreTwoPhaseFaults{}
+			if mixed {
+				none := ""
+				faults.oldDataDir.Store(&none)
+			}
+			restoreTwoPhaseFaulty(t, r, faults)
+			if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+				t.Fatal(err)
+			}
+			if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("restored contents: %v", err)
+			}
+			n := r.dataDirs(old)
+			if mixed && n != 4 {
+				t.Errorf("drives disagreed on the old data directory, but %d of 4 kept it", n)
+			}
+			if !mixed && n != 0 {
+				t.Errorf("the unanimous old data directory remained on %d drives", n)
+			}
+		})
+	}
+}
+
+// An earlier restore of a small object may have inlined its data, and the
+// expiry of that copy leaves the inline marker behind. A new restore writes
+// a data directory and must not publish the marker with it.
+func TestRestoreDropsStaleInlineMarker(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	oi, body := nativeRestoreSeed(t, r, 0, false, false)
+	disks := r.z.serverPools[0].getHashedSet(r.object).getDisks()
+	for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+		fi.SetInlineData()
+		if err := disks[i].WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+		t.Fatal(err)
+	}
+	for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+		if fi.InlineData() {
+			t.Errorf("drive %d: the restored copy is marked inline", i)
+		}
+	}
+	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+		t.Fatalf("restored contents: %v", err)
+	}
+}
+
+// A drive whose staging directory was moved away and recreated by a later
+// part misses the earlier parts: it is left out of the commit, and when too
+// few drives are left nothing is published at all.
+func TestRestoreStagingDropsRecreatedTmp(t *testing.T) {
+	for _, recreated := range []int{1, 2} {
+		t.Run(fmt.Sprintf("drives=%d", recreated), func(t *testing.T) {
+			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+			oi, body := nativeRestoreSeed(t, r, 0, false, true)
+			faults := &restoreTwoPhaseFaults{}
+			faults.recreateTmp.Store(true)
+			wrap := make([]*restoreTwoPhaseFaults, recreated)
+			for i := range wrap {
+				wrap[i] = faults
+			}
+			disks := restoreTwoPhaseFaulty(t, r, wrap...)
+			err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+			entries := restoreTwoPhaseEntries(t, r, disks, oi.VersionID)
+			if recreated == 2 {
+				if err == nil {
+					t.Fatal("a copy staged completely on two of four drives was published")
+				}
+				for i, fi := range entries {
+					if !restoreTwoPhaseUnrestored(fi) {
+						t.Errorf("drive %d after a failed staging: %q", i, fi.Metadata[xhttp.AmzRestore])
+					}
+				}
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			if restoreTwoPhaseRestored(entries[0]) {
+				t.Error("the drive with missing parts got the restored copy")
+			}
+			for i := 1; i < len(entries); i++ {
+				if !restoreTwoPhaseRestored(entries[i]) {
+					t.Errorf("drive %d: not restored", i)
+				}
+			}
+			if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+				t.Fatalf("restored contents: %v", err)
+			}
+		})
+	}
+}
+
+// A POST while the worker of an earlier POST holds the restore lock answers
+// that a restore is in progress; it starts no second worker.
+func TestRestoreSecondPOSTWhileRunning(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	initConfigSubsystem(t.Context(), r.z)
+	oi, body := restoreIsolationSeed(t, r, 0, false, true)
+	release := make(chan struct{})
+	layer := newRestoreIsolationLayer(r)
+	layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+		<-release
+		return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+	}
+	if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+		t.Fatalf("first POST: %d %s", rec.Code, rec.Body.String())
+	}
+	rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+	close(release)
+	if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "RestoreAlreadyInProgress") {
+		t.Errorf("second POST: %d %s", rec.Code, rec.Body.String())
+	}
+	if err := layer.wait(t); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case err := <-layer.done:
+		t.Fatalf("a second restore ran: %v", err)
+	default:
+	}
+	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+		t.Fatalf("restored contents: %v", err)
+	}
+}
+
+// A version deleted while its tier copy downloads is not brought back.
+func TestRestoreTargetDeletedDuringDownload(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	initConfigSubsystem(t.Context(), r.z)
+	oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+	gate := newRestoreIsolationGateTier(1, nil)
+	globalTierConfigMgr.drivercache["WARM"] = gate
+	defer gate.unblock()
+	done := make(chan error, 1)
+	go func() {
+		done <- r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+	}()
+	restoreIsolationEntered(t, gate)
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	_, err := r.z.DeleteObject(ctx, r.bucket, r.object, r.opts(oi.VersionID))
+	cancel()
+	if err != nil {
+		t.Fatalf("delete during the download: %v", err)
+	}
+	gate.unblock()
+	select {
+	case err := <-done:
+		if err == nil {
+			t.Fatal("the restore of a deleted version succeeded")
+		}
+	case <-time.After(restoreTestWait):
+		t.Fatal("restore did not finish")
+	}
+	if _, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+		t.Fatalf("deleted version after the restore: %v", err)
+	}
+}
+
+// With several pools, abandoned-data cleanup takes the pools lock that every
+// write of the object takes, also where a pool's sets have lockers of their
+// own: while a writer holds it, cleanup removes nothing, such as a data
+// directory that a commit moved in place and has not yet referenced.
+func TestCheckAbandonedPartsTakesPoolsLock(t *testing.T) {
+	r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+	nativeRestoreSeed(t, r, 1, false, false)
+	for _, set := range r.z.serverPools[1].sets {
+		previous := set.nsMutex
+		set.nsMutex = newNSLock(false)
+		t.Cleanup(func() { set.nsMutex = previous })
+	}
+	disks := r.z.serverPools[1].getHashedSet(r.object).getDisks()
+	moved := mustGetUUID()
+	for _, disk := range disks {
+		dir := filepath.Join(disk.Endpoint().Path, r.bucket, r.object, moved)
+		if err := os.MkdirAll(dir, 0o755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(filepath.Join(dir, "part.1"), []byte("staged"), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	present := func() int {
+		n := 0
+		for _, disk := range disks {
+			if _, err := os.Stat(filepath.Join(disk.Endpoint().Path, r.bucket, r.object, moved)); err == nil {
+				n++
+			}
+		}
+		return n
+	}
+	lk := r.z.NewObjectNSLock(r.bucket, r.object)
+	lkctx, err := lk.GetLock(t.Context(), globalOperationTimeout)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(t.Context(), time.Second)
+	r.z.CheckAbandonedParts(ctx, r.bucket, r.object, madmin.HealOpts{Remove: true})
+	cancel()
+	held := present()
+	lk.Unlock(lkctx)
+	if held != len(disks) {
+		t.Fatalf("abandoned-data cleanup removed a moved directory from %d drives while a writer held the object lock", len(disks)-held)
+	}
+	if err := r.z.CheckAbandonedParts(t.Context(), r.bucket, r.object, madmin.HealOpts{Remove: true}); err != nil {
+		t.Fatalf("abandoned-data cleanup without a writer: %v", err)
+	}
+	if n := present(); n != 0 {
+		t.Fatalf("abandoned-data cleanup left the unreferenced directory on %d drives", n)
+	}
+}
+
+// The header flag that tells whether a transitioned version uses its data
+// directory follows whether its restored copy has expired when a drive writes
+// the entry. A publication's own entry is recognized with either value.
+func TestRestorePublishedRevisionAcrossExpiry(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+	fi := r.fileInfo(oi.VersionID)
+	expiry := time.Now().Add(2 * time.Second).Truncate(time.Second).Add(time.Second)
+	st := &restoreStage{dataDir: mustGetUUID(), expiry: expiry}
+	published := restorePublished(fi, fi, st, 0)
+	keys, err := fileInfoRevisions(published)
+	if err != nil {
+		t.Fatal(err)
+	}
+	stored := func() revisionKey {
+		var x xlMetaV2
+		if err := x.AddVersion(published); err != nil {
+			t.Fatal(err)
+		}
+		buf, err := x.AppendTo(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var y xlMetaV2
+		if err := y.Load(buf); err != nil {
+			t.Fatal(err)
+		}
+		return versionRevision(y.versions[0], y.metaV)
+	}
+	before := stored()
+	time.Sleep(time.Until(expiry) + 100*time.Millisecond)
+	after := stored()
+	if before == after {
+		t.Fatal("fixture: the stored flag did not change at the expiry")
+	}
+	for _, got := range []revisionKey{before, after} {
+		if got != keys[0] && got != keys[1] {
+			t.Errorf("a publication's own entry is not recognized: %v not in %v", got, keys)
+		}
+	}
+}
+
+// A tier that lost the copy answers not found. That is no reason to keep the
+// version's ongoing marker: the attempt clears it, and a later POST can
+// restore the version again.
+func TestRestoreTierNotFoundClearsMarker(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	initConfigSubsystem(t.Context(), r.z)
+	oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+	tier := globalTierConfigMgr.drivercache["WARM"]
+	globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, ObjectNotFound{Bucket: r.bucket, Object: r.object}}
+	layer := newRestoreIsolationLayer(r)
+	if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+		t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+	}
+	if err := layer.wait(t); !isErrObjectNotFound(err) {
+		t.Fatalf("restore from a tier without the copy: %v", err)
+	}
+	info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+	if err != nil || info.RestoreOngoing || info.UserDefined[xhttp.AmzRestore] != "" {
+		t.Fatalf("version after the failed restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+	}
+}
+
+// newRestoreWideTest is newRestoreSafetyTest with one pool of twelve drives,
+// whose default layout is eight data and four parity shards: a version's
+// metadata reaches its read quorum on seven drives, one short of the write
+// quorum of eight that its shards need.
+func newRestoreWideTest(t *testing.T) *restoreTest {
+	t.Helper()
+	prevTier := globalTierConfigMgr
+	t.Cleanup(func() { globalTierConfigMgr = prevTier })
+	dirs, err := getRandomDisks(12)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { removeRoots(dirs) })
+	obj, _, err := initObjectLayer(t.Context(), mustGetPoolEndpoints(0, dirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	z := obj.(*erasureServerPools)
+	t.Cleanup(func() { z.Shutdown(context.Background()) })
+	// Other tests leave a storage class behind; fix the layout to 8+4.
+	saved := globalStorageClass
+	globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
+	t.Cleanup(func() { globalStorageClass.Update(saved) })
+	globalTierConfigMgr = NewTierConfigMgr()
+	globalTierConfigMgr.drivercache["WARM"] = &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
+	r := &restoreTest{t: t, z: z, bucket: getRandomBucketName(), object: "obj", labels: map[string]string{}, bodies: map[string][]byte{}}
+	if err := z.MakeBucket(t.Context(), r.bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
+		t.Fatal(err)
+	}
+	meta, err := globalBucketMetadataSys.Get(r.bucket)
+	if err != nil {
+		t.Fatal(err)
+	}
+	meta.LifecycleConfigXML = []byte(restoreSafetyRules)
+	if err := meta.Save(t.Context(), z); err != nil {
+		t.Fatal(err)
+	}
+	globalBucketMetadataSys.Set(r.bucket, meta)
+	return r
+}
+
+// On twelve drives with eight data shards, a publication that reached seven
+// drives has the metadata read quorum but not the eight shards a read needs.
+// It is not a restore: the worker brings every drive back to the unrestored
+// version, and the version reads from its tier copy.
+func TestRestorePublicationBelowWriteQuorumOnWideSet(t *testing.T) {
+	r := newRestoreWideTest(t)
+	oi, body := nativeRestoreSeed(t, r, 0, false, true)
+	fi := r.fileInfo(oi.VersionID)
+	if fi.Erasure.DataBlocks != 8 || fi.Erasure.ParityBlocks != 4 {
+		t.Fatalf("fixture layout %d+%d, want 8+4", fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks)
+	}
+	faults := &restoreTwoPhaseFaults{}
+	faults.rename.Store("fail")
+	disks := restoreTwoPhaseFaulty(t, r, faults, faults, faults, faults, faults)
+	if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err == nil {
+		t.Fatal("a publication on seven of twelve drives succeeded")
+	}
+	for i, e := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+		if !restoreTwoPhaseUnrestored(e) || e.DataDir != fi.DataDir {
+			t.Errorf("drive %d after a publication on seven drives: restore %q, remote %v", i, e.Metadata[xhttp.AmzRestore], e.IsRemote())
+		}
+	}
+	if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+		t.Fatalf("the version no longer reads from its tier copy: %v", err)
+	}
+	faults.rename.Store("")
+	if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+		t.Fatalf("restore after the drives recovered: %v", err)
+	}
+	for i, e := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+		if !restoreTwoPhaseRestored(e) {
+			t.Errorf("drive %d: not restored", i)
+		}
+	}
+}

cmd/xl-storage-format-v2.go

@@ -1372,6 +1372,26 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) {
 		}
 	}
 
+	if fi.ExpireRestored {
+		i, ver, err := x.findVersion(uv)
+		if err != nil {
+			return "", err
+		}
+		// Check the stored type and transition before any marker, legacy or
+		// replication mutation. Only completed tiered objects own a cache.
+		if ver.Type != ObjectType || string(ver.ObjectV2.MetaSys[metaTierStatus]) != lifecycle.TransitionComplete {
+			return "", nil
+		}
+		ver.ObjectV2.RemoveRestoreHdrs()
+		if err := x.setIdx(i, *ver); err != nil {
+			return "", err
+		}
+		if x.SharedDataDirCount(ver.ObjectV2.VersionID, ver.ObjectV2.DataDir) > 0 {
+			return "", nil
+		}
+		return uuid.UUID(ver.ObjectV2.DataDir).String(), nil
+	}
+
 	var ventry xlMetaV2Version
 	if fi.Deleted {
 		ventry = xlMetaV2Version{
@@ -1496,11 +1516,8 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) {
 		if err != nil {
 			return "", err
 		}
-		switch {
-		case fi.ExpireRestored:
-			ver.ObjectV2.RemoveRestoreHdrs()
-			err = x.setIdx(i, *ver)
-		case fi.TransitionStatus == lifecycle.TransitionComplete:
+		switch fi.TransitionStatus {
+		case lifecycle.TransitionComplete:
 			ver.ObjectV2.SetTransition(fi)
 			ver.ObjectV2.ResetInlineData()
 			err = x.setIdx(i, *ver)

cmd/xl-storage.go

@@ -1371,6 +1371,9 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 	}
 
 	if legacyJSON {
+		if fi.ExpireRestored {
+			return nil // Legacy objects cannot contain a completed transition.
+		}
 		// Delete the meta file, if there are no more versions the
 		// top level parent is automatically removed.
 		return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true, false)

internal/grid/handlers.go

@@ -116,6 +116,7 @@ const (
 	HandlerRenamePart
 	HandlerClearUploadID
 	HandlerCheckParts3
+	HandlerRestoreCapability
 
 	// Add more above here ^^^
 	// If all handlers are used, the type of Handler can be changed.
@@ -200,6 +201,7 @@ var handlerPrefixes = [handlerLast]string{
 	HandlerCheckParts3:                 storagePrefix,
 	HandlerRenamePart:                  storagePrefix,
 	HandlerClearUploadID:               peerPrefix,
+	HandlerRestoreCapability:           peerPrefix,
 }
 
 const (

internal/grid/handlers_string.go

@@ -86,14 +86,15 @@ func _() {
 	_ = x[HandlerRenamePart-75]
 	_ = x[HandlerClearUploadID-76]
 	_ = x[HandlerCheckParts3-77]
-	_ = x[handlerTest-78]
-	_ = x[handlerTest2-79]
-	_ = x[handlerLast-80]
+	_ = x[HandlerRestoreCapability-78]
+	_ = x[handlerTest-79]
+	_ = x[handlerTest2-80]
+	_ = x[handlerLast-81]
 }
 
-const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3handlerTesthandlerTest2handlerLast"
+const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3RestoreCapabilityhandlerTesthandlerTest2handlerLast"
 
-var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 971, 983, 994}
+var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 977, 988, 1000, 1011}
 
 func (i HandlerID) String() string {
 	idx := int(i) - 0