fix(restore): give each native restore an owner and a private upload

A native restore (POST ?restore, issue #281) marks the version ongoing,
and a background worker rehydrates it from the tier. The worker, its
upload and every status write could be reached or confused by other
callers. A public client could write into or complete the restore
upload. POST copied a whole metadata snapshot over newer protection
changes. An old or failed attempt could act on a newer one. Restore,
expiry and heal decided on whichever drive's metadata they read.

Each POST that starts a rehydration now writes an attempt ID into the
version's internal metadata, with the identity it resolved: version
ID, ETag, size and ModTime. The worker, its upload, its part writes and
its completion carry the attempt. A worker whose attempt no longer
owns the version stops without writing, completing or resetting. Every
write uses the versioning state and explicit version ID captured by
POST, so a null target stays null.

The restore upload carries an internal marker, decided over all drives.
Public multipart calls answer NoSuchUpload and listings hide it. An
unknown marker fails closed. A completion holds a commit lock from
staging its commit in the upload until the commit ends. Stale cleanup
decides on every upload past the stale age under that lock, and keeps
it until its rename ends, also past the drive deadline. Abort of a
committing upload takes the lock and removes a failed commit's
leftover only when every copy is staged and older than one hour.

POST, the worker, the reset, the ExpireRestored recheck and heal all
decide on the authoritative revision of the exact version. That is the
revision held by max(n/2+1, read quorum) drives, identified by the
version header and the normalized metadata digest. Status writes patch
only the restore fields of that revision. They write only the drives
that hold it, and put the old revision back on failure. Heal copies
restore fields only when a quorum holds the same fields.

A failed attempt resets itself with bounded in-memory retries. A later
POST takes over a stranded marker when no attempt runs. Otherwise it
answers RestoreAlreadyInProgress within a few seconds.

Older nodes do not know the marker. A new grid capability handler lets
the serving node ask every peer. POST answers 503 ServiceUnavailable
and writes nothing until all peers support native restore, and a
queued worker asks again before it creates its upload.

The single-pool lock-domain split between DELETE/CopyObject and restore
is closed by its own fix, which this change depends on.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/bucket-lifecycle-restore-boundary_test.go

@@ -137,8 +137,7 @@ func TestExpireRestoredMetadataReadKeepsMinorityVersion(t *testing.T) {
 				case "restore-cleanup":
 					// The initial read succeeded before remote download failed;
 					// cleanup must not purge a now-incomplete target on reread.
-					opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
-					err = er.updateRestoreMetadata(ctx, r.bucket, r.object, fi.ToObjectInfo(r.bucket, r.object, true), opts)
+					err = er.clearRestoreStatus(ctx, r.bucket, r.object, oi.VersionID, nil)
 				case "lookup-cleanup", "lookup-restore":
 					// Version-sensitive lookup also covers native restore routing.
 					opts.MetadataChg = true

cmd/bucket-lifecycle-restore-safety_test.go

@@ -156,7 +156,7 @@ func TestExpireRestoredRenewedOrMissing(t *testing.T) {
 		t.Fatal(err)
 	}
 	err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
-	if !isErrVersionNotFound(err) {
+	if !errors.Is(err, errLifecycleNotApplicable) {
 		t.Errorf("stale task should be rejected, got %v", err)
 	}
 	if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed || got.DataDir != before.DataDir {

cmd/bucket-lifecycle.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/event"
 	xhttp "github.com/minio/minio/internal/http"
@@ -987,8 +988,10 @@ func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string
 	}, nil
 }
 
-// set ObjectOptions for PUT call to restore temporary copy of transitioned data
-func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) {
+// set ObjectOptions for PUT call to restore temporary copy of transitioned data.
+// opts are the restore's options, with the versioning POST captured: the
+// bucket's versioning may have changed since, and must not mint a new version.
+func putRestoreOpts(rreq *RestoreObjectRequest, objInfo ObjectInfo, opts ObjectOptions) (putOpts ObjectOptions) {
 	meta := make(map[string]string)
 	sc := rreq.OutputLocation.S3.StorageClass
 	if sc == "" {
@@ -1011,8 +1014,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 			meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
 		}
 		return ObjectOptions{
-			Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
-			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+			Versioned:        opts.Versioned,
+			VersionSuspended: opts.VersionSuspended,
 			UserDefined:      meta,
 		}
 	}
@@ -1020,19 +1023,38 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 	if len(objInfo.UserTags) != 0 {
 		meta[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
-	// Set restore object status
+	// Set restore object status; a completed restore has no attempt.
 	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
 	meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
+	delete(meta, restoreAttemptKey)
+	delete(meta, restoreUploadKey)
 	return ObjectOptions{
-		Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
-		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+		Versioned:        opts.Versioned,
+		VersionSuspended: opts.VersionSuspended,
 		UserDefined:      meta,
-		VersionID:        objInfo.VersionID,
+		VersionID:        restoreVersionID(objInfo.VersionID),
 		MTime:            objInfo.ModTime,
 		Expires:          objInfo.Expires,
 	}
 }
 
+// restoredCopyExpires reports whether the restored copy of oi, as read now,
+// expires by its own reason: lifecycle expires the restored copy of this
+// version, its restore deadline is valid and has passed, its transition is
+// complete and no restore is running. Another rule falling due is no reason.
+func restoredCopyExpires(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, oi ObjectInfo) bool {
+	if !evalActionFromLifecycle(ctx, lc, lr, rcfg, oi).Action.DeleteRestored() {
+		return false
+	}
+	status, err := parseRestoreObjStatus(oi.UserDefined[xhttp.AmzRestore])
+	if err != nil {
+		return false
+	}
+	expiry, completed := status.Expiry()
+	return completed && !expiry.IsZero() && UTCNow().After(expiry) &&
+		oi.TransitionedObject.Status == lifecycle.TransitionComplete && !oi.RestoreOngoing
+}
+
 var errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
 
 // IsRemote returns true if this object version's contents are in its remote

cmd/erasure-healing.go

@@ -373,6 +373,13 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if err != nil {
 		return result, err
 	}
+	// The restore fields fence native restores and restored-copy cleanup.
+	// Heal copies them only as a quorum of the drives holds them; while the
+	// drives disagree, it waits rather than create a quorum for one side.
+	latestMeta, err = restoreHealMeta(partsMetadata, latestMeta, max(len(storageDisks)/2+1, readQuorum))
+	if err != nil {
+		return result, toObjectErr(err, bucket, object, versionID)
+	}
 
 	// No modtime quorum
 	filterDisksByETag := quorumETag != ""

cmd/erasure-multipart-cleanup_test.go

@@ -123,7 +123,20 @@ func (d *multipartCleanupDisk) Endpoint() Endpoint {
 
 func (d *multipartCleanupDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
 	d.reads++
-	return FileInfo{ModTime: d.modTime}, d.readErr
+	if d.readErr != nil {
+		return FileInfo{}, d.readErr
+	}
+	fi := newFileInfo("obj", 1, 0)
+	fi.Erasure.Index, fi.ModTime = 1, d.modTime
+	return fi, nil
+}
+
+// multipartCleanupSet is a set of one drive, which also answers the
+// metadata reads that establish the restore marker of an expired upload.
+// Its locks are local: stale cleanup decides a removal under the upload's
+// commit lock.
+func multipartCleanupSet(disk StorageAPI) erasureObjects {
+	return erasureObjects{getDisks: func() []StorageAPI { return []StorageAPI{disk} }, nsMutex: newNSLock(false)}
 }
 
 func (d *multipartCleanupDisk) StatVol(ctx context.Context, volume string) (VolInfo, error) {
@@ -143,9 +156,10 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) {
 	}{
 		{name: "recent", prefix: uuid, age: expiry / 2, readErr: errFileNotFound},
 		{name: "plaintext_base64_collision", prefix: "4eacd477-e733-4851-b821-6fd42785ff87", age: expiry / 2, readErr: errFileNotFound},
-		{name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantDeleted: true},
+		// Only an expired upload is read on every drive, for its restore marker.
+		{name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantReads: 1, wantDeleted: true},
 		{name: "legacy_recent", prefix: uuid, age: expiry / 2, legacy: true, wantReads: 1},
-		{name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 1, wantDeleted: true},
+		{name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 2, wantDeleted: true},
 		{name: "legacy_metadata_missing", prefix: uuid, age: 2 * expiry, legacy: true, readErr: errFileNotFound, wantReads: 1},
 		{name: "malformed_metadata_missing", prefix: "not-a-uuid", age: 2 * expiry, readErr: errFileNotFound, wantReads: 1},
 	} {
@@ -166,7 +180,7 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) {
 			if err := os.Chtimes(filepath.Dir(uploadDir), modTime, modTime); err != nil {
 				t.Fatal(err)
 			}
-			erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
+			multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk)
 			if disk.reads != tc.wantReads {
 				t.Errorf("ReadVersion calls = %d, want %d", disk.reads, tc.wantReads)
 			}
@@ -199,7 +213,7 @@ func TestCleanupStaleUploadsEmptyHashDirectory(t *testing.T) {
 			if err := os.Chtimes(shaDir, modTime, modTime); err != nil {
 				t.Fatal(err)
 			}
-			erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
+			multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk)
 			if _, err := os.Stat(shaDir); !os.IsNotExist(err) {
 				t.Fatalf("empty hash directory must be removed regardless of age: %v", err)
 			}
@@ -224,7 +238,7 @@ func TestCleanupStaleUploadsSharedHashDirectory(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
+	multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk)
 	if _, err := os.Stat(filepath.Join(shaDir, expiredID)); !os.IsNotExist(err) {
 		t.Errorf("expired upload must be removed: %v", err)
 	}

cmd/erasure-multipart.go

@@ -34,7 +34,6 @@ import (
 	"github.com/google/uuid"
 	"github.com/klauspost/readahead"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
@@ -72,6 +71,7 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string {
 }
 
 // checkUploadIDExists - verify if a given uploadID exists and is valid.
+// The metadata read from each drive is returned also with an error.
 func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string, write bool) (fi FileInfo, metArr []FileInfo, err error) {
 	defer func() {
 		if errors.Is(err, errFileNotFound) {
@@ -89,15 +89,15 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
 	if err != nil {
-		return fi, nil, err
+		return fi, partsMetadata, err
 	}
 
 	if readQuorum < 0 {
-		return fi, nil, errErasureReadQuorum
+		return fi, partsMetadata, errErasureReadQuorum
 	}
 
 	if writeQuorum < 0 {
-		return fi, nil, errErasureWriteQuorum
+		return fi, partsMetadata, errErasureWriteQuorum
 	}
 
 	quorum := readQuorum
@@ -114,7 +114,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 		err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
 	}
 	if err != nil {
-		return fi, nil, err
+		return fi, partsMetadata, err
 	}
 
 	// Pick one from the first valid metadata.
@@ -217,12 +217,55 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 				hasSkippedUploads = true
 				return nil
 			}
+			// A completion holds this lock from staging its commit in the
+			// upload until the commit ends: decide on a removal only while
+			// holding it, and skip an upload whose commit is running. Only
+			// uploads past the stale age pay for the lock and the read below.
+			lk := er.restoreCommitLock(uploadIDPath)
+			lkctx, err := lk.GetLock(ctx, newDynamicTimeout(time.Second, time.Second))
+			if err != nil {
+				hasSkippedUploads = true
+				return nil
+			}
+			// The rename below releases the lock when it ends, also when it
+			// outlives its deadline; every other exit releases it here.
+			renaming := false
+			defer func() {
+				if !renaming {
+					lk.Unlock(lkctx)
+				}
+			}()
+			// A native restore can run past the stale age: keep its upload
+			// while its attempt still owns the version, and keep any upload
+			// whose marker the drives cannot establish.
+			view := er.readRestoreUpload(lkctx.Context(), "", uploadIDPath)
+			if view.committing {
+				// What a commit staged, with no commit running: only a staged
+				// copy itself shows when it staged, only an old one is a
+				// leftover of a failed commit, and a copy not staged yet
+				// tells nothing.
+				local, err := disk.ReadVersion(lkctx.Context(), "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{})
+				st, serr := os.Stat(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath, xlStorageFormatFile))
+				if err != nil || serr != nil || !restoreUploadStaged(local) || time.Since(st.ModTime()) < restoreCommitGrace {
+					hasSkippedUploads = true
+					return nil
+				}
+			} else if view.live && (!view.known || (view.marker != nil && er.restoreUploadActive(lkctx.Context(), view))) {
+				hasSkippedUploads = true
+				return nil
+			}
+			renaming = true
 			w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 			return w.Run(func() error {
 				wait := deleteMultipartCleanupSleeper.Timer(ctx)
 				pathUUID := mustGetUUID()
 				targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID)
-				renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket))
+				err := renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket))
+				lk.Unlock(lkctx)
+				if err != nil {
+					// The next cleanup round tries again.
+					storageLogIf(ctx, fmt.Errorf("Unable to remove stale upload %s on %s: %w", uploadIDPath, drivePath, err))
+				}
 				wait()
 				return nil
 			})
@@ -463,6 +506,13 @@ func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket,
 		if populatedUploadIDs.Contains(uploadID) {
 			continue
 		}
+		// A native restore's upload is private to its attempt; list only an
+		// upload that the drives establish as an ordinary one.
+		metas, errs := readAllFileInfo(ctx, er.getDisks(), bucket, minioMetaMultipartBucket,
+			pathJoin(er.getMultipartSHADir(bucket, object), uploadID), "", false, false)
+		if !restoreUploadState(metas, errs).visible(ObjectOptions{}) {
+			continue
+		}
 		var fallback time.Time
 		if _, ok := multipartUploadTime(uploadID); !ok {
 			fi, err := disk.ReadVersion(ctx, bucket, minioMetaMultipartBucket,
@@ -562,6 +612,11 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	if opts.PreserveETag != "" {
 		userDefined["etag"] = opts.PreserveETag
 	}
+	// Only a native restore marks its upload private, never caller metadata.
+	delete(userDefined, restoreUploadKey)
+	if a := opts.Transition.attempt; a != nil {
+		userDefined[restoreUploadKey] = a.String()
+	}
 	onlineDisks := er.getDisks()
 
 	// Get parity and data drive count based on storage class metadata
@@ -743,7 +798,10 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 	// Validates if upload ID exists.
-	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
+	fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
+	if err == nil && !restoreUploadVisible(metas, opts) {
+		err = errUploadIDNotFound
+	}
 	if err != nil {
 		if errors.Is(err, errVolumeNotFound) {
 			return pi, toObjectErr(err, bucket)
@@ -980,7 +1038,10 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		UploadID: uploadID,
 	}
 
-	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
+	fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
+	if err == nil && !restoreUploadVisible(metas, opts) {
+		err = errUploadIDNotFound
+	}
 	if err != nil {
 		if errors.Is(err, errVolumeNotFound) {
 			return result, toObjectErr(err, bucket)
@@ -1058,7 +1119,10 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 		auditObjectErasureSet(ctx, "ListObjectParts", object, &er)
 	}
 
-	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
+	fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
+	if err == nil && !restoreUploadVisible(metas, opts) {
+		err = errUploadIDNotFound
+	}
 	if err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}
@@ -1271,7 +1335,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		auditObjectErasureSet(ctx, "CompleteMultipartUpload", object, &er)
 	}
 
-	if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile {
+	if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile || opts.Transition.attempt != nil {
 		if !opts.NoLock {
 			ns := er.NewNSLock(bucket, object)
 			lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
@@ -1304,7 +1368,24 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
+	if opts.Transition.attempt != nil {
+		// A native restore holds the upload write lock from validation
+		// through commit, after the object lock and in the lock domain of
+		// PutObjectPart, which takes a part lock and then this lock to read.
+		// NoLock only says that the caller holds the object lock.
+		uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+		ulkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return oi, err
+		}
+		ctx = ulkctx.Context()
+		defer uploadIDLock.Unlock(ulkctx)
+	}
+
 	fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
+	if err == nil && !restoreUploadVisible(partsMetadata, opts) {
+		err = errUploadIDNotFound
+	}
 	if err != nil {
 		if errors.Is(err, errVolumeNotFound) {
 			return oi, toObjectErr(err, bucket)
@@ -1618,6 +1699,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	delete(fi.Metadata, hash.MinIOMultipartChecksumType)
 	delete(fi.Metadata, multipartMetaBucket)
 	delete(fi.Metadata, multipartMetaObject)
+	delete(fi.Metadata, restoreUploadKey)
 
 	// Save the final object size and modtime.
 	fi.Size = objectSize
@@ -1697,6 +1779,20 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo, opts.Transition); err != nil {
 				return oi, toObjectErr(err, bucket, object)
 			}
+			if a := opts.Transition.attempt; a != nil {
+				// A native restore writes back only the null version it targets.
+				current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{
+					VersionID:  nullVersionID,
+					Transition: opts.Transition,
+					NoLock:     true,
+				})
+				if gerr != nil {
+					return oi, toObjectErr(gerr, bucket, object)
+				}
+				if !a.rehydrates(current, fi) {
+					return oi, toObjectErr(errAuthentication, bucket, object)
+				}
+			}
 		} else {
 			current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{
 				VersionID:        fi.VersionID,
@@ -1706,11 +1802,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 				NoLock:           true,
 			})
 			switch {
-			case gerr == nil && opts.Transition.RestoreRequest != nil &&
-				current.TransitionedObject.Status == lifecycle.TransitionComplete &&
-				current.ETag == fi.Metadata["etag"] && current.Size == fi.Size && current.ModTime.Equal(fi.ModTime):
-				// The internal restore path rehydrates this same tiered version.
-				// Ordinary multipart completion must still reject a version overwrite.
+			case gerr == nil && opts.Transition.attempt.rehydrates(current, fi):
+				// A native restore's own private upload rehydrates this same
+				// tiered version. Any other completion must still reject a
+				// version overwrite.
 			case gerr == nil, errors.As(gerr, &MethodNotAllowed{}):
 				return oi, toObjectErr(errAuthentication, bucket, object)
 			case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr):
@@ -1720,6 +1815,16 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
+	// Stale cleanup and abort judge an upload whose commit staged object
+	// metadata in it by this lock: hold it from staging through the commit.
+	commitLock := er.restoreCommitLock(uploadIDPath)
+	clkctx, err := commitLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return oi, err
+	}
+	ctx = clkctx.Context()
+	defer commitLock.Unlock(clkctx)
+
 	er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename().
 
 	defer func() {
@@ -1791,7 +1896,38 @@ func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, objec
 	if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") {
 		return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
 	}
-	defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
+	// Authorize before the unconditional cleanup below. A native restore's
+	// upload is absent to every other caller, and an upload whose marker the
+	// drives cannot establish is not removed by anyone.
+	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
+	view := er.readRestoreUpload(ctx, bucket, uploadIDPath)
+	if view.committing {
+		// What a failed commit left behind, also after stale cleanup could
+		// not remove it: only with no commit running, and once every copy
+		// shows a staging older than the grace.
+		lk := er.restoreCommitLock(uploadIDPath)
+		lkctx, err := lk.GetLock(ctx, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait))
+		if err != nil {
+			return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID)
+		}
+		defer lk.Unlock(lkctx)
+		if !er.restoreCommitAbandoned(lkctx.Context(), bucket, uploadIDPath) {
+			return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID)
+		}
+		er.deleteAll(lkctx.Context(), minioMetaMultipartBucket, uploadIDPath)
+		return true, nil
+	}
+	switch {
+	case view.visible(opts):
+	case view.known && view.marked:
+		return false, nil
+	case view.known && opts.Transition.attempt != nil:
+		// A native restore's own abort, of an upload no longer its own.
+		return false, nil
+	default:
+		return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID)
+	}
+	defer er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath)
 	_, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
 	err = toObjectErr(err, bucket, object, uploadID)
 	if _, absent := err.(InvalidUploadID); absent {

cmd/erasure-object.go

@@ -2158,6 +2158,20 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	if opts.Transition.ExpireRestored && (gerr != nil || goi.DeleteMarker) {
 		return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
 	}
+	// A restored copy expires only by its own reason, decided on the
+	// version's authoritative generation: a drive that missed a renewal or a
+	// later POST must not authorize removing the cache, nor drives that
+	// disagree on the restore status. Another lifecycle deletion falling due,
+	// or the all-pools check of an ordinary expiry, does not stand in for it.
+	if opts.Transition.ExpireRestored {
+		gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
+		if err != nil || gen.split || gen.fi.Deleted || lc == nil {
+			return goi, errLifecycleNotApplicable
+		}
+		if !restoredCopyExpires(ctx, *lc, rcfg, replcfg, gen.fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)) {
+			return goi, errLifecycleNotApplicable
+		}
+	}
 
 	if opts.EvalMetadataFn != nil {
 		dsc, err := opts.EvalMetadataFn(&goi, gerr)
@@ -2660,29 +2674,6 @@ func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket,
 	return er.restoreTransitionedObject(ctx, bucket, object, opts)
 }
 
-// update restore status header in the metadata
-func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error {
-	oi := objInfo.Clone()
-	oi.metadataOnly = true // Perform only metadata updates.
-
-	// allow retry in the case of failure to restore
-	delete(oi.UserDefined, xhttp.AmzRestore)
-
-	if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{
-		VersionID:  oi.VersionID,
-		Transition: opts.Transition,
-	}, ObjectOptions{
-		VersionID:  oi.VersionID,
-		MTime:      oi.ModTime,
-		NoLock:     opts.NoLock,
-		Transition: opts.Transition,
-	}); err != nil {
-		storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
-		return err
-	}
-	return nil
-}
-
 // getRestoreLock waits for a restore's object lock for as long as ctx lives.
 // The POST handler has already marked the version ongoing, and only the
 // restore clears that marker. A restore queued behind another version's
@@ -2698,6 +2689,11 @@ func getRestoreLock(ctx context.Context, lk RWLocker) (LockContext, error) {
 
 // restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts
 // as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta
+//
+// A POST restore publishes its attempt on the version's ongoing marker. The
+// restore rehydrates the version only while that attempt owns the marker, and
+// on failure clears the marker only while it still does. A direct caller
+// publishes no attempt; its attempt only names the private upload.
 func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error {
 	if !opts.NoLock {
 		// Single-pool and direct callers must protect the metadata snapshot
@@ -2705,30 +2701,45 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 		lk := er.NewNSLock(bucket, object)
 		lkctx, err := getRestoreLock(ctx, lk)
 		if err != nil {
-			return err
+			return restoreResetError{err}
 		}
 		ctx = lkctx.Context()
 		defer lk.Unlock(lkctx)
 		opts.NoLock = true
 	}
 
-	setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error {
-		if rerr == nil {
-			return nil // nothing to do; restore object was successful
-		}
-		er.updateRestoreMetadata(ctx, bucket, object, oi, opts)
-		return rerr
-	}
-	var oi ObjectInfo
-	// get the file info on disk for transitioned object
-	actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
+	published := opts.Transition.attempt
+	// Restore the authoritative generation of the version, never a revision
+	// that only some drives hold.
+	gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
 	if err != nil {
-		// No version was read, so there is no restore metadata to update.
-		return toObjectErr(err, bucket, object)
+		err = toObjectErr(err, bucket, object, opts.VersionID)
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			// No version was read, so there is no restore marker to clear.
+			return err
+		}
+		return restoreResetError{err}
+	}
+	actualfi := gen.fi
+	if actualfi.Deleted {
+		return toObjectErr(errMethodNotAllowed, bucket, object)
 	}
 
-	oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
-	ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi)
+	oi := actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
+	attempt := published
+	if attempt == nil {
+		attempt = newRestoreAttempt(oi)
+	} else if !attempt.owns(oi) {
+		// A later POST took the version over, or it was replaced.
+		return errRestoreSuperseded
+	}
+	opts.Transition.attempt = attempt
+	versionID := restoreVersionID(actualfi.VersionID)
+	fail := func(cause error) error {
+		return er.restoreFailed(ctx, bucket, object, versionID, published, cause)
+	}
+
+	ropts := putRestoreOpts(opts.Transition.RestoreRequest, oi, opts)
 	ropts.NoLock = opts.NoLock
 	ropts.Transition = opts.Transition
 	// Rehydration copies encoded bytes without changing their content identity.
@@ -2738,35 +2749,51 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 	if len(oi.Parts) == 1 && !oi.isMultipart() {
 		actualSize, err := oi.GetActualSize()
 		if err != nil {
-			return setRestoreHeaderFn(oi, err)
+			return fail(err)
 		}
 		ropts.IndexCB = func() []byte { return oi.Parts[0].Index }
 		var rs *HTTPRangeSpec
 		gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
 		if err != nil {
-			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
+			return fail(toObjectErr(err, bucket, object))
 		}
 		defer gr.Close()
 		hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", actualSize)
 		if err != nil {
-			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
+			return fail(toObjectErr(err, bucket, object))
 		}
 		pReader := NewPutObjReader(hashReader)
-		_, err = er.PutObject(ctx, bucket, object, pReader, ropts)
-		return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
+		if _, err = er.PutObject(ctx, bucket, object, pReader, ropts); err != nil {
+			return fail(toObjectErr(err, bucket, object))
+		}
+		return nil
 	}
 
-	res, err := er.NewMultipartUpload(ctx, bucket, object, ropts)
-	if err != nil {
-		return setRestoreHeaderFn(oi, err)
+	// Every server must keep the upload private, also a queued restore's.
+	if err := nativeRestoreAvailable(ctx); err != nil {
+		return fail(err)
 	}
+	// The upload records when it starts, which the stale-upload cleanup ages;
+	// completion still commits the version's own ModTime.
+	uopts := ropts
+	uopts.MTime = time.Time{}
+	res, err := er.NewMultipartUpload(ctx, bucket, object, uopts)
+	if err != nil {
+		return fail(err)
+	}
+	completed := false
+	defer func() {
+		if !completed {
+			er.abortRestoreUpload(bucket, object, res.UploadID, attempt)
+		}
+	}()
 
 	var uploadedParts []CompletePart
 	var rs *HTTPRangeSpec
 	// get reader from the warm backend - note that even in the case of encrypted objects, this stream is still encrypted.
 	gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
 	if err != nil {
-		return setRestoreHeaderFn(oi, err)
+		return fail(err)
 	}
 	defer gr.Close()
 
@@ -2781,16 +2808,17 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 		}
 		hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", actualSize)
 		if err != nil {
-			return setRestoreHeaderFn(oi, err)
+			return fail(err)
 		}
 		pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{
-			IndexCB: func() []byte { return partInfo.Index },
+			IndexCB:    func() []byte { return partInfo.Index },
+			Transition: TransitionOptions{attempt: attempt},
 		})
 		if err != nil {
-			return setRestoreHeaderFn(oi, err)
+			return fail(err)
 		}
 		if pInfo.Size != partInfo.Size {
-			return setRestoreHeaderFn(oi, InvalidObjectState{Bucket: bucket, Object: object})
+			return fail(InvalidObjectState{Bucket: bucket, Object: object})
 		}
 		uploadedParts = append(uploadedParts, CompletePart{
 			PartNumber: pInfo.PartNumber,
@@ -2804,7 +2832,11 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 		VersionSuspended: opts.VersionSuspended,
 		Transition:       opts.Transition,
 	})
-	return setRestoreHeaderFn(oi, err)
+	if err != nil {
+		return fail(err)
+	}
+	completed = true
+	return nil
 }
 
 // DecomTieredObject - moves tiered object to another pool during decommissioning.

cmd/erasure-server-pool.go

@@ -3369,8 +3369,16 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
 }
 
 // RestoreTransitionedObject - restore transitioned object content locally on this cluster.
-func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) (err error) {
 	object = encodeDirObject(object)
+	defer func() {
+		// The restore stopped before it could clear POST's ongoing marker.
+		var reset restoreResetError
+		if errors.As(err, &reset) {
+			z.recoverRestoreStatus(bucket, object, opts)
+			err = reset.error
+		}
+	}()
 	if z.SinglePool() {
 		return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
 	}
@@ -3380,7 +3388,7 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
 		lk := z.NewNSLock(bucket, object)
 		lkctx, err := getRestoreLock(ctx, lk)
 		if err != nil {
-			return err
+			return restoreResetError{err}
 		}
 		ctx = lkctx.Context()
 		defer lk.Unlock(lkctx)
@@ -3389,10 +3397,7 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
 	// Avoid restoring object from a pool being decommissioned.
 	opts.SkipDecommissioned = true
 	opts.NoLock = true
-	// Restore an existing version, not the latest version's write location.
-	lookupOpts := opts
-	lookupOpts.MetadataChg = true
-	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts)
+	idx, err := z.restorePoolIdx(ctx, bucket, object, opts)
 	if err != nil {
 		return err
 	}

cmd/object-api-interface.go

@@ -177,6 +177,10 @@ type TransitionOptions struct {
 	RestoreRequest *RestoreObjectRequest
 	RestoreExpiry  time.Time
 	ExpireRestored bool
+
+	// attempt is the native restore a call belongs to. Only that restore
+	// sees its private upload, see restoreUploadVisible.
+	attempt *restoreAttempt
 }
 
 // MakeBucketOptions represents bucket options for ObjectLayer bucket operations
@@ -295,6 +299,8 @@ type ObjectLayer interface {
 	DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error)
 	TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error
 	RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error
+	// restoreRequest records a POST restore on a version, see erasureServerPools.
+	restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error)
 
 	// Multipart operations.
 	ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)

cmd/object-copy-metadata_test.go

@@ -340,9 +340,9 @@ func TestCopyRewritesObjectData(t *testing.T) {
 			name: "data copy always rewrites",
 			want: true,
 		},
-		// PostRestoreObjectHandler, updateRestoreMetadata and batchKeyRotate all
-		// address the same version on both sides and never set Versioned, so they
-		// only ever reach these two cases.
+		// In-place metadata callers such as batchKeyRotate address the same
+		// version on both sides and never set Versioned, so they only ever
+		// reach these two cases.
 		{
 			name:         "unversioned in-place metadata update",
 			metadataOnly: true,

cmd/object-handlers.go

@@ -3955,7 +3955,11 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 
 	// Preserve a failed restore read even before the request body is parsed.
 	opts.Transition.RestoreRequest = &RestoreObjectRequest{}
-	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
+	// Resolve the target without the object lock: a running restore holds it
+	// for its download, and the status step bounds its own wait for it.
+	preOpts := opts
+	preOpts.NoLock = true
+	objInfo, err := getObjectInfo(ctx, bucket, object, preOpts)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -3982,48 +3986,47 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 	}
 	opts.Transition.RestoreRequest = rreq
 	statusCode := http.StatusOK
-	alreadyRestored := false
-	if err == nil {
-		if objInfo.RestoreOngoing && rreq.Type != SelectRestoreRequest {
+	// set or upgrade restore expiry
+	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
+
+	// update self with restore metadata
+	var attempt *restoreAttempt
+	if rreq.Type != SelectRestoreRequest {
+		// Native restore needs every server to keep its upload private;
+		// refuse it, writing nothing, while that is not established.
+		if err := nativeRestoreAvailable(ctx); err != nil {
+			apiErr := errorCodes.ToAPIErr(ErrBusy)
+			apiErr.Code, apiErr.Description = "ServiceUnavailable", err.Error()
+			writeErrorResponse(ctx, w, apiErr, r.URL)
+			return
+		}
+		// Decide on a fresh read under the object lock: renew a restored copy
+		// that is still there, or mark the version ongoing for a new attempt.
+		// Only the version resolved above, with its identity, is restored.
+		statusOpts := opts
+		statusOpts.Transition.RestoreExpiry = restoreExpiry
+		status, err := objectAPI.restoreRequest(GlobalContext, bucket, object, statusOpts, objInfo)
+		if errors.Is(err, errRestoreInProgress) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectRestoreAlreadyInProgress), r.URL)
 			return
 		}
-		if !objInfo.RestoreOngoing && !objInfo.RestoreExpires.IsZero() {
-			statusCode = http.StatusAccepted
-			alreadyRestored = true
-		}
-	}
-	// set or upgrade restore expiry
-	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
-	metadata := cloneMSS(objInfo.UserDefined)
-
-	// update self with restore metadata
-	if rreq.Type != SelectRestoreRequest {
-		objInfo.metadataOnly = true // Perform only metadata updates.
-		metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days)
-		metadata[xhttp.AmzRestoreRequestDate] = time.Now().UTC().Format(http.TimeFormat)
-		if alreadyRestored {
-			metadata[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
-		} else {
-			metadata[xhttp.AmzRestore] = ongoingRestoreObj().String()
-		}
-		objInfo.UserDefined = metadata
-		if _, err := objectAPI.CopyObject(GlobalContext, bucket, object, bucket, object, objInfo, ObjectOptions{
-			VersionID:  objInfo.VersionID,
-			Transition: opts.Transition,
-		}, ObjectOptions{
-			VersionID:   objInfo.VersionID,
-			Transition:  opts.Transition,
-			MTime:       objInfo.ModTime,
-			MetadataChg: true,
-		}); err != nil {
+		if errors.As(err, &InvalidObjectState{}) || errors.As(err, &MethodNotAllowed{}) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL)
 			return
 		}
-		// for previously restored object, just update the restore expiry
-		if alreadyRestored {
+		if err != nil {
+			// Such as a read quorum error while telling whether it restores.
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
+		// Events keep the version ID as this request resolved it.
+		status.objInfo.VersionID = objInfo.VersionID
+		objInfo = status.objInfo
+		// for previously restored object, just update the restore expiry
+		if status.attempt == nil {
+			return
+		}
+		attempt = status.attempt
 	}
 
 	restoreObject := mustGetUUID()
@@ -4088,12 +4091,14 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 			rreq.SelectParameters.Close()
 			return
 		}
+		// The versioning context and the explicit version POST resolved.
 		restoreOpts := ObjectOptions{
 			Transition: TransitionOptions{
 				RestoreRequest: rreq,
 				RestoreExpiry:  restoreExpiry,
+				attempt:        attempt,
 			},
-			VersionID:        objInfo.VersionID,
+			VersionID:        attempt.VersionID,
 			Versioned:        opts.Versioned,
 			VersionSuspended: opts.VersionSuspended,
 		}

cmd/peer-rest-server.go

@@ -116,6 +116,7 @@ var (
 	stopRebalanceRPC               = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerStopRebalance, grid.NewMSS, grid.NewNoPayload)
 	updateMetacacheListingRPC      = grid.NewSingleHandler[*metacache, *metacache](grid.HandlerUpdateMetacacheListing, func() *metacache { return &metacache{} }, func() *metacache { return &metacache{} })
 	cleanupUploadIDCacheMetaRPC    = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerClearUploadID, grid.NewMSS, grid.NewNoPayload)
+	restoreCapabilityRPC           = grid.NewSingleHandler[*grid.MSS, *grid.MSS](grid.HandlerRestoreCapability, grid.NewMSS, grid.NewMSS)
 
 	// STREAMS
 	// Set an output capacity of 100 for consoleLog and listenRPC
@@ -1440,6 +1441,7 @@ func registerPeerRESTHandlers(router *mux.Router, gm *grid.Manager) {
 	logger.FatalIf(stopRebalanceRPC.Register(gm, server.StopRebalanceHandler), "unable to register handler")
 	logger.FatalIf(updateMetacacheListingRPC.Register(gm, server.UpdateMetacacheListingHandler), "unable to register handler")
 	logger.FatalIf(cleanupUploadIDCacheMetaRPC.Register(gm, server.HandlerClearUploadID), "unable to register handler")
+	logger.FatalIf(restoreCapabilityRPC.Register(gm, restoreCapabilityHandler), "unable to register handler")
 
 	logger.FatalIf(gm.RegisterStreamingHandler(grid.HandlerTrace, grid.StreamHandler{
 		Handle:      server.TraceHandler,

cmd/restore-attempt.go

@@ -0,0 +1,862 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"maps"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// A native restore keeps two private records. The version that POST marks
+// ongoing names the attempt that owns the marker; that attempt's multipart
+// upload names the attempt and the version it rehydrates. Clients cannot
+// set either: requests with reserved metadata are refused, it is never
+// extracted from headers, and uploads only get the marker from the restore.
+const (
+	restoreAttemptKey = ReservedMetadataPrefixLower + "restore-attempt"
+	restoreUploadKey  = ReservedMetadataPrefixLower + "restore-upload"
+)
+
+const (
+	// restoreTakeoverWait bounds how long a POST on an ongoing version waits
+	// for the object lock. A running restore holds it for its download.
+	restoreTakeoverWait = 2 * time.Second
+	// restoreCommitGrace is how long stale cleanup keeps an upload whose
+	// completion staged object metadata in it; a commit takes seconds.
+	restoreCommitGrace = time.Hour
+	// restoreCleanupTimeout bounds each status reset and upload abort.
+	restoreCleanupTimeout = time.Minute
+)
+
+// restoreRecoveryDelays are the in-memory retries that clear the ongoing
+// marker of a restore that failed before it could clear the marker itself.
+// A restart loses them; a later POST then takes the marker over.
+var restoreRecoveryDelays = []time.Duration{0, 100 * time.Millisecond, time.Second, 5 * time.Second, 30 * time.Second, 2 * time.Minute}
+
+var (
+	// errRestoreSuperseded: the version no longer belongs to this attempt.
+	// The attempt must not rehydrate, complete or reset anything.
+	errRestoreSuperseded = errors.New("restore attempt superseded")
+	// errRestoreInProgress: a restore holds the object lock of the version.
+	errRestoreInProgress = errors.New("restore already in progress")
+	// errRestoreUnchanged: a restore status update found nothing to write.
+	errRestoreUnchanged = errors.New("restore status unchanged")
+)
+
+// restoreResetError marks a failed restore that may have left its ongoing
+// marker behind. The pools layer hands it to recoverRestoreStatus.
+type restoreResetError struct{ error }
+
+func (e restoreResetError) Unwrap() error { return e.error }
+
+// restoreVersionID is the explicit version ID of a resolved restore target.
+func restoreVersionID(versionID string) string {
+	if versionID == "" {
+		return nullVersionID
+	}
+	return versionID
+}
+
+// restoreAttempt identifies one native restore of one stored version.
+type restoreAttempt struct {
+	ID        string    `json:"id"`
+	VersionID string    `json:"v"` // "null" for the null version
+	ETag      string    `json:"e"`
+	Size      int64     `json:"s"`
+	ModTime   time.Time `json:"m"`
+}
+
+func newRestoreAttempt(oi ObjectInfo) *restoreAttempt {
+	return &restoreAttempt{
+		ID:        mustGetUUID(),
+		VersionID: restoreVersionID(oi.VersionID),
+		ETag:      oi.ETag,
+		Size:      oi.Size,
+		ModTime:   oi.ModTime.UTC(),
+	}
+}
+
+func (a *restoreAttempt) String() string {
+	b, _ := json.Marshal(a)
+	return string(b)
+}
+
+func parseRestoreAttempt(s string) (*restoreAttempt, bool) {
+	var a restoreAttempt
+	if s == "" || json.Unmarshal([]byte(s), &a) != nil || a.ID == "" || a.VersionID == "" {
+		return nil, false
+	}
+	return &a, true
+}
+
+func (a *restoreAttempt) equal(b *restoreAttempt) bool {
+	return a != nil && b != nil && a.ID == b.ID && a.VersionID == b.VersionID &&
+		a.ETag == b.ETag && a.Size == b.Size && a.ModTime.Equal(b.ModTime)
+}
+
+// targets reports whether oi is still the version the attempt was made for.
+// A replacement under the same version ID differs in ETag, size or ModTime.
+func (a *restoreAttempt) targets(oi ObjectInfo) bool {
+	return a != nil && !oi.DeleteMarker && restoreVersionID(oi.VersionID) == a.VersionID &&
+		oi.ETag == a.ETag && oi.Size == a.Size && oi.ModTime.Equal(a.ModTime) &&
+		oi.TransitionedObject.Status == lifecycle.TransitionComplete
+}
+
+// owns reports whether the version is ongoing under this attempt.
+func (a *restoreAttempt) owns(oi ObjectInfo) bool {
+	return a.targets(oi) && oi.RestoreOngoing && oi.UserDefined[restoreAttemptKey] == a.ID
+}
+
+// rehydrates reports whether completing upload fi, the attempt's private
+// upload, writes back the very version current.
+func (a *restoreAttempt) rehydrates(current ObjectInfo, fi FileInfo) bool {
+	return a.targets(current) && fi.Metadata["etag"] == a.ETag && fi.Size == a.Size && fi.ModTime.Equal(a.ModTime)
+}
+
+// restoreUploadView is the restore marker of an upload as the drives of
+// its set establish it. A marked upload is written to a write quorum, at
+// least a majority of the drives, and its copies never change their marker.
+type restoreUploadView struct {
+	// marker is the attempt of a native restore's upload: one marker that a
+	// majority of the drives hold.
+	marker *restoreAttempt
+	// marked: some readable drive holds a marker.
+	marked bool
+	// known: the marker state is established. Without it, callers fail
+	// closed: nobody lists, uses, aborts or removes the upload.
+	known bool
+	// live: enough drives hold the upload or cannot tell that a written
+	// upload may exist; otherwise only remnants of one are left.
+	live bool
+	// committing: some drive holds object metadata in the upload's place.
+	// Completion stages the committed version's object metadata there,
+	// without the marker, before it renames the upload into the object.
+	committing bool
+	// bucket and object the upload of an established marker belongs to.
+	bucket, object string
+}
+
+// restoreUploadState establishes the marker of an upload from the metadata of
+// every drive. errs may be nil, then a drive without valid metadata is one
+// that cannot tell. A marker is established on a majority of the drives; its
+// absence only when no readable drive shows one and the drives that cannot
+// tell are too few to hold a marked upload.
+func restoreUploadState(metas []FileInfo, errs []error) restoreUploadView {
+	n := len(metas)
+	majority := n/2 + 1
+	markers := make(map[string]int)
+	var unmarked, unknown, committing int
+	for i, fi := range metas {
+		if errs != nil && errs[i] != nil {
+			if !errors.Is(errs[i], errFileNotFound) && !errors.Is(errs[i], errFileVersionNotFound) &&
+				!errors.Is(errs[i], errVolumeNotFound) && !errors.Is(errs[i], errUploadIDNotFound) {
+				unknown++
+			}
+			continue
+		}
+		if !fi.IsValid() {
+			unknown++
+			continue
+		}
+		if restoreUploadStaged(fi) {
+			committing++
+			continue
+		}
+		if m := fi.Metadata[restoreUploadKey]; m != "" {
+			markers[m]++
+		} else {
+			unmarked++
+		}
+	}
+	present := unmarked
+	for _, c := range markers {
+		present += c
+	}
+	v := restoreUploadView{marked: len(markers) > 0, live: present+unknown+committing >= majority, committing: committing > 0}
+	if v.committing {
+		// Nobody but the completion may use or remove it now.
+		return v
+	}
+	for m, c := range markers {
+		if c >= majority {
+			v.marker, v.known = parseRestoreAttempt(m)
+			if v.known {
+				// The marker also names the upload's own target.
+				for _, fi := range metas {
+					if !fi.IsValid() || fi.Metadata[restoreUploadKey] != m {
+						continue
+					}
+					if restoreVersionID(fi.VersionID) != v.marker.VersionID || fi.Metadata["etag"] != v.marker.ETag {
+						v.marker, v.known = nil, false
+						break
+					}
+					v.bucket, v.object = fi.Metadata[multipartMetaBucket], fi.Metadata[multipartMetaObject]
+				}
+			}
+			return v
+		}
+	}
+	// A marker without a majority is established only as a remnant.
+	v.known = !v.live || (!v.marked && unknown < majority)
+	return v
+}
+
+// restoreUploadStaged reports whether an upload's record is object metadata
+// that a completion staged there: an upload record has no parts and no
+// other versions.
+func restoreUploadStaged(fi FileInfo) bool {
+	return fi.Deleted || len(fi.Parts) > 0 || fi.NumVersions > 1
+}
+
+// visible reports whether a caller sees the upload: a native restore its own
+// upload, any other caller only an upload established as unmarked.
+func (v restoreUploadView) visible(opts ObjectOptions) bool {
+	if a := opts.Transition.attempt; a != nil {
+		return v.marker.equal(a)
+	}
+	return v.known && !v.marked
+}
+
+// restoreUploadVisible reports whether a caller sees an upload whose metadata
+// is metas, one entry for each drive.
+func restoreUploadVisible(metas []FileInfo, opts ObjectOptions) bool {
+	return restoreUploadState(metas, nil).visible(opts)
+}
+
+// readRestoreUpload reads the metadata of an upload from every drive.
+func (er erasureObjects) readRestoreUpload(ctx context.Context, bucket, uploadIDPath string) restoreUploadView {
+	metas, errs := readAllFileInfo(ctx, er.getDisks(), bucket, minioMetaMultipartBucket, uploadIDPath, "", false, false)
+	return restoreUploadState(metas, errs)
+}
+
+// restoreCommitLock is held by a completion from staging through its commit;
+// it is named by the upload's own path, which an upload whose copies all
+// hold staged object metadata still has.
+func (er erasureObjects) restoreCommitLock(uploadIDPath string) RWLocker {
+	return er.NewNSLock(minioMetaMultipartBucket, uploadIDPath)
+}
+
+// restoreCommitAbandoned reports whether an upload holds only what a failed
+// commit left: every drive answers, and every copy that is there is staged
+// object metadata, see restoreUploadStaged, older than restoreCommitGrace by
+// that drive's own metadata file. A copy that cannot be read, or that is not
+// staged yet, may belong to a commit still running. The caller holds the
+// upload's restoreCommitLock.
+func (er erasureObjects) restoreCommitAbandoned(ctx context.Context, bucket, uploadIDPath string) bool {
+	disks := er.getDisks()
+	metas, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, uploadIDPath, "", false, false)
+	staged := 0
+	for i, fi := range metas {
+		if errs[i] != nil {
+			if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errFileVersionNotFound) {
+				continue
+			}
+			return false
+		}
+		if !fi.IsValid() || !restoreUploadStaged(fi) {
+			return false
+		}
+		st, err := disks[i].StatInfoFile(ctx, minioMetaMultipartBucket, pathJoin(uploadIDPath, xlStorageFormatFile), false)
+		if err != nil || len(st) == 0 || time.Since(st[0].ModTime) < restoreCommitGrace {
+			return false
+		}
+		staged++
+	}
+	return staged > 0
+}
+
+// restoreGeneration is the metadata generation of one version that the drives
+// of a set hold authoritatively: the revision (header and normalized metadata
+// digest, versionRevision) that at least max(n/2+1, its read quorum) drives
+// hold. drives has each drive's own copy of it, with that drive's erasure
+// index and inline data, and an invalid entry where a drive holds another or
+// no revision; disks has nil there.
+//
+// A failed restore status write can leave drives that differ only in their
+// restore fields, with no revision on a quorum. Then the generation is the
+// restore-neutral revision that a quorum holds once those fields are left
+// out: fi has no restore fields, and ongoing names the attempts whose
+// ongoing marker some of its drives hold ("" for one without an attempt).
+// Every other field, protection included, still needs the quorum.
+type restoreGeneration struct {
+	fi      FileInfo
+	drives  []FileInfo
+	disks   []StorageAPI
+	split   bool
+	ongoing map[string]bool
+}
+
+// restoreFieldKeys are the metadata keys a restore status write sets.
+var restoreFieldKeys = []string{xhttp.AmzRestore, xhttp.AmzRestoreExpiryDays, xhttp.AmzRestoreRequestDate, restoreAttemptKey}
+
+// restoreHealMeta returns latest, the metadata heal copies to drives that
+// lack the version, with the restore fields that at least quorum drives of
+// that version hold. When some drive holds restore fields and no set of
+// them has the quorum, healing the version waits.
+func restoreHealMeta(metas []FileInfo, latest FileInfo, quorum int) (FileInfo, error) {
+	counts := make(map[string]int)
+	fields := make(map[string]map[string]string)
+	restoring := false
+	for _, m := range metas {
+		if !m.IsValid() || m.Deleted || !m.ModTime.Equal(latest.ModTime) {
+			continue
+		}
+		f := make(map[string]string)
+		var key strings.Builder
+		for _, k := range restoreFieldKeys {
+			if v, ok := m.Metadata[k]; ok {
+				f[k] = v
+				fmt.Fprintf(&key, "%d:%s=%d:%s;", len(k), k, len(v), v)
+				restoring = true
+			}
+		}
+		counts[key.String()]++
+		fields[key.String()] = f
+	}
+	if !restoring {
+		return latest, nil
+	}
+	best, bestCount := "", 0
+	for key, c := range counts {
+		if c > bestCount {
+			best, bestCount = key, c
+		}
+	}
+	if bestCount < quorum {
+		return latest, errErasureReadQuorum
+	}
+	latest.Metadata = maps.Clone(latest.Metadata)
+	for _, k := range restoreFieldKeys {
+		delete(latest.Metadata, k)
+	}
+	maps.Copy(latest.Metadata, fields[best])
+	return latest, nil
+}
+
+// restoreNeutralRevision is the revision of ver without its restore fields.
+func restoreNeutralRevision(ver xlMetaV2ShallowVersion, metaV uint8) (revisionKey, bool) {
+	var v xlMetaV2Version
+	if _, err := v.unmarshalV(metaV, ver.meta); err != nil || v.Type != ObjectType || v.ObjectV2 == nil {
+		return revisionKey{}, false
+	}
+	o := *v.ObjectV2
+	o.MetaUser, o.MetaSys = maps.Clone(o.MetaUser), maps.Clone(o.MetaSys)
+	for _, k := range restoreFieldKeys {
+		delete(o.MetaUser, k)
+		delete(o.MetaSys, k)
+	}
+	v.ObjectV2 = &o
+	meta, err := v.MarshalMsg(nil)
+	if err != nil {
+		return revisionKey{}, false
+	}
+	return versionRevision(xlMetaV2ShallowVersion{header: v.header(), meta: meta}, xlMetaVersion), true
+}
+
+// readRestoreGeneration selects the authoritative generation of a version,
+// or fails closed: a revision that only some drives hold, such as one a
+// failed metadata update left behind, is never selected. An empty version ID
+// addresses the latest version, as a direct restore may.
+func (er erasureObjects) readRestoreGeneration(ctx context.Context, bucket, object, versionID string) (restoreGeneration, error) {
+	if versionID == "" {
+		fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, ObjectOptions{
+			NoLock:     true,
+			Transition: TransitionOptions{RestoreRequest: &RestoreObjectRequest{}},
+		}, false)
+		if err != nil {
+			return restoreGeneration{}, err
+		}
+		versionID = restoreVersionID(fi.VersionID)
+	}
+	var uv [16]byte
+	if versionID != nullVersionID {
+		id, err := uuid.Parse(versionID)
+		if err != nil {
+			return restoreGeneration{}, errFileVersionNotFound
+		}
+		uv = id
+	}
+	disks := er.getDisks()
+	n := len(disks)
+	majority := n/2 + 1
+	raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, true)
+	type revision struct {
+		ver   xlMetaV2ShallowVersion
+		metaV uint8
+		count int
+	}
+	keys := make([]*revisionKey, n)
+	neutral := make([]*revisionKey, n)
+	revisions := make(map[revisionKey]*revision)
+	neutrals := make(map[revisionKey]*revision)
+	absent := 0
+	for i := range disks {
+		if errs[i] != nil {
+			if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errVolumeNotFound) {
+				absent++
+			}
+			continue
+		}
+		var xl xlMetaV2
+		if xl.LoadOrConvert(raws[i].Buf) != nil {
+			continue
+		}
+		found := false
+		for _, ver := range xl.versions {
+			if ver.header.VersionID != uv || ver.header.FreeVersion() {
+				continue
+			}
+			key := versionRevision(ver, xl.metaV)
+			keys[i], found = &key, true
+			r := revisions[key]
+			if r == nil {
+				r = &revision{ver: ver, metaV: xl.metaV}
+				revisions[key] = r
+			}
+			r.count++
+			if nkey, ok := restoreNeutralRevision(ver, xl.metaV); ok {
+				neutral[i] = &nkey
+				if neutrals[nkey] == nil {
+					neutrals[nkey] = &revision{ver: ver, metaV: xl.metaV}
+				}
+				neutrals[nkey].count++
+			}
+			break
+		}
+		if !found {
+			absent++
+		}
+	}
+	accepted := func(revs map[revisionKey]*revision) (revisionKey, bool) {
+		var best *revision
+		var bestKey revisionKey
+		for key, r := range revs {
+			if best == nil || r.count > best.count {
+				best, bestKey = r, key
+			}
+		}
+		return bestKey, best != nil && best.count >= majority &&
+			best.count >= er.versionReadQuorum(ctx, bucket, object, best.ver, best.metaV, make([]FileInfo, n), make([]error, n))
+	}
+	gen := restoreGeneration{drives: make([]FileInfo, n), disks: make([]StorageAPI, n)}
+	bestKey, ok := accepted(revisions)
+	if !ok {
+		if bestKey, ok = accepted(neutrals); !ok {
+			if absent >= majority {
+				return restoreGeneration{}, errFileVersionNotFound
+			}
+			return restoreGeneration{}, errErasureReadQuorum
+		}
+		keys, gen.split, gen.ongoing = neutral, true, make(map[string]bool)
+	}
+	for i := range disks {
+		if keys[i] == nil || *keys[i] != bestKey {
+			continue
+		}
+		fi, err := getFileInfo(raws[i].Buf, bucket, object, versionID, fileInfoOpts{Data: true})
+		if err != nil {
+			continue
+		}
+		gen.drives[i], gen.disks[i] = fi, disks[i]
+		if gen.split {
+			if st, err := parseRestoreObjStatus(fi.Metadata[xhttp.AmzRestore]); err == nil && st.Ongoing() {
+				gen.ongoing[fi.Metadata[restoreAttemptKey]] = true
+			}
+		}
+		if !gen.fi.IsValid() {
+			gen.fi = fi
+		}
+	}
+	if !gen.fi.IsValid() && !gen.fi.Deleted {
+		return restoreGeneration{}, errErasureReadQuorum
+	}
+	if gen.split {
+		gen.fi.Metadata = maps.Clone(gen.fi.Metadata)
+		for _, k := range restoreFieldKeys {
+			delete(gen.fi.Metadata, k)
+		}
+	}
+	return gen, nil
+}
+
+// updateRestoreStatus rewrites the restore fields of one stored version. It
+// selects the authoritative generation of the version, see
+// readRestoreGeneration, patches only the restore fields onto exactly that
+// generation, keeping every other field and its ModTime, and writes it back
+// only to the drives that hold it. The caller holds the object lock. update
+// edits the metadata in place, or returns an error, errRestoreUnchanged to
+// write nothing; ongoing names the attempts whose ongoing marker the
+// generation holds. A failed read never removes the version, and a failed
+// write puts the generation it read back.
+func (er erasureObjects) updateRestoreStatus(ctx context.Context, bucket, object, versionID string, update func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error) (ObjectInfo, error) {
+	gen, err := er.readRestoreGeneration(ctx, bucket, object, versionID)
+	if err != nil {
+		return ObjectInfo{}, toObjectErr(err, bucket, object, versionID)
+	}
+	fi := gen.fi
+	oi := fi.ToObjectInfo(bucket, object, false)
+	if fi.Deleted {
+		return oi, toObjectErr(errMethodNotAllowed, bucket, object)
+	}
+	ongoing := gen.ongoing
+	if !gen.split && oi.RestoreOngoing {
+		ongoing = map[string]bool{oi.UserDefined[restoreAttemptKey]: true}
+	}
+	meta := maps.Clone(fi.Metadata)
+	if err := update(oi, meta, ongoing); err != nil {
+		return oi, err
+	}
+	write := func(meta func(FileInfo) map[string]string) error {
+		metaArr := make([]FileInfo, len(gen.drives))
+		for i, d := range gen.drives {
+			if gen.disks[i] == nil {
+				continue
+			}
+			if !d.InlineData() {
+				d.Data = nil
+			}
+			d.Metadata = meta(d)
+			metaArr[i] = d
+		}
+		// The write assigns each drive its slot as erasure index: order the
+		// drives by the version's distribution first.
+		disks, metaArr := shuffleDisksAndPartsMetadata(gen.disks, metaArr, fi)
+		_, err := writeUniqueFileInfo(ctx, disks, "", bucket, object, metaArr, fi.WriteQuorum(er.defaultWQuorum()))
+		return err
+	}
+	if err := write(func(FileInfo) map[string]string { return meta }); err != nil {
+		// Without the read generation back, its drives could hold two
+		// revisions and no quorum for either. A split that stays differs
+		// only in restore fields, which the next write or reset resolves.
+		if rerr := write(func(d FileInfo) map[string]string { return d.Metadata }); rerr != nil {
+			storageLogIf(ctx, fmt.Errorf("Unable to put back the restore status of %s/%s(%s): %w", bucket, object, versionID, rerr))
+		}
+		return oi, toObjectErr(err, bucket, object)
+	}
+	fi.Metadata = meta
+	return fi.ToObjectInfo(bucket, object, false), nil
+}
+
+// clearRestoreStatus makes a version retryable again while attempt a still
+// owns its ongoing marker. It never brings back an earlier completed status:
+// POST only starts a restore when that restored copy is gone.
+func (er erasureObjects) clearRestoreStatus(ctx context.Context, bucket, object, versionID string, a *restoreAttempt) error {
+	_, err := er.updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error {
+		id := ""
+		if a != nil {
+			id = a.ID
+			if !a.targets(oi) {
+				return errRestoreUnchanged
+			}
+		}
+		// Clear only this attempt's marker. Where drives disagree, another
+		// attempt's marker may be the acknowledged one: leave that to it,
+		// or to a later POST, which takes it over under the lock.
+		if !ongoing[id] || len(ongoing) != 1 {
+			return errRestoreUnchanged
+		}
+		delete(meta, xhttp.AmzRestore)
+		delete(meta, xhttp.AmzRestoreExpiryDays)
+		delete(meta, xhttp.AmzRestoreRequestDate)
+		delete(meta, restoreAttemptKey)
+		return nil
+	})
+	if errors.Is(err, errRestoreUnchanged) {
+		return nil
+	}
+	return err
+}
+
+// restoreFailed clears the ongoing marker of a failed attempt while the
+// object lock is held. A lost lock or an ended context leaves that to the
+// bounded retries of the pools layer, which take the lock afresh.
+func (er erasureObjects) restoreFailed(ctx context.Context, bucket, object, versionID string, published *restoreAttempt, cause error) error {
+	if ctx.Err() != nil {
+		return restoreResetError{cause}
+	}
+	if err := er.clearRestoreStatus(ctx, bucket, object, versionID, published); err != nil {
+		storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, versionID, err))
+		return restoreResetError{cause}
+	}
+	return cause
+}
+
+// abortRestoreUpload removes the private upload of a failed attempt, or
+// leaves it to stale-upload cleanup. The restore's own context may have
+// ended, so it uses a fresh bounded one, and the upload lock as any abort.
+// It matches the upload to its own attempt rather than to the version, which
+// a newer attempt may own by now, and it never touches the version.
+func (er erasureObjects) abortRestoreUpload(bucket, object, uploadID string, a *restoreAttempt) {
+	ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
+	defer cancel()
+	lk := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err == nil {
+		defer lk.Unlock(lkctx)
+		_, err = er.abortMultipartUpload(lkctx.Context(), bucket, object, uploadID, ObjectOptions{
+			NoAuditLog: true,
+			Transition: TransitionOptions{attempt: a},
+		})
+	}
+	if err != nil {
+		storageLogIf(ctx, fmt.Errorf("Unable to abort the restore upload of %s/%s: %w", bucket, object, err))
+	}
+}
+
+// restoreUploadActive reports whether the restore that owns a stale-aged
+// upload still runs: its attempt owns the version's ongoing marker. A long
+// download keeps its upload; one whose attempt ended loses it, and an
+// ongoing marker alone keeps nothing.
+func (er erasureObjects) restoreUploadActive(ctx context.Context, v restoreUploadView) bool {
+	gen, err := er.readRestoreGeneration(ctx, v.bucket, v.object, v.marker.VersionID)
+	if err != nil {
+		// An unreadable version may still be restoring; retry next round.
+		return !errors.Is(err, errFileVersionNotFound) && !errors.Is(err, errFileNotFound)
+	}
+	if gen.split {
+		// Drives disagree on the restore status: that the attempt ended
+		// is not established.
+		return gen.ongoing[v.marker.ID]
+	}
+	return v.marker.owns(gen.fi.ToObjectInfo(v.bucket, v.object, false))
+}
+
+// restoreLock is the object lock that restores and their status updates
+// share: with one pool the set's own lock, as single-pool writes use.
+func (z *erasureServerPools) restoreLock(bucket, object string) RWLocker {
+	if z.SinglePool() {
+		return z.serverPools[0].getHashedSet(object).NewNSLock(bucket, object)
+	}
+	return z.NewNSLock(bucket, object)
+}
+
+// restoreRequestStatus is what a POST restore recorded on its version.
+type restoreRequestStatus struct {
+	objInfo ObjectInfo
+	// attempt is the restore that must rehydrate the version; nil when a
+	// restored copy was still there and its expiry was renewed.
+	attempt *restoreAttempt
+}
+
+// restoreRequest records a POST restore on target, the version that POST
+// resolved, on a fresh read under the object lock. The version must still
+// be target: a replacement under the same version ID is refused. A copy that
+// still holds its restored data is renewed. Otherwise the version is marked
+// ongoing for a new attempt; that also takes over a marker no running
+// restore holds, as a running restore holds the object lock for its whole
+// download. Only the copy that is to be restored is written, so no copy
+// claims a restored copy it does not hold.
+func (z *erasureServerPools) restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error) {
+	object = encodeDirObject(object)
+	versionID := restoreVersionID(target.VersionID)
+	lk := z.restoreLock(bucket, object)
+	// Wait only briefly first, whatever the lockless read before saw: a
+	// restore of this version holds the lock for its whole download. Then
+	// wait as usual only if the drives show no restore of it running.
+	lkctx, err := lk.GetLock(ctx, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait))
+	if errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil {
+		running, rerr := z.restoreRunning(ctx, bucket, object, versionID)
+		switch {
+		case rerr != nil:
+			// Whether it is restoring is unknown: answer now, not after
+			// the lock frees.
+			return restoreRequestStatus{}, rerr
+		case running:
+			return restoreRequestStatus{}, errRestoreInProgress
+		}
+		lkctx, err = lk.GetLock(ctx, globalOperationTimeout)
+	}
+	if err != nil {
+		return restoreRequestStatus{}, err
+	}
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx)
+
+	expected := newRestoreAttempt(target)
+	copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{
+		VersionID:        versionID,
+		Versioned:        opts.Versioned,
+		VersionSuspended: opts.VersionSuspended,
+		Transition:       opts.Transition,
+	})
+	if err != nil {
+		return restoreRequestStatus{}, err
+	}
+	// Restore the copy reads serve first, outside pools being decommissioned,
+	// as RestoreTransitionedObject does.
+	pool := -1
+	for _, c := range copies {
+		if !z.IsSuspended(c.Index) {
+			pool = c.Index
+			break
+		}
+	}
+	if pool < 0 {
+		return restoreRequestStatus{}, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
+	}
+
+	rreq := opts.Transition.RestoreRequest
+	date := UTCNow().Format(http.TimeFormat)
+	var attempt *restoreAttempt
+	oi, err := z.serverPools[pool].getHashedSet(object).updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, _ map[string]bool) error {
+		if !expected.targets(oi) {
+			// Not the version POST resolved, or no longer transitioned.
+			return InvalidObjectState{Bucket: bucket, Object: decodeDirObject(object)}
+		}
+		attempt = nil
+		meta[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days)
+		meta[xhttp.AmzRestoreRequestDate] = date
+		if !oi.RestoreOngoing && !oi.RestoreExpires.IsZero() {
+			// Cleanup removes a restored copy together with its header.
+			meta[xhttp.AmzRestore] = completedRestoreObj(opts.Transition.RestoreExpiry).String()
+			delete(meta, restoreAttemptKey)
+			return nil
+		}
+		attempt = newRestoreAttempt(oi)
+		meta[xhttp.AmzRestore] = ongoingRestoreObj().String()
+		meta[restoreAttemptKey] = attempt.ID
+		return nil
+	})
+	if err != nil {
+		if attempt != nil {
+			// The marker may have reached some drives, or all of them.
+			z.recoverRestoreStatus(bucket, object, ObjectOptions{
+				VersionID:  attempt.VersionID,
+				Transition: TransitionOptions{attempt: attempt},
+			})
+		}
+		return restoreRequestStatus{}, err
+	}
+	return restoreRequestStatus{objInfo: oi, attempt: attempt}, nil
+}
+
+// restoreRunning reports, without the object lock and within the takeover
+// wait, whether the drives show a restore of the version ongoing in a pool.
+// An error means that this cannot be told.
+func (z *erasureServerPools) restoreRunning(ctx context.Context, bucket, object, versionID string) (bool, error) {
+	ctx, cancel := context.WithTimeout(ctx, restoreTakeoverWait)
+	defer cancel()
+	for _, pool := range z.serverPools {
+		gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, versionID)
+		if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err == nil && ctx.Err() != nil {
+			err = ctx.Err()
+		}
+		if err != nil {
+			return false, toObjectErr(err, bucket, object, versionID)
+		}
+		if len(gen.ongoing) > 0 || !gen.split && gen.fi.ToObjectInfo(bucket, object, false).RestoreOngoing {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// restorePoolIdx returns the pool whose copy the restore rehydrates: the one
+// POST marked for its attempt, or for a direct caller the existing version.
+func (z *erasureServerPools) restorePoolIdx(ctx context.Context, bucket, object string, opts ObjectOptions) (int, error) {
+	a := opts.Transition.attempt
+	if a == nil {
+		// Restore an existing version, not the latest version's write location.
+		lookupOpts := opts
+		lookupOpts.MetadataChg = true
+		idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts)
+		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return idx, restoreResetError{err}
+		}
+		return idx, err
+	}
+	found := false
+	for idx, pool := range z.serverPools {
+		gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, a.VersionID)
+		if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			return -1, restoreResetError{toObjectErr(err, bucket, object, a.VersionID)}
+		}
+		found = true
+		if !z.IsSuspended(idx) && a.owns(gen.fi.ToObjectInfo(bucket, object, false)) {
+			return idx, nil
+		}
+	}
+	if !found {
+		return -1, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: a.VersionID}
+	}
+	return -1, errRestoreSuperseded
+}
+
+// recoverRestoreStatus retries, in memory and for a bounded time, to clear the
+// ongoing marker that a failed restore left behind, so a later POST is not
+// refused. Each try takes the object lock afresh and clears the marker only
+// while it still belongs to this attempt.
+func (z *erasureServerPools) recoverRestoreStatus(bucket, object string, opts ObjectOptions) {
+	go func() {
+		var err error
+		for _, delay := range restoreRecoveryDelays {
+			select {
+			case <-GlobalContext.Done():
+				return
+			case <-time.After(delay):
+			}
+			if err = z.resetRestoreStatus(bucket, object, opts); err == nil {
+				return
+			}
+		}
+		storageLogIf(GlobalContext, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, err))
+	}()
+}
+
+// resetRestoreStatus is one try of recoverRestoreStatus.
+func (z *erasureServerPools) resetRestoreStatus(bucket, object string, opts ObjectOptions) error {
+	ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
+	defer cancel()
+	lk := z.restoreLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return err
+	}
+	defer lk.Unlock(lkctx)
+	ctx = lkctx.Context()
+
+	a := opts.Transition.attempt
+	versionID := opts.VersionID
+	if a != nil {
+		versionID = a.VersionID
+	}
+	for _, pool := range z.serverPools {
+		err := pool.getHashedSet(object).clearRestoreStatus(ctx, bucket, object, versionID, a)
+		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !isErrMethodNotAllowed(err) && !isErrBucketNotFound(err) {
+			return err
+		}
+	}
+	return nil
+}

cmd/restore-capability.go

@@ -0,0 +1,184 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"slices"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/internal/grid"
+)
+
+// A native restore keeps its multipart upload private and marks the version
+// it rehydrates (restore-attempt.go). A server that predates this ignores
+// the marker: it would list, change, abort or complete the upload. Native
+// restore therefore runs only while every server of the cluster positively
+// reports that it understands the marker. A server that does not answer,
+// or does not know the question, as an older server does not, blocks it.
+
+// nativeRestoreCapability is the native restore protocol this server speaks.
+const nativeRestoreCapability = "1"
+
+const (
+	peerRESTRestoreCapability = "restore"
+	// restoreCapabilityTTL bounds how long a positive answer is reused while
+	// no server reconnected.
+	restoreCapabilityTTL = 10 * time.Second
+	// restoreCapabilityTimeout bounds the question to each server.
+	restoreCapabilityTimeout = 5 * time.Second
+)
+
+var errNativeRestoreUnavailable = errors.New("native restore is unavailable during a rolling upgrade: not every server of the cluster supports it yet")
+
+// restoreCapabilityHandler answers the question on this server.
+func restoreCapabilityHandler(*grid.MSS) (*grid.MSS, *grid.RemoteErr) {
+	return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: nativeRestoreCapability}), nil
+}
+
+// restoreCapabilityPeer is another server of the cluster.
+type restoreCapabilityPeer interface {
+	// restoreConnection identifies the current connection to the server; it
+	// changes when the server reconnects, as a restarted one does. ok is
+	// false when the server is not connected.
+	restoreConnection() (key string, ok bool)
+	restoreCapability(ctx context.Context) (string, error)
+}
+
+type restorePeerClient struct{ *peerRESTClient }
+
+func (c restorePeerClient) restoreConnection() (string, bool) {
+	if c.peerRESTClient == nil || c.gridConn == nil {
+		return "", false
+	}
+	conn := c.gridConn()
+	if conn == nil {
+		return "", false
+	}
+	st := conn.Stats()
+	if st.Connected == 0 {
+		return "", false
+	}
+	// Keys are compared by the position of their server.
+	return fmt.Sprintf("%d/%d", st.LastConnectTime.UnixNano(), st.ReconnectCount), true
+}
+
+func (c restorePeerClient) restoreCapability(ctx context.Context) (string, error) {
+	conn := c.gridConn()
+	if conn == nil {
+		return "", errPeerOffline
+	}
+	resp, err := restoreCapabilityRPC.Call(ctx, conn, grid.NewMSS())
+	if err != nil {
+		return "", err
+	}
+	return resp.Get(peerRESTRestoreCapability), nil
+}
+
+// restoreCapabilityGate decides whether native restore may run, caching a
+// positive answer for restoreCapabilityTTL while the connections to all
+// servers stay the same. A negative answer is not cached.
+type restoreCapabilityGate struct {
+	peers func() []restoreCapabilityPeer
+
+	once    sync.Once
+	turn    chan struct{} // one check at a time
+	checked time.Time
+	keys    []string
+}
+
+var globalRestoreCapability = &restoreCapabilityGate{peers: notificationRestorePeers}
+
+func notificationRestorePeers() []restoreCapabilityPeer {
+	if globalNotificationSys == nil {
+		return nil
+	}
+	peers := make([]restoreCapabilityPeer, 0, len(globalNotificationSys.peerClients))
+	for _, client := range globalNotificationSys.peerClients {
+		peers = append(peers, restorePeerClient{client})
+	}
+	return peers
+}
+
+// nativeRestoreAvailable returns errNativeRestoreUnavailable unless every
+// other server of the cluster supports native restore.
+func nativeRestoreAvailable(ctx context.Context) error {
+	return globalRestoreCapability.check(ctx)
+}
+
+func (g *restoreCapabilityGate) check(ctx context.Context) error {
+	g.once.Do(func() { g.turn = make(chan struct{}, 1) })
+	// One deadline covers the wait for another check and the questions.
+	ctx, cancel := context.WithTimeout(ctx, restoreCapabilityTimeout)
+	defer cancel()
+	select {
+	case g.turn <- struct{}{}:
+	case <-ctx.Done():
+		return errNativeRestoreUnavailable
+	}
+	defer func() { <-g.turn }()
+
+	// Read the connections only now: one may have changed while this check
+	// waited for another.
+	peers := g.peers()
+	connections := func() ([]string, bool) {
+		keys := make([]string, len(peers))
+		for i, p := range peers {
+			key, ok := p.restoreConnection()
+			if !ok {
+				return nil, false
+			}
+			keys[i] = key
+		}
+		return keys, true
+	}
+	keys, ok := connections()
+	if !ok {
+		g.checked, g.keys = time.Time{}, nil
+		return errNativeRestoreUnavailable
+	}
+	if ctx.Err() != nil {
+		return errNativeRestoreUnavailable
+	}
+	if !g.checked.IsZero() && time.Since(g.checked) < restoreCapabilityTTL && slices.Equal(keys, g.keys) {
+		return nil
+	}
+	g.checked, g.keys = time.Time{}, nil
+	answers := make([]bool, len(peers))
+	var wg sync.WaitGroup
+	for i, p := range peers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			version, err := p.restoreCapability(ctx)
+			answers[i] = err == nil && version == nativeRestoreCapability
+		}()
+	}
+	wg.Wait()
+	// An answer counts only for the connection it was asked on: a server
+	// that reconnected meanwhile may be another binary.
+	after, ok := connections()
+	if !ok || !slices.Equal(keys, after) || slices.Contains(answers, false) || ctx.Err() != nil {
+		return errNativeRestoreUnavailable
+	}
+	g.checked, g.keys = time.Now(), keys
+	return nil
+}

cmd/restore-capability_test.go

@@ -0,0 +1,271 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"reflect"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/grid"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+// A server answers the capability question over the grid. One that does not
+// know it, as an older server, or answers something else, blocks native
+// restore, and so does one that is not connected.
+func TestNativeRestoreCapabilityGrid(t *testing.T) {
+	tg, err := grid.SetupTestGrid(4)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(tg.Cleanup)
+	if err := restoreCapabilityRPC.Register(tg.Managers[1], restoreCapabilityHandler); err != nil {
+		t.Fatal(err)
+	}
+	// Managers[2] never registers the handler.
+	if err := restoreCapabilityRPC.Register(tg.Managers[3], func(*grid.MSS) (*grid.MSS, *grid.RemoteErr) {
+		return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: "0"}), nil
+	}); err != nil {
+		t.Fatal(err)
+	}
+	peer := func(i int) restoreCapabilityPeer {
+		return restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[i]) }}}
+	}
+	offline := restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return nil }}}
+	for _, tc := range []struct {
+		name  string
+		peers []restoreCapabilityPeer
+		ok    bool
+	}{
+		{"single-server", nil, true},
+		{"all-capable", []restoreCapabilityPeer{peer(1)}, true},
+		{"older-server", []restoreCapabilityPeer{peer(1), peer(2)}, false},
+		{"other-answer", []restoreCapabilityPeer{peer(1), peer(3)}, false},
+		{"offline", []restoreCapabilityPeer{peer(1), offline}, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return tc.peers }}
+			if err := g.check(t.Context()); (err == nil) != tc.ok {
+				t.Errorf("capability: %v, want available=%v", err, tc.ok)
+			}
+		})
+	}
+}
+
+// restoreCapabilityFake is a server whose connection and answer a test sets.
+type restoreCapabilityFake struct {
+	capable   atomic.Bool
+	connected atomic.Bool
+	conn      atomic.Int64
+	asked     atomic.Int64
+}
+
+func newRestoreCapabilityFake() *restoreCapabilityFake {
+	p := &restoreCapabilityFake{}
+	p.capable.Store(true)
+	p.connected.Store(true)
+	return p
+}
+
+func (p *restoreCapabilityFake) restoreConnection() (string, bool) {
+	return strings.Repeat("c", int(p.conn.Load())), p.connected.Load()
+}
+
+func (p *restoreCapabilityFake) restoreCapability(context.Context) (string, error) {
+	p.asked.Add(1)
+	if !p.capable.Load() {
+		return "", errors.New("Invalid Handler for type: RestoreCapability")
+	}
+	return nativeRestoreCapability, nil
+}
+
+// A positive answer is reused while the connections stay the same; a
+// reconnect asks again, a disconnected server blocks without being asked,
+// and a negative answer is not reused.
+func TestNativeRestoreCapabilityCache(t *testing.T) {
+	p := newRestoreCapabilityFake()
+	g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+	for range 3 {
+		if err := g.check(t.Context()); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if n := p.asked.Load(); n != 1 {
+		t.Errorf("asked %d times while connected and capable, want 1", n)
+	}
+	// The server restarts with an older binary and reconnects.
+	p.capable.Store(false)
+	p.conn.Add(1)
+	if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) {
+		t.Errorf("after a reconnect to an older server: %v", err)
+	}
+	p.capable.Store(true)
+	if err := g.check(t.Context()); err != nil {
+		t.Errorf("a negative answer was reused: %v", err)
+	}
+	asked := p.asked.Load()
+	p.connected.Store(false)
+	if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) || p.asked.Load() != asked {
+		t.Errorf("disconnected server: %v, asked %d -> %d", err, asked, p.asked.Load())
+	}
+}
+
+// During a rolling upgrade POST restore answers 503 and writes nothing, and
+// a restore queued before it creates no upload and leaves the version
+// retryable.
+func TestNativeRestoreRollingUpgradeGate(t *testing.T) {
+	r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+	initConfigSubsystem(t.Context(), r.z)
+	oi, _ := restoreIsolationSeed(t, r, 0, false, true)
+	p := newRestoreCapabilityFake()
+	previous := globalRestoreCapability
+	globalRestoreCapability = &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+	t.Cleanup(func() { globalRestoreCapability = previous })
+
+	p.capable.Store(false)
+	before := r.fileInfo(oi.VersionID)
+	layer := newRestoreIsolationLayer(r)
+	rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+	if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "rolling upgrade") {
+		t.Errorf("POST during a rolling upgrade: %d %s", rec.Code, rec.Body.String())
+	}
+	if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before.Metadata, after.Metadata) {
+		t.Error("refused POST changed the version")
+	}
+	select {
+	case err := <-layer.done:
+		t.Fatalf("refused POST started a restore: %v", err)
+	default:
+	}
+
+	p.capable.Store(true)
+	release := make(chan struct{})
+	layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+		<-release
+		return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+	}
+	if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+		close(release)
+		t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+	}
+	// A server is replaced by an older binary while the restore is queued.
+	p.capable.Store(false)
+	p.conn.Add(1)
+	close(release)
+	if err := layer.wait(t); !errors.Is(err, errNativeRestoreUnavailable) {
+		t.Fatalf("queued restore during a rolling upgrade: %v", err)
+	}
+	if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+		t.Errorf("queued restore created an upload during a rolling upgrade: %v", uploads)
+	}
+	info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+	if err != nil || info.RestoreOngoing || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" {
+		t.Errorf("version after a refused queued restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+	}
+}
+
+// restoreCapabilityHeld is a fake server whose answer a test holds.
+type restoreCapabilityHeld struct {
+	restoreCapabilityFake
+	asking  chan struct{}
+	release chan struct{}
+}
+
+func (p *restoreCapabilityHeld) restoreCapability(ctx context.Context) (string, error) {
+	v, err := p.restoreCapabilityFake.restoreCapability(ctx)
+	p.asking <- struct{}{}
+	select {
+	case <-p.release:
+		return v, err
+	case <-ctx.Done():
+		return "", ctx.Err()
+	}
+}
+
+// A check that waits for another reads the connections once its turn
+// comes, an answer counts only if its server did not reconnect meanwhile,
+// and a waiting check ends with its context.
+func TestNativeRestoreCapabilityRevalidates(t *testing.T) {
+	t.Run("reconnect-during-check", func(t *testing.T) {
+		held := func() *restoreCapabilityHeld {
+			p := &restoreCapabilityHeld{asking: make(chan struct{}, 4), release: make(chan struct{})}
+			p.capable.Store(true)
+			p.connected.Store(true)
+			return p
+		}
+		fast, slow := held(), held()
+		g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{fast, slow} }}
+		first := make(chan error, 1)
+		go func() { first <- g.check(t.Context()) }()
+		<-fast.asking
+		<-slow.asking
+		waiting := make(chan error, 1)
+		go func() { waiting <- g.check(t.Context()) }()
+		// Both answered "capable"; then fast reconnects as an older server.
+		fast.capable.Store(false)
+		fast.conn.Add(1)
+		close(fast.release)
+		close(slow.release)
+		if err := <-first; !errors.Is(err, errNativeRestoreUnavailable) {
+			t.Errorf("check whose server reconnected while it asked: %v", err)
+		}
+		if err := <-waiting; !errors.Is(err, errNativeRestoreUnavailable) {
+			t.Errorf("waiting check accepted a reconnected older server: %v", err)
+		}
+	})
+	t.Run("canceled-cached", func(t *testing.T) {
+		p := newRestoreCapabilityFake()
+		g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+		if err := g.check(t.Context()); err != nil {
+			t.Fatal(err)
+		}
+		ctx, cancel := context.WithCancel(t.Context())
+		cancel()
+		if err := g.check(ctx); err == nil {
+			t.Error("a cached answer accepted an ended context")
+		}
+	})
+	t.Run("canceled-wait", func(t *testing.T) {
+		slow := &restoreCapabilityHeld{restoreCapabilityFake: restoreCapabilityFake{}, asking: make(chan struct{}, 4), release: make(chan struct{})}
+		slow.capable.Store(true)
+		slow.connected.Store(true)
+		defer close(slow.release)
+		g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{slow} }}
+		go g.check(t.Context())
+		<-slow.asking
+		ctx, cancel := context.WithCancel(t.Context())
+		waiting := make(chan error, 1)
+		go func() { waiting <- g.check(ctx) }()
+		cancel()
+		select {
+		case err := <-waiting:
+			if err == nil {
+				t.Error("canceled check succeeded")
+			}
+		case <-time.After(time.Second):
+			t.Error("canceled check kept waiting for another check")
+		}
+	})
+}

cmd/restore-native_test.go

@@ -375,8 +375,10 @@ func TestNativeRestorePOSTVersionIdentity(t *testing.T) {
 
 // A restore request is internal context, not an exception for normal S3
 // multipart overwrites or for a version that has never completed transition.
+// Only the upload of a native restore attempt, whose marker names the very
+// version being completed, may write that version back.
 func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
-	for _, mode := range []string{"matching-restore", "ordinary-complete", "not-tiered", "changed-time", "changed-size", "changed-etag"} {
+	for _, mode := range []string{"matching-restore", "ordinary-complete", "restore-without-attempt", "other-attempt", "not-tiered", "changed-time", "changed-size", "changed-etag"} {
 		t.Run(mode, func(t *testing.T) {
 			r := newRestoreSafetyTest(t, 1, versioning.Enabled)
 			oi, body := nativeRestoreSeed(t, r, 0, false, true)
@@ -398,8 +400,9 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			opts := putRestoreOpts(r.bucket, r.object, &RestoreObjectRequest{Days: 1}, current)
+			opts := putRestoreOpts(&RestoreObjectRequest{Days: 1}, current, r.opts(oi.VersionID))
 			opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+			opts.PreserveETag = current.ETag
 			if mode == "ordinary-complete" {
 				opts.Transition.RestoreRequest = nil
 			}
@@ -408,20 +411,26 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
 			}
 			// Hold the ETag constant for the size case, so only size can reject it.
 			if mode == "changed-size" {
-				opts.UserDefined["etag"] = current.ETag
 				body = append(body, 'x')
 			}
 			if mode == "changed-etag" {
-				opts.UserDefined["etag"] = "different-etag"
+				opts.PreserveETag = "different-etag"
 			}
-			upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, opts)
+			// The attempt claims what its upload claims; only the matching
+			// mode claims exactly the stored version.
+			if mode != "ordinary-complete" && mode != "restore-without-attempt" {
+				opts.Transition.attempt = &restoreAttempt{ID: mustGetUUID(), VersionID: oi.VersionID, ETag: opts.PreserveETag, Size: current.Size, ModTime: opts.MTime}
+			}
+			uopts := opts
+			uopts.MTime = time.Time{}
+			upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, uopts)
 			if err != nil {
 				t.Fatal(err)
 			}
 			var complete []CompletePart
 			for i, data := range [][]byte{body[:5<<20], body[5<<20:]} {
 				part, err := pool.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, i+1,
-					mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
+					mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Transition: opts.Transition})
 				if err != nil {
 					t.Fatal(err)
 				}
@@ -430,6 +439,11 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
 			if mode != "changed-size" && getCompleteMultipartMD5(complete) != current.ETag {
 				t.Fatal("fixture multipart ETag does not match original")
 			}
+			if mode == "other-attempt" {
+				other := *opts.Transition.attempt
+				other.ID = mustGetUUID()
+				opts.Transition.attempt = &other
+			}
 			before := nativeRestoreSnapshot(t, r)
 			_, err = pool.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, complete, opts)
 			if mode == "matching-restore" {
@@ -440,9 +454,16 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
 				if err != nil || !bytes.Equal(got, body) {
 					t.Fatalf("matching restore contents: %v", err)
 				}
+				if after := r.fileInfo(oi.VersionID); after.Metadata[restoreUploadKey] != "" || after.Metadata[restoreAttemptKey] != "" {
+					t.Error("restore committed its private upload or attempt marker")
+				}
 				return
 			}
-			if !errors.Is(err, errAuthentication) {
+			if mode == "other-attempt" {
+				if !errors.As(err, &InvalidUploadID{}) {
+					t.Fatalf("another attempt completed a private upload: %v", err)
+				}
+			} else if !errors.Is(err, errAuthentication) {
 				t.Fatalf("existing version overwrite: %v", err)
 			}
 			if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
@@ -721,18 +742,10 @@ func TestNativeRestoreConcurrentCompletion(t *testing.T) {
 			case <-ctx.Done():
 				t.Fatal("restore never started")
 			}
-			// A client's exact-prefix listing can discover the internal upload. It can
-			// try to complete the first part while native restore owns the object.
-			uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10)
-			if err != nil || len(uploads.Uploads) != 1 {
-				t.Fatalf("restore upload discovery: %+v, %v", uploads, err)
-			}
-			id := uploads.Uploads[0].UploadID
-			parts, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{})
-			if err != nil || len(parts.Parts) != 1 {
-				t.Fatalf("restore must have published its first part: %+v, %v", parts, err)
-			}
-			part := parts.Parts[0]
+			// A client that learns the internal upload ID, which listings no
+			// longer show, can try to complete the first part while native
+			// restore owns the object.
+			id, part := restoreIsolationUpload(t, r, 0)
 			completed := make(chan struct{})
 			var completeErr error
 			go func() {
@@ -942,17 +955,22 @@ func TestNativeRestoreCompletionReadSafety(t *testing.T) {
 	}
 }
 
+// nativeRestoreStatusHook runs afterRead once POST has read the version and
+// before POST records its status on it.
 type nativeRestoreStatusHook struct {
 	ObjectLayer
-	beforeCopy       func()
-	copyCalled       bool
+	afterRead        func()
+	readCalled       bool
 	backgroundCalled atomic.Bool
 }
 
-func (h *nativeRestoreStatusHook) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, info ObjectInfo, srcOpts, dstOpts ObjectOptions) (ObjectInfo, error) {
-	h.copyCalled = true
-	h.beforeCopy()
-	return h.ObjectLayer.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, info, srcOpts, dstOpts)
+func (h *nativeRestoreStatusHook) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+	oi, err := h.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+	if err == nil && !h.readCalled {
+		h.readCalled = true
+		h.afterRead()
+	}
+	return oi, err
 }
 
 func (h *nativeRestoreStatusHook) RestoreTransitionedObject(context.Context, string, string, ObjectOptions) error {
@@ -985,7 +1003,7 @@ func TestNativeRestorePOSTReadSafety(t *testing.T) {
 					}
 					before = restoreBoundarySnapshot(t, r)
 				}
-				hook := &nativeRestoreStatusHook{ObjectLayer: r.z, beforeCopy: mutate}
+				hook := &nativeRestoreStatusHook{ObjectLayer: r.z, afterRead: mutate}
 				if stage == "initial" {
 					mutate()
 				}
@@ -1004,7 +1022,7 @@ func TestNativeRestorePOSTReadSafety(t *testing.T) {
 				if rec.Code < 400 {
 					t.Fatalf("POST acknowledged failed read: %d %s", rec.Code, rec.Body.String())
 				}
-				if hook.copyCalled != (stage == "status-copy") || hook.backgroundCalled.Load() {
+				if hook.readCalled != (stage == "status-copy") || hook.backgroundCalled.Load() {
 					t.Fatal("request followed the wrong failure path")
 				}
 				if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {

internal/grid/handlers.go

@@ -116,6 +116,7 @@ const (
 	HandlerRenamePart
 	HandlerClearUploadID
 	HandlerCheckParts3
+	HandlerRestoreCapability
 
 	// Add more above here ^^^
 	// If all handlers are used, the type of Handler can be changed.
@@ -200,6 +201,7 @@ var handlerPrefixes = [handlerLast]string{
 	HandlerCheckParts3:                 storagePrefix,
 	HandlerRenamePart:                  storagePrefix,
 	HandlerClearUploadID:               peerPrefix,
+	HandlerRestoreCapability:           peerPrefix,
 }
 
 const (

internal/grid/handlers_string.go

@@ -86,14 +86,15 @@ func _() {
 	_ = x[HandlerRenamePart-75]
 	_ = x[HandlerClearUploadID-76]
 	_ = x[HandlerCheckParts3-77]
-	_ = x[handlerTest-78]
-	_ = x[handlerTest2-79]
-	_ = x[handlerLast-80]
+	_ = x[HandlerRestoreCapability-78]
+	_ = x[handlerTest-79]
+	_ = x[handlerTest2-80]
+	_ = x[handlerLast-81]
 }
 
-const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3handlerTesthandlerTest2handlerLast"
+const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3RestoreCapabilityhandlerTesthandlerTest2handlerLast"
 
-var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 971, 983, 994}
+var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 977, 988, 1000, 1011}
 
 func (i HandlerID) String() string {
 	idx := int(i) - 0