diff --git a/cmd/bucket-lifecycle-restore-boundary_test.go b/cmd/bucket-lifecycle-restore-boundary_test.go
index 14967cf6a..8f39342ba 100644
--- a/cmd/bucket-lifecycle-restore-boundary_test.go
+++ b/cmd/bucket-lifecycle-restore-boundary_test.go
@@ -137,8 +137,7 @@ func TestExpireRestoredMetadataReadKeepsMinorityVersion(t *testing.T) {
case "restore-cleanup":
// The initial read succeeded before remote download failed;
// cleanup must not purge a now-incomplete target on reread.
- opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
- err = er.updateRestoreMetadata(ctx, r.bucket, r.object, fi.ToObjectInfo(r.bucket, r.object, true), opts)
+ err = er.clearRestoreStatus(ctx, r.bucket, r.object, oi.VersionID, nil)
case "lookup-cleanup", "lookup-restore":
// Version-sensitive lookup also covers native restore routing.
opts.MetadataChg = true
diff --git a/cmd/bucket-lifecycle-restore-safety_test.go b/cmd/bucket-lifecycle-restore-safety_test.go
index 9facbfd0d..85a4be182 100644
--- a/cmd/bucket-lifecycle-restore-safety_test.go
+++ b/cmd/bucket-lifecycle-restore-safety_test.go
@@ -156,7 +156,7 @@ func TestExpireRestoredRenewedOrMissing(t *testing.T) {
t.Fatal(err)
}
err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
- if !isErrVersionNotFound(err) {
+ if !errors.Is(err, errLifecycleNotApplicable) {
t.Errorf("stale task should be rejected, got %v", err)
}
if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed || got.DataDir != before.DataDir {
diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go
index 5c5597587..5854fa64d 100644
--- a/cmd/bucket-lifecycle.go
+++ b/cmd/bucket-lifecycle.go
@@ -37,6 +37,7 @@ import (
"github.com/minio/minio/internal/amztime"
sse "github.com/minio/minio/internal/bucket/encryption"
"github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/replication"
"github.com/minio/minio/internal/event"
xhttp "github.com/minio/minio/internal/http"
@@ -987,8 +988,10 @@ func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string
}, nil
}
-// set ObjectOptions for PUT call to restore temporary copy of transitioned data
-func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) {
+// set ObjectOptions for PUT call to restore temporary copy of transitioned data.
+// opts are the restore's options, with the versioning POST captured: the
+// bucket's versioning may have changed since, and must not mint a new version.
+func putRestoreOpts(rreq *RestoreObjectRequest, objInfo ObjectInfo, opts ObjectOptions) (putOpts ObjectOptions) {
meta := make(map[string]string)
sc := rreq.OutputLocation.S3.StorageClass
if sc == "" {
@@ -1011,8 +1014,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
}
return ObjectOptions{
- Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, object),
- VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+ Versioned: opts.Versioned,
+ VersionSuspended: opts.VersionSuspended,
UserDefined: meta,
}
}
@@ -1020,19 +1023,38 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
if len(objInfo.UserTags) != 0 {
meta[xhttp.AmzObjectTagging] = objInfo.UserTags
}
- // Set restore object status
+ // Set restore object status; a completed restore has no attempt.
restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
+ delete(meta, restoreAttemptKey)
+ delete(meta, restoreUploadKey)
return ObjectOptions{
- Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, object),
- VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+ Versioned: opts.Versioned,
+ VersionSuspended: opts.VersionSuspended,
UserDefined: meta,
- VersionID: objInfo.VersionID,
+ VersionID: restoreVersionID(objInfo.VersionID),
MTime: objInfo.ModTime,
Expires: objInfo.Expires,
}
}
+// restoredCopyExpires reports whether the restored copy of oi, as read now,
+// expires by its own reason: lifecycle expires the restored copy of this
+// version, its restore deadline is valid and has passed, its transition is
+// complete and no restore is running. Another rule falling due is no reason.
+func restoredCopyExpires(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, oi ObjectInfo) bool {
+ if !evalActionFromLifecycle(ctx, lc, lr, rcfg, oi).Action.DeleteRestored() {
+ return false
+ }
+ status, err := parseRestoreObjStatus(oi.UserDefined[xhttp.AmzRestore])
+ if err != nil {
+ return false
+ }
+ expiry, completed := status.Expiry()
+ return completed && !expiry.IsZero() && UTCNow().After(expiry) &&
+ oi.TransitionedObject.Status == lifecycle.TransitionComplete && !oi.RestoreOngoing
+}
+
var errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
// IsRemote returns true if this object version's contents are in its remote
diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go
index 8fb371270..fad01b0cf 100644
--- a/cmd/erasure-healing.go
+++ b/cmd/erasure-healing.go
@@ -373,6 +373,13 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
if err != nil {
return result, err
}
+ // The restore fields fence native restores and restored-copy cleanup.
+ // Heal copies them only as a quorum of the drives holds them; while the
+ // drives disagree, it waits rather than create a quorum for one side.
+ latestMeta, err = restoreHealMeta(partsMetadata, latestMeta, max(len(storageDisks)/2+1, readQuorum))
+ if err != nil {
+ return result, toObjectErr(err, bucket, object, versionID)
+ }
// No modtime quorum
filterDisksByETag := quorumETag != ""
diff --git a/cmd/erasure-multipart-cleanup_test.go b/cmd/erasure-multipart-cleanup_test.go
index 26a95db53..14c012968 100644
--- a/cmd/erasure-multipart-cleanup_test.go
+++ b/cmd/erasure-multipart-cleanup_test.go
@@ -123,7 +123,20 @@ func (d *multipartCleanupDisk) Endpoint() Endpoint {
func (d *multipartCleanupDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
d.reads++
- return FileInfo{ModTime: d.modTime}, d.readErr
+ if d.readErr != nil {
+ return FileInfo{}, d.readErr
+ }
+ fi := newFileInfo("obj", 1, 0)
+ fi.Erasure.Index, fi.ModTime = 1, d.modTime
+ return fi, nil
+}
+
+// multipartCleanupSet is a set of one drive, which also answers the
+// metadata reads that establish the restore marker of an expired upload.
+// Its locks are local: stale cleanup decides a removal under the upload's
+// commit lock.
+func multipartCleanupSet(disk StorageAPI) erasureObjects {
+ return erasureObjects{getDisks: func() []StorageAPI { return []StorageAPI{disk} }, nsMutex: newNSLock(false)}
}
func (d *multipartCleanupDisk) StatVol(ctx context.Context, volume string) (VolInfo, error) {
@@ -143,9 +156,10 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) {
}{
{name: "recent", prefix: uuid, age: expiry / 2, readErr: errFileNotFound},
{name: "plaintext_base64_collision", prefix: "4eacd477-e733-4851-b821-6fd42785ff87", age: expiry / 2, readErr: errFileNotFound},
- {name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantDeleted: true},
+ // Only an expired upload is read on every drive, for its restore marker.
+ {name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantReads: 1, wantDeleted: true},
{name: "legacy_recent", prefix: uuid, age: expiry / 2, legacy: true, wantReads: 1},
- {name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 1, wantDeleted: true},
+ {name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 2, wantDeleted: true},
{name: "legacy_metadata_missing", prefix: uuid, age: 2 * expiry, legacy: true, readErr: errFileNotFound, wantReads: 1},
{name: "malformed_metadata_missing", prefix: "not-a-uuid", age: 2 * expiry, readErr: errFileNotFound, wantReads: 1},
} {
@@ -166,7 +180,7 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) {
if err := os.Chtimes(filepath.Dir(uploadDir), modTime, modTime); err != nil {
t.Fatal(err)
}
- erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
+ multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk)
if disk.reads != tc.wantReads {
t.Errorf("ReadVersion calls = %d, want %d", disk.reads, tc.wantReads)
}
@@ -199,7 +213,7 @@ func TestCleanupStaleUploadsEmptyHashDirectory(t *testing.T) {
if err := os.Chtimes(shaDir, modTime, modTime); err != nil {
t.Fatal(err)
}
- erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
+ multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk)
if _, err := os.Stat(shaDir); !os.IsNotExist(err) {
t.Fatalf("empty hash directory must be removed regardless of age: %v", err)
}
@@ -224,7 +238,7 @@ func TestCleanupStaleUploadsSharedHashDirectory(t *testing.T) {
t.Fatal(err)
}
- erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
+ multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk)
if _, err := os.Stat(filepath.Join(shaDir, expiredID)); !os.IsNotExist(err) {
t.Errorf("expired upload must be removed: %v", err)
}
diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go
index c68fbc446..c4f5d97ff 100644
--- a/cmd/erasure-multipart.go
+++ b/cmd/erasure-multipart.go
@@ -34,7 +34,6 @@ import (
"github.com/google/uuid"
"github.com/klauspost/readahead"
"github.com/minio/minio-go/v7/pkg/set"
- "github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/config/storageclass"
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/hash"
@@ -72,6 +71,7 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string {
}
// checkUploadIDExists - verify if a given uploadID exists and is valid.
+// The metadata read from each drive is returned also with an error.
func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string, write bool) (fi FileInfo, metArr []FileInfo, err error) {
defer func() {
if errors.Is(err, errFileNotFound) {
@@ -89,15 +89,15 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
if err != nil {
- return fi, nil, err
+ return fi, partsMetadata, err
}
if readQuorum < 0 {
- return fi, nil, errErasureReadQuorum
+ return fi, partsMetadata, errErasureReadQuorum
}
if writeQuorum < 0 {
- return fi, nil, errErasureWriteQuorum
+ return fi, partsMetadata, errErasureWriteQuorum
}
quorum := readQuorum
@@ -114,7 +114,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
}
if err != nil {
- return fi, nil, err
+ return fi, partsMetadata, err
}
// Pick one from the first valid metadata.
@@ -217,12 +217,55 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
hasSkippedUploads = true
return nil
}
+ // A completion holds this lock from staging its commit in the
+ // upload until the commit ends: decide on a removal only while
+ // holding it, and skip an upload whose commit is running. Only
+ // uploads past the stale age pay for the lock and the read below.
+ lk := er.restoreCommitLock(uploadIDPath)
+ lkctx, err := lk.GetLock(ctx, newDynamicTimeout(time.Second, time.Second))
+ if err != nil {
+ hasSkippedUploads = true
+ return nil
+ }
+ // The rename below releases the lock when it ends, also when it
+ // outlives its deadline; every other exit releases it here.
+ renaming := false
+ defer func() {
+ if !renaming {
+ lk.Unlock(lkctx)
+ }
+ }()
+ // A native restore can run past the stale age: keep its upload
+ // while its attempt still owns the version, and keep any upload
+ // whose marker the drives cannot establish.
+ view := er.readRestoreUpload(lkctx.Context(), "", uploadIDPath)
+ if view.committing {
+ // What a commit staged, with no commit running: only a staged
+ // copy itself shows when it staged, only an old one is a
+ // leftover of a failed commit, and a copy not staged yet
+ // tells nothing.
+ local, err := disk.ReadVersion(lkctx.Context(), "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{})
+ st, serr := os.Stat(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath, xlStorageFormatFile))
+ if err != nil || serr != nil || !restoreUploadStaged(local) || time.Since(st.ModTime()) < restoreCommitGrace {
+ hasSkippedUploads = true
+ return nil
+ }
+ } else if view.live && (!view.known || (view.marker != nil && er.restoreUploadActive(lkctx.Context(), view))) {
+ hasSkippedUploads = true
+ return nil
+ }
+ renaming = true
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error {
wait := deleteMultipartCleanupSleeper.Timer(ctx)
pathUUID := mustGetUUID()
targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID)
- renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket))
+ err := renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket))
+ lk.Unlock(lkctx)
+ if err != nil {
+ // The next cleanup round tries again.
+ storageLogIf(ctx, fmt.Errorf("Unable to remove stale upload %s on %s: %w", uploadIDPath, drivePath, err))
+ }
wait()
return nil
})
@@ -463,6 +506,13 @@ func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket,
if populatedUploadIDs.Contains(uploadID) {
continue
}
+ // A native restore's upload is private to its attempt; list only an
+ // upload that the drives establish as an ordinary one.
+ metas, errs := readAllFileInfo(ctx, er.getDisks(), bucket, minioMetaMultipartBucket,
+ pathJoin(er.getMultipartSHADir(bucket, object), uploadID), "", false, false)
+ if !restoreUploadState(metas, errs).visible(ObjectOptions{}) {
+ continue
+ }
var fallback time.Time
if _, ok := multipartUploadTime(uploadID); !ok {
fi, err := disk.ReadVersion(ctx, bucket, minioMetaMultipartBucket,
@@ -562,6 +612,11 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
if opts.PreserveETag != "" {
userDefined["etag"] = opts.PreserveETag
}
+ // Only a native restore marks its upload private, never caller metadata.
+ delete(userDefined, restoreUploadKey)
+ if a := opts.Transition.attempt; a != nil {
+ userDefined[restoreUploadKey] = a.String()
+ }
onlineDisks := er.getDisks()
// Get parity and data drive count based on storage class metadata
@@ -743,7 +798,10 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
// Validates if upload ID exists.
- fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
+ fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
+ if err == nil && !restoreUploadVisible(metas, opts) {
+ err = errUploadIDNotFound
+ }
if err != nil {
if errors.Is(err, errVolumeNotFound) {
return pi, toObjectErr(err, bucket)
@@ -980,7 +1038,10 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
UploadID: uploadID,
}
- fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
+ fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
+ if err == nil && !restoreUploadVisible(metas, opts) {
+ err = errUploadIDNotFound
+ }
if err != nil {
if errors.Is(err, errVolumeNotFound) {
return result, toObjectErr(err, bucket)
@@ -1058,7 +1119,10 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
auditObjectErasureSet(ctx, "ListObjectParts", object, &er)
}
- fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
+ fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
+ if err == nil && !restoreUploadVisible(metas, opts) {
+ err = errUploadIDNotFound
+ }
if err != nil {
return result, toObjectErr(err, bucket, object, uploadID)
}
@@ -1271,7 +1335,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
auditObjectErasureSet(ctx, "CompleteMultipartUpload", object, &er)
}
- if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile {
+ if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile || opts.Transition.attempt != nil {
if !opts.NoLock {
ns := er.NewNSLock(bucket, object)
lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
@@ -1304,7 +1368,24 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
}
}
+ if opts.Transition.attempt != nil {
+ // A native restore holds the upload write lock from validation
+ // through commit, after the object lock and in the lock domain of
+ // PutObjectPart, which takes a part lock and then this lock to read.
+ // NoLock only says that the caller holds the object lock.
+ uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+ ulkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
+ if err != nil {
+ return oi, err
+ }
+ ctx = ulkctx.Context()
+ defer uploadIDLock.Unlock(ulkctx)
+ }
+
fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
+ if err == nil && !restoreUploadVisible(partsMetadata, opts) {
+ err = errUploadIDNotFound
+ }
if err != nil {
if errors.Is(err, errVolumeNotFound) {
return oi, toObjectErr(err, bucket)
@@ -1618,6 +1699,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
delete(fi.Metadata, hash.MinIOMultipartChecksumType)
delete(fi.Metadata, multipartMetaBucket)
delete(fi.Metadata, multipartMetaObject)
+ delete(fi.Metadata, restoreUploadKey)
// Save the final object size and modtime.
fi.Size = objectSize
@@ -1697,6 +1779,20 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo, opts.Transition); err != nil {
return oi, toObjectErr(err, bucket, object)
}
+ if a := opts.Transition.attempt; a != nil {
+ // A native restore writes back only the null version it targets.
+ current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{
+ VersionID: nullVersionID,
+ Transition: opts.Transition,
+ NoLock: true,
+ })
+ if gerr != nil {
+ return oi, toObjectErr(gerr, bucket, object)
+ }
+ if !a.rehydrates(current, fi) {
+ return oi, toObjectErr(errAuthentication, bucket, object)
+ }
+ }
} else {
current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{
VersionID: fi.VersionID,
@@ -1706,11 +1802,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
NoLock: true,
})
switch {
- case gerr == nil && opts.Transition.RestoreRequest != nil &&
- current.TransitionedObject.Status == lifecycle.TransitionComplete &&
- current.ETag == fi.Metadata["etag"] && current.Size == fi.Size && current.ModTime.Equal(fi.ModTime):
- // The internal restore path rehydrates this same tiered version.
- // Ordinary multipart completion must still reject a version overwrite.
+ case gerr == nil && opts.Transition.attempt.rehydrates(current, fi):
+ // A native restore's own private upload rehydrates this same
+ // tiered version. Any other completion must still reject a
+ // version overwrite.
case gerr == nil, errors.As(gerr, &MethodNotAllowed{}):
return oi, toObjectErr(errAuthentication, bucket, object)
case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr):
@@ -1720,6 +1815,16 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
}
}
+ // Stale cleanup and abort judge an upload whose commit staged object
+ // metadata in it by this lock: hold it from staging through the commit.
+ commitLock := er.restoreCommitLock(uploadIDPath)
+ clkctx, err := commitLock.GetLock(ctx, globalOperationTimeout)
+ if err != nil {
+ return oi, err
+ }
+ ctx = clkctx.Context()
+ defer commitLock.Unlock(clkctx)
+
er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename().
defer func() {
@@ -1791,7 +1896,38 @@ func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, objec
if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") {
return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
}
- defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
+ // Authorize before the unconditional cleanup below. A native restore's
+ // upload is absent to every other caller, and an upload whose marker the
+ // drives cannot establish is not removed by anyone.
+ uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
+ view := er.readRestoreUpload(ctx, bucket, uploadIDPath)
+ if view.committing {
+ // What a failed commit left behind, also after stale cleanup could
+ // not remove it: only with no commit running, and once every copy
+ // shows a staging older than the grace.
+ lk := er.restoreCommitLock(uploadIDPath)
+ lkctx, err := lk.GetLock(ctx, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait))
+ if err != nil {
+ return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID)
+ }
+ defer lk.Unlock(lkctx)
+ if !er.restoreCommitAbandoned(lkctx.Context(), bucket, uploadIDPath) {
+ return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID)
+ }
+ er.deleteAll(lkctx.Context(), minioMetaMultipartBucket, uploadIDPath)
+ return true, nil
+ }
+ switch {
+ case view.visible(opts):
+ case view.known && view.marked:
+ return false, nil
+ case view.known && opts.Transition.attempt != nil:
+ // A native restore's own abort, of an upload no longer its own.
+ return false, nil
+ default:
+ return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID)
+ }
+ defer er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath)
_, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
err = toObjectErr(err, bucket, object, uploadID)
if _, absent := err.(InvalidUploadID); absent {
diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go
index 9307aae81..f3c8cfb42 100644
--- a/cmd/erasure-object.go
+++ b/cmd/erasure-object.go
@@ -2158,6 +2158,20 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
if opts.Transition.ExpireRestored && (gerr != nil || goi.DeleteMarker) {
return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
}
+ // A restored copy expires only by its own reason, decided on the
+ // version's authoritative generation: a drive that missed a renewal or a
+ // later POST must not authorize removing the cache, nor drives that
+ // disagree on the restore status. Another lifecycle deletion falling due,
+ // or the all-pools check of an ordinary expiry, does not stand in for it.
+ if opts.Transition.ExpireRestored {
+ gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
+ if err != nil || gen.split || gen.fi.Deleted || lc == nil {
+ return goi, errLifecycleNotApplicable
+ }
+ if !restoredCopyExpires(ctx, *lc, rcfg, replcfg, gen.fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)) {
+ return goi, errLifecycleNotApplicable
+ }
+ }
if opts.EvalMetadataFn != nil {
dsc, err := opts.EvalMetadataFn(&goi, gerr)
@@ -2660,29 +2674,6 @@ func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket,
return er.restoreTransitionedObject(ctx, bucket, object, opts)
}
-// update restore status header in the metadata
-func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error {
- oi := objInfo.Clone()
- oi.metadataOnly = true // Perform only metadata updates.
-
- // allow retry in the case of failure to restore
- delete(oi.UserDefined, xhttp.AmzRestore)
-
- if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{
- VersionID: oi.VersionID,
- Transition: opts.Transition,
- }, ObjectOptions{
- VersionID: oi.VersionID,
- MTime: oi.ModTime,
- NoLock: opts.NoLock,
- Transition: opts.Transition,
- }); err != nil {
- storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
- return err
- }
- return nil
-}
-
// getRestoreLock waits for a restore's object lock for as long as ctx lives.
// The POST handler has already marked the version ongoing, and only the
// restore clears that marker. A restore queued behind another version's
@@ -2698,6 +2689,11 @@ func getRestoreLock(ctx context.Context, lk RWLocker) (LockContext, error) {
// restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts
// as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta
+//
+// A POST restore publishes its attempt on the version's ongoing marker. The
+// restore rehydrates the version only while that attempt owns the marker, and
+// on failure clears the marker only while it still does. A direct caller
+// publishes no attempt; its attempt only names the private upload.
func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error {
if !opts.NoLock {
// Single-pool and direct callers must protect the metadata snapshot
@@ -2705,30 +2701,45 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
lk := er.NewNSLock(bucket, object)
lkctx, err := getRestoreLock(ctx, lk)
if err != nil {
- return err
+ return restoreResetError{err}
}
ctx = lkctx.Context()
defer lk.Unlock(lkctx)
opts.NoLock = true
}
- setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error {
- if rerr == nil {
- return nil // nothing to do; restore object was successful
- }
- er.updateRestoreMetadata(ctx, bucket, object, oi, opts)
- return rerr
- }
- var oi ObjectInfo
- // get the file info on disk for transitioned object
- actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
+ published := opts.Transition.attempt
+ // Restore the authoritative generation of the version, never a revision
+ // that only some drives hold.
+ gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
if err != nil {
- // No version was read, so there is no restore metadata to update.
- return toObjectErr(err, bucket, object)
+ err = toObjectErr(err, bucket, object, opts.VersionID)
+ if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+ // No version was read, so there is no restore marker to clear.
+ return err
+ }
+ return restoreResetError{err}
+ }
+ actualfi := gen.fi
+ if actualfi.Deleted {
+ return toObjectErr(errMethodNotAllowed, bucket, object)
}
- oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
- ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi)
+ oi := actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
+ attempt := published
+ if attempt == nil {
+ attempt = newRestoreAttempt(oi)
+ } else if !attempt.owns(oi) {
+ // A later POST took the version over, or it was replaced.
+ return errRestoreSuperseded
+ }
+ opts.Transition.attempt = attempt
+ versionID := restoreVersionID(actualfi.VersionID)
+ fail := func(cause error) error {
+ return er.restoreFailed(ctx, bucket, object, versionID, published, cause)
+ }
+
+ ropts := putRestoreOpts(opts.Transition.RestoreRequest, oi, opts)
ropts.NoLock = opts.NoLock
ropts.Transition = opts.Transition
// Rehydration copies encoded bytes without changing their content identity.
@@ -2738,35 +2749,51 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
if len(oi.Parts) == 1 && !oi.isMultipart() {
actualSize, err := oi.GetActualSize()
if err != nil {
- return setRestoreHeaderFn(oi, err)
+ return fail(err)
}
ropts.IndexCB = func() []byte { return oi.Parts[0].Index }
var rs *HTTPRangeSpec
gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
if err != nil {
- return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
+ return fail(toObjectErr(err, bucket, object))
}
defer gr.Close()
hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", actualSize)
if err != nil {
- return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
+ return fail(toObjectErr(err, bucket, object))
}
pReader := NewPutObjReader(hashReader)
- _, err = er.PutObject(ctx, bucket, object, pReader, ropts)
- return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
+ if _, err = er.PutObject(ctx, bucket, object, pReader, ropts); err != nil {
+ return fail(toObjectErr(err, bucket, object))
+ }
+ return nil
}
- res, err := er.NewMultipartUpload(ctx, bucket, object, ropts)
- if err != nil {
- return setRestoreHeaderFn(oi, err)
+ // Every server must keep the upload private, also a queued restore's.
+ if err := nativeRestoreAvailable(ctx); err != nil {
+ return fail(err)
}
+ // The upload records when it starts, which the stale-upload cleanup ages;
+ // completion still commits the version's own ModTime.
+ uopts := ropts
+ uopts.MTime = time.Time{}
+ res, err := er.NewMultipartUpload(ctx, bucket, object, uopts)
+ if err != nil {
+ return fail(err)
+ }
+ completed := false
+ defer func() {
+ if !completed {
+ er.abortRestoreUpload(bucket, object, res.UploadID, attempt)
+ }
+ }()
var uploadedParts []CompletePart
var rs *HTTPRangeSpec
// get reader from the warm backend - note that even in the case of encrypted objects, this stream is still encrypted.
gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
if err != nil {
- return setRestoreHeaderFn(oi, err)
+ return fail(err)
}
defer gr.Close()
@@ -2781,16 +2808,17 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
}
hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", actualSize)
if err != nil {
- return setRestoreHeaderFn(oi, err)
+ return fail(err)
}
pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{
- IndexCB: func() []byte { return partInfo.Index },
+ IndexCB: func() []byte { return partInfo.Index },
+ Transition: TransitionOptions{attempt: attempt},
})
if err != nil {
- return setRestoreHeaderFn(oi, err)
+ return fail(err)
}
if pInfo.Size != partInfo.Size {
- return setRestoreHeaderFn(oi, InvalidObjectState{Bucket: bucket, Object: object})
+ return fail(InvalidObjectState{Bucket: bucket, Object: object})
}
uploadedParts = append(uploadedParts, CompletePart{
PartNumber: pInfo.PartNumber,
@@ -2804,7 +2832,11 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
VersionSuspended: opts.VersionSuspended,
Transition: opts.Transition,
})
- return setRestoreHeaderFn(oi, err)
+ if err != nil {
+ return fail(err)
+ }
+ completed = true
+ return nil
}
// DecomTieredObject - moves tiered object to another pool during decommissioning.
diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go
index 849ad6c67..c168e22f9 100644
--- a/cmd/erasure-server-pool.go
+++ b/cmd/erasure-server-pool.go
@@ -3369,8 +3369,16 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
}
// RestoreTransitionedObject - restore transitioned object content locally on this cluster.
-func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) (err error) {
object = encodeDirObject(object)
+ defer func() {
+ // The restore stopped before it could clear POST's ongoing marker.
+ var reset restoreResetError
+ if errors.As(err, &reset) {
+ z.recoverRestoreStatus(bucket, object, opts)
+ err = reset.error
+ }
+ }()
if z.SinglePool() {
return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
}
@@ -3380,7 +3388,7 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
lk := z.NewNSLock(bucket, object)
lkctx, err := getRestoreLock(ctx, lk)
if err != nil {
- return err
+ return restoreResetError{err}
}
ctx = lkctx.Context()
defer lk.Unlock(lkctx)
@@ -3389,10 +3397,7 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
// Avoid restoring object from a pool being decommissioned.
opts.SkipDecommissioned = true
opts.NoLock = true
- // Restore an existing version, not the latest version's write location.
- lookupOpts := opts
- lookupOpts.MetadataChg = true
- idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts)
+ idx, err := z.restorePoolIdx(ctx, bucket, object, opts)
if err != nil {
return err
}
diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go
index d40ff3ce7..0f4a663ac 100644
--- a/cmd/object-api-interface.go
+++ b/cmd/object-api-interface.go
@@ -177,6 +177,10 @@ type TransitionOptions struct {
RestoreRequest *RestoreObjectRequest
RestoreExpiry time.Time
ExpireRestored bool
+
+ // attempt is the native restore a call belongs to. Only that restore
+ // sees its private upload, see restoreUploadVisible.
+ attempt *restoreAttempt
}
// MakeBucketOptions represents bucket options for ObjectLayer bucket operations
@@ -295,6 +299,8 @@ type ObjectLayer interface {
DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error)
TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error
RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error
+ // restoreRequest records a POST restore on a version, see erasureServerPools.
+ restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error)
// Multipart operations.
ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)
diff --git a/cmd/object-copy-metadata_test.go b/cmd/object-copy-metadata_test.go
index 2cb05f3eb..72c729599 100644
--- a/cmd/object-copy-metadata_test.go
+++ b/cmd/object-copy-metadata_test.go
@@ -340,9 +340,9 @@ func TestCopyRewritesObjectData(t *testing.T) {
name: "data copy always rewrites",
want: true,
},
- // PostRestoreObjectHandler, updateRestoreMetadata and batchKeyRotate all
- // address the same version on both sides and never set Versioned, so they
- // only ever reach these two cases.
+ // In-place metadata callers such as batchKeyRotate address the same
+ // version on both sides and never set Versioned, so they only ever
+ // reach these two cases.
{
name: "unversioned in-place metadata update",
metadataOnly: true,
diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go
index da9b772ac..81af341f5 100644
--- a/cmd/object-handlers.go
+++ b/cmd/object-handlers.go
@@ -3955,7 +3955,11 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
// Preserve a failed restore read even before the request body is parsed.
opts.Transition.RestoreRequest = &RestoreObjectRequest{}
- objInfo, err := getObjectInfo(ctx, bucket, object, opts)
+ // Resolve the target without the object lock: a running restore holds it
+ // for its download, and the status step bounds its own wait for it.
+ preOpts := opts
+ preOpts.NoLock = true
+ objInfo, err := getObjectInfo(ctx, bucket, object, preOpts)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -3982,48 +3986,47 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
}
opts.Transition.RestoreRequest = rreq
statusCode := http.StatusOK
- alreadyRestored := false
- if err == nil {
- if objInfo.RestoreOngoing && rreq.Type != SelectRestoreRequest {
+ // set or upgrade restore expiry
+ restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
+
+ // update self with restore metadata
+ var attempt *restoreAttempt
+ if rreq.Type != SelectRestoreRequest {
+ // Native restore needs every server to keep its upload private;
+ // refuse it, writing nothing, while that is not established.
+ if err := nativeRestoreAvailable(ctx); err != nil {
+ apiErr := errorCodes.ToAPIErr(ErrBusy)
+ apiErr.Code, apiErr.Description = "ServiceUnavailable", err.Error()
+ writeErrorResponse(ctx, w, apiErr, r.URL)
+ return
+ }
+ // Decide on a fresh read under the object lock: renew a restored copy
+ // that is still there, or mark the version ongoing for a new attempt.
+ // Only the version resolved above, with its identity, is restored.
+ statusOpts := opts
+ statusOpts.Transition.RestoreExpiry = restoreExpiry
+ status, err := objectAPI.restoreRequest(GlobalContext, bucket, object, statusOpts, objInfo)
+ if errors.Is(err, errRestoreInProgress) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectRestoreAlreadyInProgress), r.URL)
return
}
- if !objInfo.RestoreOngoing && !objInfo.RestoreExpires.IsZero() {
- statusCode = http.StatusAccepted
- alreadyRestored = true
- }
- }
- // set or upgrade restore expiry
- restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
- metadata := cloneMSS(objInfo.UserDefined)
-
- // update self with restore metadata
- if rreq.Type != SelectRestoreRequest {
- objInfo.metadataOnly = true // Perform only metadata updates.
- metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days)
- metadata[xhttp.AmzRestoreRequestDate] = time.Now().UTC().Format(http.TimeFormat)
- if alreadyRestored {
- metadata[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
- } else {
- metadata[xhttp.AmzRestore] = ongoingRestoreObj().String()
- }
- objInfo.UserDefined = metadata
- if _, err := objectAPI.CopyObject(GlobalContext, bucket, object, bucket, object, objInfo, ObjectOptions{
- VersionID: objInfo.VersionID,
- Transition: opts.Transition,
- }, ObjectOptions{
- VersionID: objInfo.VersionID,
- Transition: opts.Transition,
- MTime: objInfo.ModTime,
- MetadataChg: true,
- }); err != nil {
+ if errors.As(err, &InvalidObjectState{}) || errors.As(err, &MethodNotAllowed{}) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL)
return
}
- // for previously restored object, just update the restore expiry
- if alreadyRestored {
+ if err != nil {
+ // Such as a read quorum error while telling whether it restores.
+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
+ // Events keep the version ID as this request resolved it.
+ status.objInfo.VersionID = objInfo.VersionID
+ objInfo = status.objInfo
+ // for previously restored object, just update the restore expiry
+ if status.attempt == nil {
+ return
+ }
+ attempt = status.attempt
}
restoreObject := mustGetUUID()
@@ -4088,12 +4091,14 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
rreq.SelectParameters.Close()
return
}
+ // The versioning context and the explicit version POST resolved.
restoreOpts := ObjectOptions{
Transition: TransitionOptions{
RestoreRequest: rreq,
RestoreExpiry: restoreExpiry,
+ attempt: attempt,
},
- VersionID: objInfo.VersionID,
+ VersionID: attempt.VersionID,
Versioned: opts.Versioned,
VersionSuspended: opts.VersionSuspended,
}
diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go
index b28c8db44..6661b5fba 100644
--- a/cmd/peer-rest-server.go
+++ b/cmd/peer-rest-server.go
@@ -116,6 +116,7 @@ var (
stopRebalanceRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerStopRebalance, grid.NewMSS, grid.NewNoPayload)
updateMetacacheListingRPC = grid.NewSingleHandler[*metacache, *metacache](grid.HandlerUpdateMetacacheListing, func() *metacache { return &metacache{} }, func() *metacache { return &metacache{} })
cleanupUploadIDCacheMetaRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerClearUploadID, grid.NewMSS, grid.NewNoPayload)
+ restoreCapabilityRPC = grid.NewSingleHandler[*grid.MSS, *grid.MSS](grid.HandlerRestoreCapability, grid.NewMSS, grid.NewMSS)
// STREAMS
// Set an output capacity of 100 for consoleLog and listenRPC
@@ -1440,6 +1441,7 @@ func registerPeerRESTHandlers(router *mux.Router, gm *grid.Manager) {
logger.FatalIf(stopRebalanceRPC.Register(gm, server.StopRebalanceHandler), "unable to register handler")
logger.FatalIf(updateMetacacheListingRPC.Register(gm, server.UpdateMetacacheListingHandler), "unable to register handler")
logger.FatalIf(cleanupUploadIDCacheMetaRPC.Register(gm, server.HandlerClearUploadID), "unable to register handler")
+ logger.FatalIf(restoreCapabilityRPC.Register(gm, restoreCapabilityHandler), "unable to register handler")
logger.FatalIf(gm.RegisterStreamingHandler(grid.HandlerTrace, grid.StreamHandler{
Handle: server.TraceHandler,
diff --git a/cmd/restore-attempt.go b/cmd/restore-attempt.go
new file mode 100644
index 000000000..0df3e36fd
--- /dev/null
+++ b/cmd/restore-attempt.go
@@ -0,0 +1,862 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "maps"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// A native restore keeps two private records. The version that POST marks
+// ongoing names the attempt that owns the marker; that attempt's multipart
+// upload names the attempt and the version it rehydrates. Clients cannot
+// set either: requests with reserved metadata are refused, it is never
+// extracted from headers, and uploads only get the marker from the restore.
+const (
+ restoreAttemptKey = ReservedMetadataPrefixLower + "restore-attempt"
+ restoreUploadKey = ReservedMetadataPrefixLower + "restore-upload"
+)
+
+const (
+ // restoreTakeoverWait bounds how long a POST on an ongoing version waits
+ // for the object lock. A running restore holds it for its download.
+ restoreTakeoverWait = 2 * time.Second
+ // restoreCommitGrace is how long stale cleanup keeps an upload whose
+ // completion staged object metadata in it; a commit takes seconds.
+ restoreCommitGrace = time.Hour
+ // restoreCleanupTimeout bounds each status reset and upload abort.
+ restoreCleanupTimeout = time.Minute
+)
+
+// restoreRecoveryDelays are the in-memory retries that clear the ongoing
+// marker of a restore that failed before it could clear the marker itself.
+// A restart loses them; a later POST then takes the marker over.
+var restoreRecoveryDelays = []time.Duration{0, 100 * time.Millisecond, time.Second, 5 * time.Second, 30 * time.Second, 2 * time.Minute}
+
+var (
+ // errRestoreSuperseded: the version no longer belongs to this attempt.
+ // The attempt must not rehydrate, complete or reset anything.
+ errRestoreSuperseded = errors.New("restore attempt superseded")
+ // errRestoreInProgress: a restore holds the object lock of the version.
+ errRestoreInProgress = errors.New("restore already in progress")
+ // errRestoreUnchanged: a restore status update found nothing to write.
+ errRestoreUnchanged = errors.New("restore status unchanged")
+)
+
+// restoreResetError marks a failed restore that may have left its ongoing
+// marker behind. The pools layer hands it to recoverRestoreStatus.
+type restoreResetError struct{ error }
+
+func (e restoreResetError) Unwrap() error { return e.error }
+
+// restoreVersionID is the explicit version ID of a resolved restore target.
+func restoreVersionID(versionID string) string {
+ if versionID == "" {
+ return nullVersionID
+ }
+ return versionID
+}
+
+// restoreAttempt identifies one native restore of one stored version.
+type restoreAttempt struct {
+ ID string `json:"id"`
+ VersionID string `json:"v"` // "null" for the null version
+ ETag string `json:"e"`
+ Size int64 `json:"s"`
+ ModTime time.Time `json:"m"`
+}
+
+func newRestoreAttempt(oi ObjectInfo) *restoreAttempt {
+ return &restoreAttempt{
+ ID: mustGetUUID(),
+ VersionID: restoreVersionID(oi.VersionID),
+ ETag: oi.ETag,
+ Size: oi.Size,
+ ModTime: oi.ModTime.UTC(),
+ }
+}
+
+func (a *restoreAttempt) String() string {
+ b, _ := json.Marshal(a)
+ return string(b)
+}
+
+func parseRestoreAttempt(s string) (*restoreAttempt, bool) {
+ var a restoreAttempt
+ if s == "" || json.Unmarshal([]byte(s), &a) != nil || a.ID == "" || a.VersionID == "" {
+ return nil, false
+ }
+ return &a, true
+}
+
+func (a *restoreAttempt) equal(b *restoreAttempt) bool {
+ return a != nil && b != nil && a.ID == b.ID && a.VersionID == b.VersionID &&
+ a.ETag == b.ETag && a.Size == b.Size && a.ModTime.Equal(b.ModTime)
+}
+
+// targets reports whether oi is still the version the attempt was made for.
+// A replacement under the same version ID differs in ETag, size or ModTime.
+func (a *restoreAttempt) targets(oi ObjectInfo) bool {
+ return a != nil && !oi.DeleteMarker && restoreVersionID(oi.VersionID) == a.VersionID &&
+ oi.ETag == a.ETag && oi.Size == a.Size && oi.ModTime.Equal(a.ModTime) &&
+ oi.TransitionedObject.Status == lifecycle.TransitionComplete
+}
+
+// owns reports whether the version is ongoing under this attempt.
+func (a *restoreAttempt) owns(oi ObjectInfo) bool {
+ return a.targets(oi) && oi.RestoreOngoing && oi.UserDefined[restoreAttemptKey] == a.ID
+}
+
+// rehydrates reports whether completing upload fi, the attempt's private
+// upload, writes back the very version current.
+func (a *restoreAttempt) rehydrates(current ObjectInfo, fi FileInfo) bool {
+ return a.targets(current) && fi.Metadata["etag"] == a.ETag && fi.Size == a.Size && fi.ModTime.Equal(a.ModTime)
+}
+
+// restoreUploadView is the restore marker of an upload as the drives of
+// its set establish it. A marked upload is written to a write quorum, at
+// least a majority of the drives, and its copies never change their marker.
+type restoreUploadView struct {
+ // marker is the attempt of a native restore's upload: one marker that a
+ // majority of the drives hold.
+ marker *restoreAttempt
+ // marked: some readable drive holds a marker.
+ marked bool
+ // known: the marker state is established. Without it, callers fail
+ // closed: nobody lists, uses, aborts or removes the upload.
+ known bool
+ // live: enough drives hold the upload or cannot tell that a written
+ // upload may exist; otherwise only remnants of one are left.
+ live bool
+ // committing: some drive holds object metadata in the upload's place.
+ // Completion stages the committed version's object metadata there,
+ // without the marker, before it renames the upload into the object.
+ committing bool
+ // bucket and object the upload of an established marker belongs to.
+ bucket, object string
+}
+
+// restoreUploadState establishes the marker of an upload from the metadata of
+// every drive. errs may be nil, then a drive without valid metadata is one
+// that cannot tell. A marker is established on a majority of the drives; its
+// absence only when no readable drive shows one and the drives that cannot
+// tell are too few to hold a marked upload.
+func restoreUploadState(metas []FileInfo, errs []error) restoreUploadView {
+ n := len(metas)
+ majority := n/2 + 1
+ markers := make(map[string]int)
+ var unmarked, unknown, committing int
+ for i, fi := range metas {
+ if errs != nil && errs[i] != nil {
+ if !errors.Is(errs[i], errFileNotFound) && !errors.Is(errs[i], errFileVersionNotFound) &&
+ !errors.Is(errs[i], errVolumeNotFound) && !errors.Is(errs[i], errUploadIDNotFound) {
+ unknown++
+ }
+ continue
+ }
+ if !fi.IsValid() {
+ unknown++
+ continue
+ }
+ if restoreUploadStaged(fi) {
+ committing++
+ continue
+ }
+ if m := fi.Metadata[restoreUploadKey]; m != "" {
+ markers[m]++
+ } else {
+ unmarked++
+ }
+ }
+ present := unmarked
+ for _, c := range markers {
+ present += c
+ }
+ v := restoreUploadView{marked: len(markers) > 0, live: present+unknown+committing >= majority, committing: committing > 0}
+ if v.committing {
+ // Nobody but the completion may use or remove it now.
+ return v
+ }
+ for m, c := range markers {
+ if c >= majority {
+ v.marker, v.known = parseRestoreAttempt(m)
+ if v.known {
+ // The marker also names the upload's own target.
+ for _, fi := range metas {
+ if !fi.IsValid() || fi.Metadata[restoreUploadKey] != m {
+ continue
+ }
+ if restoreVersionID(fi.VersionID) != v.marker.VersionID || fi.Metadata["etag"] != v.marker.ETag {
+ v.marker, v.known = nil, false
+ break
+ }
+ v.bucket, v.object = fi.Metadata[multipartMetaBucket], fi.Metadata[multipartMetaObject]
+ }
+ }
+ return v
+ }
+ }
+ // A marker without a majority is established only as a remnant.
+ v.known = !v.live || (!v.marked && unknown < majority)
+ return v
+}
+
+// restoreUploadStaged reports whether an upload's record is object metadata
+// that a completion staged there: an upload record has no parts and no
+// other versions.
+func restoreUploadStaged(fi FileInfo) bool {
+ return fi.Deleted || len(fi.Parts) > 0 || fi.NumVersions > 1
+}
+
+// visible reports whether a caller sees the upload: a native restore its own
+// upload, any other caller only an upload established as unmarked.
+func (v restoreUploadView) visible(opts ObjectOptions) bool {
+ if a := opts.Transition.attempt; a != nil {
+ return v.marker.equal(a)
+ }
+ return v.known && !v.marked
+}
+
+// restoreUploadVisible reports whether a caller sees an upload whose metadata
+// is metas, one entry for each drive.
+func restoreUploadVisible(metas []FileInfo, opts ObjectOptions) bool {
+ return restoreUploadState(metas, nil).visible(opts)
+}
+
+// readRestoreUpload reads the metadata of an upload from every drive.
+func (er erasureObjects) readRestoreUpload(ctx context.Context, bucket, uploadIDPath string) restoreUploadView {
+ metas, errs := readAllFileInfo(ctx, er.getDisks(), bucket, minioMetaMultipartBucket, uploadIDPath, "", false, false)
+ return restoreUploadState(metas, errs)
+}
+
+// restoreCommitLock is held by a completion from staging through its commit;
+// it is named by the upload's own path, which an upload whose copies all
+// hold staged object metadata still has.
+func (er erasureObjects) restoreCommitLock(uploadIDPath string) RWLocker {
+ return er.NewNSLock(minioMetaMultipartBucket, uploadIDPath)
+}
+
+// restoreCommitAbandoned reports whether an upload holds only what a failed
+// commit left: every drive answers, and every copy that is there is staged
+// object metadata, see restoreUploadStaged, older than restoreCommitGrace by
+// that drive's own metadata file. A copy that cannot be read, or that is not
+// staged yet, may belong to a commit still running. The caller holds the
+// upload's restoreCommitLock.
+func (er erasureObjects) restoreCommitAbandoned(ctx context.Context, bucket, uploadIDPath string) bool {
+ disks := er.getDisks()
+ metas, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, uploadIDPath, "", false, false)
+ staged := 0
+ for i, fi := range metas {
+ if errs[i] != nil {
+ if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errFileVersionNotFound) {
+ continue
+ }
+ return false
+ }
+ if !fi.IsValid() || !restoreUploadStaged(fi) {
+ return false
+ }
+ st, err := disks[i].StatInfoFile(ctx, minioMetaMultipartBucket, pathJoin(uploadIDPath, xlStorageFormatFile), false)
+ if err != nil || len(st) == 0 || time.Since(st[0].ModTime) < restoreCommitGrace {
+ return false
+ }
+ staged++
+ }
+ return staged > 0
+}
+
+// restoreGeneration is the metadata generation of one version that the drives
+// of a set hold authoritatively: the revision (header and normalized metadata
+// digest, versionRevision) that at least max(n/2+1, its read quorum) drives
+// hold. drives has each drive's own copy of it, with that drive's erasure
+// index and inline data, and an invalid entry where a drive holds another or
+// no revision; disks has nil there.
+//
+// A failed restore status write can leave drives that differ only in their
+// restore fields, with no revision on a quorum. Then the generation is the
+// restore-neutral revision that a quorum holds once those fields are left
+// out: fi has no restore fields, and ongoing names the attempts whose
+// ongoing marker some of its drives hold ("" for one without an attempt).
+// Every other field, protection included, still needs the quorum.
+type restoreGeneration struct {
+ fi FileInfo
+ drives []FileInfo
+ disks []StorageAPI
+ split bool
+ ongoing map[string]bool
+}
+
+// restoreFieldKeys are the metadata keys a restore status write sets.
+var restoreFieldKeys = []string{xhttp.AmzRestore, xhttp.AmzRestoreExpiryDays, xhttp.AmzRestoreRequestDate, restoreAttemptKey}
+
+// restoreHealMeta returns latest, the metadata heal copies to drives that
+// lack the version, with the restore fields that at least quorum drives of
+// that version hold. When some drive holds restore fields and no set of
+// them has the quorum, healing the version waits.
+func restoreHealMeta(metas []FileInfo, latest FileInfo, quorum int) (FileInfo, error) {
+ counts := make(map[string]int)
+ fields := make(map[string]map[string]string)
+ restoring := false
+ for _, m := range metas {
+ if !m.IsValid() || m.Deleted || !m.ModTime.Equal(latest.ModTime) {
+ continue
+ }
+ f := make(map[string]string)
+ var key strings.Builder
+ for _, k := range restoreFieldKeys {
+ if v, ok := m.Metadata[k]; ok {
+ f[k] = v
+ fmt.Fprintf(&key, "%d:%s=%d:%s;", len(k), k, len(v), v)
+ restoring = true
+ }
+ }
+ counts[key.String()]++
+ fields[key.String()] = f
+ }
+ if !restoring {
+ return latest, nil
+ }
+ best, bestCount := "", 0
+ for key, c := range counts {
+ if c > bestCount {
+ best, bestCount = key, c
+ }
+ }
+ if bestCount < quorum {
+ return latest, errErasureReadQuorum
+ }
+ latest.Metadata = maps.Clone(latest.Metadata)
+ for _, k := range restoreFieldKeys {
+ delete(latest.Metadata, k)
+ }
+ maps.Copy(latest.Metadata, fields[best])
+ return latest, nil
+}
+
+// restoreNeutralRevision is the revision of ver without its restore fields.
+func restoreNeutralRevision(ver xlMetaV2ShallowVersion, metaV uint8) (revisionKey, bool) {
+ var v xlMetaV2Version
+ if _, err := v.unmarshalV(metaV, ver.meta); err != nil || v.Type != ObjectType || v.ObjectV2 == nil {
+ return revisionKey{}, false
+ }
+ o := *v.ObjectV2
+ o.MetaUser, o.MetaSys = maps.Clone(o.MetaUser), maps.Clone(o.MetaSys)
+ for _, k := range restoreFieldKeys {
+ delete(o.MetaUser, k)
+ delete(o.MetaSys, k)
+ }
+ v.ObjectV2 = &o
+ meta, err := v.MarshalMsg(nil)
+ if err != nil {
+ return revisionKey{}, false
+ }
+ return versionRevision(xlMetaV2ShallowVersion{header: v.header(), meta: meta}, xlMetaVersion), true
+}
+
+// readRestoreGeneration selects the authoritative generation of a version,
+// or fails closed: a revision that only some drives hold, such as one a
+// failed metadata update left behind, is never selected. An empty version ID
+// addresses the latest version, as a direct restore may.
+func (er erasureObjects) readRestoreGeneration(ctx context.Context, bucket, object, versionID string) (restoreGeneration, error) {
+ if versionID == "" {
+ fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, ObjectOptions{
+ NoLock: true,
+ Transition: TransitionOptions{RestoreRequest: &RestoreObjectRequest{}},
+ }, false)
+ if err != nil {
+ return restoreGeneration{}, err
+ }
+ versionID = restoreVersionID(fi.VersionID)
+ }
+ var uv [16]byte
+ if versionID != nullVersionID {
+ id, err := uuid.Parse(versionID)
+ if err != nil {
+ return restoreGeneration{}, errFileVersionNotFound
+ }
+ uv = id
+ }
+ disks := er.getDisks()
+ n := len(disks)
+ majority := n/2 + 1
+ raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, true)
+ type revision struct {
+ ver xlMetaV2ShallowVersion
+ metaV uint8
+ count int
+ }
+ keys := make([]*revisionKey, n)
+ neutral := make([]*revisionKey, n)
+ revisions := make(map[revisionKey]*revision)
+ neutrals := make(map[revisionKey]*revision)
+ absent := 0
+ for i := range disks {
+ if errs[i] != nil {
+ if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errVolumeNotFound) {
+ absent++
+ }
+ continue
+ }
+ var xl xlMetaV2
+ if xl.LoadOrConvert(raws[i].Buf) != nil {
+ continue
+ }
+ found := false
+ for _, ver := range xl.versions {
+ if ver.header.VersionID != uv || ver.header.FreeVersion() {
+ continue
+ }
+ key := versionRevision(ver, xl.metaV)
+ keys[i], found = &key, true
+ r := revisions[key]
+ if r == nil {
+ r = &revision{ver: ver, metaV: xl.metaV}
+ revisions[key] = r
+ }
+ r.count++
+ if nkey, ok := restoreNeutralRevision(ver, xl.metaV); ok {
+ neutral[i] = &nkey
+ if neutrals[nkey] == nil {
+ neutrals[nkey] = &revision{ver: ver, metaV: xl.metaV}
+ }
+ neutrals[nkey].count++
+ }
+ break
+ }
+ if !found {
+ absent++
+ }
+ }
+ accepted := func(revs map[revisionKey]*revision) (revisionKey, bool) {
+ var best *revision
+ var bestKey revisionKey
+ for key, r := range revs {
+ if best == nil || r.count > best.count {
+ best, bestKey = r, key
+ }
+ }
+ return bestKey, best != nil && best.count >= majority &&
+ best.count >= er.versionReadQuorum(ctx, bucket, object, best.ver, best.metaV, make([]FileInfo, n), make([]error, n))
+ }
+ gen := restoreGeneration{drives: make([]FileInfo, n), disks: make([]StorageAPI, n)}
+ bestKey, ok := accepted(revisions)
+ if !ok {
+ if bestKey, ok = accepted(neutrals); !ok {
+ if absent >= majority {
+ return restoreGeneration{}, errFileVersionNotFound
+ }
+ return restoreGeneration{}, errErasureReadQuorum
+ }
+ keys, gen.split, gen.ongoing = neutral, true, make(map[string]bool)
+ }
+ for i := range disks {
+ if keys[i] == nil || *keys[i] != bestKey {
+ continue
+ }
+ fi, err := getFileInfo(raws[i].Buf, bucket, object, versionID, fileInfoOpts{Data: true})
+ if err != nil {
+ continue
+ }
+ gen.drives[i], gen.disks[i] = fi, disks[i]
+ if gen.split {
+ if st, err := parseRestoreObjStatus(fi.Metadata[xhttp.AmzRestore]); err == nil && st.Ongoing() {
+ gen.ongoing[fi.Metadata[restoreAttemptKey]] = true
+ }
+ }
+ if !gen.fi.IsValid() {
+ gen.fi = fi
+ }
+ }
+ if !gen.fi.IsValid() && !gen.fi.Deleted {
+ return restoreGeneration{}, errErasureReadQuorum
+ }
+ if gen.split {
+ gen.fi.Metadata = maps.Clone(gen.fi.Metadata)
+ for _, k := range restoreFieldKeys {
+ delete(gen.fi.Metadata, k)
+ }
+ }
+ return gen, nil
+}
+
+// updateRestoreStatus rewrites the restore fields of one stored version. It
+// selects the authoritative generation of the version, see
+// readRestoreGeneration, patches only the restore fields onto exactly that
+// generation, keeping every other field and its ModTime, and writes it back
+// only to the drives that hold it. The caller holds the object lock. update
+// edits the metadata in place, or returns an error, errRestoreUnchanged to
+// write nothing; ongoing names the attempts whose ongoing marker the
+// generation holds. A failed read never removes the version, and a failed
+// write puts the generation it read back.
+func (er erasureObjects) updateRestoreStatus(ctx context.Context, bucket, object, versionID string, update func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error) (ObjectInfo, error) {
+ gen, err := er.readRestoreGeneration(ctx, bucket, object, versionID)
+ if err != nil {
+ return ObjectInfo{}, toObjectErr(err, bucket, object, versionID)
+ }
+ fi := gen.fi
+ oi := fi.ToObjectInfo(bucket, object, false)
+ if fi.Deleted {
+ return oi, toObjectErr(errMethodNotAllowed, bucket, object)
+ }
+ ongoing := gen.ongoing
+ if !gen.split && oi.RestoreOngoing {
+ ongoing = map[string]bool{oi.UserDefined[restoreAttemptKey]: true}
+ }
+ meta := maps.Clone(fi.Metadata)
+ if err := update(oi, meta, ongoing); err != nil {
+ return oi, err
+ }
+ write := func(meta func(FileInfo) map[string]string) error {
+ metaArr := make([]FileInfo, len(gen.drives))
+ for i, d := range gen.drives {
+ if gen.disks[i] == nil {
+ continue
+ }
+ if !d.InlineData() {
+ d.Data = nil
+ }
+ d.Metadata = meta(d)
+ metaArr[i] = d
+ }
+ // The write assigns each drive its slot as erasure index: order the
+ // drives by the version's distribution first.
+ disks, metaArr := shuffleDisksAndPartsMetadata(gen.disks, metaArr, fi)
+ _, err := writeUniqueFileInfo(ctx, disks, "", bucket, object, metaArr, fi.WriteQuorum(er.defaultWQuorum()))
+ return err
+ }
+ if err := write(func(FileInfo) map[string]string { return meta }); err != nil {
+ // Without the read generation back, its drives could hold two
+ // revisions and no quorum for either. A split that stays differs
+ // only in restore fields, which the next write or reset resolves.
+ if rerr := write(func(d FileInfo) map[string]string { return d.Metadata }); rerr != nil {
+ storageLogIf(ctx, fmt.Errorf("Unable to put back the restore status of %s/%s(%s): %w", bucket, object, versionID, rerr))
+ }
+ return oi, toObjectErr(err, bucket, object)
+ }
+ fi.Metadata = meta
+ return fi.ToObjectInfo(bucket, object, false), nil
+}
+
+// clearRestoreStatus makes a version retryable again while attempt a still
+// owns its ongoing marker. It never brings back an earlier completed status:
+// POST only starts a restore when that restored copy is gone.
+func (er erasureObjects) clearRestoreStatus(ctx context.Context, bucket, object, versionID string, a *restoreAttempt) error {
+ _, err := er.updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error {
+ id := ""
+ if a != nil {
+ id = a.ID
+ if !a.targets(oi) {
+ return errRestoreUnchanged
+ }
+ }
+ // Clear only this attempt's marker. Where drives disagree, another
+ // attempt's marker may be the acknowledged one: leave that to it,
+ // or to a later POST, which takes it over under the lock.
+ if !ongoing[id] || len(ongoing) != 1 {
+ return errRestoreUnchanged
+ }
+ delete(meta, xhttp.AmzRestore)
+ delete(meta, xhttp.AmzRestoreExpiryDays)
+ delete(meta, xhttp.AmzRestoreRequestDate)
+ delete(meta, restoreAttemptKey)
+ return nil
+ })
+ if errors.Is(err, errRestoreUnchanged) {
+ return nil
+ }
+ return err
+}
+
+// restoreFailed clears the ongoing marker of a failed attempt while the
+// object lock is held. A lost lock or an ended context leaves that to the
+// bounded retries of the pools layer, which take the lock afresh.
+func (er erasureObjects) restoreFailed(ctx context.Context, bucket, object, versionID string, published *restoreAttempt, cause error) error {
+ if ctx.Err() != nil {
+ return restoreResetError{cause}
+ }
+ if err := er.clearRestoreStatus(ctx, bucket, object, versionID, published); err != nil {
+ storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, versionID, err))
+ return restoreResetError{cause}
+ }
+ return cause
+}
+
+// abortRestoreUpload removes the private upload of a failed attempt, or
+// leaves it to stale-upload cleanup. The restore's own context may have
+// ended, so it uses a fresh bounded one, and the upload lock as any abort.
+// It matches the upload to its own attempt rather than to the version, which
+// a newer attempt may own by now, and it never touches the version.
+func (er erasureObjects) abortRestoreUpload(bucket, object, uploadID string, a *restoreAttempt) {
+ ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
+ defer cancel()
+ lk := er.NewNSLock(bucket, pathJoin(object, uploadID))
+ lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+ if err == nil {
+ defer lk.Unlock(lkctx)
+ _, err = er.abortMultipartUpload(lkctx.Context(), bucket, object, uploadID, ObjectOptions{
+ NoAuditLog: true,
+ Transition: TransitionOptions{attempt: a},
+ })
+ }
+ if err != nil {
+ storageLogIf(ctx, fmt.Errorf("Unable to abort the restore upload of %s/%s: %w", bucket, object, err))
+ }
+}
+
+// restoreUploadActive reports whether the restore that owns a stale-aged
+// upload still runs: its attempt owns the version's ongoing marker. A long
+// download keeps its upload; one whose attempt ended loses it, and an
+// ongoing marker alone keeps nothing.
+func (er erasureObjects) restoreUploadActive(ctx context.Context, v restoreUploadView) bool {
+ gen, err := er.readRestoreGeneration(ctx, v.bucket, v.object, v.marker.VersionID)
+ if err != nil {
+ // An unreadable version may still be restoring; retry next round.
+ return !errors.Is(err, errFileVersionNotFound) && !errors.Is(err, errFileNotFound)
+ }
+ if gen.split {
+ // Drives disagree on the restore status: that the attempt ended
+ // is not established.
+ return gen.ongoing[v.marker.ID]
+ }
+ return v.marker.owns(gen.fi.ToObjectInfo(v.bucket, v.object, false))
+}
+
+// restoreLock is the object lock that restores and their status updates
+// share: with one pool the set's own lock, as single-pool writes use.
+func (z *erasureServerPools) restoreLock(bucket, object string) RWLocker {
+ if z.SinglePool() {
+ return z.serverPools[0].getHashedSet(object).NewNSLock(bucket, object)
+ }
+ return z.NewNSLock(bucket, object)
+}
+
+// restoreRequestStatus is what a POST restore recorded on its version.
+type restoreRequestStatus struct {
+ objInfo ObjectInfo
+ // attempt is the restore that must rehydrate the version; nil when a
+ // restored copy was still there and its expiry was renewed.
+ attempt *restoreAttempt
+}
+
+// restoreRequest records a POST restore on target, the version that POST
+// resolved, on a fresh read under the object lock. The version must still
+// be target: a replacement under the same version ID is refused. A copy that
+// still holds its restored data is renewed. Otherwise the version is marked
+// ongoing for a new attempt; that also takes over a marker no running
+// restore holds, as a running restore holds the object lock for its whole
+// download. Only the copy that is to be restored is written, so no copy
+// claims a restored copy it does not hold.
+func (z *erasureServerPools) restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error) {
+ object = encodeDirObject(object)
+ versionID := restoreVersionID(target.VersionID)
+ lk := z.restoreLock(bucket, object)
+ // Wait only briefly first, whatever the lockless read before saw: a
+ // restore of this version holds the lock for its whole download. Then
+ // wait as usual only if the drives show no restore of it running.
+ lkctx, err := lk.GetLock(ctx, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait))
+ if errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil {
+ running, rerr := z.restoreRunning(ctx, bucket, object, versionID)
+ switch {
+ case rerr != nil:
+ // Whether it is restoring is unknown: answer now, not after
+ // the lock frees.
+ return restoreRequestStatus{}, rerr
+ case running:
+ return restoreRequestStatus{}, errRestoreInProgress
+ }
+ lkctx, err = lk.GetLock(ctx, globalOperationTimeout)
+ }
+ if err != nil {
+ return restoreRequestStatus{}, err
+ }
+ ctx = lkctx.Context()
+ defer lk.Unlock(lkctx)
+
+ expected := newRestoreAttempt(target)
+ copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{
+ VersionID: versionID,
+ Versioned: opts.Versioned,
+ VersionSuspended: opts.VersionSuspended,
+ Transition: opts.Transition,
+ })
+ if err != nil {
+ return restoreRequestStatus{}, err
+ }
+ // Restore the copy reads serve first, outside pools being decommissioned,
+ // as RestoreTransitionedObject does.
+ pool := -1
+ for _, c := range copies {
+ if !z.IsSuspended(c.Index) {
+ pool = c.Index
+ break
+ }
+ }
+ if pool < 0 {
+ return restoreRequestStatus{}, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
+ }
+
+ rreq := opts.Transition.RestoreRequest
+ date := UTCNow().Format(http.TimeFormat)
+ var attempt *restoreAttempt
+ oi, err := z.serverPools[pool].getHashedSet(object).updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, _ map[string]bool) error {
+ if !expected.targets(oi) {
+ // Not the version POST resolved, or no longer transitioned.
+ return InvalidObjectState{Bucket: bucket, Object: decodeDirObject(object)}
+ }
+ attempt = nil
+ meta[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days)
+ meta[xhttp.AmzRestoreRequestDate] = date
+ if !oi.RestoreOngoing && !oi.RestoreExpires.IsZero() {
+ // Cleanup removes a restored copy together with its header.
+ meta[xhttp.AmzRestore] = completedRestoreObj(opts.Transition.RestoreExpiry).String()
+ delete(meta, restoreAttemptKey)
+ return nil
+ }
+ attempt = newRestoreAttempt(oi)
+ meta[xhttp.AmzRestore] = ongoingRestoreObj().String()
+ meta[restoreAttemptKey] = attempt.ID
+ return nil
+ })
+ if err != nil {
+ if attempt != nil {
+ // The marker may have reached some drives, or all of them.
+ z.recoverRestoreStatus(bucket, object, ObjectOptions{
+ VersionID: attempt.VersionID,
+ Transition: TransitionOptions{attempt: attempt},
+ })
+ }
+ return restoreRequestStatus{}, err
+ }
+ return restoreRequestStatus{objInfo: oi, attempt: attempt}, nil
+}
+
+// restoreRunning reports, without the object lock and within the takeover
+// wait, whether the drives show a restore of the version ongoing in a pool.
+// An error means that this cannot be told.
+func (z *erasureServerPools) restoreRunning(ctx context.Context, bucket, object, versionID string) (bool, error) {
+ ctx, cancel := context.WithTimeout(ctx, restoreTakeoverWait)
+ defer cancel()
+ for _, pool := range z.serverPools {
+ gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, versionID)
+ if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) {
+ continue
+ }
+ if err == nil && ctx.Err() != nil {
+ err = ctx.Err()
+ }
+ if err != nil {
+ return false, toObjectErr(err, bucket, object, versionID)
+ }
+ if len(gen.ongoing) > 0 || !gen.split && gen.fi.ToObjectInfo(bucket, object, false).RestoreOngoing {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
+// restorePoolIdx returns the pool whose copy the restore rehydrates: the one
+// POST marked for its attempt, or for a direct caller the existing version.
+func (z *erasureServerPools) restorePoolIdx(ctx context.Context, bucket, object string, opts ObjectOptions) (int, error) {
+ a := opts.Transition.attempt
+ if a == nil {
+ // Restore an existing version, not the latest version's write location.
+ lookupOpts := opts
+ lookupOpts.MetadataChg = true
+ idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts)
+ if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+ return idx, restoreResetError{err}
+ }
+ return idx, err
+ }
+ found := false
+ for idx, pool := range z.serverPools {
+ gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, a.VersionID)
+ if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) {
+ continue
+ }
+ if err != nil {
+ return -1, restoreResetError{toObjectErr(err, bucket, object, a.VersionID)}
+ }
+ found = true
+ if !z.IsSuspended(idx) && a.owns(gen.fi.ToObjectInfo(bucket, object, false)) {
+ return idx, nil
+ }
+ }
+ if !found {
+ return -1, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: a.VersionID}
+ }
+ return -1, errRestoreSuperseded
+}
+
+// recoverRestoreStatus retries, in memory and for a bounded time, to clear the
+// ongoing marker that a failed restore left behind, so a later POST is not
+// refused. Each try takes the object lock afresh and clears the marker only
+// while it still belongs to this attempt.
+func (z *erasureServerPools) recoverRestoreStatus(bucket, object string, opts ObjectOptions) {
+ go func() {
+ var err error
+ for _, delay := range restoreRecoveryDelays {
+ select {
+ case <-GlobalContext.Done():
+ return
+ case <-time.After(delay):
+ }
+ if err = z.resetRestoreStatus(bucket, object, opts); err == nil {
+ return
+ }
+ }
+ storageLogIf(GlobalContext, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, err))
+ }()
+}
+
+// resetRestoreStatus is one try of recoverRestoreStatus.
+func (z *erasureServerPools) resetRestoreStatus(bucket, object string, opts ObjectOptions) error {
+ ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
+ defer cancel()
+ lk := z.restoreLock(bucket, object)
+ lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+ if err != nil {
+ return err
+ }
+ defer lk.Unlock(lkctx)
+ ctx = lkctx.Context()
+
+ a := opts.Transition.attempt
+ versionID := opts.VersionID
+ if a != nil {
+ versionID = a.VersionID
+ }
+ for _, pool := range z.serverPools {
+ err := pool.getHashedSet(object).clearRestoreStatus(ctx, bucket, object, versionID, a)
+ if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !isErrMethodNotAllowed(err) && !isErrBucketNotFound(err) {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/cmd/restore-capability.go b/cmd/restore-capability.go
new file mode 100644
index 000000000..3594a6dc9
--- /dev/null
+++ b/cmd/restore-capability.go
@@ -0,0 +1,184 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "slices"
+ "sync"
+ "time"
+
+ "github.com/minio/minio/internal/grid"
+)
+
+// A native restore keeps its multipart upload private and marks the version
+// it rehydrates (restore-attempt.go). A server that predates this ignores
+// the marker: it would list, change, abort or complete the upload. Native
+// restore therefore runs only while every server of the cluster positively
+// reports that it understands the marker. A server that does not answer,
+// or does not know the question, as an older server does not, blocks it.
+
+// nativeRestoreCapability is the native restore protocol this server speaks.
+const nativeRestoreCapability = "1"
+
+const (
+ peerRESTRestoreCapability = "restore"
+ // restoreCapabilityTTL bounds how long a positive answer is reused while
+ // no server reconnected.
+ restoreCapabilityTTL = 10 * time.Second
+ // restoreCapabilityTimeout bounds the question to each server.
+ restoreCapabilityTimeout = 5 * time.Second
+)
+
+var errNativeRestoreUnavailable = errors.New("native restore is unavailable during a rolling upgrade: not every server of the cluster supports it yet")
+
+// restoreCapabilityHandler answers the question on this server.
+func restoreCapabilityHandler(*grid.MSS) (*grid.MSS, *grid.RemoteErr) {
+ return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: nativeRestoreCapability}), nil
+}
+
+// restoreCapabilityPeer is another server of the cluster.
+type restoreCapabilityPeer interface {
+ // restoreConnection identifies the current connection to the server; it
+ // changes when the server reconnects, as a restarted one does. ok is
+ // false when the server is not connected.
+ restoreConnection() (key string, ok bool)
+ restoreCapability(ctx context.Context) (string, error)
+}
+
+type restorePeerClient struct{ *peerRESTClient }
+
+func (c restorePeerClient) restoreConnection() (string, bool) {
+ if c.peerRESTClient == nil || c.gridConn == nil {
+ return "", false
+ }
+ conn := c.gridConn()
+ if conn == nil {
+ return "", false
+ }
+ st := conn.Stats()
+ if st.Connected == 0 {
+ return "", false
+ }
+ // Keys are compared by the position of their server.
+ return fmt.Sprintf("%d/%d", st.LastConnectTime.UnixNano(), st.ReconnectCount), true
+}
+
+func (c restorePeerClient) restoreCapability(ctx context.Context) (string, error) {
+ conn := c.gridConn()
+ if conn == nil {
+ return "", errPeerOffline
+ }
+ resp, err := restoreCapabilityRPC.Call(ctx, conn, grid.NewMSS())
+ if err != nil {
+ return "", err
+ }
+ return resp.Get(peerRESTRestoreCapability), nil
+}
+
+// restoreCapabilityGate decides whether native restore may run, caching a
+// positive answer for restoreCapabilityTTL while the connections to all
+// servers stay the same. A negative answer is not cached.
+type restoreCapabilityGate struct {
+ peers func() []restoreCapabilityPeer
+
+ once sync.Once
+ turn chan struct{} // one check at a time
+ checked time.Time
+ keys []string
+}
+
+var globalRestoreCapability = &restoreCapabilityGate{peers: notificationRestorePeers}
+
+func notificationRestorePeers() []restoreCapabilityPeer {
+ if globalNotificationSys == nil {
+ return nil
+ }
+ peers := make([]restoreCapabilityPeer, 0, len(globalNotificationSys.peerClients))
+ for _, client := range globalNotificationSys.peerClients {
+ peers = append(peers, restorePeerClient{client})
+ }
+ return peers
+}
+
+// nativeRestoreAvailable returns errNativeRestoreUnavailable unless every
+// other server of the cluster supports native restore.
+func nativeRestoreAvailable(ctx context.Context) error {
+ return globalRestoreCapability.check(ctx)
+}
+
+func (g *restoreCapabilityGate) check(ctx context.Context) error {
+ g.once.Do(func() { g.turn = make(chan struct{}, 1) })
+ // One deadline covers the wait for another check and the questions.
+ ctx, cancel := context.WithTimeout(ctx, restoreCapabilityTimeout)
+ defer cancel()
+ select {
+ case g.turn <- struct{}{}:
+ case <-ctx.Done():
+ return errNativeRestoreUnavailable
+ }
+ defer func() { <-g.turn }()
+
+ // Read the connections only now: one may have changed while this check
+ // waited for another.
+ peers := g.peers()
+ connections := func() ([]string, bool) {
+ keys := make([]string, len(peers))
+ for i, p := range peers {
+ key, ok := p.restoreConnection()
+ if !ok {
+ return nil, false
+ }
+ keys[i] = key
+ }
+ return keys, true
+ }
+ keys, ok := connections()
+ if !ok {
+ g.checked, g.keys = time.Time{}, nil
+ return errNativeRestoreUnavailable
+ }
+ if ctx.Err() != nil {
+ return errNativeRestoreUnavailable
+ }
+ if !g.checked.IsZero() && time.Since(g.checked) < restoreCapabilityTTL && slices.Equal(keys, g.keys) {
+ return nil
+ }
+ g.checked, g.keys = time.Time{}, nil
+ answers := make([]bool, len(peers))
+ var wg sync.WaitGroup
+ for i, p := range peers {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ version, err := p.restoreCapability(ctx)
+ answers[i] = err == nil && version == nativeRestoreCapability
+ }()
+ }
+ wg.Wait()
+ // An answer counts only for the connection it was asked on: a server
+ // that reconnected meanwhile may be another binary.
+ after, ok := connections()
+ if !ok || !slices.Equal(keys, after) || slices.Contains(answers, false) || ctx.Err() != nil {
+ return errNativeRestoreUnavailable
+ }
+ g.checked, g.keys = time.Now(), keys
+ return nil
+}
diff --git a/cmd/restore-capability_test.go b/cmd/restore-capability_test.go
new file mode 100644
index 000000000..306764d12
--- /dev/null
+++ b/cmd/restore-capability_test.go
@@ -0,0 +1,271 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/versioning"
+ "github.com/minio/minio/internal/grid"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// A server answers the capability question over the grid. One that does not
+// know it, as an older server, or answers something else, blocks native
+// restore, and so does one that is not connected.
+func TestNativeRestoreCapabilityGrid(t *testing.T) {
+ tg, err := grid.SetupTestGrid(4)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(tg.Cleanup)
+ if err := restoreCapabilityRPC.Register(tg.Managers[1], restoreCapabilityHandler); err != nil {
+ t.Fatal(err)
+ }
+ // Managers[2] never registers the handler.
+ if err := restoreCapabilityRPC.Register(tg.Managers[3], func(*grid.MSS) (*grid.MSS, *grid.RemoteErr) {
+ return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: "0"}), nil
+ }); err != nil {
+ t.Fatal(err)
+ }
+ peer := func(i int) restoreCapabilityPeer {
+ return restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[i]) }}}
+ }
+ offline := restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return nil }}}
+ for _, tc := range []struct {
+ name string
+ peers []restoreCapabilityPeer
+ ok bool
+ }{
+ {"single-server", nil, true},
+ {"all-capable", []restoreCapabilityPeer{peer(1)}, true},
+ {"older-server", []restoreCapabilityPeer{peer(1), peer(2)}, false},
+ {"other-answer", []restoreCapabilityPeer{peer(1), peer(3)}, false},
+ {"offline", []restoreCapabilityPeer{peer(1), offline}, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return tc.peers }}
+ if err := g.check(t.Context()); (err == nil) != tc.ok {
+ t.Errorf("capability: %v, want available=%v", err, tc.ok)
+ }
+ })
+ }
+}
+
+// restoreCapabilityFake is a server whose connection and answer a test sets.
+type restoreCapabilityFake struct {
+ capable atomic.Bool
+ connected atomic.Bool
+ conn atomic.Int64
+ asked atomic.Int64
+}
+
+func newRestoreCapabilityFake() *restoreCapabilityFake {
+ p := &restoreCapabilityFake{}
+ p.capable.Store(true)
+ p.connected.Store(true)
+ return p
+}
+
+func (p *restoreCapabilityFake) restoreConnection() (string, bool) {
+ return strings.Repeat("c", int(p.conn.Load())), p.connected.Load()
+}
+
+func (p *restoreCapabilityFake) restoreCapability(context.Context) (string, error) {
+ p.asked.Add(1)
+ if !p.capable.Load() {
+ return "", errors.New("Invalid Handler for type: RestoreCapability")
+ }
+ return nativeRestoreCapability, nil
+}
+
+// A positive answer is reused while the connections stay the same; a
+// reconnect asks again, a disconnected server blocks without being asked,
+// and a negative answer is not reused.
+func TestNativeRestoreCapabilityCache(t *testing.T) {
+ p := newRestoreCapabilityFake()
+ g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+ for range 3 {
+ if err := g.check(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if n := p.asked.Load(); n != 1 {
+ t.Errorf("asked %d times while connected and capable, want 1", n)
+ }
+ // The server restarts with an older binary and reconnects.
+ p.capable.Store(false)
+ p.conn.Add(1)
+ if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) {
+ t.Errorf("after a reconnect to an older server: %v", err)
+ }
+ p.capable.Store(true)
+ if err := g.check(t.Context()); err != nil {
+ t.Errorf("a negative answer was reused: %v", err)
+ }
+ asked := p.asked.Load()
+ p.connected.Store(false)
+ if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) || p.asked.Load() != asked {
+ t.Errorf("disconnected server: %v, asked %d -> %d", err, asked, p.asked.Load())
+ }
+}
+
+// During a rolling upgrade POST restore answers 503 and writes nothing, and
+// a restore queued before it creates no upload and leaves the version
+// retryable.
+func TestNativeRestoreRollingUpgradeGate(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := restoreIsolationSeed(t, r, 0, false, true)
+ p := newRestoreCapabilityFake()
+ previous := globalRestoreCapability
+ globalRestoreCapability = &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+ t.Cleanup(func() { globalRestoreCapability = previous })
+
+ p.capable.Store(false)
+ before := r.fileInfo(oi.VersionID)
+ layer := newRestoreIsolationLayer(r)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "rolling upgrade") {
+ t.Errorf("POST during a rolling upgrade: %d %s", rec.Code, rec.Body.String())
+ }
+ if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before.Metadata, after.Metadata) {
+ t.Error("refused POST changed the version")
+ }
+ select {
+ case err := <-layer.done:
+ t.Fatalf("refused POST started a restore: %v", err)
+ default:
+ }
+
+ p.capable.Store(true)
+ release := make(chan struct{})
+ layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ <-release
+ return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ close(release)
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ // A server is replaced by an older binary while the restore is queued.
+ p.capable.Store(false)
+ p.conn.Add(1)
+ close(release)
+ if err := layer.wait(t); !errors.Is(err, errNativeRestoreUnavailable) {
+ t.Fatalf("queued restore during a rolling upgrade: %v", err)
+ }
+ if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+ t.Errorf("queued restore created an upload during a rolling upgrade: %v", uploads)
+ }
+ info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil || info.RestoreOngoing || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" {
+ t.Errorf("version after a refused queued restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+ }
+}
+
+// restoreCapabilityHeld is a fake server whose answer a test holds.
+type restoreCapabilityHeld struct {
+ restoreCapabilityFake
+ asking chan struct{}
+ release chan struct{}
+}
+
+func (p *restoreCapabilityHeld) restoreCapability(ctx context.Context) (string, error) {
+ v, err := p.restoreCapabilityFake.restoreCapability(ctx)
+ p.asking <- struct{}{}
+ select {
+ case <-p.release:
+ return v, err
+ case <-ctx.Done():
+ return "", ctx.Err()
+ }
+}
+
+// A check that waits for another reads the connections once its turn
+// comes, an answer counts only if its server did not reconnect meanwhile,
+// and a waiting check ends with its context.
+func TestNativeRestoreCapabilityRevalidates(t *testing.T) {
+ t.Run("reconnect-during-check", func(t *testing.T) {
+ held := func() *restoreCapabilityHeld {
+ p := &restoreCapabilityHeld{asking: make(chan struct{}, 4), release: make(chan struct{})}
+ p.capable.Store(true)
+ p.connected.Store(true)
+ return p
+ }
+ fast, slow := held(), held()
+ g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{fast, slow} }}
+ first := make(chan error, 1)
+ go func() { first <- g.check(t.Context()) }()
+ <-fast.asking
+ <-slow.asking
+ waiting := make(chan error, 1)
+ go func() { waiting <- g.check(t.Context()) }()
+ // Both answered "capable"; then fast reconnects as an older server.
+ fast.capable.Store(false)
+ fast.conn.Add(1)
+ close(fast.release)
+ close(slow.release)
+ if err := <-first; !errors.Is(err, errNativeRestoreUnavailable) {
+ t.Errorf("check whose server reconnected while it asked: %v", err)
+ }
+ if err := <-waiting; !errors.Is(err, errNativeRestoreUnavailable) {
+ t.Errorf("waiting check accepted a reconnected older server: %v", err)
+ }
+ })
+ t.Run("canceled-cached", func(t *testing.T) {
+ p := newRestoreCapabilityFake()
+ g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+ if err := g.check(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ if err := g.check(ctx); err == nil {
+ t.Error("a cached answer accepted an ended context")
+ }
+ })
+ t.Run("canceled-wait", func(t *testing.T) {
+ slow := &restoreCapabilityHeld{restoreCapabilityFake: restoreCapabilityFake{}, asking: make(chan struct{}, 4), release: make(chan struct{})}
+ slow.capable.Store(true)
+ slow.connected.Store(true)
+ defer close(slow.release)
+ g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{slow} }}
+ go g.check(t.Context())
+ <-slow.asking
+ ctx, cancel := context.WithCancel(t.Context())
+ waiting := make(chan error, 1)
+ go func() { waiting <- g.check(ctx) }()
+ cancel()
+ select {
+ case err := <-waiting:
+ if err == nil {
+ t.Error("canceled check succeeded")
+ }
+ case <-time.After(time.Second):
+ t.Error("canceled check kept waiting for another check")
+ }
+ })
+}
diff --git a/cmd/restore-native-generation_test.go b/cmd/restore-native-generation_test.go
new file mode 100644
index 000000000..d79bc1030
--- /dev/null
+++ b/cmd/restore-native-generation_test.go
@@ -0,0 +1,1418 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "maps"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "reflect"
+ "runtime"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/versioning"
+ "github.com/minio/minio/internal/config/drive"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// restoreGenerationDisk fails metadata writes or updates of the object, or
+// metadata reads of uploads, while its toggles are set.
+type restoreGenerationDisk struct {
+ StorageAPI
+ bucket, object string
+ write, update, uploadRead *atomic.Bool
+}
+
+func (d restoreGenerationDisk) WriteMetadata(ctx context.Context, orig, vol, path string, fi FileInfo) error {
+ if d.write != nil && d.write.Load() && vol == d.bucket && path == d.object {
+ return errFaultyDisk
+ }
+ return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi)
+}
+
+func (d restoreGenerationDisk) UpdateMetadata(ctx context.Context, vol, path string, fi FileInfo, opts UpdateMetadataOpts) error {
+ if d.update != nil && d.update.Load() && vol == d.bucket && path == d.object {
+ return errFaultyDisk
+ }
+ return d.StorageAPI.UpdateMetadata(ctx, vol, path, fi, opts)
+}
+
+func (d restoreGenerationDisk) ReadVersion(ctx context.Context, orig, vol, path, version string, opts ReadOptions) (FileInfo, error) {
+ if d.uploadRead != nil && d.uploadRead.Load() && vol == minioMetaMultipartBucket {
+ return FileInfo{}, errFaultyDisk
+ }
+ return d.StorageAPI.ReadVersion(ctx, orig, vol, path, version, opts)
+}
+
+// restoreGenerationOffline makes a drive's object metadata unreadable and
+// unwritable while set, its uploads still readable.
+type restoreGenerationOffline struct {
+ StorageAPI
+ bucket, object string
+ fail *atomic.Bool
+}
+
+func (d restoreGenerationOffline) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
+ if d.fail.Load() && bucket == d.bucket && object == d.object {
+ return RawFileInfo{}, errFaultyDisk
+ }
+ return d.StorageAPI.ReadXL(ctx, bucket, object, data)
+}
+
+func (d restoreGenerationOffline) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
+ if d.fail.Load() && bucket == d.bucket && object == d.object {
+ return FileInfo{}, errFaultyDisk
+ }
+ return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
+}
+
+func (d restoreGenerationOffline) WriteMetadata(ctx context.Context, orig, bucket, object string, fi FileInfo) error {
+ if d.fail.Load() && bucket == d.bucket && object == d.object {
+ return errFaultyDisk
+ }
+ return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi)
+}
+
+// restoreGenerationMetadata returns the version's metadata on each drive.
+func restoreGenerationMetadata(t *testing.T, disks []StorageAPI, r *restoreTest, versionID string) []map[string]string {
+ t.Helper()
+ metas := make([]map[string]string, len(disks))
+ for i, d := range disks {
+ fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, versionID, ReadOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ metas[i] = fi.Metadata
+ }
+ return metas
+}
+
+// An Object Lock update acknowledged on a quorum, which one drive missed,
+// survives POST restore: POST patches the restore fields onto the quorum's
+// generation and never writes the stale drive's metadata anywhere.
+func TestRestoreStatusUsesQuorumGeneration(t *testing.T) {
+ for _, kind := range []string{"legal-hold", "retention"} {
+ t.Run(kind, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ enableBucketObjectLock(t, r.bucket)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ initial := []byte(`OFF`)
+ updated := []byte(`ON`)
+ key := strings.ToLower(xhttp.AmzObjectLockLegalHold)
+ if kind == "retention" {
+ initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour))
+ updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))
+ key = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
+ }
+ if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ var fail atomic.Bool
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, update: &fail}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ fail.Store(true)
+ if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK {
+ t.Fatalf("protection update: %d %s", rec.Code, rec.Body.String())
+ }
+ fail.Store(false)
+ before := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+ acknowledged := before[1][key]
+ if before[0][key] == acknowledged {
+ t.Fatal("fixture: drive 0 did not miss the update")
+ }
+ for i := 1; i < len(disks); i++ {
+ if before[i][key] != acknowledged {
+ t.Fatal("fixture: the update did not reach the other drives")
+ }
+ }
+ if rec := restoreIsolationPOST(t, r, r.z, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ after := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+ for i := 1; i < len(disks); i++ {
+ if after[i][key] != acknowledged {
+ t.Errorf("drive %d: POST wrote %q over the acknowledged %q", i, after[i][key], acknowledged)
+ }
+ if after[i][xhttp.AmzRestoreRequestDate] == before[i][xhttp.AmzRestoreRequestDate] {
+ t.Errorf("drive %d: POST did not record its request", i)
+ }
+ }
+ // The stale drive is left to heal; reads that pick it are not
+ // changed here.
+ if after[0][key] != before[0][key] || after[0][xhttp.AmzRestoreRequestDate] != before[0][xhttp.AmzRestoreRequestDate] {
+ t.Error("POST wrote the stale drive instead of leaving it to heal")
+ }
+ })
+ }
+}
+
+// The attempt is part of a version's metadata generation. A POST whose
+// marker reached all drives but one is acknowledged: its restore owns the
+// version, and the reset of an older attempt leaves the newer marker alone.
+func TestRestoreAttemptGenerationFencing(t *testing.T) {
+ for _, mode := range []string{"worker", "old-reset", "degraded-old-reset"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ var fail, offline atomic.Bool
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, write: &fail}
+ wrapped[1] = restoreGenerationOffline{StorageAPI: disks[1], bucket: r.bucket, object: r.object, fail: &offline}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ captured := make(chan ObjectOptions, 2)
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error {
+ captured <- opts
+ return nil
+ }
+ var older ObjectOptions
+ if mode != "worker" {
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ older = <-captured
+ layer.wait(t)
+ }
+ fail.Store(true)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ fail.Store(false)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("POST on all drives but one: %d %s", rec.Code, rec.Body.String())
+ }
+ newer := <-captured
+ layer.wait(t)
+ id := newer.Transition.attempt.ID
+ if mode == "worker" {
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, newer); err != nil {
+ t.Fatalf("restore of the acknowledged attempt: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ return
+ }
+ if mode == "degraded-old-reset" {
+ // One drive of the newer attempt is unreadable: the drives
+ // that answer no longer show which attempt is acknowledged.
+ offline.Store(true)
+ }
+ if err := r.z.resetRestoreStatus(r.bucket, r.object, older); err != nil {
+ t.Fatal(err)
+ }
+ offline.Store(false)
+ after := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+ for i := 1; i < len(disks); i++ {
+ if after[i][restoreAttemptKey] != id || !strings.Contains(after[i][xhttp.AmzRestore], "true") {
+ t.Errorf("drive %d: the older attempt's reset cleared the newer marker: %q %q", i, after[i][restoreAttemptKey], after[i][xhttp.AmzRestore])
+ }
+ }
+ if after[0][restoreAttemptKey] != older.Transition.attempt.ID {
+ t.Error("the reset wrote the stale drive")
+ }
+ // The newer attempt still owns the version and restores it.
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, newer); err != nil {
+ t.Fatalf("restore of the newer attempt after the older reset: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ })
+ }
+}
+
+// restoreGenerationUploadPath is the stored path of an upload.
+func restoreGenerationUploadPath(t *testing.T, r *restoreTest, id string) string {
+ t.Helper()
+ b, err := base64.RawURLEncoding.DecodeString(id)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, uploadUUID, ok := strings.Cut(string(b), ".")
+ if !ok {
+ t.Fatalf("upload ID without UUID: %q", id)
+ }
+ return pathJoin(r.z.serverPools[0].getHashedSet(r.object).getMultipartSHADir(r.bucket, r.object), uploadUUID)
+}
+
+// The restore marker is established on a majority of the drives. One drive
+// whose copy of the upload lacks it does not open the upload to clients,
+// and the restore still completes.
+func TestRestoreUploadMarkerQuorum(t *testing.T) {
+ r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ tier := newRestoreIsolationGateTier(5<<20, nil)
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ defer tier.unblock()
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ restoreIsolationEntered(t, tier)
+ id, _ := restoreIsolationUpload(t, r, 0)
+ disk := r.z.serverPools[0].getHashedSet(r.object).getDisks()[0]
+ path := restoreGenerationUploadPath(t, r, id)
+ fi, err := disk.ReadVersion(t.Context(), "", minioMetaMultipartBucket, path, "", ReadOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ delete(fi.Metadata, restoreUploadKey)
+ if err := disk.WriteMetadata(t.Context(), "", minioMetaMultipartBucket, path, fi); err != nil {
+ t.Fatal(err)
+ }
+ poison := bytes.Repeat([]byte("p"), 5<<20)
+ if _, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, id, 1,
+ mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", ""), ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) {
+ t.Errorf("public part with one unmarked copy of the upload: %v, want NoSuchUpload", err)
+ }
+ for _, op := range []func() error{
+ func() error {
+ _, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{})
+ return err
+ },
+ func() error {
+ _, err := r.z.GetMultipartInfo(t.Context(), r.bucket, r.object, id, ObjectOptions{})
+ return err
+ },
+ } {
+ if err := op(); !errors.As(err, &InvalidUploadID{}) {
+ t.Errorf("public read with one unmarked copy of the upload: %v, want NoSuchUpload", err)
+ }
+ }
+ if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10); err != nil || len(uploads.Uploads) != 0 {
+ t.Errorf("listing with one unmarked copy of the upload: %+v, %v", uploads.Uploads, err)
+ }
+ tier.unblock()
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restore contents: %v", err)
+ }
+}
+
+// When the drives cannot tell an upload's marker, no client lists it, no
+// abort removes it and stale cleanup keeps it; the restore completes once
+// the drives answer again.
+func TestRestoreUploadUnreadableFailsClosed(t *testing.T) {
+ for _, mode := range []string{"list", "abort", "stale"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ var fail atomic.Bool
+ // Stale cleanup with a short age also removes in-flight temporary
+ // parts: hold that restore in completion, after its last part and
+ // before its commit, and the others in their download, outside
+ // the upload lock.
+ gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})}
+ defer gate.unblock()
+ tier := newRestoreIsolationGateTier(5<<20, nil)
+ defer tier.unblock()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ disk := d
+ if mode == "stale" {
+ disk = restoreIsolationValidationDisk{d, gate}
+ }
+ wrapped[i] = restoreGenerationDisk{StorageAPI: disk, uploadRead: &fail}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ if mode != "stale" {
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ }
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ if mode == "stale" {
+ select {
+ case <-gate.entered:
+ case <-time.After(10 * time.Second):
+ t.Fatal("restore never reached completion")
+ }
+ } else {
+ restoreIsolationEntered(t, tier)
+ }
+ stored := restoreIsolationUploads(t, r, 0)
+ if len(stored) != 1 {
+ t.Fatalf("restore upload on disk: %v", stored)
+ }
+ var id string
+ for uploadUUID := range stored {
+ id = base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID))
+ }
+ fail.Store(true)
+ switch mode {
+ case "list":
+ if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 100); len(uploads.Uploads) != 0 {
+ t.Errorf("an upload with an unknown marker is listed: %+v, %v", uploads.Uploads, err)
+ }
+ case "abort":
+ if err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); err == nil {
+ t.Error("abort of an upload with an unknown marker succeeded")
+ }
+ case "stale":
+ globalAPIConfig.mu.Lock()
+ prev := globalAPIConfig.staleUploadsExpiry
+ globalAPIConfig.staleUploadsExpiry = time.Nanosecond
+ globalAPIConfig.mu.Unlock()
+ for _, disk := range wrapped {
+ er.cleanupStaleUploadsOnDisk(t.Context(), disk)
+ }
+ globalAPIConfig.mu.Lock()
+ globalAPIConfig.staleUploadsExpiry = prev
+ globalAPIConfig.mu.Unlock()
+ }
+ fail.Store(false)
+ if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 1 {
+ t.Errorf("%s removed an upload with an unknown marker: %v", mode, uploads)
+ }
+ gate.unblock()
+ tier.unblock()
+ if err := layer.wait(t); err != nil {
+ t.Fatalf("restore after %s: %v", mode, err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restore contents: %v", err)
+ }
+ })
+ }
+}
+
+// POST restores the version it resolved with that version's identity. A
+// replacement under the same version ID between its read and its status
+// write is refused, and nothing restores it.
+func TestRestorePOSTRejectsReplacedTarget(t *testing.T) {
+ for _, state := range []versioning.State{"", versioning.Suspended} {
+ t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, state)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := restoreIsolationSeed(t, r, 0, true, false)
+ layer := newRestoreIsolationLayer(r)
+ var replacement ObjectInfo
+ layer.afterRead = func() {
+ data := []byte("replacement after POST resolved its target")
+ var err error
+ replacement, err = r.z.PutObject(t.Context(), r.bucket, r.object,
+ mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: state == versioning.Suspended})
+ if err != nil {
+ t.Fatal(err)
+ }
+ opts := r.opts(nullVersionID)
+ opts.MTime = replacement.ModTime
+ opts.Transition = TransitionOptions{Tier: "WARM", ETag: replacement.ETag}
+ if err := r.z.serverPools[0].TransitionObject(t.Context(), r.bucket, r.object, opts); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code < 400 {
+ layer.wait(t)
+ t.Errorf("POST accepted a replacement of its target: %d", rec.Code)
+ }
+ after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(nullVersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after.ETag != replacement.ETag || after.UserDefined[xhttp.AmzRestore] != "" || after.ETag == oi.ETag {
+ t.Errorf("replacement %q changed by POST: etag %q restore %q", replacement.ETag, after.ETag, after.UserDefined[xhttp.AmzRestore])
+ }
+ })
+ }
+}
+
+// A POST on a version whose restore holds the object lock answers within
+// the two-second takeover bound: its first read does not wait for the lock.
+func TestRestorePOSTOngoingBound(t *testing.T) {
+ for _, mode := range []string{"pools=1", "pools=2", "stale-minority", "failed-activity-read"} {
+ t.Run(mode, func(t *testing.T) {
+ pools := 1
+ if mode == "pools=2" {
+ pools = 2
+ }
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+ tier := newRestoreIsolationGateTier(0, nil)
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ defer tier.unblock()
+ var fail atomic.Bool
+ if mode == "stale-minority" {
+ // The first read may pick a drive that missed the marker.
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, write: &fail}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ fail.Store(true)
+ }
+ layer := newRestoreIsolationLayer(r)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ fail.Store(false)
+ if rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ restoreIsolationEntered(t, tier)
+ var unreadable atomic.Bool
+ second := newRestoreIsolationLayer(r)
+ if mode == "failed-activity-read" {
+ // Once POST resolved its target, nothing tells whether it
+ // is restoring.
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ wrapped[i] = restoreGenerationOffline{StorageAPI: d, bucket: r.bucket, object: r.object, fail: &unreadable}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ second.afterRead = func() { unreadable.Store(true) }
+ }
+ done := make(chan *httptest.ResponseRecorder, 1)
+ start := time.Now()
+ go func() { done <- restoreIsolationPOST(t, r, second, oi.VersionID) }()
+ select {
+ case rec := <-done:
+ want := rec.Code == http.StatusConflict
+ if mode == "failed-activity-read" {
+ want = rec.Code >= 500
+ }
+ if !want || time.Since(start) > restoreTakeoverWait+time.Second {
+ t.Errorf("POST during a running restore: %d after %s, want a bounded refusal", rec.Code, time.Since(start))
+ }
+ case <-time.After(restoreTakeoverWait + 2*time.Second):
+ t.Error("POST during a running restore waited for the object lock")
+ unreadable.Store(false)
+ tier.unblock()
+ <-done
+ }
+ unreadable.Store(false)
+ tier.unblock()
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+// A POST whose status write fails on drives that stay unwritten puts the
+// generation it read back, and a split that a failed put-back leaves
+// differs only in restore fields: the attempt's reset and a later POST both
+// resolve it, while every other field still needs the quorum.
+func TestRestorePOSTStatusWriteSplit(t *testing.T) {
+ for _, mode := range []string{"put-back", "split-reset", "split-post"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ var fail atomic.Bool
+ wrapped := append([]StorageAPI(nil), disks...)
+ for i := range 2 {
+ wrapped[i] = restoreGenerationDisk{StorageAPI: disks[i], bucket: r.bucket, object: r.object, write: &fail}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ before := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+ var attempt *restoreAttempt
+ if mode == "put-back" {
+ fail.Store(true)
+ rec := restoreIsolationPOST(t, r, r.z, oi.VersionID)
+ fail.Store(false)
+ if rec.Code < 400 {
+ t.Fatalf("POST acknowledged a status write on two of four drives: %d", rec.Code)
+ }
+ if after := restoreGenerationMetadata(t, disks, r, oi.VersionID); !reflect.DeepEqual(before, after) {
+ t.Error("failed status write was not put back")
+ }
+ } else {
+ // A status write and its put-back both failed on two drives.
+ attempt = newRestoreAttempt(oi)
+ for _, d := range disks[2:] {
+ fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ fi.Metadata[xhttp.AmzRestore] = ongoingRestoreObj().String()
+ fi.Metadata[restoreAttemptKey] = attempt.ID
+ if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if _, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); err != nil {
+ t.Fatalf("a split in restore fields only has no generation: %v", err)
+ }
+ }
+ if mode == "split-reset" {
+ opts := nativeRestoreOptions(r, oi.VersionID)
+ opts.Transition.attempt = attempt
+ if err := r.z.resetRestoreStatus(r.bucket, r.object, opts); err != nil {
+ t.Fatalf("reset of the split attempt: %v", err)
+ }
+ for i, meta := range restoreGenerationMetadata(t, disks, r, oi.VersionID) {
+ if meta[xhttp.AmzRestore] != "" || meta[restoreAttemptKey] != "" {
+ t.Errorf("drive %d keeps the split attempt: %q %q", i, meta[xhttp.AmzRestore], meta[restoreAttemptKey])
+ }
+ }
+ }
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST after the failed status write: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restore contents: %v", err)
+ }
+ })
+ }
+}
+
+// A split in any other field is not resolved: the generation fails closed.
+func TestRestoreGenerationSplitNeedsQuorum(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ for i, d := range er.getDisks()[2:] {
+ fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ fi.Metadata[xhttp.AmzRestore] = ongoingRestoreObj().String()
+ fi.Metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = fmt.Sprint("ON", i)
+ if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if _, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); !errors.Is(err, errErasureReadQuorum) {
+ t.Fatalf("split outside the restore fields: %v, want read quorum", err)
+ }
+ if rec := restoreIsolationPOST(t, r, r.z, oi.VersionID); rec.Code < 400 {
+ t.Errorf("POST over a split outside the restore fields: %d", rec.Code)
+ }
+}
+
+// restoreGenerationStageDisk holds a completion that is staging: as
+// xlStorage.RenameData does, a drive that stages writes the committed
+// object metadata into the upload before it renames the upload into the
+// object.
+type restoreGenerationStageDisk struct {
+ StorageAPI
+ bucket, object string
+ stage bool
+ arrived *atomic.Int32
+ entered chan struct{}
+ release chan struct{}
+ disks int32
+}
+
+func (d restoreGenerationStageDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
+ if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object {
+ if d.stage {
+ if err := d.StorageAPI.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil {
+ return RenameDataResp{}, err
+ }
+ }
+ if d.arrived.Add(1) == d.disks {
+ close(d.entered)
+ }
+ select {
+ case <-d.release:
+ case <-ctx.Done():
+ return RenameDataResp{}, ctx.Err()
+ }
+ }
+ return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
+}
+
+// While a completion stages its commit, the upload holds object metadata
+// without the marker on the drives that staged. Nobody lists, aborts or
+// removes it then, also when the copies not staged yet are old.
+func TestRestoreUploadStagingFailsClosed(t *testing.T) {
+ for _, mode := range []string{"staged", "partly-staged", "stalled-commit"} {
+ t.Run(mode, func(t *testing.T) { testRestoreUploadStaging(t, mode) })
+ }
+}
+
+func testRestoreUploadStaging(t *testing.T, mode string) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ var arrived atomic.Int32
+ entered, release := make(chan struct{}), make(chan struct{})
+ released := false
+ defer func() {
+ if !released {
+ close(release)
+ }
+ }()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ wrapped[i] = restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: mode != "partly-staged" || i == 0,
+ arrived: &arrived, entered: entered, release: release, disks: int32(len(disks))}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ select {
+ case <-entered:
+ case <-time.After(10 * time.Second):
+ t.Fatal("completion never staged its commit")
+ }
+ stored := restoreIsolationUploads(t, r, 0)
+ var id string
+ for uploadUUID := range stored {
+ id = base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID))
+ if mode != "staged" {
+ // partly-staged: a long restore, the copies not staged yet are
+ // old. stalled-commit: drives that stall the running commit
+ // past the grace make its staged copies look abandoned.
+ old := UTCNow().Add(-48 * time.Hour)
+ for _, d := range disks[1:] {
+ if err := os.Chtimes(pathJoin(d.Endpoint().Path, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), uploadUUID, xlStorageFormatFile), old, old); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ }
+ if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10); err != nil || len(uploads.Uploads) != 0 {
+ t.Errorf("a staged restore upload is listed: %+v, %v", uploads.Uploads, err)
+ }
+ // The set's own abort: the pools layer would wait for the upload lock
+ // the completion holds.
+ if err := er.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); err == nil {
+ t.Error("abort of a staged restore upload succeeded")
+ }
+ globalAPIConfig.mu.Lock()
+ prev := globalAPIConfig.staleUploadsExpiry
+ globalAPIConfig.staleUploadsExpiry = time.Nanosecond
+ globalAPIConfig.mu.Unlock()
+ for _, disk := range wrapped {
+ er.cleanupStaleUploadsOnDisk(t.Context(), disk)
+ }
+ globalAPIConfig.mu.Lock()
+ globalAPIConfig.staleUploadsExpiry = prev
+ globalAPIConfig.mu.Unlock()
+ if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) {
+ t.Errorf("a staged restore upload was removed: %v -> %v", stored, after)
+ }
+ released = true
+ close(release)
+ if err := layer.wait(t); err != nil {
+ t.Fatalf("restore after its staging was attacked: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restore contents: %v", err)
+ }
+}
+
+// When a drive of an acknowledged attempt becomes unreadable, the drives that
+// answer disagree on the restore status. That the attempt ended is then not
+// established, and stale cleanup keeps its upload.
+func TestRestoreSplitKeepsRunningUpload(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ var missed, offline atomic.Bool
+ gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})}
+ defer gate.unblock()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ var disk StorageAPI = restoreIsolationValidationDisk{d, gate}
+ switch i {
+ case 0:
+ disk = restoreGenerationDisk{StorageAPI: disk, bucket: r.bucket, object: r.object, write: &missed}
+ case 1:
+ disk = restoreGenerationOffline{StorageAPI: disk, bucket: r.bucket, object: r.object, fail: &offline}
+ }
+ wrapped[i] = disk
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ layer := newRestoreIsolationLayer(r)
+ missed.Store(true)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ missed.Store(false)
+ if rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ select {
+ case <-gate.entered:
+ case <-time.After(10 * time.Second):
+ t.Fatal("restore never reached completion")
+ }
+ stored := restoreIsolationUploads(t, r, 0)
+ offline.Store(true)
+ if gen, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); err != nil || !gen.split {
+ t.Fatalf("fixture: no split generation: %v", err)
+ }
+ globalAPIConfig.mu.Lock()
+ prev := globalAPIConfig.staleUploadsExpiry
+ globalAPIConfig.staleUploadsExpiry = time.Nanosecond
+ globalAPIConfig.mu.Unlock()
+ for _, disk := range wrapped {
+ er.cleanupStaleUploadsOnDisk(t.Context(), disk)
+ }
+ globalAPIConfig.mu.Lock()
+ globalAPIConfig.staleUploadsExpiry = prev
+ globalAPIConfig.mu.Unlock()
+ offline.Store(false)
+ if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) {
+ t.Errorf("stale cleanup removed a running restore's upload: %v -> %v", stored, after)
+ }
+ gate.unblock()
+ if err := layer.wait(t); err != nil {
+ t.Fatalf("restore after the split: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restore contents: %v", err)
+ }
+}
+
+// restoreGenerationCleanupDisk misses restored-copy cleanups and metadata
+// writes of the object while its toggles are set.
+type restoreGenerationCleanupDisk struct {
+ StorageAPI
+ bucket, object string
+ cleanup, metaWrites *atomic.Bool
+}
+
+func (d restoreGenerationCleanupDisk) DeleteVersion(ctx context.Context, bucket, object string, fi FileInfo, force bool, opts DeleteOptions) error {
+ if bucket == d.bucket && object == d.object && fi.ExpireRestored && d.cleanup.Load() {
+ return errFaultyDisk
+ }
+ return d.StorageAPI.DeleteVersion(ctx, bucket, object, fi, force, opts)
+}
+
+func (d restoreGenerationCleanupDisk) WriteMetadata(ctx context.Context, orig, bucket, object string, fi FileInfo) error {
+ if bucket == d.bucket && object == d.object && d.metaWrites.Load() {
+ return errFaultyDisk
+ }
+ return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi)
+}
+
+// Restored-copy expiry decides on the version's authoritative generation. A
+// drive that missed a cleanup and the POST after it keeps an expired header,
+// which must not clear the quorum's ongoing restore; drives split between
+// an expired and a renewed header authorize nothing.
+func TestExpireRestoredUsesQuorumGeneration(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, mode := range []string{"after-partial-cleanup", "split-renewal"} {
+ t.Run(fmt.Sprintf("pools=%d/%s", pools, mode), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi := r.put("cache", false, true)
+ r.restore(oi)
+ stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ expire := func() error {
+ return expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+ }
+ if mode == "split-renewal" {
+ // A renewal reached two drives; the other two keep the
+ // expired header.
+ dataDir := r.fileInfo(oi.VersionID).DataDir
+ for _, d := range disks[2:] {
+ fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ fi.Metadata[xhttp.AmzRestore] = completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()
+ if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := expire(); err == nil || r.dataDirs(dataDir) != len(disks) {
+ t.Errorf("expiry on drives split between expired and renewed: %v, %d/%d cache dirs", err, r.dataDirs(dataDir), len(disks))
+ }
+ return
+ }
+ var cleanup, metaWrites atomic.Bool
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[0] = restoreGenerationCleanupDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, cleanup: &cleanup, metaWrites: &metaWrites}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ cleanup.Store(true)
+ err = expire()
+ cleanup.Store(false)
+ if err != nil {
+ t.Fatalf("cleanup on all drives but one: %v", err)
+ }
+ captured := make(chan ObjectOptions, 1)
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error {
+ captured <- opts
+ return nil
+ }
+ metaWrites.Store(true)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ metaWrites.Store(false)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("POST after the cleanup: %d %s", rec.Code, rec.Body.String())
+ }
+ opts := <-captured
+ layer.wait(t)
+ // The same cleanup, queued twice, runs again.
+ if err := expire(); err == nil {
+ t.Error("a drive's expired header cleared the quorum's ongoing restore")
+ }
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, opts); err != nil {
+ t.Fatalf("restore of the acknowledged attempt: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, r.bodies["cache"]) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ })
+ }
+ }
+}
+
+// restoreGenerationHidden hides a drive's copy of the object version while
+// set, as a drive that lost it and that heal writes again.
+type restoreGenerationHidden struct {
+ StorageAPI
+ bucket, object string
+ hidden *atomic.Bool
+}
+
+func (d restoreGenerationHidden) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
+ if d.hidden.Load() && bucket == d.bucket && object == d.object {
+ return FileInfo{}, errFileNotFound
+ }
+ return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
+}
+
+// Heal copies the restore fields to a drive only as a quorum of the drives
+// holds them. While they disagree, healing the version waits: it must not
+// create a quorum for an expired header over a renewal, nor for a
+// superseded attempt. Fields a quorum agrees on are copied.
+func TestRestoreHealNeedsRestoreQuorum(t *testing.T) {
+ for _, mode := range []string{"expired-over-renewal", "superseded-attempt", "agreed"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ setRestore := func(versionID string, drives []StorageAPI, fields map[string]string) {
+ for _, d := range drives {
+ fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, versionID, ReadOptions{ReadData: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ maps.Copy(fi.Metadata, fields)
+ if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ var versionID string
+ var stale ObjectInfo
+ var older ObjectOptions
+ hide := 3
+ switch mode {
+ case "expired-over-renewal", "agreed":
+ oi := r.put("cache", false, true)
+ r.restore(oi)
+ versionID = oi.VersionID
+ var err error
+ if stale, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(versionID)); err != nil {
+ t.Fatal(err)
+ }
+ renewed := map[string]string{xhttp.AmzRestore: completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()}
+ if mode == "agreed" {
+ // Three drives renewed, the hidden one expired.
+ setRestore(versionID, disks[:3], renewed)
+ } else {
+ // No quorum: two drives expired an hour ago, one renewed,
+ // and the hidden one expired a day ago.
+ setRestore(versionID, disks[:2], map[string]string{xhttp.AmzRestore: completedRestoreObj(UTCNow().Add(-time.Hour)).String()})
+ setRestore(versionID, disks[2:3], renewed)
+ }
+ case "superseded-attempt":
+ oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+ versionID = oi.VersionID
+ captured := make(chan ObjectOptions, 1)
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error {
+ captured <- opts
+ return nil
+ }
+ if rec := restoreIsolationPOST(t, r, layer, versionID); rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ older = <-captured
+ layer.wait(t)
+ // A newer attempt reached two drives.
+ setRestore(versionID, disks[2:], map[string]string{xhttp.AmzRestore: ongoingRestoreObj().String(), restoreAttemptKey: newRestoreAttempt(oi).ID})
+ hide = 2
+ }
+ before := restoreGenerationMetadata(t, disks, r, versionID)
+ var hidden atomic.Bool
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[hide] = restoreGenerationHidden{StorageAPI: disks[hide], bucket: r.bucket, object: r.object, hidden: &hidden}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ hidden.Store(true)
+ _, healErr := er.HealObject(t.Context(), r.bucket, r.object, versionID, madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+ hidden.Store(false)
+ er.getDisks = func() []StorageAPI { return disks }
+ after := restoreGenerationMetadata(t, disks, r, versionID)
+ if mode == "agreed" {
+ if healErr != nil || after[hide][xhttp.AmzRestore] != after[0][xhttp.AmzRestore] {
+ t.Errorf("heal did not copy the restore fields a quorum holds: %v, %q", healErr, after[hide][xhttp.AmzRestore])
+ }
+ return
+ }
+ if healErr == nil {
+ t.Error("heal proceeded on drives that disagree on the restore fields")
+ }
+ if after[hide][xhttp.AmzRestore] != before[hide][xhttp.AmzRestore] || after[hide][restoreAttemptKey] != before[hide][restoreAttemptKey] {
+ t.Errorf("heal wrote a minority restore state: %q -> %q", before[hide][xhttp.AmzRestore], after[hide][xhttp.AmzRestore])
+ }
+ if mode == "expired-over-renewal" {
+ dataDir := r.fileInfo(versionID).DataDir
+ if err := expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err == nil || r.dataDirs(dataDir) != len(disks) {
+ t.Errorf("expiry after heal: %v, %d/%d cache dirs", err, r.dataDirs(dataDir), len(disks))
+ }
+ return
+ }
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, older); !errors.Is(err, errRestoreSuperseded) {
+ t.Errorf("superseded attempt after heal: %v", err)
+ }
+ })
+ }
+}
+
+// restoreGenerationStageFails fails the commit before it stages.
+type restoreGenerationStageFails struct {
+ StorageAPI
+ bucket, object string
+}
+
+func (d restoreGenerationStageFails) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
+ if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object {
+ return RenameDataResp{}, errFaultyDisk
+ }
+ return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
+}
+
+// restoreGenerationFailedCommit stages the committed metadata into the
+// upload, as RenameData does, and then fails the commit.
+type restoreGenerationFailedCommit struct {
+ StorageAPI
+ bucket, object string
+}
+
+func (d restoreGenerationFailedCommit) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
+ if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object {
+ if err := d.StorageAPI.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil {
+ return RenameDataResp{}, err
+ }
+ return RenameDataResp{}, errFaultyDisk
+ }
+ return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
+}
+
+// restoreGenerationLeftover runs a restore whose commit fails after it
+// staged on every drive, or on every drive but drive 0, which fails before
+// it stages. It returns the upload left behind and its on-disk UUID.
+func restoreGenerationLeftover(t *testing.T, r *restoreTest, unstaged bool) (map[string]int, string) {
+ t.Helper()
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := restoreIsolationSeed(t, r, 0, false, true)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ if unstaged && i == 0 {
+ wrapped[i] = restoreGenerationStageFails{StorageAPI: d, bucket: r.bucket, object: r.object}
+ continue
+ }
+ wrapped[i] = restoreGenerationFailedCommit{StorageAPI: d, bucket: r.bucket, object: r.object}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ if err := layer.wait(t); err == nil {
+ t.Fatal("fixture: the failed commit succeeded")
+ }
+ er.getDisks = func() []StorageAPI { return disks }
+ stored := restoreIsolationUploads(t, r, 0)
+ if len(stored) != 1 {
+ t.Fatalf("fixture: no leftover of the failed commit: %v", stored)
+ }
+ for uploadUUID := range stored {
+ return stored, uploadUUID
+ }
+ return nil, ""
+}
+
+// restoreGenerationAgeUpload makes every drive's copy of an upload's
+// metadata look written d ago.
+func restoreGenerationAgeUpload(t *testing.T, r *restoreTest, uploadUUID string, d time.Duration) {
+ t.Helper()
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ old := UTCNow().Add(-d)
+ for _, disk := range er.getDisks() {
+ if err := os.Chtimes(pathJoin(disk.Endpoint().Path, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), uploadUUID, xlStorageFormatFile), old, old); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+// What a failed commit leaves in an upload can always be removed once every
+// copy is staged and older than the commit grace: abort removes it, also
+// when stale cleanup could not. A recent staging, a copy that cannot be read
+// and a copy not staged yet are refused, and nothing is removed then.
+func TestRestoreAbandonedCommitAbort(t *testing.T) {
+ for _, age := range []string{"recent", "abandoned", "unreadable-copy", "unstaged-copy"} {
+ t.Run(age, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ stored, uploadUUID := restoreGenerationLeftover(t, r, age == "unstaged-copy")
+ if age != "recent" {
+ restoreGenerationAgeUpload(t, r, uploadUUID, 2*restoreCommitGrace)
+ }
+ id := base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID))
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ var unreadable atomic.Bool
+ if age == "unreadable-copy" {
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], uploadRead: &unreadable}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ unreadable.Store(true)
+ }
+ err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{})
+ unreadable.Store(false)
+ left := restoreIsolationUploads(t, r, 0)
+ if age != "abandoned" {
+ if err == nil || !reflect.DeepEqual(left, stored) {
+ t.Errorf("abort of an upload that may still commit (%s): %v, left %v", age, err, left)
+ }
+ return
+ }
+ if err != nil || len(left) != 0 {
+ t.Errorf("abort of an abandoned commit: %v, left %v", err, left)
+ }
+ })
+ }
+}
+
+// With no commit running, stale cleanup removes what a failed commit left
+// drive by drive: a staged copy older than the commit grace. A recent
+// staging and a copy that never staged stay.
+func TestRestoreFailedCommitStaleCleanup(t *testing.T) {
+ for _, mode := range []string{"recent", "abandoned", "unstaged-copy"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ stored, uploadUUID := restoreGenerationLeftover(t, r, mode == "unstaged-copy")
+ if mode != "recent" {
+ // The copy that never staged is as old as the others.
+ restoreGenerationAgeUpload(t, r, uploadUUID, 2*restoreCommitGrace)
+ }
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ globalAPIConfig.mu.Lock()
+ prev := globalAPIConfig.staleUploadsExpiry
+ globalAPIConfig.staleUploadsExpiry = time.Nanosecond
+ globalAPIConfig.mu.Unlock()
+ for _, disk := range er.getDisks() {
+ er.cleanupStaleUploadsOnDisk(t.Context(), disk)
+ }
+ globalAPIConfig.mu.Lock()
+ globalAPIConfig.staleUploadsExpiry = prev
+ globalAPIConfig.mu.Unlock()
+ want := map[string]int{}
+ switch mode {
+ case "recent":
+ want = stored
+ case "unstaged-copy":
+ want = map[string]int{uploadUUID: 1}
+ }
+ if left := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(left, want) {
+ t.Errorf("stale cleanup of a failed commit's leftover: %v -> %v, want %v", stored, left, want)
+ }
+ })
+ }
+}
+
+// restoreGenerationSnapshotKey marks the context of a stale cleanup whose
+// reads of upload metadata restoreGenerationSnapshotDisk holds.
+type restoreGenerationSnapshotKey struct{}
+
+// restoreGenerationSnapshotDisk reads upload metadata at once, and hands the
+// reply to a marked stale cleanup only on release.
+type restoreGenerationSnapshotDisk struct {
+ StorageAPI
+ reads *atomic.Int32
+ release chan struct{}
+}
+
+func (d restoreGenerationSnapshotDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) {
+ fi, err := d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
+ if volume == minioMetaMultipartBucket && ctx.Value(restoreGenerationSnapshotKey{}) != nil {
+ d.reads.Add(1)
+ select {
+ case <-d.release:
+ case <-ctx.Done():
+ return FileInfo{}, ctx.Err()
+ }
+ }
+ return fi, err
+}
+
+// Stale cleanup decides on a removal under the commit lock. A completion
+// that starts while cleanup reads an upload past the stale age waits for
+// it, and a cleanup that comes later skips the running commit: the upload,
+// an ordinary one, still commits with its contents.
+func TestCleanupStaleUploadsSkipsRunningCommit(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ mp, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true, MTime: UTCNow().Add(-48 * time.Hour)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ body := []byte("an ordinary upload past the stale age")
+ part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ disks := er.getDisks()
+ var reads, staged atomic.Int32
+ readRelease := make(chan struct{})
+ stageEntered, stageRelease := make(chan struct{}), make(chan struct{})
+ var readOnce, stageOnce sync.Once
+ defer readOnce.Do(func() { close(readRelease) })
+ defer stageOnce.Do(func() { close(stageRelease) })
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ stage := restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: true,
+ arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks))}
+ wrapped[i] = restoreGenerationSnapshotDisk{StorageAPI: stage, reads: &reads, release: readRelease}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ globalAPIConfig.mu.Lock()
+ prev := globalAPIConfig.staleUploadsExpiry
+ globalAPIConfig.staleUploadsExpiry = time.Nanosecond
+ globalAPIConfig.mu.Unlock()
+ defer func() {
+ globalAPIConfig.mu.Lock()
+ globalAPIConfig.staleUploadsExpiry = prev
+ globalAPIConfig.mu.Unlock()
+ }()
+ // Every drive's cleanup at once, each reading the upload on every drive.
+ ctx := context.WithValue(t.Context(), restoreGenerationSnapshotKey{}, true)
+ var cleanups sync.WaitGroup
+ for _, disk := range wrapped {
+ cleanups.Go(func() { er.cleanupStaleUploadsOnDisk(ctx, disk) })
+ }
+ cleanupDone := make(chan struct{})
+ go func() { cleanups.Wait(); close(cleanupDone) }()
+ // The completion starts once every cleanup that may decide holds its
+ // read of the upload from before the commit.
+ for deadline := time.Now().Add(2 * time.Second); reads.Load() < int32(len(disks)*len(disks)) && time.Now().Before(deadline); {
+ time.Sleep(10 * time.Millisecond)
+ }
+ completion := make(chan ObjectInfo, 1)
+ completionErr := make(chan error, 1)
+ go func() {
+ oi, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true})
+ completion <- oi
+ completionErr <- err
+ }()
+ select {
+ case <-stageEntered:
+ case <-time.After(3 * time.Second):
+ }
+ readOnce.Do(func() { close(readRelease) })
+ select {
+ case <-cleanupDone:
+ case <-time.After(10 * time.Second):
+ t.Fatal("stale cleanup never returned")
+ }
+ stageOnce.Do(func() { close(stageRelease) })
+ oi := <-completion
+ if err := <-completionErr; err != nil {
+ t.Fatalf("completion of an upload stale cleanup read: %v", err)
+ }
+ er.getDisks = func() []StorageAPI { return disks }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("completed contents: %q, %v", got, err)
+ }
+}
+
+// restoreGenerationRand draws UUIDs from crypto/rand for the whole test
+// binary, installed before any goroutine runs. While a pause is set, it holds
+// the UUID that stale cleanup draws in its deadline-bound worker, for the
+// trash path right before its rename, until the pause is released.
+var restoreGenerationRand = &restoreGenerationRandReader{}
+
+func init() { uuid.SetRand(restoreGenerationRand) }
+
+type restoreGenerationRandReader struct {
+ pause atomic.Pointer[restoreGenerationPause]
+}
+
+type restoreGenerationPause struct {
+ entered, release chan struct{}
+}
+
+func (r *restoreGenerationRandReader) Read(b []byte) (int, error) {
+ if p := r.pause.Load(); p != nil && restoreGenerationInCleanupRename() && r.pause.CompareAndSwap(p, nil) {
+ close(p.entered)
+ <-p.release
+ }
+ return rand.Read(b)
+}
+
+// restoreGenerationInCleanupRename reports whether the caller runs in stale
+// cleanup's deadline-bound worker.
+func restoreGenerationInCleanupRename() bool {
+ pcs := make([]uintptr, 32)
+ frames := runtime.CallersFrames(pcs[:runtime.Callers(3, pcs)])
+ cleanup, worker := false, false
+ for {
+ f, more := frames.Next()
+ cleanup = cleanup || strings.Contains(f.Function, "cleanupStaleUploadsOnDisk")
+ worker = worker || strings.Contains(f.Function, "ioutil.WithDeadline")
+ if !more {
+ return cleanup && worker
+ }
+ }
+}
+
+// Stale cleanup holds the commit lock until its rename ends, also when the
+// rename outlives the drive deadline and cleanup has moved on: a completion
+// does not stage while that rename can still remove its upload.
+func TestCleanupStaleUploadsHoldsLockThroughRename(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ mp, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true, MTime: UTCNow().Add(-48 * time.Hour)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ body := []byte("an upload whose stale cleanup outlives the drive deadline")
+ part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ disks := er.getDisks()
+ stored := restoreIsolationUploads(t, r, 0)
+ prev := globalDriveConfig.GetMaxTimeout()
+ // The shortest drive deadline the configuration takes.
+ globalDriveConfig.Update(drive.Config{MaxTimeout: 1100 * time.Millisecond})
+ defer globalDriveConfig.Update(drive.Config{MaxTimeout: prev})
+ pause := &restoreGenerationPause{entered: make(chan struct{}), release: make(chan struct{})}
+ var paused sync.Once
+ defer paused.Do(func() { close(pause.release) })
+ restoreGenerationRand.pause.Store(pause)
+ defer restoreGenerationRand.pause.CompareAndSwap(pause, nil)
+
+ cleanupDone := make(chan struct{})
+ go func() {
+ er.cleanupStaleUploadsOnDisk(t.Context(), disks[0])
+ close(cleanupDone)
+ }()
+ select {
+ case <-pause.entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("stale cleanup never reached its rename")
+ }
+ select {
+ case <-cleanupDone:
+ case <-time.After(5 * time.Second):
+ t.Fatal("stale cleanup did not move on after the drive deadline")
+ }
+ if left := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(left, stored) {
+ t.Fatalf("fixture: the held rename already ran: %v -> %v", stored, left)
+ }
+
+ var staged atomic.Int32
+ stageEntered, stageRelease := make(chan struct{}), make(chan struct{})
+ var stageOnce sync.Once
+ defer stageOnce.Do(func() { close(stageRelease) })
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ wrapped[i] = restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: true,
+ arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks))}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ completion := make(chan ObjectInfo, 1)
+ completionErr := make(chan error, 1)
+ go func() {
+ oi, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true})
+ completion <- oi
+ completionErr <- err
+ }()
+ select {
+ case <-stageEntered:
+ t.Error("a completion staged while stale cleanup's rename could still run")
+ case <-time.After(2 * time.Second):
+ }
+ paused.Do(func() { close(pause.release) })
+ select {
+ case <-stageEntered:
+ case <-time.After(10 * time.Second):
+ t.Fatal("completion never staged after the rename ended")
+ }
+ stageOnce.Do(func() { close(stageRelease) })
+ oi := <-completion
+ if err := <-completionErr; err != nil {
+ t.Fatalf("completion after the rename ended: %v", err)
+ }
+ er.getDisks = func() []StorageAPI { return disks }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("completed contents: %q, %v", got, err)
+ }
+}
diff --git a/cmd/restore-native-isolation_test.go b/cmd/restore-native-isolation_test.go
new file mode 100644
index 000000000..232eb4323
--- /dev/null
+++ b/cmd/restore-native-isolation_test.go
@@ -0,0 +1,1397 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "reflect"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/versioning"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/minio/mux"
+)
+
+// The private metadata of a native restore, as stored. Literal names keep
+// these regressions meaningful on a tree without the keys.
+const (
+ restoreIsolationAttemptKey = "x-minio-internal-restore-attempt"
+ restoreIsolationUploadKey = "x-minio-internal-restore-upload"
+)
+
+// restoreIsolationLayer hooks the object layer a POST restore handler uses.
+// afterRead runs once POST has read the version, before POST records its
+// status. restore, when set, replaces the background restore; every
+// background restore result is sent to done.
+type restoreIsolationLayer struct {
+ ObjectLayer
+ afterRead func()
+ once sync.Once
+ restore func(ctx context.Context, bucket, object string, opts ObjectOptions) error
+ done chan error
+}
+
+func newRestoreIsolationLayer(r *restoreTest) *restoreIsolationLayer {
+ return &restoreIsolationLayer{ObjectLayer: r.z, done: make(chan error, 4)}
+}
+
+func (l *restoreIsolationLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+ oi, err := l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+ if err == nil && l.afterRead != nil {
+ l.once.Do(l.afterRead)
+ }
+ return oi, err
+}
+
+func (l *restoreIsolationLayer) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ var err error
+ if l.restore != nil {
+ err = l.restore(ctx, bucket, object, opts)
+ } else {
+ err = l.ObjectLayer.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ l.done <- err
+ return err
+}
+
+// wait returns the result of the next background restore.
+func (l *restoreIsolationLayer) wait(t *testing.T) error {
+ t.Helper()
+ select {
+ case err := <-l.done:
+ return err
+ case <-time.After(10 * time.Second):
+ t.Fatal("background restore did not finish")
+ return nil
+ }
+}
+
+func restoreIsolationPOST(t *testing.T, r *restoreTest, layer ObjectLayer, versionID string) *httptest.ResponseRecorder {
+ t.Helper()
+ body := []byte(`1`)
+ path := "/" + r.bucket + "/" + r.object + "?restore"
+ if versionID != "" {
+ path += "&versionId=" + url.QueryEscape(versionID)
+ }
+ req, err := newTestSignedRequestV4(http.MethodPost, path, int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := req.ParseForm(); err != nil {
+ t.Fatal(err)
+ }
+ req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object})
+ rec := httptest.NewRecorder()
+ api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return layer }}
+ api.PostRestoreObjectHandler(rec, req)
+ return rec
+}
+
+// restoreIsolationSeed is a transitioned version without the ongoing marker
+// that nativeRestoreSeed leaves for direct restores: POST sets its own.
+func restoreIsolationSeed(t *testing.T, r *restoreTest, pool int, null, multipart bool) (ObjectInfo, []byte) {
+ t.Helper()
+ oi, body := nativeRestoreSeed(t, r, pool, null, multipart)
+ opts := r.opts(oi.VersionID)
+ opts.MTime = oi.ModTime
+ opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+ o.UserDefined[xhttp.AmzRestore] = ""
+ return ReplicateDecision{}, nil
+ }
+ if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil {
+ t.Fatal(err)
+ }
+ return oi, body
+}
+
+// restoreIsolationSettled waits until the version is no longer ongoing.
+func restoreIsolationSettled(t *testing.T, r *restoreTest, versionID string) ObjectInfo {
+ t.Helper()
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ oi, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(versionID))
+ if err == nil && !oi.RestoreOngoing {
+ return oi
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("version stays ongoing, refusing every later POST: %q, %v", oi.UserDefined[xhttp.AmzRestore], err)
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+}
+
+// restoreIsolationUploads returns the upload directories of the object on
+// each drive of a pool, as stored, whatever listings show.
+func restoreIsolationUploads(t *testing.T, r *restoreTest, pool int) map[string]int {
+ t.Helper()
+ er := r.z.serverPools[pool].getHashedSet(r.object)
+ found := make(map[string]int)
+ for _, disk := range er.getDisks() {
+ entries, err := disk.ListDir(t.Context(), r.bucket, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), -1)
+ if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
+ continue
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, entry := range entries {
+ found[strings.TrimSuffix(entry, SlashSeparator)]++
+ }
+ }
+ return found
+}
+
+// restoreIsolationUpload finds the single upload of the object in a pool on
+// disk, with its first part, and returns its public upload ID.
+func restoreIsolationUpload(t *testing.T, r *restoreTest, pool int) (string, ObjectPartInfo) {
+ t.Helper()
+ uploads := restoreIsolationUploads(t, r, pool)
+ if len(uploads) != 1 {
+ t.Fatalf("want one upload on disk, found %v", uploads)
+ }
+ var uploadUUID string
+ for uploadUUID = range uploads {
+ }
+ er := r.z.serverPools[pool].getHashedSet(r.object)
+ uploadPath := pathJoin(er.getMultipartSHADir(r.bucket, r.object), uploadUUID)
+ fi, err := er.getDisks()[0].ReadVersion(t.Context(), "", minioMetaMultipartBucket, uploadPath, "", ReadOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ parts, err := readParts(t.Context(), er.getDisks(), minioMetaMultipartBucket, []string{pathJoin(uploadPath, fi.DataDir, "part.1.meta")}, []int{1}, 1)
+ if err != nil || len(parts) != 1 || parts[0].Error != "" {
+ t.Fatalf("restore must have published its first part: %+v, %v", parts, err)
+ }
+ return base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)), parts[0]
+}
+
+// restoreIsolationGateTier holds a download after its first bytes and can
+// fail it once released.
+type restoreIsolationGateTier struct {
+ WarmBackend
+ first int
+ entered chan struct{}
+ release chan struct{}
+ failure error
+ enter sync.Once
+ open sync.Once
+}
+
+func newRestoreIsolationGateTier(first int, failure error) *restoreIsolationGateTier {
+ return &restoreIsolationGateTier{
+ WarmBackend: globalTierConfigMgr.drivercache["WARM"],
+ first: first, failure: failure,
+ entered: make(chan struct{}), release: make(chan struct{}),
+ }
+}
+
+func (b *restoreIsolationGateTier) unblock() { b.open.Do(func() { close(b.release) }) }
+
+func (b *restoreIsolationGateTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+ rd, err := b.WarmBackend.Get(ctx, object, version, opts)
+ if err != nil {
+ return nil, err
+ }
+ return &restoreIsolationGateReader{ReadCloser: rd, tier: b, ctx: ctx, remaining: b.first}, nil
+}
+
+type restoreIsolationGateReader struct {
+ io.ReadCloser
+ tier *restoreIsolationGateTier
+ ctx context.Context
+ remaining int
+}
+
+func (r *restoreIsolationGateReader) Read(p []byte) (int, error) {
+ if r.remaining > 0 {
+ if len(p) > r.remaining {
+ p = p[:r.remaining]
+ }
+ n, err := r.ReadCloser.Read(p)
+ r.remaining -= n
+ return n, err
+ }
+ r.tier.enter.Do(func() { close(r.tier.entered) })
+ select {
+ case <-r.tier.release:
+ case <-r.ctx.Done():
+ return 0, r.ctx.Err()
+ }
+ if r.tier.failure != nil {
+ return 0, r.tier.failure
+ }
+ return r.ReadCloser.Read(p)
+}
+
+func restoreIsolationEntered(t *testing.T, tier *restoreIsolationGateTier) {
+ t.Helper()
+ select {
+ case <-tier.entered:
+ case <-time.After(10 * time.Second):
+ t.Fatal("restore never reached its download gate")
+ }
+}
+
+// restoreIsolationFaultDisk fails metadata reads of the object while
+// failReads is set, and reports writes of its metadata as failed, after
+// they were made, while failWrites is set. Toggles keep the wrapper in
+// place, so no restore retry races a swap of the drives.
+type restoreIsolationFaultDisk struct {
+ StorageAPI
+ bucket, object string
+ failReads *atomic.Bool
+ failWrites *atomic.Bool
+}
+
+func (d restoreIsolationFaultDisk) ReadVersion(ctx context.Context, origvolume, volume, path, version string, opts ReadOptions) (FileInfo, error) {
+ if d.failReads.Load() && volume == d.bucket && path == d.object {
+ return FileInfo{}, errDiskNotFound
+ }
+ return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, version, opts)
+}
+
+func (d restoreIsolationFaultDisk) ReadXL(ctx context.Context, volume, path string, readData bool) (RawFileInfo, error) {
+ if d.failReads.Load() && volume == d.bucket && path == d.object {
+ return RawFileInfo{}, errDiskNotFound
+ }
+ return d.StorageAPI.ReadXL(ctx, volume, path, readData)
+}
+
+func (d restoreIsolationFaultDisk) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error {
+ err := d.StorageAPI.WriteMetadata(ctx, origvolume, volume, path, fi)
+ if err == nil && d.failWrites != nil && d.failWrites.Load() && volume == d.bucket && path == d.object {
+ return errFaultyDisk
+ }
+ return err
+}
+
+// installRestoreIsolationFaults wraps the drives of the object's set in a
+// pool for the rest of the test; failing marks the drives whose writes fail.
+func installRestoreIsolationFaults(r *restoreTest, pool int, failing int) (reads, writes *atomic.Bool) {
+ reads, writes = new(atomic.Bool), new(atomic.Bool)
+ er := r.z.serverPools[pool].getHashedSet(r.object)
+ disks := er.getDisks()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, disk := range disks {
+ d := restoreIsolationFaultDisk{StorageAPI: disk, bucket: r.bucket, object: r.object, failReads: reads}
+ if i < failing {
+ d.failWrites = writes
+ }
+ wrapped[i] = d
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ return reads, writes
+}
+
+// A public client must not replace a part of a native restore's upload,
+// before or after the restore validated it: the bytes would become the
+// COMPLIANCE-protected version under its original ETag.
+func TestRestoreUploadIsolatedFromPublicParts(t *testing.T) {
+ r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ enableBucketObjectLock(t, r.bucket)
+ previous := newObjectLayerFn()
+ t.Cleanup(func() { setObjectLayer(previous) })
+ router := initTestAPIEndPoints(r.z, []string{"PutObjectPart", "GetObject"})
+ oi, body := nativeRestoreSeed(t, r, 0, false, true)
+ if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, "retention", retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))); rec.Code != http.StatusOK {
+ t.Fatalf("lock: %d %s", rec.Code, rec.Body.String())
+ }
+ // Hold completion after it validated the parts, before it commits.
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ original := er.getDisks
+ gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})}
+ defer gate.unblock()
+ disks := original()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ wrapped[i] = restoreIsolationCompletionDisk{d, gate}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ defer func() { er.getDisks = original }()
+ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
+ defer cancel()
+ restored := make(chan error, 1)
+ go func() {
+ restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ }()
+ select {
+ case <-gate.entered:
+ case err := <-restored:
+ t.Fatalf("restore stopped before validation: %v", err)
+ case <-ctx.Done():
+ t.Fatal("no completion gate")
+ }
+ id, _ := restoreIsolationUpload(t, r, 0)
+ poison := bytes.Repeat([]byte("z"), 5<<20)
+ // The request may wait for the upload lock that completion holds.
+ uploaded := make(chan *httptest.ResponseRecorder, 1)
+ go func() {
+ uploaded <- attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodPut, "/"+r.bucket+"/"+r.object+"?uploadId="+url.QueryEscape(id)+"&partNumber=1", poison, nil)
+ }()
+ var rec *httptest.ResponseRecorder
+ select {
+ case rec = <-uploaded:
+ case <-time.After(500 * time.Millisecond):
+ }
+ gate.unblock()
+ if err := <-restored; err != nil {
+ t.Fatalf("restore: %v", err)
+ }
+ if rec == nil {
+ rec = <-uploaded
+ }
+ if rec.Code != http.StatusNotFound || !strings.Contains(rec.Body.String(), "NoSuchUpload") {
+ t.Errorf("public UploadPart on a native restore upload: %d %s, want NoSuchUpload", rec.Code, rec.Body.String())
+ }
+ got := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, "/"+r.bucket+"/"+r.object+"?versionId="+oi.VersionID, nil, nil)
+ if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), body) {
+ t.Errorf("COMPLIANCE version holds other bytes after restore: GET=%d poisoned=%v", got.Code, bytes.HasPrefix(got.Body.Bytes(), poison))
+ }
+}
+
+type restoreIsolationCompletionGate struct {
+ entered, release chan struct{}
+ enter, open sync.Once
+}
+
+func (g *restoreIsolationCompletionGate) unblock() { g.open.Do(func() { close(g.release) }) }
+
+type restoreIsolationCompletionDisk struct {
+ StorageAPI
+ gate *restoreIsolationCompletionGate
+}
+
+// Completion removes the part metadata it validated just before it renames
+// the upload into place.
+func (d restoreIsolationCompletionDisk) DeleteBulk(ctx context.Context, bucket string, paths ...string) error {
+ if bucket == minioMetaMultipartBucket && len(paths) == 2 && strings.HasSuffix(paths[0], ".meta") && strings.HasSuffix(paths[1], ".meta") {
+ d.gate.enter.Do(func() { close(d.gate.entered) })
+ select {
+ case <-d.gate.release:
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+ return d.StorageAPI.DeleteBulk(ctx, bucket, paths...)
+}
+
+// restoreIsolationValidationDisk holds a completion where it reads the parts
+// it validates: after its object and upload locks, before its commit lock.
+// Stale cleanup then decides on the upload itself, not on a running commit.
+type restoreIsolationValidationDisk struct {
+ StorageAPI
+ gate *restoreIsolationCompletionGate
+}
+
+func (d restoreIsolationValidationDisk) ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error) {
+ if bucket == minioMetaMultipartBucket {
+ d.gate.enter.Do(func() { close(d.gate.entered) })
+ select {
+ case <-d.gate.release:
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+ }
+ return d.StorageAPI.ReadParts(ctx, bucket, partMetaPaths...)
+}
+
+// Every public multipart entry point finds no native restore upload, and
+// none of them can change it: the restore still completes with its own data.
+func TestRestoreUploadHiddenFromPublicEntryPoints(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ oi, body := nativeRestoreSeed(t, r, 0, false, true)
+ tier := newRestoreIsolationGateTier(5<<20, nil)
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ defer tier.unblock()
+ ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second)
+ defer cancel()
+ restored := make(chan error, 1)
+ go func() {
+ restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ }()
+ restoreIsolationEntered(t, tier)
+ id, part := restoreIsolationUpload(t, r, 0)
+ stored := restoreIsolationUploads(t, r, 0)
+
+ for _, prefix := range []string{r.object, ""} {
+ uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, prefix, "", "", "", 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, u := range uploads.Uploads {
+ if u.UploadID == id {
+ t.Errorf("ListMultipartUploads(prefix=%q) shows the native restore upload", prefix)
+ }
+ }
+ }
+ poison := bytes.Repeat([]byte("z"), 5<<20)
+ noUpload := func(op string, err error) {
+ t.Helper()
+ if !errors.As(err, &InvalidUploadID{}) {
+ t.Errorf("%s on a native restore upload: %v, want NoSuchUpload", op, err)
+ }
+ }
+ // With one pool, part writes first wait for the object lock the
+ // restore holds; with more they reach the upload at once.
+ written := make(chan error, 2)
+ go func() {
+ _, err := r.z.PutObjectPart(ctx, r.bucket, r.object, id, 1, mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", ""), ObjectOptions{})
+ written <- err
+ }()
+ go func() {
+ src := ObjectInfo{PutObjReader: mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", "")}
+ _, err := r.z.CopyObjectPart(ctx, r.bucket, r.object, r.bucket, r.object, id, 1, 0, int64(len(poison)), src, ObjectOptions{}, ObjectOptions{})
+ written <- err
+ }()
+ var writes []error
+ if pools > 1 {
+ writes = append(writes, <-written, <-written)
+ }
+ _, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{})
+ noUpload("ListObjectParts", err)
+ _, err = r.z.GetMultipartInfo(t.Context(), r.bucket, r.object, id, ObjectOptions{})
+ noUpload("GetMultipartInfo", err)
+ err = r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{})
+ noUpload("AbortMultipartUpload", err)
+ if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) {
+ t.Errorf("public calls changed the restore upload on disk: %v -> %v", stored, after)
+ }
+ // Completion waits for the object lock the restore holds.
+ completed := make(chan error, 1)
+ go func() {
+ _, err := r.z.CompleteMultipartUpload(ctx, r.bucket, r.object, id, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, r.opts(oi.VersionID))
+ completed <- err
+ }()
+ tier.unblock()
+ if err := <-restored; err != nil {
+ t.Fatalf("restore after public calls: %v", err)
+ }
+ for len(writes) < 2 {
+ writes = append(writes, <-written)
+ }
+ for _, err := range writes {
+ noUpload("PutObjectPart/CopyObjectPart", err)
+ }
+ noUpload("CompleteMultipartUpload", <-completed)
+ _, got, err := r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restore contents after public calls: %v", err)
+ }
+ })
+ }
+}
+
+// POST records its status on a fresh read under the object lock, so an
+// Object Lock update it did not see survives.
+func TestRestorePOSTKeepsConcurrentProtection(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, kind := range []string{"legal-hold", "retention"} {
+ t.Run(fmt.Sprintf("pools=%d/%s", pools, kind), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ enableBucketObjectLock(t, r.bucket)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ initial := []byte(`OFF`)
+ updated := []byte(`ON`)
+ key := strings.ToLower(xhttp.AmzObjectLockLegalHold)
+ if kind == "retention" {
+ initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour))
+ updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))
+ key = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
+ }
+ if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK {
+ t.Fatalf("initial protection: %d %s", rec.Code, rec.Body.String())
+ }
+ var accepted ObjectInfo
+ layer := newRestoreIsolationLayer(r)
+ layer.afterRead = func() {
+ if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK {
+ t.Fatalf("update: %d %s", rec.Code, rec.Body.String())
+ }
+ var err error
+ if accepted, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code >= 400 {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if accepted.UserDefined[key] == "" || after.UserDefined[key] != accepted.UserDefined[key] {
+ t.Errorf("POST restore overwrote acknowledged Object Lock metadata: %q -> %q", accepted.UserDefined[key], after.UserDefined[key])
+ }
+ if !after.ModTime.Equal(oi.ModTime) || after.ETag != oi.ETag {
+ t.Error("POST restore changed the version identity")
+ }
+ })
+ }
+ }
+}
+
+// A restored copy removed between POST's read and its status write leads to
+// a new restore, never to a completed header without the restored data.
+func TestRestorePOSTRacesCacheExpiry(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi := r.put("restored", false, true)
+ r.restore(oi)
+ layer := newRestoreIsolationLayer(r)
+ layer.afterRead = func() {
+ if err := expireTransitionedObject(t.Context(), r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code >= 400 {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ select {
+ case err := <-layer.done:
+ if err != nil {
+ t.Fatalf("rehydration: %v", err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Error("POST renewed a removed restored copy instead of restoring it")
+ }
+ _, body, err := r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(body, r.bodies["restored"]) {
+ t.Errorf("version unreadable after POST raced cache expiry: %v", err)
+ }
+ })
+ }
+}
+
+// A restore queued with the versioning POST captured writes back the null
+// version it resolved, even after versioning was enabled in between.
+func TestRestoreKeepsCapturedNullIdentity(t *testing.T) {
+ enable := func(t *testing.T, r *restoreTest) {
+ meta, err := globalBucketMetadataSys.Get(r.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ meta.VersioningConfigXML = []byte(`Enabled`)
+ if err := meta.Save(t.Context(), r.z); err != nil {
+ t.Fatal(err)
+ }
+ globalBucketMetadataSys.Set(r.bucket, meta)
+ }
+ for _, mode := range []string{"direct", "post"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, "")
+ initConfigSubsystem(t.Context(), r.z)
+ var oi ObjectInfo
+ var body []byte
+ if mode == "direct" {
+ oi, body = nativeRestoreSeed(t, r, 0, true, false)
+ // These are the options that POST captured while versioning was off.
+ queued := nativeRestoreOptions(r, "")
+ enable(t, r)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, queued); err != nil {
+ t.Fatal(err)
+ }
+ } else {
+ oi, body = restoreIsolationSeed(t, r, 0, true, false)
+ release := make(chan struct{})
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ <-release
+ return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ enable(t, r)
+ close(release)
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ }
+ versions := nativeRestoreSnapshot(t, r)[0]
+ if len(versions) != 1 || versions[0].VersionID != "" {
+ t.Errorf("restore created a UUID alias of the queued null version: %d versions", len(versions))
+ }
+ _, got, err := r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("null original: %v", err)
+ }
+ })
+ }
+}
+
+// A restore that fails before it reads the version, on its lock wait or on
+// its first read, must not leave POST's ongoing marker for every later POST
+// to refuse: bounded retries clear it once the version is readable again.
+func TestRestoreRecoversStrandedMarker(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, stage := range []string{"lock-wait", "initial-read"} {
+ t.Run(fmt.Sprintf("pools=%d/%s", pools, stage), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+ var err error
+ if stage == "lock-wait" {
+ lk := r.z.NewNSLock(r.bucket, r.object)
+ if pools == 1 {
+ lk = r.z.serverPools[0].getHashedSet(r.object).NewNSLock(r.bucket, r.object)
+ }
+ lkctx, lerr := lk.GetLock(t.Context(), globalOperationTimeout)
+ if lerr != nil {
+ t.Fatal(lerr)
+ }
+ ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+ err = r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ cancel()
+ lk.Unlock(lkctx)
+ } else {
+ reads, _ := installRestoreIsolationFaults(r, 0, 0)
+ reads.Store(true)
+ err = r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ reads.Store(false)
+ }
+ if err == nil {
+ t.Fatal("restore succeeded without its lock or its version")
+ }
+ after := restoreIsolationSettled(t, r, oi.VersionID)
+ if !after.IsRemote() || !after.ModTime.Equal(oi.ModTime) {
+ t.Error("recovery changed more than the restore status")
+ }
+ })
+ }
+ }
+}
+
+// A later POST takes over a marker that no running restore holds, and the
+// superseded attempt then neither downloads nor clears the newer status,
+// also when both POSTs share a second, the precision of the request date.
+func TestRestoreSupersededAttempt(t *testing.T) {
+ for _, mode := range []string{"newer-completes", "newer-fails"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ release := make(chan struct{})
+ var releaseOnce sync.Once
+ unblock := func() { releaseOnce.Do(func() { close(release) }) }
+ t.Cleanup(unblock)
+ older := newRestoreIsolationLayer(r)
+ older.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ <-release
+ return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ newer := newRestoreIsolationLayer(r)
+ time.Sleep(time.Until(time.Now().Truncate(time.Second).Add(time.Second + 10*time.Millisecond)))
+ if rec := restoreIsolationPOST(t, r, older, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("first POST: %d %s", rec.Code, rec.Body.String())
+ }
+ first := r.fileInfo(oi.VersionID).Metadata[xhttp.AmzRestoreRequestDate]
+ if mode == "newer-fails" {
+ globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, errors.New("injected remote failure")}
+ }
+ if rec := restoreIsolationPOST(t, r, newer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST over a marker no restore holds: %d %s", rec.Code, rec.Body.String())
+ }
+ second := r.fileInfo(oi.VersionID).Metadata[xhttp.AmzRestoreRequestDate]
+ t.Logf("request dates %q %q, same second: %v", first, second, first == second)
+ newerErr := newer.wait(t)
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ before := r.fileInfo(oi.VersionID)
+ gets := tier.gets.Load()
+ unblock()
+ if err := older.wait(t); err == nil {
+ t.Error("superseded restore reported success")
+ }
+ if tier.gets.Load() != gets {
+ t.Error("superseded restore downloaded the version again")
+ }
+ after := r.fileInfo(oi.VersionID)
+ if !reflect.DeepEqual(before.Metadata, after.Metadata) || before.DataDir != after.DataDir {
+ t.Errorf("superseded restore changed the version: %q -> %q", before.Metadata[xhttp.AmzRestore], after.Metadata[xhttp.AmzRestore])
+ }
+ info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil || info.RestoreOngoing {
+ t.Fatalf("version left ongoing: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+ }
+ if mode == "newer-completes" {
+ if newerErr != nil || info.IsRemote() {
+ t.Fatalf("newer restore: %v remote=%v", newerErr, info.IsRemote())
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) || tier.gets.Load() != gets {
+ t.Errorf("restored contents: %v", err)
+ }
+ return
+ }
+ if newerErr == nil || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" {
+ t.Fatalf("failed newer restore is not retryable: %v %q", newerErr, info.UserDefined[xhttp.AmzRestore])
+ }
+ if rec := restoreIsolationPOST(t, r, newer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("retry POST: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := newer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Errorf("retried restore contents: %v", err)
+ }
+ })
+ }
+}
+
+// A restart loses the restore that POST started; a POST to the restarted
+// server takes the stranded marker over and restores the version.
+func TestRestoreRestartTakeover(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ lost := newRestoreIsolationLayer(r)
+ lost.restore = func(context.Context, string, string, ObjectOptions) error { return nil }
+ if rec := restoreIsolationPOST(t, r, lost, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST before restart: %d %s", rec.Code, rec.Body.String())
+ }
+ lost.wait(t)
+ if info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err != nil || !info.RestoreOngoing {
+ t.Fatalf("fixture: %q %v", info.UserDefined[xhttp.AmzRestore], err)
+ }
+
+ endpoints := make(EndpointServerPools, len(r.z.serverPools))
+ for i, pool := range r.z.serverPools {
+ endpoints[i] = pool.endpoints
+ }
+ obj, err := newErasureServerPools(t.Context(), endpoints)
+ if err != nil {
+ t.Fatal(err)
+ }
+ restarted := obj.(*erasureServerPools)
+ t.Cleanup(func() { restarted.Shutdown(context.Background()) })
+ r.z = restarted
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST after restart: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ info := restoreIsolationSettled(t, r, oi.VersionID)
+ if info.IsRemote() || info.UserDefined[restoreIsolationAttemptKey] != "" {
+ t.Errorf("restore after restart incomplete: remote=%v attempt=%q", info.IsRemote(), info.UserDefined[restoreIsolationAttemptKey])
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Errorf("contents after restart: %v", err)
+ }
+}
+
+// A null version replaced while its restore was queued is another version:
+// the restore must not rehydrate it, even if it is transitioned as well.
+func TestRestoreReplacementNullObject(t *testing.T) {
+ for _, state := range []versioning.State{"", versioning.Suspended} {
+ t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, state)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := restoreIsolationSeed(t, r, 0, true, false)
+ release := make(chan struct{})
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ <-release
+ return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code != http.StatusOK {
+ close(release)
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ replacement := []byte("replacement null object")
+ repl, err := r.z.serverPools[0].PutObject(t.Context(), r.bucket, r.object,
+ mustGetPutObjReader(t, bytes.NewReader(replacement), int64(len(replacement)), "", ""),
+ ObjectOptions{VersionSuspended: state == versioning.Suspended})
+ if err != nil {
+ close(release)
+ t.Fatal(err)
+ }
+ topts := r.opts(nullVersionID)
+ topts.MTime = repl.ModTime
+ topts.Transition = TransitionOptions{Tier: "WARM", ETag: repl.ETag}
+ if err := r.z.serverPools[0].TransitionObject(t.Context(), r.bucket, r.object, topts); err != nil {
+ close(release)
+ t.Fatal(err)
+ }
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ before := r.fileInfo(nullVersionID)
+ gets := tier.gets.Load()
+ close(release)
+ layer.wait(t)
+ after := r.fileInfo(nullVersionID)
+ if tier.gets.Load() != gets || after.Metadata[xhttp.AmzRestore] != "" || !reflect.DeepEqual(before.Metadata, after.Metadata) {
+ t.Errorf("queued restore rehydrated the replacement null object: %q", after.Metadata[xhttp.AmzRestore])
+ }
+ if after.ModTime.Equal(oi.ModTime) || !after.ModTime.Equal(repl.ModTime) {
+ t.Fatal("fixture did not replace the null version")
+ }
+ if _, got, err := r.read(nullVersionID); err != nil || !bytes.Equal(got, replacement) {
+ t.Errorf("replacement contents: %v", err)
+ }
+ })
+ }
+}
+
+// POST's own status write can fail after its ongoing marker reached the
+// drives; the marker must not stay for every later POST to refuse.
+func TestRestorePOSTStatusWritePartialFailure(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ _, writes := installRestoreIsolationFaults(r, 0, 2)
+ layer := newRestoreIsolationLayer(r)
+ writes.Store(true)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ writes.Store(false)
+ if rec.Code < 400 {
+ t.Fatalf("POST acknowledged a failed status write: %d %s", rec.Code, rec.Body.String())
+ }
+ if info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err == nil && !info.RestoreOngoing {
+ t.Log("the failed status write left no marker in quorum")
+ }
+ restoreIsolationSettled(t, r, oi.VersionID)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST after a failed status write: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Errorf("restore after a failed status write: %v", err)
+ }
+}
+
+// Duplicate copies of a version in two pools can differ in whether they
+// hold a restored copy. A completed restore header must never reach a copy
+// without one; POST renews or restores only the copy that restores serve.
+func TestRestoreDuplicatePoolCacheStates(t *testing.T) {
+ for _, mode := range []string{"suspended-owner", "unrestored-first", "restored-first"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ // The duplicate is the same stored version, metadata only.
+ src := r.z.serverPools[0].getHashedSet(r.object).getDisks()
+ dst := r.z.serverPools[1].getHashedSet(r.object).getDisks()
+ for i := range src {
+ raw, err := src[i].ReadAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := dst[i].WriteAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile), raw); err != nil {
+ t.Fatal(err)
+ }
+ }
+ owner := 0
+ if mode == "unrestored-first" {
+ owner = 1
+ }
+ if err := r.z.serverPools[owner].RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ if mode == "suspended-owner" {
+ r.z.poolMetaMutex.Lock()
+ r.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+ r.z.poolMetaMutex.Unlock()
+ t.Cleanup(func() {
+ r.z.poolMetaMutex.Lock()
+ r.z.poolMeta.Pools[0].Decommission = nil
+ r.z.poolMetaMutex.Unlock()
+ })
+ }
+ before := nativeRestoreSnapshot(t, r)
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ if mode != "restored-first" {
+ select {
+ case err := <-layer.done:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Error("POST did not restore the copy without restored data")
+ }
+ }
+ after := nativeRestoreSnapshot(t, r)
+ for pool := range r.z.serverPools {
+ gr, err := r.z.serverPools[pool].GetObjectNInfo(t.Context(), r.bucket, r.object, nil, nil, ObjectOptions{VersionID: oi.VersionID})
+ if err != nil {
+ t.Errorf("pool %d copy unreadable, header %q: %v", pool, after[pool][0].Metadata[xhttp.AmzRestore], err)
+ continue
+ }
+ got, err := io.ReadAll(gr)
+ gr.Close()
+ if err != nil || !bytes.Equal(got, body) {
+ t.Errorf("pool %d copy contents: %v", pool, err)
+ }
+ }
+ // The copy that is neither renewed nor restored stays as it was.
+ untouched := 1
+ if mode == "suspended-owner" {
+ untouched = 0
+ }
+ if !reflect.DeepEqual(before[untouched], after[untouched]) {
+ t.Errorf("POST changed the pool %d copy: %q -> %q", untouched, before[untouched][0].Metadata[xhttp.AmzRestore], after[untouched][0].Metadata[xhttp.AmzRestore])
+ }
+ })
+ }
+}
+
+// Stale-upload cleanup ages an upload by when it started, not by the
+// version's ModTime, keeps the upload of a running restore beyond the stale
+// age, and still removes ordinary stale uploads and those of ended attempts.
+func TestRestoreUploadOutlivesStaleAge(t *testing.T) {
+ cleanup := func(t *testing.T, r *restoreTest) {
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ for _, disk := range er.getDisks() {
+ er.cleanupStaleUploadsOnDisk(t.Context(), disk)
+ }
+ }
+ shortExpiry := func(t *testing.T) {
+ globalAPIConfig.mu.Lock()
+ prev := globalAPIConfig.staleUploadsExpiry
+ globalAPIConfig.staleUploadsExpiry = time.Nanosecond
+ globalAPIConfig.mu.Unlock()
+ t.Cleanup(func() {
+ globalAPIConfig.mu.Lock()
+ globalAPIConfig.staleUploadsExpiry = prev
+ globalAPIConfig.mu.Unlock()
+ })
+ }
+ for _, mode := range []string{"old-version", "past-stale-age", "public-abort", "ordinary-stale", "unowned-marker"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ if mode == "ordinary-stale" {
+ // Control: an abandoned client upload, old by its upload ID.
+ if _, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{MTime: UTCNow().Add(-72 * time.Hour)}); err != nil {
+ t.Fatal(err)
+ }
+ cleanup(t, r)
+ if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+ t.Errorf("ordinary stale upload survived cleanup: %v", uploads)
+ }
+ return
+ }
+ var oi ObjectInfo
+ var body []byte
+ if mode == "past-stale-age" {
+ oi, body = restoreIsolationSeed(t, r, 0, false, true)
+ } else {
+ oi, body = nativeRestoreSeed(t, r, 0, false, true)
+ }
+ old := r.opts(oi.VersionID)
+ old.MTime = UTCNow().Add(-72 * time.Hour)
+ old.EvalMetadataFn = func(*ObjectInfo, error) (ReplicateDecision, error) { return ReplicateDecision{}, nil }
+ if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, old); err != nil {
+ t.Fatal(err)
+ }
+ // The cleanup with a short stale age also removes in-flight
+ // temporary parts; hold such restores in completion instead,
+ // before their commit.
+ gated := mode == "past-stale-age" || mode == "unowned-marker"
+ tier := newRestoreIsolationGateTier(5<<20, nil)
+ if !gated {
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ }
+ defer tier.unblock()
+ gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})}
+ defer gate.unblock()
+ if gated {
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ original := er.getDisks
+ disks := original()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ wrapped[i] = restoreIsolationValidationDisk{d, gate}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ defer func() { er.getDisks = original }()
+ }
+ result := make(chan error, 1)
+ layer := newRestoreIsolationLayer(r)
+ if mode == "past-stale-age" {
+ // A POST publishes the attempt that a running restore owns.
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ } else {
+ go func() {
+ result <- r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ }()
+ }
+ if gated {
+ select {
+ case <-gate.entered:
+ case <-time.After(10 * time.Second):
+ t.Fatal("restore never reached completion")
+ }
+ } else {
+ restoreIsolationEntered(t, tier)
+ }
+ switch mode {
+ case "past-stale-age", "unowned-marker":
+ shortExpiry(t)
+ cleanup(t, r)
+ kept := len(restoreIsolationUploads(t, r, 0)) == 1
+ if mode == "unowned-marker" && kept {
+ // The version is ongoing, but not for this direct restore's
+ // upload: an ongoing marker alone does not keep an upload.
+ t.Error("stale cleanup kept an upload no attempt owns")
+ }
+ if mode == "past-stale-age" && !kept {
+ t.Error("stale cleanup removed the upload of a running restore")
+ }
+ case "old-version":
+ cleanup(t, r)
+ case "public-abort":
+ id, _ := restoreIsolationUpload(t, r, 0)
+ if err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) {
+ t.Errorf("public abort of a native restore upload: %v, want NoSuchUpload", err)
+ }
+ }
+ tier.unblock()
+ gate.unblock()
+ var err error
+ if mode == "past-stale-age" {
+ err = layer.wait(t)
+ } else {
+ err = <-result
+ }
+ if mode == "unowned-marker" {
+ if err == nil {
+ t.Error("restore completed without its upload")
+ }
+ return
+ }
+ if err != nil {
+ t.Fatalf("restore lost its upload: %v", err)
+ }
+ _, got, err := r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restore contents: %v", err)
+ }
+ })
+ }
+}
+
+// A failed multipart restore removes its own upload from the drives, also
+// when a newer attempt owns the version by then, and leaves that newer
+// attempt's marker alone.
+func TestRestoreFailureAbortsUpload(t *testing.T) {
+ for _, mode := range []string{"download-failure", "superseded-mid-download"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ failure := errors.New("injected remote failure")
+ if mode == "download-failure" {
+ oi, _ := nativeRestoreSeed(t, r, 0, false, true)
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, failure}
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); !errors.Is(err, failure) {
+ t.Fatalf("restore error=%v, want %v", err, failure)
+ }
+ if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+ t.Errorf("failed native restore left its upload on disk: %v", uploads)
+ }
+ return
+ }
+ oi, _ := restoreIsolationSeed(t, r, 0, false, true)
+ tier := newRestoreIsolationGateTier(5<<20, failure)
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ defer tier.unblock()
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ restoreIsolationEntered(t, tier)
+ if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 1 {
+ t.Fatalf("restore upload on disk: %v", uploads)
+ }
+ // Model a newer generation that owns the version by now, such as
+ // one that took over after this restore lost its lock.
+ newer := r.opts(oi.VersionID)
+ newer.MTime = oi.ModTime
+ newer.NoLock = true
+ newer.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+ o.UserDefined[restoreIsolationAttemptKey] = "newer-generation"
+ return ReplicateDecision{}, nil
+ }
+ if _, err := r.z.serverPools[0].getHashedSet(r.object).PutObjectMetadata(t.Context(), r.bucket, r.object, newer); err != nil {
+ t.Fatal(err)
+ }
+ tier.unblock()
+ if err := layer.wait(t); !errors.Is(err, failure) {
+ t.Fatalf("restore error=%v, want %v", err, failure)
+ }
+ if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+ t.Errorf("superseded restore left its upload on disk: %v", uploads)
+ }
+ after := r.fileInfo(oi.VersionID)
+ if !strings.Contains(after.Metadata[xhttp.AmzRestore], "true") || after.Metadata[restoreIsolationAttemptKey] != "newer-generation" {
+ t.Errorf("failed restore cleared a newer attempt's marker: %q %q", after.Metadata[xhttp.AmzRestore], after.Metadata[restoreIsolationAttemptKey])
+ }
+ })
+ }
+}
+
+// A restored copy expires only by its own reason, on a fresh read: its
+// restore deadline passed and nothing else. A renewed deadline survives
+// another rule falling due and an expiry checked across all pools.
+func TestExpireRestoredNeedsOwnReason(t *testing.T) {
+ for _, mode := range []string{"other-expiry-due", "checked-all-pools", "ongoing", "renewed", "expired"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi := r.put("restored", false, true)
+ r.restore(oi)
+ before := r.fileInfo(oi.VersionID)
+ if mode == "other-expiry-due" {
+ meta, err := globalBucketMetadataSys.Get(r.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ meta.LifecycleConfigXML = []byte(`deleteEnabled2020-01-01T00:00:00Z`)
+ if err := meta.Save(t.Context(), r.z); err != nil {
+ t.Fatal(err)
+ }
+ globalBucketMetadataSys.Set(r.bucket, meta)
+ }
+ header := completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()
+ switch mode {
+ case "ongoing":
+ header = ongoingRestoreObj().String()
+ case "expired":
+ header = ""
+ }
+ if header != "" {
+ opts := r.opts(oi.VersionID)
+ opts.MTime = oi.ModTime
+ opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+ o.UserDefined[xhttp.AmzRestore] = header
+ return ReplicateDecision{}, nil
+ }
+ if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil {
+ t.Fatal(err)
+ }
+ }
+ var err error
+ if mode == "checked-all-pools" {
+ opts := r.opts(oi.VersionID)
+ opts.Expiration = ExpirationOptions{Expire: true, CheckedAllPools: true}
+ opts.Transition.ExpireRestored = true
+ _, err = r.z.serverPools[0].getHashedSet(r.object).DeleteObject(t.Context(), r.bucket, r.object, opts)
+ } else {
+ err = expireTransitionedObject(t.Context(), r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+ }
+ after := r.fileInfo(oi.VersionID)
+ if mode == "expired" {
+ // Control: an elapsed deadline is the reason itself.
+ if err != nil || after.Metadata[xhttp.AmzRestore] != "" || r.dataDirs(before.DataDir) != 0 {
+ t.Errorf("expired restored copy not removed: %v", err)
+ }
+ return
+ }
+ t.Logf("cleanup=%v header=%q", err, after.Metadata[xhttp.AmzRestore])
+ if err == nil || after.Metadata[xhttp.AmzRestore] != header || r.dataDirs(before.DataDir) != len(r.z.serverPools[0].getHashedSet(r.object).getDisks()) {
+ t.Error("cache cleanup removed a restored copy whose own deadline has not passed")
+ }
+ })
+ }
+}
+
+// Caller metadata cannot make an upload a native restore upload: the marker
+// is dropped, the upload stays an ordinary one and nothing commits it.
+func TestRestoreUploadMarkerSpoofing(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, true)
+ forged := fmt.Sprintf(`{"id":%q,"v":%q,"e":%q,"s":%d,"m":%q}`, mustGetUUID(), oi.VersionID, oi.ETag, oi.Size, oi.ModTime.UTC().Format(time.RFC3339Nano))
+ for _, target := range []string{"existing-version", "new-object"} {
+ t.Run(target, func(t *testing.T) {
+ opts := ObjectOptions{Versioned: true, UserDefined: map[string]string{restoreIsolationUploadKey: forged}}
+ object := r.object
+ if target == "existing-version" {
+ opts.VersionID = oi.VersionID
+ opts.MTime = oi.ModTime
+ opts.PreserveETag = oi.ETag
+ } else {
+ object = "spoofed-" + r.object
+ }
+ upload, err := r.z.NewMultipartUpload(t.Context(), r.bucket, object, opts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, object, "", "", "", 100)
+ if err != nil || len(uploads.Uploads) != 1 || uploads.Uploads[0].UploadID != upload.UploadID {
+ t.Fatalf("a caller's upload must stay listed: %+v, %v", uploads.Uploads, err)
+ }
+ data := []byte("spoofed part")
+ part, err := r.z.PutObjectPart(t.Context(), r.bucket, object, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ before := nativeRestoreSnapshot(t, r)
+ done, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, object, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true, MTime: opts.MTime})
+ if target == "existing-version" {
+ if !errors.Is(err, errAuthentication) {
+ t.Errorf("spoofed upload overwrote the transitioned version: %v", err)
+ }
+ if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
+ t.Error("rejected spoofed completion changed the version")
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ info, err := r.z.GetObjectInfo(t.Context(), r.bucket, object, ObjectOptions{VersionID: done.VersionID})
+ if err != nil || info.UserDefined[restoreIsolationUploadKey] != "" {
+ t.Errorf("caller metadata committed a restore upload marker: %q, %v", info.UserDefined[restoreIsolationUploadKey], err)
+ }
+ })
+ }
+}
+
+// Controls: an ordinary POST restore, a renewal, and a client's multipart
+// upload of the same key while a restore runs.
+func TestRestorePOSTControls(t *testing.T) {
+ t.Run("restore", func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil || info.IsRemote() || info.RestoreOngoing || info.UserDefined[restoreIsolationAttemptKey] != "" {
+ t.Fatalf("restored status: %q %q, %v", info.UserDefined[xhttp.AmzRestore], info.UserDefined[restoreIsolationAttemptKey], err)
+ }
+ if !info.ModTime.Equal(oi.ModTime) || info.ETag != oi.ETag || info.UserDefined["x-amz-meta-preserve"] != "keep-me" {
+ t.Error("restore changed the version identity or metadata")
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Errorf("restored contents: %v", err)
+ }
+ if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+ t.Errorf("completed restore left its upload: %v", uploads)
+ }
+ })
+ t.Run("renewal", func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := nativeRestoreSeed(t, r, 0, false, false)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ before := r.fileInfo(oi.VersionID)
+ gets := tier.gets.Load()
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ select {
+ case err := <-layer.done:
+ t.Fatalf("renewal restored again: %v", err)
+ case <-time.After(200 * time.Millisecond):
+ }
+ after := r.fileInfo(oi.VersionID)
+ if after.DataDir != before.DataDir || !after.ModTime.Equal(before.ModTime) || after.Metadata[xhttp.AmzRestore] == "" {
+ t.Error("renewal changed the restored copy")
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) || tier.gets.Load() != gets {
+ t.Errorf("renewed copy not served locally: %v", err)
+ }
+ })
+ t.Run("public-multipart-same-key", func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ tier := newRestoreIsolationGateTier(5<<20, nil)
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ defer tier.unblock()
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ restoreIsolationEntered(t, tier)
+ upload, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 100)
+ listed := false
+ for _, u := range uploads.Uploads {
+ listed = listed || u.UploadID == upload.UploadID
+ }
+ if err != nil || !listed {
+ t.Fatalf("client upload not listed: %+v, %v", uploads.Uploads, err)
+ }
+ tier.unblock()
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ // Single-pool part writes wait for the object lock of a running restore.
+ data := []byte("client data")
+ part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ parts, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, upload.UploadID, 0, 10, ObjectOptions{})
+ if err != nil || len(parts.Parts) != 1 {
+ t.Fatalf("client parts: %+v, %v", parts, err)
+ }
+ newer, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, got, err := r.read(newer.VersionID); err != nil || !bytes.Equal(got, data) {
+ t.Errorf("client version: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Errorf("restored version: %v", err)
+ }
+ })
+}
diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go
index 884b88d24..44676ac7c 100644
--- a/cmd/restore-native_test.go
+++ b/cmd/restore-native_test.go
@@ -375,8 +375,10 @@ func TestNativeRestorePOSTVersionIdentity(t *testing.T) {
// A restore request is internal context, not an exception for normal S3
// multipart overwrites or for a version that has never completed transition.
+// Only the upload of a native restore attempt, whose marker names the very
+// version being completed, may write that version back.
func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
- for _, mode := range []string{"matching-restore", "ordinary-complete", "not-tiered", "changed-time", "changed-size", "changed-etag"} {
+ for _, mode := range []string{"matching-restore", "ordinary-complete", "restore-without-attempt", "other-attempt", "not-tiered", "changed-time", "changed-size", "changed-etag"} {
t.Run(mode, func(t *testing.T) {
r := newRestoreSafetyTest(t, 1, versioning.Enabled)
oi, body := nativeRestoreSeed(t, r, 0, false, true)
@@ -398,8 +400,9 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- opts := putRestoreOpts(r.bucket, r.object, &RestoreObjectRequest{Days: 1}, current)
+ opts := putRestoreOpts(&RestoreObjectRequest{Days: 1}, current, r.opts(oi.VersionID))
opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+ opts.PreserveETag = current.ETag
if mode == "ordinary-complete" {
opts.Transition.RestoreRequest = nil
}
@@ -408,20 +411,26 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
}
// Hold the ETag constant for the size case, so only size can reject it.
if mode == "changed-size" {
- opts.UserDefined["etag"] = current.ETag
body = append(body, 'x')
}
if mode == "changed-etag" {
- opts.UserDefined["etag"] = "different-etag"
+ opts.PreserveETag = "different-etag"
}
- upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, opts)
+ // The attempt claims what its upload claims; only the matching
+ // mode claims exactly the stored version.
+ if mode != "ordinary-complete" && mode != "restore-without-attempt" {
+ opts.Transition.attempt = &restoreAttempt{ID: mustGetUUID(), VersionID: oi.VersionID, ETag: opts.PreserveETag, Size: current.Size, ModTime: opts.MTime}
+ }
+ uopts := opts
+ uopts.MTime = time.Time{}
+ upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, uopts)
if err != nil {
t.Fatal(err)
}
var complete []CompletePart
for i, data := range [][]byte{body[:5<<20], body[5<<20:]} {
part, err := pool.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, i+1,
- mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
+ mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Transition: opts.Transition})
if err != nil {
t.Fatal(err)
}
@@ -430,6 +439,11 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
if mode != "changed-size" && getCompleteMultipartMD5(complete) != current.ETag {
t.Fatal("fixture multipart ETag does not match original")
}
+ if mode == "other-attempt" {
+ other := *opts.Transition.attempt
+ other.ID = mustGetUUID()
+ opts.Transition.attempt = &other
+ }
before := nativeRestoreSnapshot(t, r)
_, err = pool.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, complete, opts)
if mode == "matching-restore" {
@@ -440,9 +454,16 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
if err != nil || !bytes.Equal(got, body) {
t.Fatalf("matching restore contents: %v", err)
}
+ if after := r.fileInfo(oi.VersionID); after.Metadata[restoreUploadKey] != "" || after.Metadata[restoreAttemptKey] != "" {
+ t.Error("restore committed its private upload or attempt marker")
+ }
return
}
- if !errors.Is(err, errAuthentication) {
+ if mode == "other-attempt" {
+ if !errors.As(err, &InvalidUploadID{}) {
+ t.Fatalf("another attempt completed a private upload: %v", err)
+ }
+ } else if !errors.Is(err, errAuthentication) {
t.Fatalf("existing version overwrite: %v", err)
}
if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
@@ -721,18 +742,10 @@ func TestNativeRestoreConcurrentCompletion(t *testing.T) {
case <-ctx.Done():
t.Fatal("restore never started")
}
- // A client's exact-prefix listing can discover the internal upload. It can
- // try to complete the first part while native restore owns the object.
- uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10)
- if err != nil || len(uploads.Uploads) != 1 {
- t.Fatalf("restore upload discovery: %+v, %v", uploads, err)
- }
- id := uploads.Uploads[0].UploadID
- parts, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{})
- if err != nil || len(parts.Parts) != 1 {
- t.Fatalf("restore must have published its first part: %+v, %v", parts, err)
- }
- part := parts.Parts[0]
+ // A client that learns the internal upload ID, which listings no
+ // longer show, can try to complete the first part while native
+ // restore owns the object.
+ id, part := restoreIsolationUpload(t, r, 0)
completed := make(chan struct{})
var completeErr error
go func() {
@@ -942,17 +955,22 @@ func TestNativeRestoreCompletionReadSafety(t *testing.T) {
}
}
+// nativeRestoreStatusHook runs afterRead once POST has read the version and
+// before POST records its status on it.
type nativeRestoreStatusHook struct {
ObjectLayer
- beforeCopy func()
- copyCalled bool
+ afterRead func()
+ readCalled bool
backgroundCalled atomic.Bool
}
-func (h *nativeRestoreStatusHook) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, info ObjectInfo, srcOpts, dstOpts ObjectOptions) (ObjectInfo, error) {
- h.copyCalled = true
- h.beforeCopy()
- return h.ObjectLayer.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, info, srcOpts, dstOpts)
+func (h *nativeRestoreStatusHook) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+ oi, err := h.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+ if err == nil && !h.readCalled {
+ h.readCalled = true
+ h.afterRead()
+ }
+ return oi, err
}
func (h *nativeRestoreStatusHook) RestoreTransitionedObject(context.Context, string, string, ObjectOptions) error {
@@ -985,7 +1003,7 @@ func TestNativeRestorePOSTReadSafety(t *testing.T) {
}
before = restoreBoundarySnapshot(t, r)
}
- hook := &nativeRestoreStatusHook{ObjectLayer: r.z, beforeCopy: mutate}
+ hook := &nativeRestoreStatusHook{ObjectLayer: r.z, afterRead: mutate}
if stage == "initial" {
mutate()
}
@@ -1004,7 +1022,7 @@ func TestNativeRestorePOSTReadSafety(t *testing.T) {
if rec.Code < 400 {
t.Fatalf("POST acknowledged failed read: %d %s", rec.Code, rec.Body.String())
}
- if hook.copyCalled != (stage == "status-copy") || hook.backgroundCalled.Load() {
+ if hook.readCalled != (stage == "status-copy") || hook.backgroundCalled.Load() {
t.Fatal("request followed the wrong failure path")
}
if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
diff --git a/internal/grid/handlers.go b/internal/grid/handlers.go
index 1d20fa9b9..0f2c39d7e 100644
--- a/internal/grid/handlers.go
+++ b/internal/grid/handlers.go
@@ -116,6 +116,7 @@ const (
HandlerRenamePart
HandlerClearUploadID
HandlerCheckParts3
+ HandlerRestoreCapability
// Add more above here ^^^
// If all handlers are used, the type of Handler can be changed.
@@ -200,6 +201,7 @@ var handlerPrefixes = [handlerLast]string{
HandlerCheckParts3: storagePrefix,
HandlerRenamePart: storagePrefix,
HandlerClearUploadID: peerPrefix,
+ HandlerRestoreCapability: peerPrefix,
}
const (
diff --git a/internal/grid/handlers_string.go b/internal/grid/handlers_string.go
index d6c6b4c38..aea3f45fb 100644
--- a/internal/grid/handlers_string.go
+++ b/internal/grid/handlers_string.go
@@ -86,14 +86,15 @@ func _() {
_ = x[HandlerRenamePart-75]
_ = x[HandlerClearUploadID-76]
_ = x[HandlerCheckParts3-77]
- _ = x[handlerTest-78]
- _ = x[handlerTest2-79]
- _ = x[handlerLast-80]
+ _ = x[HandlerRestoreCapability-78]
+ _ = x[handlerTest-79]
+ _ = x[handlerTest2-80]
+ _ = x[handlerLast-81]
}
-const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3handlerTesthandlerTest2handlerLast"
+const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3RestoreCapabilityhandlerTesthandlerTest2handlerLast"
-var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 971, 983, 994}
+var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 977, 988, 1000, 1011}
func (i HandlerID) String() string {
idx := int(i) - 0