From 7f38bf6282c11d6ad272313b7b1dff1d2ed81280 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 06:47:38 +0800 Subject: [PATCH] fix(restore): give each native restore an owner and a private upload A native restore (POST ?restore, issue #281) marks the version ongoing, and a background worker rehydrates it from the tier. The worker, its upload and every status write could be reached or confused by other callers. A public client could write into or complete the restore upload. POST copied a whole metadata snapshot over newer protection changes. An old or failed attempt could act on a newer one. Restore, expiry and heal decided on whichever drive's metadata they read. Each POST that starts a rehydration now writes an attempt ID into the version's internal metadata, with the identity it resolved: version ID, ETag, size and ModTime. The worker, its upload, its part writes and its completion carry the attempt. A worker whose attempt no longer owns the version stops without writing, completing or resetting. Every write uses the versioning state and explicit version ID captured by POST, so a null target stays null. The restore upload carries an internal marker, decided over all drives. Public multipart calls answer NoSuchUpload and listings hide it. An unknown marker fails closed. A completion holds a commit lock from staging its commit in the upload until the commit ends. Stale cleanup decides on every upload past the stale age under that lock, and keeps it until its rename ends, also past the drive deadline. Abort of a committing upload takes the lock and removes a failed commit's leftover only when every copy is staged and older than one hour. POST, the worker, the reset, the ExpireRestored recheck and heal all decide on the authoritative revision of the exact version. That is the revision held by max(n/2+1, read quorum) drives, identified by the version header and the normalized metadata digest. Status writes patch only the restore fields of that revision. They write only the drives that hold it, and put the old revision back on failure. Heal copies restore fields only when a quorum holds the same fields. A failed attempt resets itself with bounded in-memory retries. A later POST takes over a stranded marker when no attempt runs. Otherwise it answers RestoreAlreadyInProgress within a few seconds. Older nodes do not know the marker. A new grid capability handler lets the serving node ask every peer. POST answers 503 ServiceUnavailable and writes nothing until all peers support native restore, and a queued worker asks again before it creates its upload. The single-pool lock-domain split between DELETE/CopyObject and restore is closed by its own fix, which this change depends on. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/bucket-lifecycle-restore-boundary_test.go | 3 +- cmd/bucket-lifecycle-restore-safety_test.go | 2 +- cmd/bucket-lifecycle.go | 38 +- cmd/erasure-healing.go | 7 + cmd/erasure-multipart-cleanup_test.go | 26 +- cmd/erasure-multipart.go | 168 +- cmd/erasure-object.go | 136 +- cmd/erasure-server-pool.go | 17 +- cmd/object-api-interface.go | 6 + cmd/object-copy-metadata_test.go | 6 +- cmd/object-handlers.go | 77 +- cmd/peer-rest-server.go | 2 + cmd/restore-attempt.go | 862 ++++++++++ cmd/restore-capability.go | 184 +++ cmd/restore-capability_test.go | 271 ++++ cmd/restore-native-generation_test.go | 1418 +++++++++++++++++ cmd/restore-native-isolation_test.go | 1397 ++++++++++++++++ cmd/restore-native_test.go | 72 +- internal/grid/handlers.go | 2 + internal/grid/handlers_string.go | 11 +- 20 files changed, 4543 insertions(+), 162 deletions(-) create mode 100644 cmd/restore-attempt.go create mode 100644 cmd/restore-capability.go create mode 100644 cmd/restore-capability_test.go create mode 100644 cmd/restore-native-generation_test.go create mode 100644 cmd/restore-native-isolation_test.go diff --git a/cmd/bucket-lifecycle-restore-boundary_test.go b/cmd/bucket-lifecycle-restore-boundary_test.go index 14967cf6a..8f39342ba 100644 --- a/cmd/bucket-lifecycle-restore-boundary_test.go +++ b/cmd/bucket-lifecycle-restore-boundary_test.go @@ -137,8 +137,7 @@ func TestExpireRestoredMetadataReadKeepsMinorityVersion(t *testing.T) { case "restore-cleanup": // The initial read succeeded before remote download failed; // cleanup must not purge a now-incomplete target on reread. - opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1} - err = er.updateRestoreMetadata(ctx, r.bucket, r.object, fi.ToObjectInfo(r.bucket, r.object, true), opts) + err = er.clearRestoreStatus(ctx, r.bucket, r.object, oi.VersionID, nil) case "lookup-cleanup", "lookup-restore": // Version-sensitive lookup also covers native restore routing. opts.MetadataChg = true diff --git a/cmd/bucket-lifecycle-restore-safety_test.go b/cmd/bucket-lifecycle-restore-safety_test.go index 9facbfd0d..85a4be182 100644 --- a/cmd/bucket-lifecycle-restore-safety_test.go +++ b/cmd/bucket-lifecycle-restore-safety_test.go @@ -156,7 +156,7 @@ func TestExpireRestoredRenewedOrMissing(t *testing.T) { t.Fatal(err) } err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner) - if !isErrVersionNotFound(err) { + if !errors.Is(err, errLifecycleNotApplicable) { t.Errorf("stale task should be rejected, got %v", err) } if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed || got.DataDir != before.DataDir { diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go index 5c5597587..5854fa64d 100644 --- a/cmd/bucket-lifecycle.go +++ b/cmd/bucket-lifecycle.go @@ -37,6 +37,7 @@ import ( "github.com/minio/minio/internal/amztime" sse "github.com/minio/minio/internal/bucket/encryption" "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/object/lock" "github.com/minio/minio/internal/bucket/replication" "github.com/minio/minio/internal/event" xhttp "github.com/minio/minio/internal/http" @@ -987,8 +988,10 @@ func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string }, nil } -// set ObjectOptions for PUT call to restore temporary copy of transitioned data -func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) { +// set ObjectOptions for PUT call to restore temporary copy of transitioned data. +// opts are the restore's options, with the versioning POST captured: the +// bucket's versioning may have changed since, and must not mint a new version. +func putRestoreOpts(rreq *RestoreObjectRequest, objInfo ObjectInfo, opts ObjectOptions) (putOpts ObjectOptions) { meta := make(map[string]string) sc := rreq.OutputLocation.S3.StorageClass if sc == "" { @@ -1011,8 +1014,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES } return ObjectOptions{ - Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, object), - VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object), + Versioned: opts.Versioned, + VersionSuspended: opts.VersionSuspended, UserDefined: meta, } } @@ -1020,19 +1023,38 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O if len(objInfo.UserTags) != 0 { meta[xhttp.AmzObjectTagging] = objInfo.UserTags } - // Set restore object status + // Set restore object status; a completed restore has no attempt. restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days) meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String() + delete(meta, restoreAttemptKey) + delete(meta, restoreUploadKey) return ObjectOptions{ - Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, object), - VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object), + Versioned: opts.Versioned, + VersionSuspended: opts.VersionSuspended, UserDefined: meta, - VersionID: objInfo.VersionID, + VersionID: restoreVersionID(objInfo.VersionID), MTime: objInfo.ModTime, Expires: objInfo.Expires, } } +// restoredCopyExpires reports whether the restored copy of oi, as read now, +// expires by its own reason: lifecycle expires the restored copy of this +// version, its restore deadline is valid and has passed, its transition is +// complete and no restore is running. Another rule falling due is no reason. +func restoredCopyExpires(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, oi ObjectInfo) bool { + if !evalActionFromLifecycle(ctx, lc, lr, rcfg, oi).Action.DeleteRestored() { + return false + } + status, err := parseRestoreObjStatus(oi.UserDefined[xhttp.AmzRestore]) + if err != nil { + return false + } + expiry, completed := status.Expiry() + return completed && !expiry.IsZero() && UTCNow().After(expiry) && + oi.TransitionedObject.Status == lifecycle.TransitionComplete && !oi.RestoreOngoing +} + var errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed") // IsRemote returns true if this object version's contents are in its remote diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go index 8fb371270..fad01b0cf 100644 --- a/cmd/erasure-healing.go +++ b/cmd/erasure-healing.go @@ -373,6 +373,13 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object if err != nil { return result, err } + // The restore fields fence native restores and restored-copy cleanup. + // Heal copies them only as a quorum of the drives holds them; while the + // drives disagree, it waits rather than create a quorum for one side. + latestMeta, err = restoreHealMeta(partsMetadata, latestMeta, max(len(storageDisks)/2+1, readQuorum)) + if err != nil { + return result, toObjectErr(err, bucket, object, versionID) + } // No modtime quorum filterDisksByETag := quorumETag != "" diff --git a/cmd/erasure-multipart-cleanup_test.go b/cmd/erasure-multipart-cleanup_test.go index 26a95db53..14c012968 100644 --- a/cmd/erasure-multipart-cleanup_test.go +++ b/cmd/erasure-multipart-cleanup_test.go @@ -123,7 +123,20 @@ func (d *multipartCleanupDisk) Endpoint() Endpoint { func (d *multipartCleanupDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) { d.reads++ - return FileInfo{ModTime: d.modTime}, d.readErr + if d.readErr != nil { + return FileInfo{}, d.readErr + } + fi := newFileInfo("obj", 1, 0) + fi.Erasure.Index, fi.ModTime = 1, d.modTime + return fi, nil +} + +// multipartCleanupSet is a set of one drive, which also answers the +// metadata reads that establish the restore marker of an expired upload. +// Its locks are local: stale cleanup decides a removal under the upload's +// commit lock. +func multipartCleanupSet(disk StorageAPI) erasureObjects { + return erasureObjects{getDisks: func() []StorageAPI { return []StorageAPI{disk} }, nsMutex: newNSLock(false)} } func (d *multipartCleanupDisk) StatVol(ctx context.Context, volume string) (VolInfo, error) { @@ -143,9 +156,10 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) { }{ {name: "recent", prefix: uuid, age: expiry / 2, readErr: errFileNotFound}, {name: "plaintext_base64_collision", prefix: "4eacd477-e733-4851-b821-6fd42785ff87", age: expiry / 2, readErr: errFileNotFound}, - {name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantDeleted: true}, + // Only an expired upload is read on every drive, for its restore marker. + {name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantReads: 1, wantDeleted: true}, {name: "legacy_recent", prefix: uuid, age: expiry / 2, legacy: true, wantReads: 1}, - {name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 1, wantDeleted: true}, + {name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 2, wantDeleted: true}, {name: "legacy_metadata_missing", prefix: uuid, age: 2 * expiry, legacy: true, readErr: errFileNotFound, wantReads: 1}, {name: "malformed_metadata_missing", prefix: "not-a-uuid", age: 2 * expiry, readErr: errFileNotFound, wantReads: 1}, } { @@ -166,7 +180,7 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) { if err := os.Chtimes(filepath.Dir(uploadDir), modTime, modTime); err != nil { t.Fatal(err) } - erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) + multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk) if disk.reads != tc.wantReads { t.Errorf("ReadVersion calls = %d, want %d", disk.reads, tc.wantReads) } @@ -199,7 +213,7 @@ func TestCleanupStaleUploadsEmptyHashDirectory(t *testing.T) { if err := os.Chtimes(shaDir, modTime, modTime); err != nil { t.Fatal(err) } - erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) + multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk) if _, err := os.Stat(shaDir); !os.IsNotExist(err) { t.Fatalf("empty hash directory must be removed regardless of age: %v", err) } @@ -224,7 +238,7 @@ func TestCleanupStaleUploadsSharedHashDirectory(t *testing.T) { t.Fatal(err) } - erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) + multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk) if _, err := os.Stat(filepath.Join(shaDir, expiredID)); !os.IsNotExist(err) { t.Errorf("expired upload must be removed: %v", err) } diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index c68fbc446..c4f5d97ff 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -34,7 +34,6 @@ import ( "github.com/google/uuid" "github.com/klauspost/readahead" "github.com/minio/minio-go/v7/pkg/set" - "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/config/storageclass" "github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/hash" @@ -72,6 +71,7 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string { } // checkUploadIDExists - verify if a given uploadID exists and is valid. +// The metadata read from each drive is returned also with an error. func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string, write bool) (fi FileInfo, metArr []FileInfo, err error) { defer func() { if errors.Is(err, errFileNotFound) { @@ -89,15 +89,15 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount) if err != nil { - return fi, nil, err + return fi, partsMetadata, err } if readQuorum < 0 { - return fi, nil, errErasureReadQuorum + return fi, partsMetadata, errErasureReadQuorum } if writeQuorum < 0 { - return fi, nil, errErasureWriteQuorum + return fi, partsMetadata, errErasureWriteQuorum } quorum := readQuorum @@ -114,7 +114,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum) } if err != nil { - return fi, nil, err + return fi, partsMetadata, err } // Pick one from the first valid metadata. @@ -217,12 +217,55 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto hasSkippedUploads = true return nil } + // A completion holds this lock from staging its commit in the + // upload until the commit ends: decide on a removal only while + // holding it, and skip an upload whose commit is running. Only + // uploads past the stale age pay for the lock and the read below. + lk := er.restoreCommitLock(uploadIDPath) + lkctx, err := lk.GetLock(ctx, newDynamicTimeout(time.Second, time.Second)) + if err != nil { + hasSkippedUploads = true + return nil + } + // The rename below releases the lock when it ends, also when it + // outlives its deadline; every other exit releases it here. + renaming := false + defer func() { + if !renaming { + lk.Unlock(lkctx) + } + }() + // A native restore can run past the stale age: keep its upload + // while its attempt still owns the version, and keep any upload + // whose marker the drives cannot establish. + view := er.readRestoreUpload(lkctx.Context(), "", uploadIDPath) + if view.committing { + // What a commit staged, with no commit running: only a staged + // copy itself shows when it staged, only an old one is a + // leftover of a failed commit, and a copy not staged yet + // tells nothing. + local, err := disk.ReadVersion(lkctx.Context(), "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{}) + st, serr := os.Stat(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath, xlStorageFormatFile)) + if err != nil || serr != nil || !restoreUploadStaged(local) || time.Since(st.ModTime()) < restoreCommitGrace { + hasSkippedUploads = true + return nil + } + } else if view.live && (!view.known || (view.marker != nil && er.restoreUploadActive(lkctx.Context(), view))) { + hasSkippedUploads = true + return nil + } + renaming = true w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) return w.Run(func() error { wait := deleteMultipartCleanupSleeper.Timer(ctx) pathUUID := mustGetUUID() targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID) - renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket)) + err := renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket)) + lk.Unlock(lkctx) + if err != nil { + // The next cleanup round tries again. + storageLogIf(ctx, fmt.Errorf("Unable to remove stale upload %s on %s: %w", uploadIDPath, drivePath, err)) + } wait() return nil }) @@ -463,6 +506,13 @@ func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, if populatedUploadIDs.Contains(uploadID) { continue } + // A native restore's upload is private to its attempt; list only an + // upload that the drives establish as an ordinary one. + metas, errs := readAllFileInfo(ctx, er.getDisks(), bucket, minioMetaMultipartBucket, + pathJoin(er.getMultipartSHADir(bucket, object), uploadID), "", false, false) + if !restoreUploadState(metas, errs).visible(ObjectOptions{}) { + continue + } var fallback time.Time if _, ok := multipartUploadTime(uploadID); !ok { fi, err := disk.ReadVersion(ctx, bucket, minioMetaMultipartBucket, @@ -562,6 +612,11 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string, if opts.PreserveETag != "" { userDefined["etag"] = opts.PreserveETag } + // Only a native restore marks its upload private, never caller metadata. + delete(userDefined, restoreUploadKey) + if a := opts.Transition.attempt; a != nil { + userDefined[restoreUploadKey] = a.String() + } onlineDisks := er.getDisks() // Get parity and data drive count based on storage class metadata @@ -743,7 +798,10 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) // Validates if upload ID exists. - fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) + fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) + if err == nil && !restoreUploadVisible(metas, opts) { + err = errUploadIDNotFound + } if err != nil { if errors.Is(err, errVolumeNotFound) { return pi, toObjectErr(err, bucket) @@ -980,7 +1038,10 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u UploadID: uploadID, } - fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + if err == nil && !restoreUploadVisible(metas, opts) { + err = errUploadIDNotFound + } if err != nil { if errors.Is(err, errVolumeNotFound) { return result, toObjectErr(err, bucket) @@ -1058,7 +1119,10 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up auditObjectErasureSet(ctx, "ListObjectParts", object, &er) } - fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + if err == nil && !restoreUploadVisible(metas, opts) { + err = errUploadIDNotFound + } if err != nil { return result, toObjectErr(err, bucket, object, uploadID) } @@ -1271,7 +1335,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str auditObjectErasureSet(ctx, "CompleteMultipartUpload", object, &er) } - if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile { + if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile || opts.Transition.attempt != nil { if !opts.NoLock { ns := er.NewNSLock(bucket, object) lkctx, err := ns.GetLock(ctx, globalOperationTimeout) @@ -1304,7 +1368,24 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str } } + if opts.Transition.attempt != nil { + // A native restore holds the upload write lock from validation + // through commit, after the object lock and in the lock domain of + // PutObjectPart, which takes a part lock and then this lock to read. + // NoLock only says that the caller holds the object lock. + uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID)) + ulkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + return oi, err + } + ctx = ulkctx.Context() + defer uploadIDLock.Unlock(ulkctx) + } + fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) + if err == nil && !restoreUploadVisible(partsMetadata, opts) { + err = errUploadIDNotFound + } if err != nil { if errors.Is(err, errVolumeNotFound) { return oi, toObjectErr(err, bucket) @@ -1618,6 +1699,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str delete(fi.Metadata, hash.MinIOMultipartChecksumType) delete(fi.Metadata, multipartMetaBucket) delete(fi.Metadata, multipartMetaObject) + delete(fi.Metadata, restoreUploadKey) // Save the final object size and modtime. fi.Size = objectSize @@ -1697,6 +1779,20 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo, opts.Transition); err != nil { return oi, toObjectErr(err, bucket, object) } + if a := opts.Transition.attempt; a != nil { + // A native restore writes back only the null version it targets. + current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{ + VersionID: nullVersionID, + Transition: opts.Transition, + NoLock: true, + }) + if gerr != nil { + return oi, toObjectErr(gerr, bucket, object) + } + if !a.rehydrates(current, fi) { + return oi, toObjectErr(errAuthentication, bucket, object) + } + } } else { current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{ VersionID: fi.VersionID, @@ -1706,11 +1802,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str NoLock: true, }) switch { - case gerr == nil && opts.Transition.RestoreRequest != nil && - current.TransitionedObject.Status == lifecycle.TransitionComplete && - current.ETag == fi.Metadata["etag"] && current.Size == fi.Size && current.ModTime.Equal(fi.ModTime): - // The internal restore path rehydrates this same tiered version. - // Ordinary multipart completion must still reject a version overwrite. + case gerr == nil && opts.Transition.attempt.rehydrates(current, fi): + // A native restore's own private upload rehydrates this same + // tiered version. Any other completion must still reject a + // version overwrite. case gerr == nil, errors.As(gerr, &MethodNotAllowed{}): return oi, toObjectErr(errAuthentication, bucket, object) case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr): @@ -1720,6 +1815,16 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str } } + // Stale cleanup and abort judge an upload whose commit staged object + // metadata in it by this lock: hold it from staging through the commit. + commitLock := er.restoreCommitLock(uploadIDPath) + clkctx, err := commitLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + return oi, err + } + ctx = clkctx.Context() + defer commitLock.Unlock(clkctx) + er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename(). defer func() { @@ -1791,7 +1896,38 @@ func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, objec if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") { return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID} } - defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID)) + // Authorize before the unconditional cleanup below. A native restore's + // upload is absent to every other caller, and an upload whose marker the + // drives cannot establish is not removed by anyone. + uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) + view := er.readRestoreUpload(ctx, bucket, uploadIDPath) + if view.committing { + // What a failed commit left behind, also after stale cleanup could + // not remove it: only with no commit running, and once every copy + // shows a staging older than the grace. + lk := er.restoreCommitLock(uploadIDPath) + lkctx, err := lk.GetLock(ctx, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait)) + if err != nil { + return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID) + } + defer lk.Unlock(lkctx) + if !er.restoreCommitAbandoned(lkctx.Context(), bucket, uploadIDPath) { + return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID) + } + er.deleteAll(lkctx.Context(), minioMetaMultipartBucket, uploadIDPath) + return true, nil + } + switch { + case view.visible(opts): + case view.known && view.marked: + return false, nil + case view.known && opts.Transition.attempt != nil: + // A native restore's own abort, of an upload no longer its own. + return false, nil + default: + return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID) + } + defer er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath) _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false) err = toObjectErr(err, bucket, object, uploadID) if _, absent := err.(InvalidUploadID); absent { diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 9307aae81..f3c8cfb42 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2158,6 +2158,20 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string if opts.Transition.ExpireRestored && (gerr != nil || goi.DeleteMarker) { return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID} } + // A restored copy expires only by its own reason, decided on the + // version's authoritative generation: a drive that missed a renewal or a + // later POST must not authorize removing the cache, nor drives that + // disagree on the restore status. Another lifecycle deletion falling due, + // or the all-pools check of an ordinary expiry, does not stand in for it. + if opts.Transition.ExpireRestored { + gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID) + if err != nil || gen.split || gen.fi.Deleted || lc == nil { + return goi, errLifecycleNotApplicable + } + if !restoredCopyExpires(ctx, *lc, rcfg, replcfg, gen.fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)) { + return goi, errLifecycleNotApplicable + } + } if opts.EvalMetadataFn != nil { dsc, err := opts.EvalMetadataFn(&goi, gerr) @@ -2660,29 +2674,6 @@ func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket, return er.restoreTransitionedObject(ctx, bucket, object, opts) } -// update restore status header in the metadata -func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error { - oi := objInfo.Clone() - oi.metadataOnly = true // Perform only metadata updates. - - // allow retry in the case of failure to restore - delete(oi.UserDefined, xhttp.AmzRestore) - - if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{ - VersionID: oi.VersionID, - Transition: opts.Transition, - }, ObjectOptions{ - VersionID: oi.VersionID, - MTime: oi.ModTime, - NoLock: opts.NoLock, - Transition: opts.Transition, - }); err != nil { - storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err)) - return err - } - return nil -} - // getRestoreLock waits for a restore's object lock for as long as ctx lives. // The POST handler has already marked the version ongoing, and only the // restore clears that marker. A restore queued behind another version's @@ -2698,6 +2689,11 @@ func getRestoreLock(ctx context.Context, lk RWLocker) (LockContext, error) { // restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts // as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta +// +// A POST restore publishes its attempt on the version's ongoing marker. The +// restore rehydrates the version only while that attempt owns the marker, and +// on failure clears the marker only while it still does. A direct caller +// publishes no attempt; its attempt only names the private upload. func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error { if !opts.NoLock { // Single-pool and direct callers must protect the metadata snapshot @@ -2705,30 +2701,45 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s lk := er.NewNSLock(bucket, object) lkctx, err := getRestoreLock(ctx, lk) if err != nil { - return err + return restoreResetError{err} } ctx = lkctx.Context() defer lk.Unlock(lkctx) opts.NoLock = true } - setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error { - if rerr == nil { - return nil // nothing to do; restore object was successful - } - er.updateRestoreMetadata(ctx, bucket, object, oi, opts) - return rerr - } - var oi ObjectInfo - // get the file info on disk for transitioned object - actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false) + published := opts.Transition.attempt + // Restore the authoritative generation of the version, never a revision + // that only some drives hold. + gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID) if err != nil { - // No version was read, so there is no restore metadata to update. - return toObjectErr(err, bucket, object) + err = toObjectErr(err, bucket, object, opts.VersionID) + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + // No version was read, so there is no restore marker to clear. + return err + } + return restoreResetError{err} + } + actualfi := gen.fi + if actualfi.Deleted { + return toObjectErr(errMethodNotAllowed, bucket, object) } - oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) - ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi) + oi := actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) + attempt := published + if attempt == nil { + attempt = newRestoreAttempt(oi) + } else if !attempt.owns(oi) { + // A later POST took the version over, or it was replaced. + return errRestoreSuperseded + } + opts.Transition.attempt = attempt + versionID := restoreVersionID(actualfi.VersionID) + fail := func(cause error) error { + return er.restoreFailed(ctx, bucket, object, versionID, published, cause) + } + + ropts := putRestoreOpts(opts.Transition.RestoreRequest, oi, opts) ropts.NoLock = opts.NoLock ropts.Transition = opts.Transition // Rehydration copies encoded bytes without changing their content identity. @@ -2738,35 +2749,51 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s if len(oi.Parts) == 1 && !oi.isMultipart() { actualSize, err := oi.GetActualSize() if err != nil { - return setRestoreHeaderFn(oi, err) + return fail(err) } ropts.IndexCB = func() []byte { return oi.Parts[0].Index } var rs *HTTPRangeSpec gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts) if err != nil { - return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + return fail(toObjectErr(err, bucket, object)) } defer gr.Close() hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", actualSize) if err != nil { - return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + return fail(toObjectErr(err, bucket, object)) } pReader := NewPutObjReader(hashReader) - _, err = er.PutObject(ctx, bucket, object, pReader, ropts) - return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + if _, err = er.PutObject(ctx, bucket, object, pReader, ropts); err != nil { + return fail(toObjectErr(err, bucket, object)) + } + return nil } - res, err := er.NewMultipartUpload(ctx, bucket, object, ropts) - if err != nil { - return setRestoreHeaderFn(oi, err) + // Every server must keep the upload private, also a queued restore's. + if err := nativeRestoreAvailable(ctx); err != nil { + return fail(err) } + // The upload records when it starts, which the stale-upload cleanup ages; + // completion still commits the version's own ModTime. + uopts := ropts + uopts.MTime = time.Time{} + res, err := er.NewMultipartUpload(ctx, bucket, object, uopts) + if err != nil { + return fail(err) + } + completed := false + defer func() { + if !completed { + er.abortRestoreUpload(bucket, object, res.UploadID, attempt) + } + }() var uploadedParts []CompletePart var rs *HTTPRangeSpec // get reader from the warm backend - note that even in the case of encrypted objects, this stream is still encrypted. gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts) if err != nil { - return setRestoreHeaderFn(oi, err) + return fail(err) } defer gr.Close() @@ -2781,16 +2808,17 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s } hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", actualSize) if err != nil { - return setRestoreHeaderFn(oi, err) + return fail(err) } pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{ - IndexCB: func() []byte { return partInfo.Index }, + IndexCB: func() []byte { return partInfo.Index }, + Transition: TransitionOptions{attempt: attempt}, }) if err != nil { - return setRestoreHeaderFn(oi, err) + return fail(err) } if pInfo.Size != partInfo.Size { - return setRestoreHeaderFn(oi, InvalidObjectState{Bucket: bucket, Object: object}) + return fail(InvalidObjectState{Bucket: bucket, Object: object}) } uploadedParts = append(uploadedParts, CompletePart{ PartNumber: pInfo.PartNumber, @@ -2804,7 +2832,11 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s VersionSuspended: opts.VersionSuspended, Transition: opts.Transition, }) - return setRestoreHeaderFn(oi, err) + if err != nil { + return fail(err) + } + completed = true + return nil } // DecomTieredObject - moves tiered object to another pool during decommissioning. diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 849ad6c67..c168e22f9 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -3369,8 +3369,16 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec } // RestoreTransitionedObject - restore transitioned object content locally on this cluster. -func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { +func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) (err error) { object = encodeDirObject(object) + defer func() { + // The restore stopped before it could clear POST's ongoing marker. + var reset restoreResetError + if errors.As(err, &reset) { + z.recoverRestoreStatus(bucket, object, opts) + err = reset.error + } + }() if z.SinglePool() { return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts) } @@ -3380,7 +3388,7 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck lk := z.NewNSLock(bucket, object) lkctx, err := getRestoreLock(ctx, lk) if err != nil { - return err + return restoreResetError{err} } ctx = lkctx.Context() defer lk.Unlock(lkctx) @@ -3389,10 +3397,7 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck // Avoid restoring object from a pool being decommissioned. opts.SkipDecommissioned = true opts.NoLock = true - // Restore an existing version, not the latest version's write location. - lookupOpts := opts - lookupOpts.MetadataChg = true - idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts) + idx, err := z.restorePoolIdx(ctx, bucket, object, opts) if err != nil { return err } diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index d40ff3ce7..0f4a663ac 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -177,6 +177,10 @@ type TransitionOptions struct { RestoreRequest *RestoreObjectRequest RestoreExpiry time.Time ExpireRestored bool + + // attempt is the native restore a call belongs to. Only that restore + // sees its private upload, see restoreUploadVisible. + attempt *restoreAttempt } // MakeBucketOptions represents bucket options for ObjectLayer bucket operations @@ -295,6 +299,8 @@ type ObjectLayer interface { DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error + // restoreRequest records a POST restore on a version, see erasureServerPools. + restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error) // Multipart operations. ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) diff --git a/cmd/object-copy-metadata_test.go b/cmd/object-copy-metadata_test.go index 2cb05f3eb..72c729599 100644 --- a/cmd/object-copy-metadata_test.go +++ b/cmd/object-copy-metadata_test.go @@ -340,9 +340,9 @@ func TestCopyRewritesObjectData(t *testing.T) { name: "data copy always rewrites", want: true, }, - // PostRestoreObjectHandler, updateRestoreMetadata and batchKeyRotate all - // address the same version on both sides and never set Versioned, so they - // only ever reach these two cases. + // In-place metadata callers such as batchKeyRotate address the same + // version on both sides and never set Versioned, so they only ever + // reach these two cases. { name: "unversioned in-place metadata update", metadataOnly: true, diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index da9b772ac..81af341f5 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -3955,7 +3955,11 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * // Preserve a failed restore read even before the request body is parsed. opts.Transition.RestoreRequest = &RestoreObjectRequest{} - objInfo, err := getObjectInfo(ctx, bucket, object, opts) + // Resolve the target without the object lock: a running restore holds it + // for its download, and the status step bounds its own wait for it. + preOpts := opts + preOpts.NoLock = true + objInfo, err := getObjectInfo(ctx, bucket, object, preOpts) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return @@ -3982,48 +3986,47 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * } opts.Transition.RestoreRequest = rreq statusCode := http.StatusOK - alreadyRestored := false - if err == nil { - if objInfo.RestoreOngoing && rreq.Type != SelectRestoreRequest { + // set or upgrade restore expiry + restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days) + + // update self with restore metadata + var attempt *restoreAttempt + if rreq.Type != SelectRestoreRequest { + // Native restore needs every server to keep its upload private; + // refuse it, writing nothing, while that is not established. + if err := nativeRestoreAvailable(ctx); err != nil { + apiErr := errorCodes.ToAPIErr(ErrBusy) + apiErr.Code, apiErr.Description = "ServiceUnavailable", err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + // Decide on a fresh read under the object lock: renew a restored copy + // that is still there, or mark the version ongoing for a new attempt. + // Only the version resolved above, with its identity, is restored. + statusOpts := opts + statusOpts.Transition.RestoreExpiry = restoreExpiry + status, err := objectAPI.restoreRequest(GlobalContext, bucket, object, statusOpts, objInfo) + if errors.Is(err, errRestoreInProgress) { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectRestoreAlreadyInProgress), r.URL) return } - if !objInfo.RestoreOngoing && !objInfo.RestoreExpires.IsZero() { - statusCode = http.StatusAccepted - alreadyRestored = true - } - } - // set or upgrade restore expiry - restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days) - metadata := cloneMSS(objInfo.UserDefined) - - // update self with restore metadata - if rreq.Type != SelectRestoreRequest { - objInfo.metadataOnly = true // Perform only metadata updates. - metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days) - metadata[xhttp.AmzRestoreRequestDate] = time.Now().UTC().Format(http.TimeFormat) - if alreadyRestored { - metadata[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String() - } else { - metadata[xhttp.AmzRestore] = ongoingRestoreObj().String() - } - objInfo.UserDefined = metadata - if _, err := objectAPI.CopyObject(GlobalContext, bucket, object, bucket, object, objInfo, ObjectOptions{ - VersionID: objInfo.VersionID, - Transition: opts.Transition, - }, ObjectOptions{ - VersionID: objInfo.VersionID, - Transition: opts.Transition, - MTime: objInfo.ModTime, - MetadataChg: true, - }); err != nil { + if errors.As(err, &InvalidObjectState{}) || errors.As(err, &MethodNotAllowed{}) { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL) return } - // for previously restored object, just update the restore expiry - if alreadyRestored { + if err != nil { + // Such as a read quorum error while telling whether it restores. + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } + // Events keep the version ID as this request resolved it. + status.objInfo.VersionID = objInfo.VersionID + objInfo = status.objInfo + // for previously restored object, just update the restore expiry + if status.attempt == nil { + return + } + attempt = status.attempt } restoreObject := mustGetUUID() @@ -4088,12 +4091,14 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * rreq.SelectParameters.Close() return } + // The versioning context and the explicit version POST resolved. restoreOpts := ObjectOptions{ Transition: TransitionOptions{ RestoreRequest: rreq, RestoreExpiry: restoreExpiry, + attempt: attempt, }, - VersionID: objInfo.VersionID, + VersionID: attempt.VersionID, Versioned: opts.Versioned, VersionSuspended: opts.VersionSuspended, } diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index b28c8db44..6661b5fba 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -116,6 +116,7 @@ var ( stopRebalanceRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerStopRebalance, grid.NewMSS, grid.NewNoPayload) updateMetacacheListingRPC = grid.NewSingleHandler[*metacache, *metacache](grid.HandlerUpdateMetacacheListing, func() *metacache { return &metacache{} }, func() *metacache { return &metacache{} }) cleanupUploadIDCacheMetaRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerClearUploadID, grid.NewMSS, grid.NewNoPayload) + restoreCapabilityRPC = grid.NewSingleHandler[*grid.MSS, *grid.MSS](grid.HandlerRestoreCapability, grid.NewMSS, grid.NewMSS) // STREAMS // Set an output capacity of 100 for consoleLog and listenRPC @@ -1440,6 +1441,7 @@ func registerPeerRESTHandlers(router *mux.Router, gm *grid.Manager) { logger.FatalIf(stopRebalanceRPC.Register(gm, server.StopRebalanceHandler), "unable to register handler") logger.FatalIf(updateMetacacheListingRPC.Register(gm, server.UpdateMetacacheListingHandler), "unable to register handler") logger.FatalIf(cleanupUploadIDCacheMetaRPC.Register(gm, server.HandlerClearUploadID), "unable to register handler") + logger.FatalIf(restoreCapabilityRPC.Register(gm, restoreCapabilityHandler), "unable to register handler") logger.FatalIf(gm.RegisterStreamingHandler(grid.HandlerTrace, grid.StreamHandler{ Handle: server.TraceHandler, diff --git a/cmd/restore-attempt.go b/cmd/restore-attempt.go new file mode 100644 index 000000000..0df3e36fd --- /dev/null +++ b/cmd/restore-attempt.go @@ -0,0 +1,862 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "net/http" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/minio/minio/internal/bucket/lifecycle" + xhttp "github.com/minio/minio/internal/http" +) + +// A native restore keeps two private records. The version that POST marks +// ongoing names the attempt that owns the marker; that attempt's multipart +// upload names the attempt and the version it rehydrates. Clients cannot +// set either: requests with reserved metadata are refused, it is never +// extracted from headers, and uploads only get the marker from the restore. +const ( + restoreAttemptKey = ReservedMetadataPrefixLower + "restore-attempt" + restoreUploadKey = ReservedMetadataPrefixLower + "restore-upload" +) + +const ( + // restoreTakeoverWait bounds how long a POST on an ongoing version waits + // for the object lock. A running restore holds it for its download. + restoreTakeoverWait = 2 * time.Second + // restoreCommitGrace is how long stale cleanup keeps an upload whose + // completion staged object metadata in it; a commit takes seconds. + restoreCommitGrace = time.Hour + // restoreCleanupTimeout bounds each status reset and upload abort. + restoreCleanupTimeout = time.Minute +) + +// restoreRecoveryDelays are the in-memory retries that clear the ongoing +// marker of a restore that failed before it could clear the marker itself. +// A restart loses them; a later POST then takes the marker over. +var restoreRecoveryDelays = []time.Duration{0, 100 * time.Millisecond, time.Second, 5 * time.Second, 30 * time.Second, 2 * time.Minute} + +var ( + // errRestoreSuperseded: the version no longer belongs to this attempt. + // The attempt must not rehydrate, complete or reset anything. + errRestoreSuperseded = errors.New("restore attempt superseded") + // errRestoreInProgress: a restore holds the object lock of the version. + errRestoreInProgress = errors.New("restore already in progress") + // errRestoreUnchanged: a restore status update found nothing to write. + errRestoreUnchanged = errors.New("restore status unchanged") +) + +// restoreResetError marks a failed restore that may have left its ongoing +// marker behind. The pools layer hands it to recoverRestoreStatus. +type restoreResetError struct{ error } + +func (e restoreResetError) Unwrap() error { return e.error } + +// restoreVersionID is the explicit version ID of a resolved restore target. +func restoreVersionID(versionID string) string { + if versionID == "" { + return nullVersionID + } + return versionID +} + +// restoreAttempt identifies one native restore of one stored version. +type restoreAttempt struct { + ID string `json:"id"` + VersionID string `json:"v"` // "null" for the null version + ETag string `json:"e"` + Size int64 `json:"s"` + ModTime time.Time `json:"m"` +} + +func newRestoreAttempt(oi ObjectInfo) *restoreAttempt { + return &restoreAttempt{ + ID: mustGetUUID(), + VersionID: restoreVersionID(oi.VersionID), + ETag: oi.ETag, + Size: oi.Size, + ModTime: oi.ModTime.UTC(), + } +} + +func (a *restoreAttempt) String() string { + b, _ := json.Marshal(a) + return string(b) +} + +func parseRestoreAttempt(s string) (*restoreAttempt, bool) { + var a restoreAttempt + if s == "" || json.Unmarshal([]byte(s), &a) != nil || a.ID == "" || a.VersionID == "" { + return nil, false + } + return &a, true +} + +func (a *restoreAttempt) equal(b *restoreAttempt) bool { + return a != nil && b != nil && a.ID == b.ID && a.VersionID == b.VersionID && + a.ETag == b.ETag && a.Size == b.Size && a.ModTime.Equal(b.ModTime) +} + +// targets reports whether oi is still the version the attempt was made for. +// A replacement under the same version ID differs in ETag, size or ModTime. +func (a *restoreAttempt) targets(oi ObjectInfo) bool { + return a != nil && !oi.DeleteMarker && restoreVersionID(oi.VersionID) == a.VersionID && + oi.ETag == a.ETag && oi.Size == a.Size && oi.ModTime.Equal(a.ModTime) && + oi.TransitionedObject.Status == lifecycle.TransitionComplete +} + +// owns reports whether the version is ongoing under this attempt. +func (a *restoreAttempt) owns(oi ObjectInfo) bool { + return a.targets(oi) && oi.RestoreOngoing && oi.UserDefined[restoreAttemptKey] == a.ID +} + +// rehydrates reports whether completing upload fi, the attempt's private +// upload, writes back the very version current. +func (a *restoreAttempt) rehydrates(current ObjectInfo, fi FileInfo) bool { + return a.targets(current) && fi.Metadata["etag"] == a.ETag && fi.Size == a.Size && fi.ModTime.Equal(a.ModTime) +} + +// restoreUploadView is the restore marker of an upload as the drives of +// its set establish it. A marked upload is written to a write quorum, at +// least a majority of the drives, and its copies never change their marker. +type restoreUploadView struct { + // marker is the attempt of a native restore's upload: one marker that a + // majority of the drives hold. + marker *restoreAttempt + // marked: some readable drive holds a marker. + marked bool + // known: the marker state is established. Without it, callers fail + // closed: nobody lists, uses, aborts or removes the upload. + known bool + // live: enough drives hold the upload or cannot tell that a written + // upload may exist; otherwise only remnants of one are left. + live bool + // committing: some drive holds object metadata in the upload's place. + // Completion stages the committed version's object metadata there, + // without the marker, before it renames the upload into the object. + committing bool + // bucket and object the upload of an established marker belongs to. + bucket, object string +} + +// restoreUploadState establishes the marker of an upload from the metadata of +// every drive. errs may be nil, then a drive without valid metadata is one +// that cannot tell. A marker is established on a majority of the drives; its +// absence only when no readable drive shows one and the drives that cannot +// tell are too few to hold a marked upload. +func restoreUploadState(metas []FileInfo, errs []error) restoreUploadView { + n := len(metas) + majority := n/2 + 1 + markers := make(map[string]int) + var unmarked, unknown, committing int + for i, fi := range metas { + if errs != nil && errs[i] != nil { + if !errors.Is(errs[i], errFileNotFound) && !errors.Is(errs[i], errFileVersionNotFound) && + !errors.Is(errs[i], errVolumeNotFound) && !errors.Is(errs[i], errUploadIDNotFound) { + unknown++ + } + continue + } + if !fi.IsValid() { + unknown++ + continue + } + if restoreUploadStaged(fi) { + committing++ + continue + } + if m := fi.Metadata[restoreUploadKey]; m != "" { + markers[m]++ + } else { + unmarked++ + } + } + present := unmarked + for _, c := range markers { + present += c + } + v := restoreUploadView{marked: len(markers) > 0, live: present+unknown+committing >= majority, committing: committing > 0} + if v.committing { + // Nobody but the completion may use or remove it now. + return v + } + for m, c := range markers { + if c >= majority { + v.marker, v.known = parseRestoreAttempt(m) + if v.known { + // The marker also names the upload's own target. + for _, fi := range metas { + if !fi.IsValid() || fi.Metadata[restoreUploadKey] != m { + continue + } + if restoreVersionID(fi.VersionID) != v.marker.VersionID || fi.Metadata["etag"] != v.marker.ETag { + v.marker, v.known = nil, false + break + } + v.bucket, v.object = fi.Metadata[multipartMetaBucket], fi.Metadata[multipartMetaObject] + } + } + return v + } + } + // A marker without a majority is established only as a remnant. + v.known = !v.live || (!v.marked && unknown < majority) + return v +} + +// restoreUploadStaged reports whether an upload's record is object metadata +// that a completion staged there: an upload record has no parts and no +// other versions. +func restoreUploadStaged(fi FileInfo) bool { + return fi.Deleted || len(fi.Parts) > 0 || fi.NumVersions > 1 +} + +// visible reports whether a caller sees the upload: a native restore its own +// upload, any other caller only an upload established as unmarked. +func (v restoreUploadView) visible(opts ObjectOptions) bool { + if a := opts.Transition.attempt; a != nil { + return v.marker.equal(a) + } + return v.known && !v.marked +} + +// restoreUploadVisible reports whether a caller sees an upload whose metadata +// is metas, one entry for each drive. +func restoreUploadVisible(metas []FileInfo, opts ObjectOptions) bool { + return restoreUploadState(metas, nil).visible(opts) +} + +// readRestoreUpload reads the metadata of an upload from every drive. +func (er erasureObjects) readRestoreUpload(ctx context.Context, bucket, uploadIDPath string) restoreUploadView { + metas, errs := readAllFileInfo(ctx, er.getDisks(), bucket, minioMetaMultipartBucket, uploadIDPath, "", false, false) + return restoreUploadState(metas, errs) +} + +// restoreCommitLock is held by a completion from staging through its commit; +// it is named by the upload's own path, which an upload whose copies all +// hold staged object metadata still has. +func (er erasureObjects) restoreCommitLock(uploadIDPath string) RWLocker { + return er.NewNSLock(minioMetaMultipartBucket, uploadIDPath) +} + +// restoreCommitAbandoned reports whether an upload holds only what a failed +// commit left: every drive answers, and every copy that is there is staged +// object metadata, see restoreUploadStaged, older than restoreCommitGrace by +// that drive's own metadata file. A copy that cannot be read, or that is not +// staged yet, may belong to a commit still running. The caller holds the +// upload's restoreCommitLock. +func (er erasureObjects) restoreCommitAbandoned(ctx context.Context, bucket, uploadIDPath string) bool { + disks := er.getDisks() + metas, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, uploadIDPath, "", false, false) + staged := 0 + for i, fi := range metas { + if errs[i] != nil { + if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errFileVersionNotFound) { + continue + } + return false + } + if !fi.IsValid() || !restoreUploadStaged(fi) { + return false + } + st, err := disks[i].StatInfoFile(ctx, minioMetaMultipartBucket, pathJoin(uploadIDPath, xlStorageFormatFile), false) + if err != nil || len(st) == 0 || time.Since(st[0].ModTime) < restoreCommitGrace { + return false + } + staged++ + } + return staged > 0 +} + +// restoreGeneration is the metadata generation of one version that the drives +// of a set hold authoritatively: the revision (header and normalized metadata +// digest, versionRevision) that at least max(n/2+1, its read quorum) drives +// hold. drives has each drive's own copy of it, with that drive's erasure +// index and inline data, and an invalid entry where a drive holds another or +// no revision; disks has nil there. +// +// A failed restore status write can leave drives that differ only in their +// restore fields, with no revision on a quorum. Then the generation is the +// restore-neutral revision that a quorum holds once those fields are left +// out: fi has no restore fields, and ongoing names the attempts whose +// ongoing marker some of its drives hold ("" for one without an attempt). +// Every other field, protection included, still needs the quorum. +type restoreGeneration struct { + fi FileInfo + drives []FileInfo + disks []StorageAPI + split bool + ongoing map[string]bool +} + +// restoreFieldKeys are the metadata keys a restore status write sets. +var restoreFieldKeys = []string{xhttp.AmzRestore, xhttp.AmzRestoreExpiryDays, xhttp.AmzRestoreRequestDate, restoreAttemptKey} + +// restoreHealMeta returns latest, the metadata heal copies to drives that +// lack the version, with the restore fields that at least quorum drives of +// that version hold. When some drive holds restore fields and no set of +// them has the quorum, healing the version waits. +func restoreHealMeta(metas []FileInfo, latest FileInfo, quorum int) (FileInfo, error) { + counts := make(map[string]int) + fields := make(map[string]map[string]string) + restoring := false + for _, m := range metas { + if !m.IsValid() || m.Deleted || !m.ModTime.Equal(latest.ModTime) { + continue + } + f := make(map[string]string) + var key strings.Builder + for _, k := range restoreFieldKeys { + if v, ok := m.Metadata[k]; ok { + f[k] = v + fmt.Fprintf(&key, "%d:%s=%d:%s;", len(k), k, len(v), v) + restoring = true + } + } + counts[key.String()]++ + fields[key.String()] = f + } + if !restoring { + return latest, nil + } + best, bestCount := "", 0 + for key, c := range counts { + if c > bestCount { + best, bestCount = key, c + } + } + if bestCount < quorum { + return latest, errErasureReadQuorum + } + latest.Metadata = maps.Clone(latest.Metadata) + for _, k := range restoreFieldKeys { + delete(latest.Metadata, k) + } + maps.Copy(latest.Metadata, fields[best]) + return latest, nil +} + +// restoreNeutralRevision is the revision of ver without its restore fields. +func restoreNeutralRevision(ver xlMetaV2ShallowVersion, metaV uint8) (revisionKey, bool) { + var v xlMetaV2Version + if _, err := v.unmarshalV(metaV, ver.meta); err != nil || v.Type != ObjectType || v.ObjectV2 == nil { + return revisionKey{}, false + } + o := *v.ObjectV2 + o.MetaUser, o.MetaSys = maps.Clone(o.MetaUser), maps.Clone(o.MetaSys) + for _, k := range restoreFieldKeys { + delete(o.MetaUser, k) + delete(o.MetaSys, k) + } + v.ObjectV2 = &o + meta, err := v.MarshalMsg(nil) + if err != nil { + return revisionKey{}, false + } + return versionRevision(xlMetaV2ShallowVersion{header: v.header(), meta: meta}, xlMetaVersion), true +} + +// readRestoreGeneration selects the authoritative generation of a version, +// or fails closed: a revision that only some drives hold, such as one a +// failed metadata update left behind, is never selected. An empty version ID +// addresses the latest version, as a direct restore may. +func (er erasureObjects) readRestoreGeneration(ctx context.Context, bucket, object, versionID string) (restoreGeneration, error) { + if versionID == "" { + fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, ObjectOptions{ + NoLock: true, + Transition: TransitionOptions{RestoreRequest: &RestoreObjectRequest{}}, + }, false) + if err != nil { + return restoreGeneration{}, err + } + versionID = restoreVersionID(fi.VersionID) + } + var uv [16]byte + if versionID != nullVersionID { + id, err := uuid.Parse(versionID) + if err != nil { + return restoreGeneration{}, errFileVersionNotFound + } + uv = id + } + disks := er.getDisks() + n := len(disks) + majority := n/2 + 1 + raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, true) + type revision struct { + ver xlMetaV2ShallowVersion + metaV uint8 + count int + } + keys := make([]*revisionKey, n) + neutral := make([]*revisionKey, n) + revisions := make(map[revisionKey]*revision) + neutrals := make(map[revisionKey]*revision) + absent := 0 + for i := range disks { + if errs[i] != nil { + if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errVolumeNotFound) { + absent++ + } + continue + } + var xl xlMetaV2 + if xl.LoadOrConvert(raws[i].Buf) != nil { + continue + } + found := false + for _, ver := range xl.versions { + if ver.header.VersionID != uv || ver.header.FreeVersion() { + continue + } + key := versionRevision(ver, xl.metaV) + keys[i], found = &key, true + r := revisions[key] + if r == nil { + r = &revision{ver: ver, metaV: xl.metaV} + revisions[key] = r + } + r.count++ + if nkey, ok := restoreNeutralRevision(ver, xl.metaV); ok { + neutral[i] = &nkey + if neutrals[nkey] == nil { + neutrals[nkey] = &revision{ver: ver, metaV: xl.metaV} + } + neutrals[nkey].count++ + } + break + } + if !found { + absent++ + } + } + accepted := func(revs map[revisionKey]*revision) (revisionKey, bool) { + var best *revision + var bestKey revisionKey + for key, r := range revs { + if best == nil || r.count > best.count { + best, bestKey = r, key + } + } + return bestKey, best != nil && best.count >= majority && + best.count >= er.versionReadQuorum(ctx, bucket, object, best.ver, best.metaV, make([]FileInfo, n), make([]error, n)) + } + gen := restoreGeneration{drives: make([]FileInfo, n), disks: make([]StorageAPI, n)} + bestKey, ok := accepted(revisions) + if !ok { + if bestKey, ok = accepted(neutrals); !ok { + if absent >= majority { + return restoreGeneration{}, errFileVersionNotFound + } + return restoreGeneration{}, errErasureReadQuorum + } + keys, gen.split, gen.ongoing = neutral, true, make(map[string]bool) + } + for i := range disks { + if keys[i] == nil || *keys[i] != bestKey { + continue + } + fi, err := getFileInfo(raws[i].Buf, bucket, object, versionID, fileInfoOpts{Data: true}) + if err != nil { + continue + } + gen.drives[i], gen.disks[i] = fi, disks[i] + if gen.split { + if st, err := parseRestoreObjStatus(fi.Metadata[xhttp.AmzRestore]); err == nil && st.Ongoing() { + gen.ongoing[fi.Metadata[restoreAttemptKey]] = true + } + } + if !gen.fi.IsValid() { + gen.fi = fi + } + } + if !gen.fi.IsValid() && !gen.fi.Deleted { + return restoreGeneration{}, errErasureReadQuorum + } + if gen.split { + gen.fi.Metadata = maps.Clone(gen.fi.Metadata) + for _, k := range restoreFieldKeys { + delete(gen.fi.Metadata, k) + } + } + return gen, nil +} + +// updateRestoreStatus rewrites the restore fields of one stored version. It +// selects the authoritative generation of the version, see +// readRestoreGeneration, patches only the restore fields onto exactly that +// generation, keeping every other field and its ModTime, and writes it back +// only to the drives that hold it. The caller holds the object lock. update +// edits the metadata in place, or returns an error, errRestoreUnchanged to +// write nothing; ongoing names the attempts whose ongoing marker the +// generation holds. A failed read never removes the version, and a failed +// write puts the generation it read back. +func (er erasureObjects) updateRestoreStatus(ctx context.Context, bucket, object, versionID string, update func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error) (ObjectInfo, error) { + gen, err := er.readRestoreGeneration(ctx, bucket, object, versionID) + if err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object, versionID) + } + fi := gen.fi + oi := fi.ToObjectInfo(bucket, object, false) + if fi.Deleted { + return oi, toObjectErr(errMethodNotAllowed, bucket, object) + } + ongoing := gen.ongoing + if !gen.split && oi.RestoreOngoing { + ongoing = map[string]bool{oi.UserDefined[restoreAttemptKey]: true} + } + meta := maps.Clone(fi.Metadata) + if err := update(oi, meta, ongoing); err != nil { + return oi, err + } + write := func(meta func(FileInfo) map[string]string) error { + metaArr := make([]FileInfo, len(gen.drives)) + for i, d := range gen.drives { + if gen.disks[i] == nil { + continue + } + if !d.InlineData() { + d.Data = nil + } + d.Metadata = meta(d) + metaArr[i] = d + } + // The write assigns each drive its slot as erasure index: order the + // drives by the version's distribution first. + disks, metaArr := shuffleDisksAndPartsMetadata(gen.disks, metaArr, fi) + _, err := writeUniqueFileInfo(ctx, disks, "", bucket, object, metaArr, fi.WriteQuorum(er.defaultWQuorum())) + return err + } + if err := write(func(FileInfo) map[string]string { return meta }); err != nil { + // Without the read generation back, its drives could hold two + // revisions and no quorum for either. A split that stays differs + // only in restore fields, which the next write or reset resolves. + if rerr := write(func(d FileInfo) map[string]string { return d.Metadata }); rerr != nil { + storageLogIf(ctx, fmt.Errorf("Unable to put back the restore status of %s/%s(%s): %w", bucket, object, versionID, rerr)) + } + return oi, toObjectErr(err, bucket, object) + } + fi.Metadata = meta + return fi.ToObjectInfo(bucket, object, false), nil +} + +// clearRestoreStatus makes a version retryable again while attempt a still +// owns its ongoing marker. It never brings back an earlier completed status: +// POST only starts a restore when that restored copy is gone. +func (er erasureObjects) clearRestoreStatus(ctx context.Context, bucket, object, versionID string, a *restoreAttempt) error { + _, err := er.updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error { + id := "" + if a != nil { + id = a.ID + if !a.targets(oi) { + return errRestoreUnchanged + } + } + // Clear only this attempt's marker. Where drives disagree, another + // attempt's marker may be the acknowledged one: leave that to it, + // or to a later POST, which takes it over under the lock. + if !ongoing[id] || len(ongoing) != 1 { + return errRestoreUnchanged + } + delete(meta, xhttp.AmzRestore) + delete(meta, xhttp.AmzRestoreExpiryDays) + delete(meta, xhttp.AmzRestoreRequestDate) + delete(meta, restoreAttemptKey) + return nil + }) + if errors.Is(err, errRestoreUnchanged) { + return nil + } + return err +} + +// restoreFailed clears the ongoing marker of a failed attempt while the +// object lock is held. A lost lock or an ended context leaves that to the +// bounded retries of the pools layer, which take the lock afresh. +func (er erasureObjects) restoreFailed(ctx context.Context, bucket, object, versionID string, published *restoreAttempt, cause error) error { + if ctx.Err() != nil { + return restoreResetError{cause} + } + if err := er.clearRestoreStatus(ctx, bucket, object, versionID, published); err != nil { + storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, versionID, err)) + return restoreResetError{cause} + } + return cause +} + +// abortRestoreUpload removes the private upload of a failed attempt, or +// leaves it to stale-upload cleanup. The restore's own context may have +// ended, so it uses a fresh bounded one, and the upload lock as any abort. +// It matches the upload to its own attempt rather than to the version, which +// a newer attempt may own by now, and it never touches the version. +func (er erasureObjects) abortRestoreUpload(bucket, object, uploadID string, a *restoreAttempt) { + ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout) + defer cancel() + lk := er.NewNSLock(bucket, pathJoin(object, uploadID)) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err == nil { + defer lk.Unlock(lkctx) + _, err = er.abortMultipartUpload(lkctx.Context(), bucket, object, uploadID, ObjectOptions{ + NoAuditLog: true, + Transition: TransitionOptions{attempt: a}, + }) + } + if err != nil { + storageLogIf(ctx, fmt.Errorf("Unable to abort the restore upload of %s/%s: %w", bucket, object, err)) + } +} + +// restoreUploadActive reports whether the restore that owns a stale-aged +// upload still runs: its attempt owns the version's ongoing marker. A long +// download keeps its upload; one whose attempt ended loses it, and an +// ongoing marker alone keeps nothing. +func (er erasureObjects) restoreUploadActive(ctx context.Context, v restoreUploadView) bool { + gen, err := er.readRestoreGeneration(ctx, v.bucket, v.object, v.marker.VersionID) + if err != nil { + // An unreadable version may still be restoring; retry next round. + return !errors.Is(err, errFileVersionNotFound) && !errors.Is(err, errFileNotFound) + } + if gen.split { + // Drives disagree on the restore status: that the attempt ended + // is not established. + return gen.ongoing[v.marker.ID] + } + return v.marker.owns(gen.fi.ToObjectInfo(v.bucket, v.object, false)) +} + +// restoreLock is the object lock that restores and their status updates +// share: with one pool the set's own lock, as single-pool writes use. +func (z *erasureServerPools) restoreLock(bucket, object string) RWLocker { + if z.SinglePool() { + return z.serverPools[0].getHashedSet(object).NewNSLock(bucket, object) + } + return z.NewNSLock(bucket, object) +} + +// restoreRequestStatus is what a POST restore recorded on its version. +type restoreRequestStatus struct { + objInfo ObjectInfo + // attempt is the restore that must rehydrate the version; nil when a + // restored copy was still there and its expiry was renewed. + attempt *restoreAttempt +} + +// restoreRequest records a POST restore on target, the version that POST +// resolved, on a fresh read under the object lock. The version must still +// be target: a replacement under the same version ID is refused. A copy that +// still holds its restored data is renewed. Otherwise the version is marked +// ongoing for a new attempt; that also takes over a marker no running +// restore holds, as a running restore holds the object lock for its whole +// download. Only the copy that is to be restored is written, so no copy +// claims a restored copy it does not hold. +func (z *erasureServerPools) restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error) { + object = encodeDirObject(object) + versionID := restoreVersionID(target.VersionID) + lk := z.restoreLock(bucket, object) + // Wait only briefly first, whatever the lockless read before saw: a + // restore of this version holds the lock for its whole download. Then + // wait as usual only if the drives show no restore of it running. + lkctx, err := lk.GetLock(ctx, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait)) + if errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil { + running, rerr := z.restoreRunning(ctx, bucket, object, versionID) + switch { + case rerr != nil: + // Whether it is restoring is unknown: answer now, not after + // the lock frees. + return restoreRequestStatus{}, rerr + case running: + return restoreRequestStatus{}, errRestoreInProgress + } + lkctx, err = lk.GetLock(ctx, globalOperationTimeout) + } + if err != nil { + return restoreRequestStatus{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + + expected := newRestoreAttempt(target) + copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{ + VersionID: versionID, + Versioned: opts.Versioned, + VersionSuspended: opts.VersionSuspended, + Transition: opts.Transition, + }) + if err != nil { + return restoreRequestStatus{}, err + } + // Restore the copy reads serve first, outside pools being decommissioned, + // as RestoreTransitionedObject does. + pool := -1 + for _, c := range copies { + if !z.IsSuspended(c.Index) { + pool = c.Index + break + } + } + if pool < 0 { + return restoreRequestStatus{}, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID} + } + + rreq := opts.Transition.RestoreRequest + date := UTCNow().Format(http.TimeFormat) + var attempt *restoreAttempt + oi, err := z.serverPools[pool].getHashedSet(object).updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, _ map[string]bool) error { + if !expected.targets(oi) { + // Not the version POST resolved, or no longer transitioned. + return InvalidObjectState{Bucket: bucket, Object: decodeDirObject(object)} + } + attempt = nil + meta[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days) + meta[xhttp.AmzRestoreRequestDate] = date + if !oi.RestoreOngoing && !oi.RestoreExpires.IsZero() { + // Cleanup removes a restored copy together with its header. + meta[xhttp.AmzRestore] = completedRestoreObj(opts.Transition.RestoreExpiry).String() + delete(meta, restoreAttemptKey) + return nil + } + attempt = newRestoreAttempt(oi) + meta[xhttp.AmzRestore] = ongoingRestoreObj().String() + meta[restoreAttemptKey] = attempt.ID + return nil + }) + if err != nil { + if attempt != nil { + // The marker may have reached some drives, or all of them. + z.recoverRestoreStatus(bucket, object, ObjectOptions{ + VersionID: attempt.VersionID, + Transition: TransitionOptions{attempt: attempt}, + }) + } + return restoreRequestStatus{}, err + } + return restoreRequestStatus{objInfo: oi, attempt: attempt}, nil +} + +// restoreRunning reports, without the object lock and within the takeover +// wait, whether the drives show a restore of the version ongoing in a pool. +// An error means that this cannot be told. +func (z *erasureServerPools) restoreRunning(ctx context.Context, bucket, object, versionID string) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, restoreTakeoverWait) + defer cancel() + for _, pool := range z.serverPools { + gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, versionID) + if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) { + continue + } + if err == nil && ctx.Err() != nil { + err = ctx.Err() + } + if err != nil { + return false, toObjectErr(err, bucket, object, versionID) + } + if len(gen.ongoing) > 0 || !gen.split && gen.fi.ToObjectInfo(bucket, object, false).RestoreOngoing { + return true, nil + } + } + return false, nil +} + +// restorePoolIdx returns the pool whose copy the restore rehydrates: the one +// POST marked for its attempt, or for a direct caller the existing version. +func (z *erasureServerPools) restorePoolIdx(ctx context.Context, bucket, object string, opts ObjectOptions) (int, error) { + a := opts.Transition.attempt + if a == nil { + // Restore an existing version, not the latest version's write location. + lookupOpts := opts + lookupOpts.MetadataChg = true + idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts) + if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + return idx, restoreResetError{err} + } + return idx, err + } + found := false + for idx, pool := range z.serverPools { + gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, a.VersionID) + if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) { + continue + } + if err != nil { + return -1, restoreResetError{toObjectErr(err, bucket, object, a.VersionID)} + } + found = true + if !z.IsSuspended(idx) && a.owns(gen.fi.ToObjectInfo(bucket, object, false)) { + return idx, nil + } + } + if !found { + return -1, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: a.VersionID} + } + return -1, errRestoreSuperseded +} + +// recoverRestoreStatus retries, in memory and for a bounded time, to clear the +// ongoing marker that a failed restore left behind, so a later POST is not +// refused. Each try takes the object lock afresh and clears the marker only +// while it still belongs to this attempt. +func (z *erasureServerPools) recoverRestoreStatus(bucket, object string, opts ObjectOptions) { + go func() { + var err error + for _, delay := range restoreRecoveryDelays { + select { + case <-GlobalContext.Done(): + return + case <-time.After(delay): + } + if err = z.resetRestoreStatus(bucket, object, opts); err == nil { + return + } + } + storageLogIf(GlobalContext, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, err)) + }() +} + +// resetRestoreStatus is one try of recoverRestoreStatus. +func (z *erasureServerPools) resetRestoreStatus(bucket, object string, opts ObjectOptions) error { + ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout) + defer cancel() + lk := z.restoreLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return err + } + defer lk.Unlock(lkctx) + ctx = lkctx.Context() + + a := opts.Transition.attempt + versionID := opts.VersionID + if a != nil { + versionID = a.VersionID + } + for _, pool := range z.serverPools { + err := pool.getHashedSet(object).clearRestoreStatus(ctx, bucket, object, versionID, a) + if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !isErrMethodNotAllowed(err) && !isErrBucketNotFound(err) { + return err + } + } + return nil +} diff --git a/cmd/restore-capability.go b/cmd/restore-capability.go new file mode 100644 index 000000000..3594a6dc9 --- /dev/null +++ b/cmd/restore-capability.go @@ -0,0 +1,184 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" + "time" + + "github.com/minio/minio/internal/grid" +) + +// A native restore keeps its multipart upload private and marks the version +// it rehydrates (restore-attempt.go). A server that predates this ignores +// the marker: it would list, change, abort or complete the upload. Native +// restore therefore runs only while every server of the cluster positively +// reports that it understands the marker. A server that does not answer, +// or does not know the question, as an older server does not, blocks it. + +// nativeRestoreCapability is the native restore protocol this server speaks. +const nativeRestoreCapability = "1" + +const ( + peerRESTRestoreCapability = "restore" + // restoreCapabilityTTL bounds how long a positive answer is reused while + // no server reconnected. + restoreCapabilityTTL = 10 * time.Second + // restoreCapabilityTimeout bounds the question to each server. + restoreCapabilityTimeout = 5 * time.Second +) + +var errNativeRestoreUnavailable = errors.New("native restore is unavailable during a rolling upgrade: not every server of the cluster supports it yet") + +// restoreCapabilityHandler answers the question on this server. +func restoreCapabilityHandler(*grid.MSS) (*grid.MSS, *grid.RemoteErr) { + return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: nativeRestoreCapability}), nil +} + +// restoreCapabilityPeer is another server of the cluster. +type restoreCapabilityPeer interface { + // restoreConnection identifies the current connection to the server; it + // changes when the server reconnects, as a restarted one does. ok is + // false when the server is not connected. + restoreConnection() (key string, ok bool) + restoreCapability(ctx context.Context) (string, error) +} + +type restorePeerClient struct{ *peerRESTClient } + +func (c restorePeerClient) restoreConnection() (string, bool) { + if c.peerRESTClient == nil || c.gridConn == nil { + return "", false + } + conn := c.gridConn() + if conn == nil { + return "", false + } + st := conn.Stats() + if st.Connected == 0 { + return "", false + } + // Keys are compared by the position of their server. + return fmt.Sprintf("%d/%d", st.LastConnectTime.UnixNano(), st.ReconnectCount), true +} + +func (c restorePeerClient) restoreCapability(ctx context.Context) (string, error) { + conn := c.gridConn() + if conn == nil { + return "", errPeerOffline + } + resp, err := restoreCapabilityRPC.Call(ctx, conn, grid.NewMSS()) + if err != nil { + return "", err + } + return resp.Get(peerRESTRestoreCapability), nil +} + +// restoreCapabilityGate decides whether native restore may run, caching a +// positive answer for restoreCapabilityTTL while the connections to all +// servers stay the same. A negative answer is not cached. +type restoreCapabilityGate struct { + peers func() []restoreCapabilityPeer + + once sync.Once + turn chan struct{} // one check at a time + checked time.Time + keys []string +} + +var globalRestoreCapability = &restoreCapabilityGate{peers: notificationRestorePeers} + +func notificationRestorePeers() []restoreCapabilityPeer { + if globalNotificationSys == nil { + return nil + } + peers := make([]restoreCapabilityPeer, 0, len(globalNotificationSys.peerClients)) + for _, client := range globalNotificationSys.peerClients { + peers = append(peers, restorePeerClient{client}) + } + return peers +} + +// nativeRestoreAvailable returns errNativeRestoreUnavailable unless every +// other server of the cluster supports native restore. +func nativeRestoreAvailable(ctx context.Context) error { + return globalRestoreCapability.check(ctx) +} + +func (g *restoreCapabilityGate) check(ctx context.Context) error { + g.once.Do(func() { g.turn = make(chan struct{}, 1) }) + // One deadline covers the wait for another check and the questions. + ctx, cancel := context.WithTimeout(ctx, restoreCapabilityTimeout) + defer cancel() + select { + case g.turn <- struct{}{}: + case <-ctx.Done(): + return errNativeRestoreUnavailable + } + defer func() { <-g.turn }() + + // Read the connections only now: one may have changed while this check + // waited for another. + peers := g.peers() + connections := func() ([]string, bool) { + keys := make([]string, len(peers)) + for i, p := range peers { + key, ok := p.restoreConnection() + if !ok { + return nil, false + } + keys[i] = key + } + return keys, true + } + keys, ok := connections() + if !ok { + g.checked, g.keys = time.Time{}, nil + return errNativeRestoreUnavailable + } + if ctx.Err() != nil { + return errNativeRestoreUnavailable + } + if !g.checked.IsZero() && time.Since(g.checked) < restoreCapabilityTTL && slices.Equal(keys, g.keys) { + return nil + } + g.checked, g.keys = time.Time{}, nil + answers := make([]bool, len(peers)) + var wg sync.WaitGroup + for i, p := range peers { + wg.Add(1) + go func() { + defer wg.Done() + version, err := p.restoreCapability(ctx) + answers[i] = err == nil && version == nativeRestoreCapability + }() + } + wg.Wait() + // An answer counts only for the connection it was asked on: a server + // that reconnected meanwhile may be another binary. + after, ok := connections() + if !ok || !slices.Equal(keys, after) || slices.Contains(answers, false) || ctx.Err() != nil { + return errNativeRestoreUnavailable + } + g.checked, g.keys = time.Now(), keys + return nil +} diff --git a/cmd/restore-capability_test.go b/cmd/restore-capability_test.go new file mode 100644 index 000000000..306764d12 --- /dev/null +++ b/cmd/restore-capability_test.go @@ -0,0 +1,271 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "net/http" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/grid" + xhttp "github.com/minio/minio/internal/http" +) + +// A server answers the capability question over the grid. One that does not +// know it, as an older server, or answers something else, blocks native +// restore, and so does one that is not connected. +func TestNativeRestoreCapabilityGrid(t *testing.T) { + tg, err := grid.SetupTestGrid(4) + if err != nil { + t.Fatal(err) + } + t.Cleanup(tg.Cleanup) + if err := restoreCapabilityRPC.Register(tg.Managers[1], restoreCapabilityHandler); err != nil { + t.Fatal(err) + } + // Managers[2] never registers the handler. + if err := restoreCapabilityRPC.Register(tg.Managers[3], func(*grid.MSS) (*grid.MSS, *grid.RemoteErr) { + return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: "0"}), nil + }); err != nil { + t.Fatal(err) + } + peer := func(i int) restoreCapabilityPeer { + return restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[i]) }}} + } + offline := restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return nil }}} + for _, tc := range []struct { + name string + peers []restoreCapabilityPeer + ok bool + }{ + {"single-server", nil, true}, + {"all-capable", []restoreCapabilityPeer{peer(1)}, true}, + {"older-server", []restoreCapabilityPeer{peer(1), peer(2)}, false}, + {"other-answer", []restoreCapabilityPeer{peer(1), peer(3)}, false}, + {"offline", []restoreCapabilityPeer{peer(1), offline}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return tc.peers }} + if err := g.check(t.Context()); (err == nil) != tc.ok { + t.Errorf("capability: %v, want available=%v", err, tc.ok) + } + }) + } +} + +// restoreCapabilityFake is a server whose connection and answer a test sets. +type restoreCapabilityFake struct { + capable atomic.Bool + connected atomic.Bool + conn atomic.Int64 + asked atomic.Int64 +} + +func newRestoreCapabilityFake() *restoreCapabilityFake { + p := &restoreCapabilityFake{} + p.capable.Store(true) + p.connected.Store(true) + return p +} + +func (p *restoreCapabilityFake) restoreConnection() (string, bool) { + return strings.Repeat("c", int(p.conn.Load())), p.connected.Load() +} + +func (p *restoreCapabilityFake) restoreCapability(context.Context) (string, error) { + p.asked.Add(1) + if !p.capable.Load() { + return "", errors.New("Invalid Handler for type: RestoreCapability") + } + return nativeRestoreCapability, nil +} + +// A positive answer is reused while the connections stay the same; a +// reconnect asks again, a disconnected server blocks without being asked, +// and a negative answer is not reused. +func TestNativeRestoreCapabilityCache(t *testing.T) { + p := newRestoreCapabilityFake() + g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }} + for range 3 { + if err := g.check(t.Context()); err != nil { + t.Fatal(err) + } + } + if n := p.asked.Load(); n != 1 { + t.Errorf("asked %d times while connected and capable, want 1", n) + } + // The server restarts with an older binary and reconnects. + p.capable.Store(false) + p.conn.Add(1) + if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) { + t.Errorf("after a reconnect to an older server: %v", err) + } + p.capable.Store(true) + if err := g.check(t.Context()); err != nil { + t.Errorf("a negative answer was reused: %v", err) + } + asked := p.asked.Load() + p.connected.Store(false) + if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) || p.asked.Load() != asked { + t.Errorf("disconnected server: %v, asked %d -> %d", err, asked, p.asked.Load()) + } +} + +// During a rolling upgrade POST restore answers 503 and writes nothing, and +// a restore queued before it creates no upload and leaves the version +// retryable. +func TestNativeRestoreRollingUpgradeGate(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, false, true) + p := newRestoreCapabilityFake() + previous := globalRestoreCapability + globalRestoreCapability = &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }} + t.Cleanup(func() { globalRestoreCapability = previous }) + + p.capable.Store(false) + before := r.fileInfo(oi.VersionID) + layer := newRestoreIsolationLayer(r) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "rolling upgrade") { + t.Errorf("POST during a rolling upgrade: %d %s", rec.Code, rec.Body.String()) + } + if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before.Metadata, after.Metadata) { + t.Error("refused POST changed the version") + } + select { + case err := <-layer.done: + t.Fatalf("refused POST started a restore: %v", err) + default: + } + + p.capable.Store(true) + release := make(chan struct{}) + layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + <-release + return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) + } + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + close(release) + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + // A server is replaced by an older binary while the restore is queued. + p.capable.Store(false) + p.conn.Add(1) + close(release) + if err := layer.wait(t); !errors.Is(err, errNativeRestoreUnavailable) { + t.Fatalf("queued restore during a rolling upgrade: %v", err) + } + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("queued restore created an upload during a rolling upgrade: %v", uploads) + } + info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil || info.RestoreOngoing || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" { + t.Errorf("version after a refused queued restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err) + } +} + +// restoreCapabilityHeld is a fake server whose answer a test holds. +type restoreCapabilityHeld struct { + restoreCapabilityFake + asking chan struct{} + release chan struct{} +} + +func (p *restoreCapabilityHeld) restoreCapability(ctx context.Context) (string, error) { + v, err := p.restoreCapabilityFake.restoreCapability(ctx) + p.asking <- struct{}{} + select { + case <-p.release: + return v, err + case <-ctx.Done(): + return "", ctx.Err() + } +} + +// A check that waits for another reads the connections once its turn +// comes, an answer counts only if its server did not reconnect meanwhile, +// and a waiting check ends with its context. +func TestNativeRestoreCapabilityRevalidates(t *testing.T) { + t.Run("reconnect-during-check", func(t *testing.T) { + held := func() *restoreCapabilityHeld { + p := &restoreCapabilityHeld{asking: make(chan struct{}, 4), release: make(chan struct{})} + p.capable.Store(true) + p.connected.Store(true) + return p + } + fast, slow := held(), held() + g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{fast, slow} }} + first := make(chan error, 1) + go func() { first <- g.check(t.Context()) }() + <-fast.asking + <-slow.asking + waiting := make(chan error, 1) + go func() { waiting <- g.check(t.Context()) }() + // Both answered "capable"; then fast reconnects as an older server. + fast.capable.Store(false) + fast.conn.Add(1) + close(fast.release) + close(slow.release) + if err := <-first; !errors.Is(err, errNativeRestoreUnavailable) { + t.Errorf("check whose server reconnected while it asked: %v", err) + } + if err := <-waiting; !errors.Is(err, errNativeRestoreUnavailable) { + t.Errorf("waiting check accepted a reconnected older server: %v", err) + } + }) + t.Run("canceled-cached", func(t *testing.T) { + p := newRestoreCapabilityFake() + g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }} + if err := g.check(t.Context()); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := g.check(ctx); err == nil { + t.Error("a cached answer accepted an ended context") + } + }) + t.Run("canceled-wait", func(t *testing.T) { + slow := &restoreCapabilityHeld{restoreCapabilityFake: restoreCapabilityFake{}, asking: make(chan struct{}, 4), release: make(chan struct{})} + slow.capable.Store(true) + slow.connected.Store(true) + defer close(slow.release) + g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{slow} }} + go g.check(t.Context()) + <-slow.asking + ctx, cancel := context.WithCancel(t.Context()) + waiting := make(chan error, 1) + go func() { waiting <- g.check(ctx) }() + cancel() + select { + case err := <-waiting: + if err == nil { + t.Error("canceled check succeeded") + } + case <-time.After(time.Second): + t.Error("canceled check kept waiting for another check") + } + }) +} diff --git a/cmd/restore-native-generation_test.go b/cmd/restore-native-generation_test.go new file mode 100644 index 000000000..d79bc1030 --- /dev/null +++ b/cmd/restore-native-generation_test.go @@ -0,0 +1,1418 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "os" + "reflect" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/config/drive" + xhttp "github.com/minio/minio/internal/http" +) + +// restoreGenerationDisk fails metadata writes or updates of the object, or +// metadata reads of uploads, while its toggles are set. +type restoreGenerationDisk struct { + StorageAPI + bucket, object string + write, update, uploadRead *atomic.Bool +} + +func (d restoreGenerationDisk) WriteMetadata(ctx context.Context, orig, vol, path string, fi FileInfo) error { + if d.write != nil && d.write.Load() && vol == d.bucket && path == d.object { + return errFaultyDisk + } + return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi) +} + +func (d restoreGenerationDisk) UpdateMetadata(ctx context.Context, vol, path string, fi FileInfo, opts UpdateMetadataOpts) error { + if d.update != nil && d.update.Load() && vol == d.bucket && path == d.object { + return errFaultyDisk + } + return d.StorageAPI.UpdateMetadata(ctx, vol, path, fi, opts) +} + +func (d restoreGenerationDisk) ReadVersion(ctx context.Context, orig, vol, path, version string, opts ReadOptions) (FileInfo, error) { + if d.uploadRead != nil && d.uploadRead.Load() && vol == minioMetaMultipartBucket { + return FileInfo{}, errFaultyDisk + } + return d.StorageAPI.ReadVersion(ctx, orig, vol, path, version, opts) +} + +// restoreGenerationOffline makes a drive's object metadata unreadable and +// unwritable while set, its uploads still readable. +type restoreGenerationOffline struct { + StorageAPI + bucket, object string + fail *atomic.Bool +} + +func (d restoreGenerationOffline) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) { + if d.fail.Load() && bucket == d.bucket && object == d.object { + return RawFileInfo{}, errFaultyDisk + } + return d.StorageAPI.ReadXL(ctx, bucket, object, data) +} + +func (d restoreGenerationOffline) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) { + if d.fail.Load() && bucket == d.bucket && object == d.object { + return FileInfo{}, errFaultyDisk + } + return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts) +} + +func (d restoreGenerationOffline) WriteMetadata(ctx context.Context, orig, bucket, object string, fi FileInfo) error { + if d.fail.Load() && bucket == d.bucket && object == d.object { + return errFaultyDisk + } + return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi) +} + +// restoreGenerationMetadata returns the version's metadata on each drive. +func restoreGenerationMetadata(t *testing.T, disks []StorageAPI, r *restoreTest, versionID string) []map[string]string { + t.Helper() + metas := make([]map[string]string, len(disks)) + for i, d := range disks { + fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, versionID, ReadOptions{}) + if err != nil { + t.Fatal(err) + } + metas[i] = fi.Metadata + } + return metas +} + +// An Object Lock update acknowledged on a quorum, which one drive missed, +// survives POST restore: POST patches the restore fields onto the quorum's +// generation and never writes the stale drive's metadata anywhere. +func TestRestoreStatusUsesQuorumGeneration(t *testing.T) { + for _, kind := range []string{"legal-hold", "retention"} { + t.Run(kind, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + enableBucketObjectLock(t, r.bucket) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + initial := []byte(`OFF`) + updated := []byte(`ON`) + key := strings.ToLower(xhttp.AmzObjectLockLegalHold) + if kind == "retention" { + initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour)) + updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour)) + key = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate) + } + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var fail atomic.Bool + wrapped := append([]StorageAPI(nil), disks...) + wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, update: &fail} + er.getDisks = func() []StorageAPI { return wrapped } + fail.Store(true) + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK { + t.Fatalf("protection update: %d %s", rec.Code, rec.Body.String()) + } + fail.Store(false) + before := restoreGenerationMetadata(t, disks, r, oi.VersionID) + acknowledged := before[1][key] + if before[0][key] == acknowledged { + t.Fatal("fixture: drive 0 did not miss the update") + } + for i := 1; i < len(disks); i++ { + if before[i][key] != acknowledged { + t.Fatal("fixture: the update did not reach the other drives") + } + } + if rec := restoreIsolationPOST(t, r, r.z, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + after := restoreGenerationMetadata(t, disks, r, oi.VersionID) + for i := 1; i < len(disks); i++ { + if after[i][key] != acknowledged { + t.Errorf("drive %d: POST wrote %q over the acknowledged %q", i, after[i][key], acknowledged) + } + if after[i][xhttp.AmzRestoreRequestDate] == before[i][xhttp.AmzRestoreRequestDate] { + t.Errorf("drive %d: POST did not record its request", i) + } + } + // The stale drive is left to heal; reads that pick it are not + // changed here. + if after[0][key] != before[0][key] || after[0][xhttp.AmzRestoreRequestDate] != before[0][xhttp.AmzRestoreRequestDate] { + t.Error("POST wrote the stale drive instead of leaving it to heal") + } + }) + } +} + +// The attempt is part of a version's metadata generation. A POST whose +// marker reached all drives but one is acknowledged: its restore owns the +// version, and the reset of an older attempt leaves the newer marker alone. +func TestRestoreAttemptGenerationFencing(t *testing.T) { + for _, mode := range []string{"worker", "old-reset", "degraded-old-reset"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var fail, offline atomic.Bool + wrapped := append([]StorageAPI(nil), disks...) + wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, write: &fail} + wrapped[1] = restoreGenerationOffline{StorageAPI: disks[1], bucket: r.bucket, object: r.object, fail: &offline} + er.getDisks = func() []StorageAPI { return wrapped } + captured := make(chan ObjectOptions, 2) + layer := newRestoreIsolationLayer(r) + layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error { + captured <- opts + return nil + } + var older ObjectOptions + if mode != "worker" { + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + older = <-captured + layer.wait(t) + } + fail.Store(true) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + fail.Store(false) + if rec.Code != http.StatusOK { + t.Fatalf("POST on all drives but one: %d %s", rec.Code, rec.Body.String()) + } + newer := <-captured + layer.wait(t) + id := newer.Transition.attempt.ID + if mode == "worker" { + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, newer); err != nil { + t.Fatalf("restore of the acknowledged attempt: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } + return + } + if mode == "degraded-old-reset" { + // One drive of the newer attempt is unreadable: the drives + // that answer no longer show which attempt is acknowledged. + offline.Store(true) + } + if err := r.z.resetRestoreStatus(r.bucket, r.object, older); err != nil { + t.Fatal(err) + } + offline.Store(false) + after := restoreGenerationMetadata(t, disks, r, oi.VersionID) + for i := 1; i < len(disks); i++ { + if after[i][restoreAttemptKey] != id || !strings.Contains(after[i][xhttp.AmzRestore], "true") { + t.Errorf("drive %d: the older attempt's reset cleared the newer marker: %q %q", i, after[i][restoreAttemptKey], after[i][xhttp.AmzRestore]) + } + } + if after[0][restoreAttemptKey] != older.Transition.attempt.ID { + t.Error("the reset wrote the stale drive") + } + // The newer attempt still owns the version and restores it. + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, newer); err != nil { + t.Fatalf("restore of the newer attempt after the older reset: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } + }) + } +} + +// restoreGenerationUploadPath is the stored path of an upload. +func restoreGenerationUploadPath(t *testing.T, r *restoreTest, id string) string { + t.Helper() + b, err := base64.RawURLEncoding.DecodeString(id) + if err != nil { + t.Fatal(err) + } + _, uploadUUID, ok := strings.Cut(string(b), ".") + if !ok { + t.Fatalf("upload ID without UUID: %q", id) + } + return pathJoin(r.z.serverPools[0].getHashedSet(r.object).getMultipartSHADir(r.bucket, r.object), uploadUUID) +} + +// The restore marker is established on a majority of the drives. One drive +// whose copy of the upload lacks it does not open the upload to clients, +// and the restore still completes. +func TestRestoreUploadMarkerQuorum(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + tier := newRestoreIsolationGateTier(5<<20, nil) + globalTierConfigMgr.drivercache["WARM"] = tier + defer tier.unblock() + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + restoreIsolationEntered(t, tier) + id, _ := restoreIsolationUpload(t, r, 0) + disk := r.z.serverPools[0].getHashedSet(r.object).getDisks()[0] + path := restoreGenerationUploadPath(t, r, id) + fi, err := disk.ReadVersion(t.Context(), "", minioMetaMultipartBucket, path, "", ReadOptions{}) + if err != nil { + t.Fatal(err) + } + delete(fi.Metadata, restoreUploadKey) + if err := disk.WriteMetadata(t.Context(), "", minioMetaMultipartBucket, path, fi); err != nil { + t.Fatal(err) + } + poison := bytes.Repeat([]byte("p"), 5<<20) + if _, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, id, 1, + mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", ""), ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) { + t.Errorf("public part with one unmarked copy of the upload: %v, want NoSuchUpload", err) + } + for _, op := range []func() error{ + func() error { + _, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{}) + return err + }, + func() error { + _, err := r.z.GetMultipartInfo(t.Context(), r.bucket, r.object, id, ObjectOptions{}) + return err + }, + } { + if err := op(); !errors.As(err, &InvalidUploadID{}) { + t.Errorf("public read with one unmarked copy of the upload: %v, want NoSuchUpload", err) + } + } + if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10); err != nil || len(uploads.Uploads) != 0 { + t.Errorf("listing with one unmarked copy of the upload: %+v, %v", uploads.Uploads, err) + } + tier.unblock() + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } +} + +// When the drives cannot tell an upload's marker, no client lists it, no +// abort removes it and stale cleanup keeps it; the restore completes once +// the drives answer again. +func TestRestoreUploadUnreadableFailsClosed(t *testing.T) { + for _, mode := range []string{"list", "abort", "stale"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var fail atomic.Bool + // Stale cleanup with a short age also removes in-flight temporary + // parts: hold that restore in completion, after its last part and + // before its commit, and the others in their download, outside + // the upload lock. + gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} + defer gate.unblock() + tier := newRestoreIsolationGateTier(5<<20, nil) + defer tier.unblock() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + disk := d + if mode == "stale" { + disk = restoreIsolationValidationDisk{d, gate} + } + wrapped[i] = restoreGenerationDisk{StorageAPI: disk, uploadRead: &fail} + } + er.getDisks = func() []StorageAPI { return wrapped } + if mode != "stale" { + globalTierConfigMgr.drivercache["WARM"] = tier + } + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + if mode == "stale" { + select { + case <-gate.entered: + case <-time.After(10 * time.Second): + t.Fatal("restore never reached completion") + } + } else { + restoreIsolationEntered(t, tier) + } + stored := restoreIsolationUploads(t, r, 0) + if len(stored) != 1 { + t.Fatalf("restore upload on disk: %v", stored) + } + var id string + for uploadUUID := range stored { + id = base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)) + } + fail.Store(true) + switch mode { + case "list": + if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 100); len(uploads.Uploads) != 0 { + t.Errorf("an upload with an unknown marker is listed: %+v, %v", uploads.Uploads, err) + } + case "abort": + if err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); err == nil { + t.Error("abort of an upload with an unknown marker succeeded") + } + case "stale": + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + for _, disk := range wrapped { + er.cleanupStaleUploadsOnDisk(t.Context(), disk) + } + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + } + fail.Store(false) + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 1 { + t.Errorf("%s removed an upload with an unknown marker: %v", mode, uploads) + } + gate.unblock() + tier.unblock() + if err := layer.wait(t); err != nil { + t.Fatalf("restore after %s: %v", mode, err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } + }) + } +} + +// POST restores the version it resolved with that version's identity. A +// replacement under the same version ID between its read and its status +// write is refused, and nothing restores it. +func TestRestorePOSTRejectsReplacedTarget(t *testing.T) { + for _, state := range []versioning.State{"", versioning.Suspended} { + t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, state) + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, true, false) + layer := newRestoreIsolationLayer(r) + var replacement ObjectInfo + layer.afterRead = func() { + data := []byte("replacement after POST resolved its target") + var err error + replacement, err = r.z.PutObject(t.Context(), r.bucket, r.object, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: state == versioning.Suspended}) + if err != nil { + t.Fatal(err) + } + opts := r.opts(nullVersionID) + opts.MTime = replacement.ModTime + opts.Transition = TransitionOptions{Tier: "WARM", ETag: replacement.ETag} + if err := r.z.serverPools[0].TransitionObject(t.Context(), r.bucket, r.object, opts); err != nil { + t.Fatal(err) + } + } + if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code < 400 { + layer.wait(t) + t.Errorf("POST accepted a replacement of its target: %d", rec.Code) + } + after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(nullVersionID)) + if err != nil { + t.Fatal(err) + } + if after.ETag != replacement.ETag || after.UserDefined[xhttp.AmzRestore] != "" || after.ETag == oi.ETag { + t.Errorf("replacement %q changed by POST: etag %q restore %q", replacement.ETag, after.ETag, after.UserDefined[xhttp.AmzRestore]) + } + }) + } +} + +// A POST on a version whose restore holds the object lock answers within +// the two-second takeover bound: its first read does not wait for the lock. +func TestRestorePOSTOngoingBound(t *testing.T) { + for _, mode := range []string{"pools=1", "pools=2", "stale-minority", "failed-activity-read"} { + t.Run(mode, func(t *testing.T) { + pools := 1 + if mode == "pools=2" { + pools = 2 + } + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, false, false) + tier := newRestoreIsolationGateTier(0, nil) + globalTierConfigMgr.drivercache["WARM"] = tier + defer tier.unblock() + var fail atomic.Bool + if mode == "stale-minority" { + // The first read may pick a drive that missed the marker. + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + wrapped := append([]StorageAPI(nil), disks...) + wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, write: &fail} + er.getDisks = func() []StorageAPI { return wrapped } + fail.Store(true) + } + layer := newRestoreIsolationLayer(r) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + fail.Store(false) + if rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + restoreIsolationEntered(t, tier) + var unreadable atomic.Bool + second := newRestoreIsolationLayer(r) + if mode == "failed-activity-read" { + // Once POST resolved its target, nothing tells whether it + // is restoring. + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreGenerationOffline{StorageAPI: d, bucket: r.bucket, object: r.object, fail: &unreadable} + } + er.getDisks = func() []StorageAPI { return wrapped } + second.afterRead = func() { unreadable.Store(true) } + } + done := make(chan *httptest.ResponseRecorder, 1) + start := time.Now() + go func() { done <- restoreIsolationPOST(t, r, second, oi.VersionID) }() + select { + case rec := <-done: + want := rec.Code == http.StatusConflict + if mode == "failed-activity-read" { + want = rec.Code >= 500 + } + if !want || time.Since(start) > restoreTakeoverWait+time.Second { + t.Errorf("POST during a running restore: %d after %s, want a bounded refusal", rec.Code, time.Since(start)) + } + case <-time.After(restoreTakeoverWait + 2*time.Second): + t.Error("POST during a running restore waited for the object lock") + unreadable.Store(false) + tier.unblock() + <-done + } + unreadable.Store(false) + tier.unblock() + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + }) + } +} + +// A POST whose status write fails on drives that stay unwritten puts the +// generation it read back, and a split that a failed put-back leaves +// differs only in restore fields: the attempt's reset and a later POST both +// resolve it, while every other field still needs the quorum. +func TestRestorePOSTStatusWriteSplit(t *testing.T) { + for _, mode := range []string{"put-back", "split-reset", "split-post"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var fail atomic.Bool + wrapped := append([]StorageAPI(nil), disks...) + for i := range 2 { + wrapped[i] = restoreGenerationDisk{StorageAPI: disks[i], bucket: r.bucket, object: r.object, write: &fail} + } + er.getDisks = func() []StorageAPI { return wrapped } + before := restoreGenerationMetadata(t, disks, r, oi.VersionID) + var attempt *restoreAttempt + if mode == "put-back" { + fail.Store(true) + rec := restoreIsolationPOST(t, r, r.z, oi.VersionID) + fail.Store(false) + if rec.Code < 400 { + t.Fatalf("POST acknowledged a status write on two of four drives: %d", rec.Code) + } + if after := restoreGenerationMetadata(t, disks, r, oi.VersionID); !reflect.DeepEqual(before, after) { + t.Error("failed status write was not put back") + } + } else { + // A status write and its put-back both failed on two drives. + attempt = newRestoreAttempt(oi) + for _, d := range disks[2:] { + fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true}) + if err != nil { + t.Fatal(err) + } + fi.Metadata[xhttp.AmzRestore] = ongoingRestoreObj().String() + fi.Metadata[restoreAttemptKey] = attempt.ID + if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + if _, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); err != nil { + t.Fatalf("a split in restore fields only has no generation: %v", err) + } + } + if mode == "split-reset" { + opts := nativeRestoreOptions(r, oi.VersionID) + opts.Transition.attempt = attempt + if err := r.z.resetRestoreStatus(r.bucket, r.object, opts); err != nil { + t.Fatalf("reset of the split attempt: %v", err) + } + for i, meta := range restoreGenerationMetadata(t, disks, r, oi.VersionID) { + if meta[xhttp.AmzRestore] != "" || meta[restoreAttemptKey] != "" { + t.Errorf("drive %d keeps the split attempt: %q %q", i, meta[xhttp.AmzRestore], meta[restoreAttemptKey]) + } + } + } + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST after the failed status write: %d %s", rec.Code, rec.Body.String()) + } + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } + }) + } +} + +// A split in any other field is not resolved: the generation fails closed. +func TestRestoreGenerationSplitNeedsQuorum(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, _ := restoreIsolationSeed(t, r, 0, false, false) + er := r.z.serverPools[0].getHashedSet(r.object) + for i, d := range er.getDisks()[2:] { + fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true}) + if err != nil { + t.Fatal(err) + } + fi.Metadata[xhttp.AmzRestore] = ongoingRestoreObj().String() + fi.Metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = fmt.Sprint("ON", i) + if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + if _, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); !errors.Is(err, errErasureReadQuorum) { + t.Fatalf("split outside the restore fields: %v, want read quorum", err) + } + if rec := restoreIsolationPOST(t, r, r.z, oi.VersionID); rec.Code < 400 { + t.Errorf("POST over a split outside the restore fields: %d", rec.Code) + } +} + +// restoreGenerationStageDisk holds a completion that is staging: as +// xlStorage.RenameData does, a drive that stages writes the committed +// object metadata into the upload before it renames the upload into the +// object. +type restoreGenerationStageDisk struct { + StorageAPI + bucket, object string + stage bool + arrived *atomic.Int32 + entered chan struct{} + release chan struct{} + disks int32 +} + +func (d restoreGenerationStageDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { + if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { + if d.stage { + if err := d.StorageAPI.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil { + return RenameDataResp{}, err + } + } + if d.arrived.Add(1) == d.disks { + close(d.entered) + } + select { + case <-d.release: + case <-ctx.Done(): + return RenameDataResp{}, ctx.Err() + } + } + return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) +} + +// While a completion stages its commit, the upload holds object metadata +// without the marker on the drives that staged. Nobody lists, aborts or +// removes it then, also when the copies not staged yet are old. +func TestRestoreUploadStagingFailsClosed(t *testing.T) { + for _, mode := range []string{"staged", "partly-staged", "stalled-commit"} { + t.Run(mode, func(t *testing.T) { testRestoreUploadStaging(t, mode) }) + } +} + +func testRestoreUploadStaging(t *testing.T, mode string) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var arrived atomic.Int32 + entered, release := make(chan struct{}), make(chan struct{}) + released := false + defer func() { + if !released { + close(release) + } + }() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: mode != "partly-staged" || i == 0, + arrived: &arrived, entered: entered, release: release, disks: int32(len(disks))} + } + er.getDisks = func() []StorageAPI { return wrapped } + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("completion never staged its commit") + } + stored := restoreIsolationUploads(t, r, 0) + var id string + for uploadUUID := range stored { + id = base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)) + if mode != "staged" { + // partly-staged: a long restore, the copies not staged yet are + // old. stalled-commit: drives that stall the running commit + // past the grace make its staged copies look abandoned. + old := UTCNow().Add(-48 * time.Hour) + for _, d := range disks[1:] { + if err := os.Chtimes(pathJoin(d.Endpoint().Path, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), uploadUUID, xlStorageFormatFile), old, old); err != nil { + t.Fatal(err) + } + } + } + } + if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10); err != nil || len(uploads.Uploads) != 0 { + t.Errorf("a staged restore upload is listed: %+v, %v", uploads.Uploads, err) + } + // The set's own abort: the pools layer would wait for the upload lock + // the completion holds. + if err := er.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); err == nil { + t.Error("abort of a staged restore upload succeeded") + } + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + for _, disk := range wrapped { + er.cleanupStaleUploadsOnDisk(t.Context(), disk) + } + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) { + t.Errorf("a staged restore upload was removed: %v -> %v", stored, after) + } + released = true + close(release) + if err := layer.wait(t); err != nil { + t.Fatalf("restore after its staging was attacked: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } +} + +// When a drive of an acknowledged attempt becomes unreadable, the drives that +// answer disagree on the restore status. That the attempt ended is then not +// established, and stale cleanup keeps its upload. +func TestRestoreSplitKeepsRunningUpload(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var missed, offline atomic.Bool + gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} + defer gate.unblock() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + var disk StorageAPI = restoreIsolationValidationDisk{d, gate} + switch i { + case 0: + disk = restoreGenerationDisk{StorageAPI: disk, bucket: r.bucket, object: r.object, write: &missed} + case 1: + disk = restoreGenerationOffline{StorageAPI: disk, bucket: r.bucket, object: r.object, fail: &offline} + } + wrapped[i] = disk + } + er.getDisks = func() []StorageAPI { return wrapped } + layer := newRestoreIsolationLayer(r) + missed.Store(true) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + missed.Store(false) + if rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + select { + case <-gate.entered: + case <-time.After(10 * time.Second): + t.Fatal("restore never reached completion") + } + stored := restoreIsolationUploads(t, r, 0) + offline.Store(true) + if gen, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); err != nil || !gen.split { + t.Fatalf("fixture: no split generation: %v", err) + } + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + for _, disk := range wrapped { + er.cleanupStaleUploadsOnDisk(t.Context(), disk) + } + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + offline.Store(false) + if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) { + t.Errorf("stale cleanup removed a running restore's upload: %v -> %v", stored, after) + } + gate.unblock() + if err := layer.wait(t); err != nil { + t.Fatalf("restore after the split: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } +} + +// restoreGenerationCleanupDisk misses restored-copy cleanups and metadata +// writes of the object while its toggles are set. +type restoreGenerationCleanupDisk struct { + StorageAPI + bucket, object string + cleanup, metaWrites *atomic.Bool +} + +func (d restoreGenerationCleanupDisk) DeleteVersion(ctx context.Context, bucket, object string, fi FileInfo, force bool, opts DeleteOptions) error { + if bucket == d.bucket && object == d.object && fi.ExpireRestored && d.cleanup.Load() { + return errFaultyDisk + } + return d.StorageAPI.DeleteVersion(ctx, bucket, object, fi, force, opts) +} + +func (d restoreGenerationCleanupDisk) WriteMetadata(ctx context.Context, orig, bucket, object string, fi FileInfo) error { + if bucket == d.bucket && object == d.object && d.metaWrites.Load() { + return errFaultyDisk + } + return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi) +} + +// Restored-copy expiry decides on the version's authoritative generation. A +// drive that missed a cleanup and the POST after it keeps an expired header, +// which must not clear the quorum's ongoing restore; drives split between +// an expired and a renewed header authorize nothing. +func TestExpireRestoredUsesQuorumGeneration(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, mode := range []string{"after-partial-cleanup", "split-renewal"} { + t.Run(fmt.Sprintf("pools=%d/%s", pools, mode), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi := r.put("cache", false, true) + r.restore(oi) + stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil { + t.Fatal(err) + } + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + expire := func() error { + return expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner) + } + if mode == "split-renewal" { + // A renewal reached two drives; the other two keep the + // expired header. + dataDir := r.fileInfo(oi.VersionID).DataDir + for _, d := range disks[2:] { + fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true}) + if err != nil { + t.Fatal(err) + } + fi.Metadata[xhttp.AmzRestore] = completedRestoreObj(UTCNow().Add(48 * time.Hour)).String() + if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + if err := expire(); err == nil || r.dataDirs(dataDir) != len(disks) { + t.Errorf("expiry on drives split between expired and renewed: %v, %d/%d cache dirs", err, r.dataDirs(dataDir), len(disks)) + } + return + } + var cleanup, metaWrites atomic.Bool + wrapped := append([]StorageAPI(nil), disks...) + wrapped[0] = restoreGenerationCleanupDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, cleanup: &cleanup, metaWrites: &metaWrites} + er.getDisks = func() []StorageAPI { return wrapped } + cleanup.Store(true) + err = expire() + cleanup.Store(false) + if err != nil { + t.Fatalf("cleanup on all drives but one: %v", err) + } + captured := make(chan ObjectOptions, 1) + layer := newRestoreIsolationLayer(r) + layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error { + captured <- opts + return nil + } + metaWrites.Store(true) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + metaWrites.Store(false) + if rec.Code != http.StatusOK { + t.Fatalf("POST after the cleanup: %d %s", rec.Code, rec.Body.String()) + } + opts := <-captured + layer.wait(t) + // The same cleanup, queued twice, runs again. + if err := expire(); err == nil { + t.Error("a drive's expired header cleared the quorum's ongoing restore") + } + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, opts); err != nil { + t.Fatalf("restore of the acknowledged attempt: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, r.bodies["cache"]) { + t.Fatalf("restored contents: %v", err) + } + }) + } + } +} + +// restoreGenerationHidden hides a drive's copy of the object version while +// set, as a drive that lost it and that heal writes again. +type restoreGenerationHidden struct { + StorageAPI + bucket, object string + hidden *atomic.Bool +} + +func (d restoreGenerationHidden) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) { + if d.hidden.Load() && bucket == d.bucket && object == d.object { + return FileInfo{}, errFileNotFound + } + return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts) +} + +// Heal copies the restore fields to a drive only as a quorum of the drives +// holds them. While they disagree, healing the version waits: it must not +// create a quorum for an expired header over a renewal, nor for a +// superseded attempt. Fields a quorum agrees on are copied. +func TestRestoreHealNeedsRestoreQuorum(t *testing.T) { + for _, mode := range []string{"expired-over-renewal", "superseded-attempt", "agreed"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + setRestore := func(versionID string, drives []StorageAPI, fields map[string]string) { + for _, d := range drives { + fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, versionID, ReadOptions{ReadData: true}) + if err != nil { + t.Fatal(err) + } + maps.Copy(fi.Metadata, fields) + if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + } + var versionID string + var stale ObjectInfo + var older ObjectOptions + hide := 3 + switch mode { + case "expired-over-renewal", "agreed": + oi := r.put("cache", false, true) + r.restore(oi) + versionID = oi.VersionID + var err error + if stale, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(versionID)); err != nil { + t.Fatal(err) + } + renewed := map[string]string{xhttp.AmzRestore: completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()} + if mode == "agreed" { + // Three drives renewed, the hidden one expired. + setRestore(versionID, disks[:3], renewed) + } else { + // No quorum: two drives expired an hour ago, one renewed, + // and the hidden one expired a day ago. + setRestore(versionID, disks[:2], map[string]string{xhttp.AmzRestore: completedRestoreObj(UTCNow().Add(-time.Hour)).String()}) + setRestore(versionID, disks[2:3], renewed) + } + case "superseded-attempt": + oi, _ := restoreIsolationSeed(t, r, 0, false, false) + versionID = oi.VersionID + captured := make(chan ObjectOptions, 1) + layer := newRestoreIsolationLayer(r) + layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error { + captured <- opts + return nil + } + if rec := restoreIsolationPOST(t, r, layer, versionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + older = <-captured + layer.wait(t) + // A newer attempt reached two drives. + setRestore(versionID, disks[2:], map[string]string{xhttp.AmzRestore: ongoingRestoreObj().String(), restoreAttemptKey: newRestoreAttempt(oi).ID}) + hide = 2 + } + before := restoreGenerationMetadata(t, disks, r, versionID) + var hidden atomic.Bool + wrapped := append([]StorageAPI(nil), disks...) + wrapped[hide] = restoreGenerationHidden{StorageAPI: disks[hide], bucket: r.bucket, object: r.object, hidden: &hidden} + er.getDisks = func() []StorageAPI { return wrapped } + hidden.Store(true) + _, healErr := er.HealObject(t.Context(), r.bucket, r.object, versionID, madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + hidden.Store(false) + er.getDisks = func() []StorageAPI { return disks } + after := restoreGenerationMetadata(t, disks, r, versionID) + if mode == "agreed" { + if healErr != nil || after[hide][xhttp.AmzRestore] != after[0][xhttp.AmzRestore] { + t.Errorf("heal did not copy the restore fields a quorum holds: %v, %q", healErr, after[hide][xhttp.AmzRestore]) + } + return + } + if healErr == nil { + t.Error("heal proceeded on drives that disagree on the restore fields") + } + if after[hide][xhttp.AmzRestore] != before[hide][xhttp.AmzRestore] || after[hide][restoreAttemptKey] != before[hide][restoreAttemptKey] { + t.Errorf("heal wrote a minority restore state: %q -> %q", before[hide][xhttp.AmzRestore], after[hide][xhttp.AmzRestore]) + } + if mode == "expired-over-renewal" { + dataDir := r.fileInfo(versionID).DataDir + if err := expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err == nil || r.dataDirs(dataDir) != len(disks) { + t.Errorf("expiry after heal: %v, %d/%d cache dirs", err, r.dataDirs(dataDir), len(disks)) + } + return + } + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, older); !errors.Is(err, errRestoreSuperseded) { + t.Errorf("superseded attempt after heal: %v", err) + } + }) + } +} + +// restoreGenerationStageFails fails the commit before it stages. +type restoreGenerationStageFails struct { + StorageAPI + bucket, object string +} + +func (d restoreGenerationStageFails) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { + if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { + return RenameDataResp{}, errFaultyDisk + } + return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) +} + +// restoreGenerationFailedCommit stages the committed metadata into the +// upload, as RenameData does, and then fails the commit. +type restoreGenerationFailedCommit struct { + StorageAPI + bucket, object string +} + +func (d restoreGenerationFailedCommit) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { + if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { + if err := d.StorageAPI.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil { + return RenameDataResp{}, err + } + return RenameDataResp{}, errFaultyDisk + } + return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) +} + +// restoreGenerationLeftover runs a restore whose commit fails after it +// staged on every drive, or on every drive but drive 0, which fails before +// it stages. It returns the upload left behind and its on-disk UUID. +func restoreGenerationLeftover(t *testing.T, r *restoreTest, unstaged bool) (map[string]int, string) { + t.Helper() + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, false, true) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + if unstaged && i == 0 { + wrapped[i] = restoreGenerationStageFails{StorageAPI: d, bucket: r.bucket, object: r.object} + continue + } + wrapped[i] = restoreGenerationFailedCommit{StorageAPI: d, bucket: r.bucket, object: r.object} + } + er.getDisks = func() []StorageAPI { return wrapped } + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + if err := layer.wait(t); err == nil { + t.Fatal("fixture: the failed commit succeeded") + } + er.getDisks = func() []StorageAPI { return disks } + stored := restoreIsolationUploads(t, r, 0) + if len(stored) != 1 { + t.Fatalf("fixture: no leftover of the failed commit: %v", stored) + } + for uploadUUID := range stored { + return stored, uploadUUID + } + return nil, "" +} + +// restoreGenerationAgeUpload makes every drive's copy of an upload's +// metadata look written d ago. +func restoreGenerationAgeUpload(t *testing.T, r *restoreTest, uploadUUID string, d time.Duration) { + t.Helper() + er := r.z.serverPools[0].getHashedSet(r.object) + old := UTCNow().Add(-d) + for _, disk := range er.getDisks() { + if err := os.Chtimes(pathJoin(disk.Endpoint().Path, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), uploadUUID, xlStorageFormatFile), old, old); err != nil { + t.Fatal(err) + } + } +} + +// What a failed commit leaves in an upload can always be removed once every +// copy is staged and older than the commit grace: abort removes it, also +// when stale cleanup could not. A recent staging, a copy that cannot be read +// and a copy not staged yet are refused, and nothing is removed then. +func TestRestoreAbandonedCommitAbort(t *testing.T) { + for _, age := range []string{"recent", "abandoned", "unreadable-copy", "unstaged-copy"} { + t.Run(age, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + stored, uploadUUID := restoreGenerationLeftover(t, r, age == "unstaged-copy") + if age != "recent" { + restoreGenerationAgeUpload(t, r, uploadUUID, 2*restoreCommitGrace) + } + id := base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var unreadable atomic.Bool + if age == "unreadable-copy" { + wrapped := append([]StorageAPI(nil), disks...) + wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], uploadRead: &unreadable} + er.getDisks = func() []StorageAPI { return wrapped } + unreadable.Store(true) + } + err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}) + unreadable.Store(false) + left := restoreIsolationUploads(t, r, 0) + if age != "abandoned" { + if err == nil || !reflect.DeepEqual(left, stored) { + t.Errorf("abort of an upload that may still commit (%s): %v, left %v", age, err, left) + } + return + } + if err != nil || len(left) != 0 { + t.Errorf("abort of an abandoned commit: %v, left %v", err, left) + } + }) + } +} + +// With no commit running, stale cleanup removes what a failed commit left +// drive by drive: a staged copy older than the commit grace. A recent +// staging and a copy that never staged stay. +func TestRestoreFailedCommitStaleCleanup(t *testing.T) { + for _, mode := range []string{"recent", "abandoned", "unstaged-copy"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + stored, uploadUUID := restoreGenerationLeftover(t, r, mode == "unstaged-copy") + if mode != "recent" { + // The copy that never staged is as old as the others. + restoreGenerationAgeUpload(t, r, uploadUUID, 2*restoreCommitGrace) + } + er := r.z.serverPools[0].getHashedSet(r.object) + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + for _, disk := range er.getDisks() { + er.cleanupStaleUploadsOnDisk(t.Context(), disk) + } + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + want := map[string]int{} + switch mode { + case "recent": + want = stored + case "unstaged-copy": + want = map[string]int{uploadUUID: 1} + } + if left := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(left, want) { + t.Errorf("stale cleanup of a failed commit's leftover: %v -> %v, want %v", stored, left, want) + } + }) + } +} + +// restoreGenerationSnapshotKey marks the context of a stale cleanup whose +// reads of upload metadata restoreGenerationSnapshotDisk holds. +type restoreGenerationSnapshotKey struct{} + +// restoreGenerationSnapshotDisk reads upload metadata at once, and hands the +// reply to a marked stale cleanup only on release. +type restoreGenerationSnapshotDisk struct { + StorageAPI + reads *atomic.Int32 + release chan struct{} +} + +func (d restoreGenerationSnapshotDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) { + fi, err := d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts) + if volume == minioMetaMultipartBucket && ctx.Value(restoreGenerationSnapshotKey{}) != nil { + d.reads.Add(1) + select { + case <-d.release: + case <-ctx.Done(): + return FileInfo{}, ctx.Err() + } + } + return fi, err +} + +// Stale cleanup decides on a removal under the commit lock. A completion +// that starts while cleanup reads an upload past the stale age waits for +// it, and a cleanup that comes later skips the running commit: the upload, +// an ordinary one, still commits with its contents. +func TestCleanupStaleUploadsSkipsRunningCommit(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + er := r.z.serverPools[0].getHashedSet(r.object) + mp, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true, MTime: UTCNow().Add(-48 * time.Hour)}) + if err != nil { + t.Fatal(err) + } + body := []byte("an ordinary upload past the stale age") + part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + disks := er.getDisks() + var reads, staged atomic.Int32 + readRelease := make(chan struct{}) + stageEntered, stageRelease := make(chan struct{}), make(chan struct{}) + var readOnce, stageOnce sync.Once + defer readOnce.Do(func() { close(readRelease) }) + defer stageOnce.Do(func() { close(stageRelease) }) + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + stage := restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: true, + arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks))} + wrapped[i] = restoreGenerationSnapshotDisk{StorageAPI: stage, reads: &reads, release: readRelease} + } + er.getDisks = func() []StorageAPI { return wrapped } + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + defer func() { + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + }() + // Every drive's cleanup at once, each reading the upload on every drive. + ctx := context.WithValue(t.Context(), restoreGenerationSnapshotKey{}, true) + var cleanups sync.WaitGroup + for _, disk := range wrapped { + cleanups.Go(func() { er.cleanupStaleUploadsOnDisk(ctx, disk) }) + } + cleanupDone := make(chan struct{}) + go func() { cleanups.Wait(); close(cleanupDone) }() + // The completion starts once every cleanup that may decide holds its + // read of the upload from before the commit. + for deadline := time.Now().Add(2 * time.Second); reads.Load() < int32(len(disks)*len(disks)) && time.Now().Before(deadline); { + time.Sleep(10 * time.Millisecond) + } + completion := make(chan ObjectInfo, 1) + completionErr := make(chan error, 1) + go func() { + oi, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true}) + completion <- oi + completionErr <- err + }() + select { + case <-stageEntered: + case <-time.After(3 * time.Second): + } + readOnce.Do(func() { close(readRelease) }) + select { + case <-cleanupDone: + case <-time.After(10 * time.Second): + t.Fatal("stale cleanup never returned") + } + stageOnce.Do(func() { close(stageRelease) }) + oi := <-completion + if err := <-completionErr; err != nil { + t.Fatalf("completion of an upload stale cleanup read: %v", err) + } + er.getDisks = func() []StorageAPI { return disks } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("completed contents: %q, %v", got, err) + } +} + +// restoreGenerationRand draws UUIDs from crypto/rand for the whole test +// binary, installed before any goroutine runs. While a pause is set, it holds +// the UUID that stale cleanup draws in its deadline-bound worker, for the +// trash path right before its rename, until the pause is released. +var restoreGenerationRand = &restoreGenerationRandReader{} + +func init() { uuid.SetRand(restoreGenerationRand) } + +type restoreGenerationRandReader struct { + pause atomic.Pointer[restoreGenerationPause] +} + +type restoreGenerationPause struct { + entered, release chan struct{} +} + +func (r *restoreGenerationRandReader) Read(b []byte) (int, error) { + if p := r.pause.Load(); p != nil && restoreGenerationInCleanupRename() && r.pause.CompareAndSwap(p, nil) { + close(p.entered) + <-p.release + } + return rand.Read(b) +} + +// restoreGenerationInCleanupRename reports whether the caller runs in stale +// cleanup's deadline-bound worker. +func restoreGenerationInCleanupRename() bool { + pcs := make([]uintptr, 32) + frames := runtime.CallersFrames(pcs[:runtime.Callers(3, pcs)]) + cleanup, worker := false, false + for { + f, more := frames.Next() + cleanup = cleanup || strings.Contains(f.Function, "cleanupStaleUploadsOnDisk") + worker = worker || strings.Contains(f.Function, "ioutil.WithDeadline") + if !more { + return cleanup && worker + } + } +} + +// Stale cleanup holds the commit lock until its rename ends, also when the +// rename outlives the drive deadline and cleanup has moved on: a completion +// does not stage while that rename can still remove its upload. +func TestCleanupStaleUploadsHoldsLockThroughRename(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + er := r.z.serverPools[0].getHashedSet(r.object) + mp, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true, MTime: UTCNow().Add(-48 * time.Hour)}) + if err != nil { + t.Fatal(err) + } + body := []byte("an upload whose stale cleanup outlives the drive deadline") + part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + disks := er.getDisks() + stored := restoreIsolationUploads(t, r, 0) + prev := globalDriveConfig.GetMaxTimeout() + // The shortest drive deadline the configuration takes. + globalDriveConfig.Update(drive.Config{MaxTimeout: 1100 * time.Millisecond}) + defer globalDriveConfig.Update(drive.Config{MaxTimeout: prev}) + pause := &restoreGenerationPause{entered: make(chan struct{}), release: make(chan struct{})} + var paused sync.Once + defer paused.Do(func() { close(pause.release) }) + restoreGenerationRand.pause.Store(pause) + defer restoreGenerationRand.pause.CompareAndSwap(pause, nil) + + cleanupDone := make(chan struct{}) + go func() { + er.cleanupStaleUploadsOnDisk(t.Context(), disks[0]) + close(cleanupDone) + }() + select { + case <-pause.entered: + case <-time.After(5 * time.Second): + t.Fatal("stale cleanup never reached its rename") + } + select { + case <-cleanupDone: + case <-time.After(5 * time.Second): + t.Fatal("stale cleanup did not move on after the drive deadline") + } + if left := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(left, stored) { + t.Fatalf("fixture: the held rename already ran: %v -> %v", stored, left) + } + + var staged atomic.Int32 + stageEntered, stageRelease := make(chan struct{}), make(chan struct{}) + var stageOnce sync.Once + defer stageOnce.Do(func() { close(stageRelease) }) + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: true, + arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks))} + } + er.getDisks = func() []StorageAPI { return wrapped } + completion := make(chan ObjectInfo, 1) + completionErr := make(chan error, 1) + go func() { + oi, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true}) + completion <- oi + completionErr <- err + }() + select { + case <-stageEntered: + t.Error("a completion staged while stale cleanup's rename could still run") + case <-time.After(2 * time.Second): + } + paused.Do(func() { close(pause.release) }) + select { + case <-stageEntered: + case <-time.After(10 * time.Second): + t.Fatal("completion never staged after the rename ended") + } + stageOnce.Do(func() { close(stageRelease) }) + oi := <-completion + if err := <-completionErr; err != nil { + t.Fatalf("completion after the rename ended: %v", err) + } + er.getDisks = func() []StorageAPI { return disks } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("completed contents: %q, %v", got, err) + } +} diff --git a/cmd/restore-native-isolation_test.go b/cmd/restore-native-isolation_test.go new file mode 100644 index 000000000..232eb4323 --- /dev/null +++ b/cmd/restore-native-isolation_test.go @@ -0,0 +1,1397 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/versioning" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/mux" +) + +// The private metadata of a native restore, as stored. Literal names keep +// these regressions meaningful on a tree without the keys. +const ( + restoreIsolationAttemptKey = "x-minio-internal-restore-attempt" + restoreIsolationUploadKey = "x-minio-internal-restore-upload" +) + +// restoreIsolationLayer hooks the object layer a POST restore handler uses. +// afterRead runs once POST has read the version, before POST records its +// status. restore, when set, replaces the background restore; every +// background restore result is sent to done. +type restoreIsolationLayer struct { + ObjectLayer + afterRead func() + once sync.Once + restore func(ctx context.Context, bucket, object string, opts ObjectOptions) error + done chan error +} + +func newRestoreIsolationLayer(r *restoreTest) *restoreIsolationLayer { + return &restoreIsolationLayer{ObjectLayer: r.z, done: make(chan error, 4)} +} + +func (l *restoreIsolationLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + oi, err := l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts) + if err == nil && l.afterRead != nil { + l.once.Do(l.afterRead) + } + return oi, err +} + +func (l *restoreIsolationLayer) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + var err error + if l.restore != nil { + err = l.restore(ctx, bucket, object, opts) + } else { + err = l.ObjectLayer.RestoreTransitionedObject(ctx, bucket, object, opts) + } + l.done <- err + return err +} + +// wait returns the result of the next background restore. +func (l *restoreIsolationLayer) wait(t *testing.T) error { + t.Helper() + select { + case err := <-l.done: + return err + case <-time.After(10 * time.Second): + t.Fatal("background restore did not finish") + return nil + } +} + +func restoreIsolationPOST(t *testing.T, r *restoreTest, layer ObjectLayer, versionID string) *httptest.ResponseRecorder { + t.Helper() + body := []byte(`1`) + path := "/" + r.bucket + "/" + r.object + "?restore" + if versionID != "" { + path += "&versionId=" + url.QueryEscape(versionID) + } + req, err := newTestSignedRequestV4(http.MethodPost, path, int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + if err := req.ParseForm(); err != nil { + t.Fatal(err) + } + req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object}) + rec := httptest.NewRecorder() + api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return layer }} + api.PostRestoreObjectHandler(rec, req) + return rec +} + +// restoreIsolationSeed is a transitioned version without the ongoing marker +// that nativeRestoreSeed leaves for direct restores: POST sets its own. +func restoreIsolationSeed(t *testing.T, r *restoreTest, pool int, null, multipart bool) (ObjectInfo, []byte) { + t.Helper() + oi, body := nativeRestoreSeed(t, r, pool, null, multipart) + opts := r.opts(oi.VersionID) + opts.MTime = oi.ModTime + opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[xhttp.AmzRestore] = "" + return ReplicateDecision{}, nil + } + if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil { + t.Fatal(err) + } + return oi, body +} + +// restoreIsolationSettled waits until the version is no longer ongoing. +func restoreIsolationSettled(t *testing.T, r *restoreTest, versionID string) ObjectInfo { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + oi, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(versionID)) + if err == nil && !oi.RestoreOngoing { + return oi + } + if time.Now().After(deadline) { + t.Fatalf("version stays ongoing, refusing every later POST: %q, %v", oi.UserDefined[xhttp.AmzRestore], err) + } + time.Sleep(20 * time.Millisecond) + } +} + +// restoreIsolationUploads returns the upload directories of the object on +// each drive of a pool, as stored, whatever listings show. +func restoreIsolationUploads(t *testing.T, r *restoreTest, pool int) map[string]int { + t.Helper() + er := r.z.serverPools[pool].getHashedSet(r.object) + found := make(map[string]int) + for _, disk := range er.getDisks() { + entries, err := disk.ListDir(t.Context(), r.bucket, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), -1) + if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) { + continue + } + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + found[strings.TrimSuffix(entry, SlashSeparator)]++ + } + } + return found +} + +// restoreIsolationUpload finds the single upload of the object in a pool on +// disk, with its first part, and returns its public upload ID. +func restoreIsolationUpload(t *testing.T, r *restoreTest, pool int) (string, ObjectPartInfo) { + t.Helper() + uploads := restoreIsolationUploads(t, r, pool) + if len(uploads) != 1 { + t.Fatalf("want one upload on disk, found %v", uploads) + } + var uploadUUID string + for uploadUUID = range uploads { + } + er := r.z.serverPools[pool].getHashedSet(r.object) + uploadPath := pathJoin(er.getMultipartSHADir(r.bucket, r.object), uploadUUID) + fi, err := er.getDisks()[0].ReadVersion(t.Context(), "", minioMetaMultipartBucket, uploadPath, "", ReadOptions{}) + if err != nil { + t.Fatal(err) + } + parts, err := readParts(t.Context(), er.getDisks(), minioMetaMultipartBucket, []string{pathJoin(uploadPath, fi.DataDir, "part.1.meta")}, []int{1}, 1) + if err != nil || len(parts) != 1 || parts[0].Error != "" { + t.Fatalf("restore must have published its first part: %+v, %v", parts, err) + } + return base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)), parts[0] +} + +// restoreIsolationGateTier holds a download after its first bytes and can +// fail it once released. +type restoreIsolationGateTier struct { + WarmBackend + first int + entered chan struct{} + release chan struct{} + failure error + enter sync.Once + open sync.Once +} + +func newRestoreIsolationGateTier(first int, failure error) *restoreIsolationGateTier { + return &restoreIsolationGateTier{ + WarmBackend: globalTierConfigMgr.drivercache["WARM"], + first: first, failure: failure, + entered: make(chan struct{}), release: make(chan struct{}), + } +} + +func (b *restoreIsolationGateTier) unblock() { b.open.Do(func() { close(b.release) }) } + +func (b *restoreIsolationGateTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) { + rd, err := b.WarmBackend.Get(ctx, object, version, opts) + if err != nil { + return nil, err + } + return &restoreIsolationGateReader{ReadCloser: rd, tier: b, ctx: ctx, remaining: b.first}, nil +} + +type restoreIsolationGateReader struct { + io.ReadCloser + tier *restoreIsolationGateTier + ctx context.Context + remaining int +} + +func (r *restoreIsolationGateReader) Read(p []byte) (int, error) { + if r.remaining > 0 { + if len(p) > r.remaining { + p = p[:r.remaining] + } + n, err := r.ReadCloser.Read(p) + r.remaining -= n + return n, err + } + r.tier.enter.Do(func() { close(r.tier.entered) }) + select { + case <-r.tier.release: + case <-r.ctx.Done(): + return 0, r.ctx.Err() + } + if r.tier.failure != nil { + return 0, r.tier.failure + } + return r.ReadCloser.Read(p) +} + +func restoreIsolationEntered(t *testing.T, tier *restoreIsolationGateTier) { + t.Helper() + select { + case <-tier.entered: + case <-time.After(10 * time.Second): + t.Fatal("restore never reached its download gate") + } +} + +// restoreIsolationFaultDisk fails metadata reads of the object while +// failReads is set, and reports writes of its metadata as failed, after +// they were made, while failWrites is set. Toggles keep the wrapper in +// place, so no restore retry races a swap of the drives. +type restoreIsolationFaultDisk struct { + StorageAPI + bucket, object string + failReads *atomic.Bool + failWrites *atomic.Bool +} + +func (d restoreIsolationFaultDisk) ReadVersion(ctx context.Context, origvolume, volume, path, version string, opts ReadOptions) (FileInfo, error) { + if d.failReads.Load() && volume == d.bucket && path == d.object { + return FileInfo{}, errDiskNotFound + } + return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, version, opts) +} + +func (d restoreIsolationFaultDisk) ReadXL(ctx context.Context, volume, path string, readData bool) (RawFileInfo, error) { + if d.failReads.Load() && volume == d.bucket && path == d.object { + return RawFileInfo{}, errDiskNotFound + } + return d.StorageAPI.ReadXL(ctx, volume, path, readData) +} + +func (d restoreIsolationFaultDisk) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error { + err := d.StorageAPI.WriteMetadata(ctx, origvolume, volume, path, fi) + if err == nil && d.failWrites != nil && d.failWrites.Load() && volume == d.bucket && path == d.object { + return errFaultyDisk + } + return err +} + +// installRestoreIsolationFaults wraps the drives of the object's set in a +// pool for the rest of the test; failing marks the drives whose writes fail. +func installRestoreIsolationFaults(r *restoreTest, pool int, failing int) (reads, writes *atomic.Bool) { + reads, writes = new(atomic.Bool), new(atomic.Bool) + er := r.z.serverPools[pool].getHashedSet(r.object) + disks := er.getDisks() + wrapped := make([]StorageAPI, len(disks)) + for i, disk := range disks { + d := restoreIsolationFaultDisk{StorageAPI: disk, bucket: r.bucket, object: r.object, failReads: reads} + if i < failing { + d.failWrites = writes + } + wrapped[i] = d + } + er.getDisks = func() []StorageAPI { return wrapped } + return reads, writes +} + +// A public client must not replace a part of a native restore's upload, +// before or after the restore validated it: the bytes would become the +// COMPLIANCE-protected version under its original ETag. +func TestRestoreUploadIsolatedFromPublicParts(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + enableBucketObjectLock(t, r.bucket) + previous := newObjectLayerFn() + t.Cleanup(func() { setObjectLayer(previous) }) + router := initTestAPIEndPoints(r.z, []string{"PutObjectPart", "GetObject"}) + oi, body := nativeRestoreSeed(t, r, 0, false, true) + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, "retention", retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))); rec.Code != http.StatusOK { + t.Fatalf("lock: %d %s", rec.Code, rec.Body.String()) + } + // Hold completion after it validated the parts, before it commits. + er := r.z.serverPools[0].getHashedSet(r.object) + original := er.getDisks + gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} + defer gate.unblock() + disks := original() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreIsolationCompletionDisk{d, gate} + } + er.getDisks = func() []StorageAPI { return wrapped } + defer func() { er.getDisks = original }() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + restored := make(chan error, 1) + go func() { + restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + select { + case <-gate.entered: + case err := <-restored: + t.Fatalf("restore stopped before validation: %v", err) + case <-ctx.Done(): + t.Fatal("no completion gate") + } + id, _ := restoreIsolationUpload(t, r, 0) + poison := bytes.Repeat([]byte("z"), 5<<20) + // The request may wait for the upload lock that completion holds. + uploaded := make(chan *httptest.ResponseRecorder, 1) + go func() { + uploaded <- attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodPut, "/"+r.bucket+"/"+r.object+"?uploadId="+url.QueryEscape(id)+"&partNumber=1", poison, nil) + }() + var rec *httptest.ResponseRecorder + select { + case rec = <-uploaded: + case <-time.After(500 * time.Millisecond): + } + gate.unblock() + if err := <-restored; err != nil { + t.Fatalf("restore: %v", err) + } + if rec == nil { + rec = <-uploaded + } + if rec.Code != http.StatusNotFound || !strings.Contains(rec.Body.String(), "NoSuchUpload") { + t.Errorf("public UploadPart on a native restore upload: %d %s, want NoSuchUpload", rec.Code, rec.Body.String()) + } + got := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, "/"+r.bucket+"/"+r.object+"?versionId="+oi.VersionID, nil, nil) + if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), body) { + t.Errorf("COMPLIANCE version holds other bytes after restore: GET=%d poisoned=%v", got.Code, bytes.HasPrefix(got.Body.Bytes(), poison)) + } +} + +type restoreIsolationCompletionGate struct { + entered, release chan struct{} + enter, open sync.Once +} + +func (g *restoreIsolationCompletionGate) unblock() { g.open.Do(func() { close(g.release) }) } + +type restoreIsolationCompletionDisk struct { + StorageAPI + gate *restoreIsolationCompletionGate +} + +// Completion removes the part metadata it validated just before it renames +// the upload into place. +func (d restoreIsolationCompletionDisk) DeleteBulk(ctx context.Context, bucket string, paths ...string) error { + if bucket == minioMetaMultipartBucket && len(paths) == 2 && strings.HasSuffix(paths[0], ".meta") && strings.HasSuffix(paths[1], ".meta") { + d.gate.enter.Do(func() { close(d.gate.entered) }) + select { + case <-d.gate.release: + case <-ctx.Done(): + return ctx.Err() + } + } + return d.StorageAPI.DeleteBulk(ctx, bucket, paths...) +} + +// restoreIsolationValidationDisk holds a completion where it reads the parts +// it validates: after its object and upload locks, before its commit lock. +// Stale cleanup then decides on the upload itself, not on a running commit. +type restoreIsolationValidationDisk struct { + StorageAPI + gate *restoreIsolationCompletionGate +} + +func (d restoreIsolationValidationDisk) ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error) { + if bucket == minioMetaMultipartBucket { + d.gate.enter.Do(func() { close(d.gate.entered) }) + select { + case <-d.gate.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return d.StorageAPI.ReadParts(ctx, bucket, partMetaPaths...) +} + +// Every public multipart entry point finds no native restore upload, and +// none of them can change it: the restore still completes with its own data. +func TestRestoreUploadHiddenFromPublicEntryPoints(t *testing.T) { + for _, pools := range []int{1, 2} { + t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, 0, false, true) + tier := newRestoreIsolationGateTier(5<<20, nil) + globalTierConfigMgr.drivercache["WARM"] = tier + defer tier.unblock() + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + restored := make(chan error, 1) + go func() { + restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + restoreIsolationEntered(t, tier) + id, part := restoreIsolationUpload(t, r, 0) + stored := restoreIsolationUploads(t, r, 0) + + for _, prefix := range []string{r.object, ""} { + uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, prefix, "", "", "", 100) + if err != nil { + t.Fatal(err) + } + for _, u := range uploads.Uploads { + if u.UploadID == id { + t.Errorf("ListMultipartUploads(prefix=%q) shows the native restore upload", prefix) + } + } + } + poison := bytes.Repeat([]byte("z"), 5<<20) + noUpload := func(op string, err error) { + t.Helper() + if !errors.As(err, &InvalidUploadID{}) { + t.Errorf("%s on a native restore upload: %v, want NoSuchUpload", op, err) + } + } + // With one pool, part writes first wait for the object lock the + // restore holds; with more they reach the upload at once. + written := make(chan error, 2) + go func() { + _, err := r.z.PutObjectPart(ctx, r.bucket, r.object, id, 1, mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", ""), ObjectOptions{}) + written <- err + }() + go func() { + src := ObjectInfo{PutObjReader: mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", "")} + _, err := r.z.CopyObjectPart(ctx, r.bucket, r.object, r.bucket, r.object, id, 1, 0, int64(len(poison)), src, ObjectOptions{}, ObjectOptions{}) + written <- err + }() + var writes []error + if pools > 1 { + writes = append(writes, <-written, <-written) + } + _, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{}) + noUpload("ListObjectParts", err) + _, err = r.z.GetMultipartInfo(t.Context(), r.bucket, r.object, id, ObjectOptions{}) + noUpload("GetMultipartInfo", err) + err = r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}) + noUpload("AbortMultipartUpload", err) + if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) { + t.Errorf("public calls changed the restore upload on disk: %v -> %v", stored, after) + } + // Completion waits for the object lock the restore holds. + completed := make(chan error, 1) + go func() { + _, err := r.z.CompleteMultipartUpload(ctx, r.bucket, r.object, id, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, r.opts(oi.VersionID)) + completed <- err + }() + tier.unblock() + if err := <-restored; err != nil { + t.Fatalf("restore after public calls: %v", err) + } + for len(writes) < 2 { + writes = append(writes, <-written) + } + for _, err := range writes { + noUpload("PutObjectPart/CopyObjectPart", err) + } + noUpload("CompleteMultipartUpload", <-completed) + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents after public calls: %v", err) + } + }) + } +} + +// POST records its status on a fresh read under the object lock, so an +// Object Lock update it did not see survives. +func TestRestorePOSTKeepsConcurrentProtection(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, kind := range []string{"legal-hold", "retention"} { + t.Run(fmt.Sprintf("pools=%d/%s", pools, kind), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + enableBucketObjectLock(t, r.bucket) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + initial := []byte(`OFF`) + updated := []byte(`ON`) + key := strings.ToLower(xhttp.AmzObjectLockLegalHold) + if kind == "retention" { + initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour)) + updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour)) + key = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate) + } + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK { + t.Fatalf("initial protection: %d %s", rec.Code, rec.Body.String()) + } + var accepted ObjectInfo + layer := newRestoreIsolationLayer(r) + layer.afterRead = func() { + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK { + t.Fatalf("update: %d %s", rec.Code, rec.Body.String()) + } + var err error + if accepted, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err != nil { + t.Fatal(err) + } + } + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code >= 400 { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil { + t.Fatal(err) + } + if accepted.UserDefined[key] == "" || after.UserDefined[key] != accepted.UserDefined[key] { + t.Errorf("POST restore overwrote acknowledged Object Lock metadata: %q -> %q", accepted.UserDefined[key], after.UserDefined[key]) + } + if !after.ModTime.Equal(oi.ModTime) || after.ETag != oi.ETag { + t.Error("POST restore changed the version identity") + } + }) + } + } +} + +// A restored copy removed between POST's read and its status write leads to +// a new restore, never to a completed header without the restored data. +func TestRestorePOSTRacesCacheExpiry(t *testing.T) { + for _, pools := range []int{1, 2} { + t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi := r.put("restored", false, true) + r.restore(oi) + layer := newRestoreIsolationLayer(r) + layer.afterRead = func() { + if err := expireTransitionedObject(t.Context(), r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err != nil { + t.Fatal(err) + } + } + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code >= 400 { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + select { + case err := <-layer.done: + if err != nil { + t.Fatalf("rehydration: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("POST renewed a removed restored copy instead of restoring it") + } + _, body, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(body, r.bodies["restored"]) { + t.Errorf("version unreadable after POST raced cache expiry: %v", err) + } + }) + } +} + +// A restore queued with the versioning POST captured writes back the null +// version it resolved, even after versioning was enabled in between. +func TestRestoreKeepsCapturedNullIdentity(t *testing.T) { + enable := func(t *testing.T, r *restoreTest) { + meta, err := globalBucketMetadataSys.Get(r.bucket) + if err != nil { + t.Fatal(err) + } + meta.VersioningConfigXML = []byte(`Enabled`) + if err := meta.Save(t.Context(), r.z); err != nil { + t.Fatal(err) + } + globalBucketMetadataSys.Set(r.bucket, meta) + } + for _, mode := range []string{"direct", "post"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, "") + initConfigSubsystem(t.Context(), r.z) + var oi ObjectInfo + var body []byte + if mode == "direct" { + oi, body = nativeRestoreSeed(t, r, 0, true, false) + // These are the options that POST captured while versioning was off. + queued := nativeRestoreOptions(r, "") + enable(t, r) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, queued); err != nil { + t.Fatal(err) + } + } else { + oi, body = restoreIsolationSeed(t, r, 0, true, false) + release := make(chan struct{}) + layer := newRestoreIsolationLayer(r) + layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + <-release + return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) + } + if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + enable(t, r) + close(release) + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + } + versions := nativeRestoreSnapshot(t, r)[0] + if len(versions) != 1 || versions[0].VersionID != "" { + t.Errorf("restore created a UUID alias of the queued null version: %d versions", len(versions)) + } + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("null original: %v", err) + } + }) + } +} + +// A restore that fails before it reads the version, on its lock wait or on +// its first read, must not leave POST's ongoing marker for every later POST +// to refuse: bounded retries clear it once the version is readable again. +func TestRestoreRecoversStrandedMarker(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, stage := range []string{"lock-wait", "initial-read"} { + t.Run(fmt.Sprintf("pools=%d/%s", pools, stage), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + var err error + if stage == "lock-wait" { + lk := r.z.NewNSLock(r.bucket, r.object) + if pools == 1 { + lk = r.z.serverPools[0].getHashedSet(r.object).NewNSLock(r.bucket, r.object) + } + lkctx, lerr := lk.GetLock(t.Context(), globalOperationTimeout) + if lerr != nil { + t.Fatal(lerr) + } + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + err = r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + cancel() + lk.Unlock(lkctx) + } else { + reads, _ := installRestoreIsolationFaults(r, 0, 0) + reads.Store(true) + err = r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + reads.Store(false) + } + if err == nil { + t.Fatal("restore succeeded without its lock or its version") + } + after := restoreIsolationSettled(t, r, oi.VersionID) + if !after.IsRemote() || !after.ModTime.Equal(oi.ModTime) { + t.Error("recovery changed more than the restore status") + } + }) + } + } +} + +// A later POST takes over a marker that no running restore holds, and the +// superseded attempt then neither downloads nor clears the newer status, +// also when both POSTs share a second, the precision of the request date. +func TestRestoreSupersededAttempt(t *testing.T) { + for _, mode := range []string{"newer-completes", "newer-fails"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + release := make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + older := newRestoreIsolationLayer(r) + older.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + <-release + return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) + } + newer := newRestoreIsolationLayer(r) + time.Sleep(time.Until(time.Now().Truncate(time.Second).Add(time.Second + 10*time.Millisecond))) + if rec := restoreIsolationPOST(t, r, older, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("first POST: %d %s", rec.Code, rec.Body.String()) + } + first := r.fileInfo(oi.VersionID).Metadata[xhttp.AmzRestoreRequestDate] + if mode == "newer-fails" { + globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, errors.New("injected remote failure")} + } + if rec := restoreIsolationPOST(t, r, newer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST over a marker no restore holds: %d %s", rec.Code, rec.Body.String()) + } + second := r.fileInfo(oi.VersionID).Metadata[xhttp.AmzRestoreRequestDate] + t.Logf("request dates %q %q, same second: %v", first, second, first == second) + newerErr := newer.wait(t) + globalTierConfigMgr.drivercache["WARM"] = tier + before := r.fileInfo(oi.VersionID) + gets := tier.gets.Load() + unblock() + if err := older.wait(t); err == nil { + t.Error("superseded restore reported success") + } + if tier.gets.Load() != gets { + t.Error("superseded restore downloaded the version again") + } + after := r.fileInfo(oi.VersionID) + if !reflect.DeepEqual(before.Metadata, after.Metadata) || before.DataDir != after.DataDir { + t.Errorf("superseded restore changed the version: %q -> %q", before.Metadata[xhttp.AmzRestore], after.Metadata[xhttp.AmzRestore]) + } + info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil || info.RestoreOngoing { + t.Fatalf("version left ongoing: %q, %v", info.UserDefined[xhttp.AmzRestore], err) + } + if mode == "newer-completes" { + if newerErr != nil || info.IsRemote() { + t.Fatalf("newer restore: %v remote=%v", newerErr, info.IsRemote()) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) || tier.gets.Load() != gets { + t.Errorf("restored contents: %v", err) + } + return + } + if newerErr == nil || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" { + t.Fatalf("failed newer restore is not retryable: %v %q", newerErr, info.UserDefined[xhttp.AmzRestore]) + } + if rec := restoreIsolationPOST(t, r, newer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("retry POST: %d %s", rec.Code, rec.Body.String()) + } + if err := newer.wait(t); err != nil { + t.Fatal(err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Errorf("retried restore contents: %v", err) + } + }) + } +} + +// A restart loses the restore that POST started; a POST to the restarted +// server takes the stranded marker over and restores the version. +func TestRestoreRestartTakeover(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + lost := newRestoreIsolationLayer(r) + lost.restore = func(context.Context, string, string, ObjectOptions) error { return nil } + if rec := restoreIsolationPOST(t, r, lost, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST before restart: %d %s", rec.Code, rec.Body.String()) + } + lost.wait(t) + if info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err != nil || !info.RestoreOngoing { + t.Fatalf("fixture: %q %v", info.UserDefined[xhttp.AmzRestore], err) + } + + endpoints := make(EndpointServerPools, len(r.z.serverPools)) + for i, pool := range r.z.serverPools { + endpoints[i] = pool.endpoints + } + obj, err := newErasureServerPools(t.Context(), endpoints) + if err != nil { + t.Fatal(err) + } + restarted := obj.(*erasureServerPools) + t.Cleanup(func() { restarted.Shutdown(context.Background()) }) + r.z = restarted + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST after restart: %d %s", rec.Code, rec.Body.String()) + } + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + info := restoreIsolationSettled(t, r, oi.VersionID) + if info.IsRemote() || info.UserDefined[restoreIsolationAttemptKey] != "" { + t.Errorf("restore after restart incomplete: remote=%v attempt=%q", info.IsRemote(), info.UserDefined[restoreIsolationAttemptKey]) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Errorf("contents after restart: %v", err) + } +} + +// A null version replaced while its restore was queued is another version: +// the restore must not rehydrate it, even if it is transitioned as well. +func TestRestoreReplacementNullObject(t *testing.T) { + for _, state := range []versioning.State{"", versioning.Suspended} { + t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, state) + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, true, false) + release := make(chan struct{}) + layer := newRestoreIsolationLayer(r) + layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + <-release + return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) + } + if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code != http.StatusOK { + close(release) + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + replacement := []byte("replacement null object") + repl, err := r.z.serverPools[0].PutObject(t.Context(), r.bucket, r.object, + mustGetPutObjReader(t, bytes.NewReader(replacement), int64(len(replacement)), "", ""), + ObjectOptions{VersionSuspended: state == versioning.Suspended}) + if err != nil { + close(release) + t.Fatal(err) + } + topts := r.opts(nullVersionID) + topts.MTime = repl.ModTime + topts.Transition = TransitionOptions{Tier: "WARM", ETag: repl.ETag} + if err := r.z.serverPools[0].TransitionObject(t.Context(), r.bucket, r.object, topts); err != nil { + close(release) + t.Fatal(err) + } + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + before := r.fileInfo(nullVersionID) + gets := tier.gets.Load() + close(release) + layer.wait(t) + after := r.fileInfo(nullVersionID) + if tier.gets.Load() != gets || after.Metadata[xhttp.AmzRestore] != "" || !reflect.DeepEqual(before.Metadata, after.Metadata) { + t.Errorf("queued restore rehydrated the replacement null object: %q", after.Metadata[xhttp.AmzRestore]) + } + if after.ModTime.Equal(oi.ModTime) || !after.ModTime.Equal(repl.ModTime) { + t.Fatal("fixture did not replace the null version") + } + if _, got, err := r.read(nullVersionID); err != nil || !bytes.Equal(got, replacement) { + t.Errorf("replacement contents: %v", err) + } + }) + } +} + +// POST's own status write can fail after its ongoing marker reached the +// drives; the marker must not stay for every later POST to refuse. +func TestRestorePOSTStatusWritePartialFailure(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + _, writes := installRestoreIsolationFaults(r, 0, 2) + layer := newRestoreIsolationLayer(r) + writes.Store(true) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + writes.Store(false) + if rec.Code < 400 { + t.Fatalf("POST acknowledged a failed status write: %d %s", rec.Code, rec.Body.String()) + } + if info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err == nil && !info.RestoreOngoing { + t.Log("the failed status write left no marker in quorum") + } + restoreIsolationSettled(t, r, oi.VersionID) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST after a failed status write: %d %s", rec.Code, rec.Body.String()) + } + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Errorf("restore after a failed status write: %v", err) + } +} + +// Duplicate copies of a version in two pools can differ in whether they +// hold a restored copy. A completed restore header must never reach a copy +// without one; POST renews or restores only the copy that restores serve. +func TestRestoreDuplicatePoolCacheStates(t *testing.T) { + for _, mode := range []string{"suspended-owner", "unrestored-first", "restored-first"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + // The duplicate is the same stored version, metadata only. + src := r.z.serverPools[0].getHashedSet(r.object).getDisks() + dst := r.z.serverPools[1].getHashedSet(r.object).getDisks() + for i := range src { + raw, err := src[i].ReadAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile)) + if err != nil { + t.Fatal(err) + } + if err := dst[i].WriteAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile), raw); err != nil { + t.Fatal(err) + } + } + owner := 0 + if mode == "unrestored-first" { + owner = 1 + } + if err := r.z.serverPools[owner].RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + if mode == "suspended-owner" { + r.z.poolMetaMutex.Lock() + r.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{} + r.z.poolMetaMutex.Unlock() + t.Cleanup(func() { + r.z.poolMetaMutex.Lock() + r.z.poolMeta.Pools[0].Decommission = nil + r.z.poolMetaMutex.Unlock() + }) + } + before := nativeRestoreSnapshot(t, r) + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + if mode != "restored-first" { + select { + case err := <-layer.done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Error("POST did not restore the copy without restored data") + } + } + after := nativeRestoreSnapshot(t, r) + for pool := range r.z.serverPools { + gr, err := r.z.serverPools[pool].GetObjectNInfo(t.Context(), r.bucket, r.object, nil, nil, ObjectOptions{VersionID: oi.VersionID}) + if err != nil { + t.Errorf("pool %d copy unreadable, header %q: %v", pool, after[pool][0].Metadata[xhttp.AmzRestore], err) + continue + } + got, err := io.ReadAll(gr) + gr.Close() + if err != nil || !bytes.Equal(got, body) { + t.Errorf("pool %d copy contents: %v", pool, err) + } + } + // The copy that is neither renewed nor restored stays as it was. + untouched := 1 + if mode == "suspended-owner" { + untouched = 0 + } + if !reflect.DeepEqual(before[untouched], after[untouched]) { + t.Errorf("POST changed the pool %d copy: %q -> %q", untouched, before[untouched][0].Metadata[xhttp.AmzRestore], after[untouched][0].Metadata[xhttp.AmzRestore]) + } + }) + } +} + +// Stale-upload cleanup ages an upload by when it started, not by the +// version's ModTime, keeps the upload of a running restore beyond the stale +// age, and still removes ordinary stale uploads and those of ended attempts. +func TestRestoreUploadOutlivesStaleAge(t *testing.T) { + cleanup := func(t *testing.T, r *restoreTest) { + er := r.z.serverPools[0].getHashedSet(r.object) + for _, disk := range er.getDisks() { + er.cleanupStaleUploadsOnDisk(t.Context(), disk) + } + } + shortExpiry := func(t *testing.T) { + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + t.Cleanup(func() { + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + }) + } + for _, mode := range []string{"old-version", "past-stale-age", "public-abort", "ordinary-stale", "unowned-marker"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + if mode == "ordinary-stale" { + // Control: an abandoned client upload, old by its upload ID. + if _, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{MTime: UTCNow().Add(-72 * time.Hour)}); err != nil { + t.Fatal(err) + } + cleanup(t, r) + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("ordinary stale upload survived cleanup: %v", uploads) + } + return + } + var oi ObjectInfo + var body []byte + if mode == "past-stale-age" { + oi, body = restoreIsolationSeed(t, r, 0, false, true) + } else { + oi, body = nativeRestoreSeed(t, r, 0, false, true) + } + old := r.opts(oi.VersionID) + old.MTime = UTCNow().Add(-72 * time.Hour) + old.EvalMetadataFn = func(*ObjectInfo, error) (ReplicateDecision, error) { return ReplicateDecision{}, nil } + if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, old); err != nil { + t.Fatal(err) + } + // The cleanup with a short stale age also removes in-flight + // temporary parts; hold such restores in completion instead, + // before their commit. + gated := mode == "past-stale-age" || mode == "unowned-marker" + tier := newRestoreIsolationGateTier(5<<20, nil) + if !gated { + globalTierConfigMgr.drivercache["WARM"] = tier + } + defer tier.unblock() + gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} + defer gate.unblock() + if gated { + er := r.z.serverPools[0].getHashedSet(r.object) + original := er.getDisks + disks := original() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreIsolationValidationDisk{d, gate} + } + er.getDisks = func() []StorageAPI { return wrapped } + defer func() { er.getDisks = original }() + } + result := make(chan error, 1) + layer := newRestoreIsolationLayer(r) + if mode == "past-stale-age" { + // A POST publishes the attempt that a running restore owns. + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + } else { + go func() { + result <- r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + } + if gated { + select { + case <-gate.entered: + case <-time.After(10 * time.Second): + t.Fatal("restore never reached completion") + } + } else { + restoreIsolationEntered(t, tier) + } + switch mode { + case "past-stale-age", "unowned-marker": + shortExpiry(t) + cleanup(t, r) + kept := len(restoreIsolationUploads(t, r, 0)) == 1 + if mode == "unowned-marker" && kept { + // The version is ongoing, but not for this direct restore's + // upload: an ongoing marker alone does not keep an upload. + t.Error("stale cleanup kept an upload no attempt owns") + } + if mode == "past-stale-age" && !kept { + t.Error("stale cleanup removed the upload of a running restore") + } + case "old-version": + cleanup(t, r) + case "public-abort": + id, _ := restoreIsolationUpload(t, r, 0) + if err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) { + t.Errorf("public abort of a native restore upload: %v, want NoSuchUpload", err) + } + } + tier.unblock() + gate.unblock() + var err error + if mode == "past-stale-age" { + err = layer.wait(t) + } else { + err = <-result + } + if mode == "unowned-marker" { + if err == nil { + t.Error("restore completed without its upload") + } + return + } + if err != nil { + t.Fatalf("restore lost its upload: %v", err) + } + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } + }) + } +} + +// A failed multipart restore removes its own upload from the drives, also +// when a newer attempt owns the version by then, and leaves that newer +// attempt's marker alone. +func TestRestoreFailureAbortsUpload(t *testing.T) { + for _, mode := range []string{"download-failure", "superseded-mid-download"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + failure := errors.New("injected remote failure") + if mode == "download-failure" { + oi, _ := nativeRestoreSeed(t, r, 0, false, true) + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, failure} + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); !errors.Is(err, failure) { + t.Fatalf("restore error=%v, want %v", err, failure) + } + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("failed native restore left its upload on disk: %v", uploads) + } + return + } + oi, _ := restoreIsolationSeed(t, r, 0, false, true) + tier := newRestoreIsolationGateTier(5<<20, failure) + globalTierConfigMgr.drivercache["WARM"] = tier + defer tier.unblock() + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + restoreIsolationEntered(t, tier) + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 1 { + t.Fatalf("restore upload on disk: %v", uploads) + } + // Model a newer generation that owns the version by now, such as + // one that took over after this restore lost its lock. + newer := r.opts(oi.VersionID) + newer.MTime = oi.ModTime + newer.NoLock = true + newer.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[restoreIsolationAttemptKey] = "newer-generation" + return ReplicateDecision{}, nil + } + if _, err := r.z.serverPools[0].getHashedSet(r.object).PutObjectMetadata(t.Context(), r.bucket, r.object, newer); err != nil { + t.Fatal(err) + } + tier.unblock() + if err := layer.wait(t); !errors.Is(err, failure) { + t.Fatalf("restore error=%v, want %v", err, failure) + } + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("superseded restore left its upload on disk: %v", uploads) + } + after := r.fileInfo(oi.VersionID) + if !strings.Contains(after.Metadata[xhttp.AmzRestore], "true") || after.Metadata[restoreIsolationAttemptKey] != "newer-generation" { + t.Errorf("failed restore cleared a newer attempt's marker: %q %q", after.Metadata[xhttp.AmzRestore], after.Metadata[restoreIsolationAttemptKey]) + } + }) + } +} + +// A restored copy expires only by its own reason, on a fresh read: its +// restore deadline passed and nothing else. A renewed deadline survives +// another rule falling due and an expiry checked across all pools. +func TestExpireRestoredNeedsOwnReason(t *testing.T) { + for _, mode := range []string{"other-expiry-due", "checked-all-pools", "ongoing", "renewed", "expired"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi := r.put("restored", false, true) + r.restore(oi) + before := r.fileInfo(oi.VersionID) + if mode == "other-expiry-due" { + meta, err := globalBucketMetadataSys.Get(r.bucket) + if err != nil { + t.Fatal(err) + } + meta.LifecycleConfigXML = []byte(`deleteEnabled2020-01-01T00:00:00Z`) + if err := meta.Save(t.Context(), r.z); err != nil { + t.Fatal(err) + } + globalBucketMetadataSys.Set(r.bucket, meta) + } + header := completedRestoreObj(UTCNow().Add(48 * time.Hour)).String() + switch mode { + case "ongoing": + header = ongoingRestoreObj().String() + case "expired": + header = "" + } + if header != "" { + opts := r.opts(oi.VersionID) + opts.MTime = oi.ModTime + opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[xhttp.AmzRestore] = header + return ReplicateDecision{}, nil + } + if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil { + t.Fatal(err) + } + } + var err error + if mode == "checked-all-pools" { + opts := r.opts(oi.VersionID) + opts.Expiration = ExpirationOptions{Expire: true, CheckedAllPools: true} + opts.Transition.ExpireRestored = true + _, err = r.z.serverPools[0].getHashedSet(r.object).DeleteObject(t.Context(), r.bucket, r.object, opts) + } else { + err = expireTransitionedObject(t.Context(), r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner) + } + after := r.fileInfo(oi.VersionID) + if mode == "expired" { + // Control: an elapsed deadline is the reason itself. + if err != nil || after.Metadata[xhttp.AmzRestore] != "" || r.dataDirs(before.DataDir) != 0 { + t.Errorf("expired restored copy not removed: %v", err) + } + return + } + t.Logf("cleanup=%v header=%q", err, after.Metadata[xhttp.AmzRestore]) + if err == nil || after.Metadata[xhttp.AmzRestore] != header || r.dataDirs(before.DataDir) != len(r.z.serverPools[0].getHashedSet(r.object).getDisks()) { + t.Error("cache cleanup removed a restored copy whose own deadline has not passed") + } + }) + } +} + +// Caller metadata cannot make an upload a native restore upload: the marker +// is dropped, the upload stays an ordinary one and nothing commits it. +func TestRestoreUploadMarkerSpoofing(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, _ := nativeRestoreSeed(t, r, 0, false, true) + forged := fmt.Sprintf(`{"id":%q,"v":%q,"e":%q,"s":%d,"m":%q}`, mustGetUUID(), oi.VersionID, oi.ETag, oi.Size, oi.ModTime.UTC().Format(time.RFC3339Nano)) + for _, target := range []string{"existing-version", "new-object"} { + t.Run(target, func(t *testing.T) { + opts := ObjectOptions{Versioned: true, UserDefined: map[string]string{restoreIsolationUploadKey: forged}} + object := r.object + if target == "existing-version" { + opts.VersionID = oi.VersionID + opts.MTime = oi.ModTime + opts.PreserveETag = oi.ETag + } else { + object = "spoofed-" + r.object + } + upload, err := r.z.NewMultipartUpload(t.Context(), r.bucket, object, opts) + if err != nil { + t.Fatal(err) + } + uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, object, "", "", "", 100) + if err != nil || len(uploads.Uploads) != 1 || uploads.Uploads[0].UploadID != upload.UploadID { + t.Fatalf("a caller's upload must stay listed: %+v, %v", uploads.Uploads, err) + } + data := []byte("spoofed part") + part, err := r.z.PutObjectPart(t.Context(), r.bucket, object, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + before := nativeRestoreSnapshot(t, r) + done, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, object, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true, MTime: opts.MTime}) + if target == "existing-version" { + if !errors.Is(err, errAuthentication) { + t.Errorf("spoofed upload overwrote the transitioned version: %v", err) + } + if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) { + t.Error("rejected spoofed completion changed the version") + } + return + } + if err != nil { + t.Fatal(err) + } + info, err := r.z.GetObjectInfo(t.Context(), r.bucket, object, ObjectOptions{VersionID: done.VersionID}) + if err != nil || info.UserDefined[restoreIsolationUploadKey] != "" { + t.Errorf("caller metadata committed a restore upload marker: %q, %v", info.UserDefined[restoreIsolationUploadKey], err) + } + }) + } +} + +// Controls: an ordinary POST restore, a renewal, and a client's multipart +// upload of the same key while a restore runs. +func TestRestorePOSTControls(t *testing.T) { + t.Run("restore", func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil || info.IsRemote() || info.RestoreOngoing || info.UserDefined[restoreIsolationAttemptKey] != "" { + t.Fatalf("restored status: %q %q, %v", info.UserDefined[xhttp.AmzRestore], info.UserDefined[restoreIsolationAttemptKey], err) + } + if !info.ModTime.Equal(oi.ModTime) || info.ETag != oi.ETag || info.UserDefined["x-amz-meta-preserve"] != "keep-me" { + t.Error("restore changed the version identity or metadata") + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Errorf("restored contents: %v", err) + } + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("completed restore left its upload: %v", uploads) + } + }) + t.Run("renewal", func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := nativeRestoreSeed(t, r, 0, false, false) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + before := r.fileInfo(oi.VersionID) + gets := tier.gets.Load() + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + select { + case err := <-layer.done: + t.Fatalf("renewal restored again: %v", err) + case <-time.After(200 * time.Millisecond): + } + after := r.fileInfo(oi.VersionID) + if after.DataDir != before.DataDir || !after.ModTime.Equal(before.ModTime) || after.Metadata[xhttp.AmzRestore] == "" { + t.Error("renewal changed the restored copy") + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) || tier.gets.Load() != gets { + t.Errorf("renewed copy not served locally: %v", err) + } + }) + t.Run("public-multipart-same-key", func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + tier := newRestoreIsolationGateTier(5<<20, nil) + globalTierConfigMgr.drivercache["WARM"] = tier + defer tier.unblock() + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + restoreIsolationEntered(t, tier) + upload, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true}) + if err != nil { + t.Fatal(err) + } + uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 100) + listed := false + for _, u := range uploads.Uploads { + listed = listed || u.UploadID == upload.UploadID + } + if err != nil || !listed { + t.Fatalf("client upload not listed: %+v, %v", uploads.Uploads, err) + } + tier.unblock() + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + // Single-pool part writes wait for the object lock of a running restore. + data := []byte("client data") + part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + parts, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, upload.UploadID, 0, 10, ObjectOptions{}) + if err != nil || len(parts.Parts) != 1 { + t.Fatalf("client parts: %+v, %v", parts, err) + } + newer, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true}) + if err != nil { + t.Fatal(err) + } + if _, got, err := r.read(newer.VersionID); err != nil || !bytes.Equal(got, data) { + t.Errorf("client version: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Errorf("restored version: %v", err) + } + }) +} diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go index 884b88d24..44676ac7c 100644 --- a/cmd/restore-native_test.go +++ b/cmd/restore-native_test.go @@ -375,8 +375,10 @@ func TestNativeRestorePOSTVersionIdentity(t *testing.T) { // A restore request is internal context, not an exception for normal S3 // multipart overwrites or for a version that has never completed transition. +// Only the upload of a native restore attempt, whose marker names the very +// version being completed, may write that version back. func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { - for _, mode := range []string{"matching-restore", "ordinary-complete", "not-tiered", "changed-time", "changed-size", "changed-etag"} { + for _, mode := range []string{"matching-restore", "ordinary-complete", "restore-without-attempt", "other-attempt", "not-tiered", "changed-time", "changed-size", "changed-etag"} { t.Run(mode, func(t *testing.T) { r := newRestoreSafetyTest(t, 1, versioning.Enabled) oi, body := nativeRestoreSeed(t, r, 0, false, true) @@ -398,8 +400,9 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { if err != nil { t.Fatal(err) } - opts := putRestoreOpts(r.bucket, r.object, &RestoreObjectRequest{Days: 1}, current) + opts := putRestoreOpts(&RestoreObjectRequest{Days: 1}, current, r.opts(oi.VersionID)) opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1} + opts.PreserveETag = current.ETag if mode == "ordinary-complete" { opts.Transition.RestoreRequest = nil } @@ -408,20 +411,26 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { } // Hold the ETag constant for the size case, so only size can reject it. if mode == "changed-size" { - opts.UserDefined["etag"] = current.ETag body = append(body, 'x') } if mode == "changed-etag" { - opts.UserDefined["etag"] = "different-etag" + opts.PreserveETag = "different-etag" } - upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, opts) + // The attempt claims what its upload claims; only the matching + // mode claims exactly the stored version. + if mode != "ordinary-complete" && mode != "restore-without-attempt" { + opts.Transition.attempt = &restoreAttempt{ID: mustGetUUID(), VersionID: oi.VersionID, ETag: opts.PreserveETag, Size: current.Size, ModTime: opts.MTime} + } + uopts := opts + uopts.MTime = time.Time{} + upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, uopts) if err != nil { t.Fatal(err) } var complete []CompletePart for i, data := range [][]byte{body[:5<<20], body[5<<20:]} { part, err := pool.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, i+1, - mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Transition: opts.Transition}) if err != nil { t.Fatal(err) } @@ -430,6 +439,11 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { if mode != "changed-size" && getCompleteMultipartMD5(complete) != current.ETag { t.Fatal("fixture multipart ETag does not match original") } + if mode == "other-attempt" { + other := *opts.Transition.attempt + other.ID = mustGetUUID() + opts.Transition.attempt = &other + } before := nativeRestoreSnapshot(t, r) _, err = pool.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, complete, opts) if mode == "matching-restore" { @@ -440,9 +454,16 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { if err != nil || !bytes.Equal(got, body) { t.Fatalf("matching restore contents: %v", err) } + if after := r.fileInfo(oi.VersionID); after.Metadata[restoreUploadKey] != "" || after.Metadata[restoreAttemptKey] != "" { + t.Error("restore committed its private upload or attempt marker") + } return } - if !errors.Is(err, errAuthentication) { + if mode == "other-attempt" { + if !errors.As(err, &InvalidUploadID{}) { + t.Fatalf("another attempt completed a private upload: %v", err) + } + } else if !errors.Is(err, errAuthentication) { t.Fatalf("existing version overwrite: %v", err) } if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) { @@ -721,18 +742,10 @@ func TestNativeRestoreConcurrentCompletion(t *testing.T) { case <-ctx.Done(): t.Fatal("restore never started") } - // A client's exact-prefix listing can discover the internal upload. It can - // try to complete the first part while native restore owns the object. - uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10) - if err != nil || len(uploads.Uploads) != 1 { - t.Fatalf("restore upload discovery: %+v, %v", uploads, err) - } - id := uploads.Uploads[0].UploadID - parts, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{}) - if err != nil || len(parts.Parts) != 1 { - t.Fatalf("restore must have published its first part: %+v, %v", parts, err) - } - part := parts.Parts[0] + // A client that learns the internal upload ID, which listings no + // longer show, can try to complete the first part while native + // restore owns the object. + id, part := restoreIsolationUpload(t, r, 0) completed := make(chan struct{}) var completeErr error go func() { @@ -942,17 +955,22 @@ func TestNativeRestoreCompletionReadSafety(t *testing.T) { } } +// nativeRestoreStatusHook runs afterRead once POST has read the version and +// before POST records its status on it. type nativeRestoreStatusHook struct { ObjectLayer - beforeCopy func() - copyCalled bool + afterRead func() + readCalled bool backgroundCalled atomic.Bool } -func (h *nativeRestoreStatusHook) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, info ObjectInfo, srcOpts, dstOpts ObjectOptions) (ObjectInfo, error) { - h.copyCalled = true - h.beforeCopy() - return h.ObjectLayer.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, info, srcOpts, dstOpts) +func (h *nativeRestoreStatusHook) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + oi, err := h.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts) + if err == nil && !h.readCalled { + h.readCalled = true + h.afterRead() + } + return oi, err } func (h *nativeRestoreStatusHook) RestoreTransitionedObject(context.Context, string, string, ObjectOptions) error { @@ -985,7 +1003,7 @@ func TestNativeRestorePOSTReadSafety(t *testing.T) { } before = restoreBoundarySnapshot(t, r) } - hook := &nativeRestoreStatusHook{ObjectLayer: r.z, beforeCopy: mutate} + hook := &nativeRestoreStatusHook{ObjectLayer: r.z, afterRead: mutate} if stage == "initial" { mutate() } @@ -1004,7 +1022,7 @@ func TestNativeRestorePOSTReadSafety(t *testing.T) { if rec.Code < 400 { t.Fatalf("POST acknowledged failed read: %d %s", rec.Code, rec.Body.String()) } - if hook.copyCalled != (stage == "status-copy") || hook.backgroundCalled.Load() { + if hook.readCalled != (stage == "status-copy") || hook.backgroundCalled.Load() { t.Fatal("request followed the wrong failure path") } if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) { diff --git a/internal/grid/handlers.go b/internal/grid/handlers.go index 1d20fa9b9..0f2c39d7e 100644 --- a/internal/grid/handlers.go +++ b/internal/grid/handlers.go @@ -116,6 +116,7 @@ const ( HandlerRenamePart HandlerClearUploadID HandlerCheckParts3 + HandlerRestoreCapability // Add more above here ^^^ // If all handlers are used, the type of Handler can be changed. @@ -200,6 +201,7 @@ var handlerPrefixes = [handlerLast]string{ HandlerCheckParts3: storagePrefix, HandlerRenamePart: storagePrefix, HandlerClearUploadID: peerPrefix, + HandlerRestoreCapability: peerPrefix, } const ( diff --git a/internal/grid/handlers_string.go b/internal/grid/handlers_string.go index d6c6b4c38..aea3f45fb 100644 --- a/internal/grid/handlers_string.go +++ b/internal/grid/handlers_string.go @@ -86,14 +86,15 @@ func _() { _ = x[HandlerRenamePart-75] _ = x[HandlerClearUploadID-76] _ = x[HandlerCheckParts3-77] - _ = x[handlerTest-78] - _ = x[handlerTest2-79] - _ = x[handlerLast-80] + _ = x[HandlerRestoreCapability-78] + _ = x[handlerTest-79] + _ = x[handlerTest2-80] + _ = x[handlerLast-81] } -const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3handlerTesthandlerTest2handlerLast" +const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3RestoreCapabilityhandlerTesthandlerTest2handlerLast" -var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 971, 983, 994} +var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 977, 988, 1000, 1011} func (i HandlerID) String() string { idx := int(i) - 0