diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 2bf27c930..79ea533b5 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -96,10 +96,15 @@ jobs:
- name: Run multi-pool delete consistency tests under race detector
run: go test -race ./cmd -run '^Test(Pools(Delete|BulkDelete|RetiringCopyPreservesSharedTierObject)|BulkDeleteQuorumEvidence)' -count=1 -timeout=5m
+ - name: Run lifecycle expiry and native restore tests under race detector
+ env:
+ MINIO_API_REQUESTS_MAX: "10000"
+ run: go test -race -tags kqueue,dev ./cmd -run '^Test(PoolsLifecycle|PoolsMigrationLifecycle|ExpireRestored|NativeRestore)' -count=1 -timeout=5m
+
- name: Run multipart listing, cancellation and cleanup tests under race detector
env:
MINIO_API_REQUESTS_MAX: "10000"
- run: go test -race ./cmd -run '^Test(Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m
+ run: go test -race ./cmd -run '^Test(ObjectCompleteMultipartUpload|Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m
- name: Run listing metadata recovery, pagination and cancellation tests under race detector
run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity|ListObjectVersionsCrossParityNull|MigrationCrossParity)' -count=1 -timeout=5m
diff --git a/buildscripts/rebrand-guard/compat-baseline.json b/buildscripts/rebrand-guard/compat-baseline.json
index 2ff232340..620cbe847 100644
--- a/buildscripts/rebrand-guard/compat-baseline.json
+++ b/buildscripts/rebrand-guard/compat-baseline.json
@@ -594,11 +594,13 @@
"x-minio-from-peer",
"x-minio-healing-drives",
"x-minio-internal",
+ "x-minio-internal-a",
"x-minio-internal-actual-object-size",
"x-minio-internal-actual-size",
"x-minio-internal-archive-info",
"x-minio-internal-archive-type",
"x-minio-internal-b",
+ "x-minio-internal-c",
"x-minio-internal-compression",
"x-minio-internal-encrypted-multipart",
"x-minio-internal-encryptedmultipart",
@@ -611,6 +613,8 @@
"x-minio-internal-replication-reset-arn",
"x-minio-internal-replication-status",
"x-minio-internal-replication-timestamp",
+ "x-minio-internal-restore-attempt",
+ "x-minio-internal-restore-upload",
"x-minio-internal-server-side-encryption",
"x-minio-internal-server-side-encryption-context",
"x-minio-internal-server-side-encryption-iv",
diff --git a/cmd/batch-rotate.go b/cmd/batch-rotate.go
index a238af08b..3ebbc844d 100644
--- a/cmd/batch-rotate.go
+++ b/cmd/batch-rotate.go
@@ -180,7 +180,7 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
- lock := api.NewNSLock(r.Bucket, objInfo.Name)
+ lock := api.NewObjectNSLock(r.Bucket, objInfo.Name)
lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
if err != nil {
return err
diff --git a/cmd/bucket-lifecycle-restore-boundary_test.go b/cmd/bucket-lifecycle-restore-boundary_test.go
new file mode 100644
index 000000000..8f39342ba
--- /dev/null
+++ b/cmd/bucket-lifecycle-restore-boundary_test.go
@@ -0,0 +1,277 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/versioning"
+)
+
+// Read the actual metadata from every disk without object-layer read repair.
+func restoreBoundarySnapshot(t *testing.T, r *restoreTest) [][]byte {
+ t.Helper()
+ var result [][]byte
+ for _, pool := range r.z.serverPools {
+ for _, disk := range pool.getHashedSet(r.object).getDisks() {
+ raw, err := disk.ReadXL(t.Context(), r.bucket, r.object, true)
+ if err != nil && err != errFileNotFound {
+ t.Fatal(err)
+ }
+ result = append(result, bytes.Clone(raw.Buf))
+ }
+ }
+ return result
+}
+
+func TestExpireRestoredKeepsReplacementDeleteMarker(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Suspended)
+ r.put("old", false, false)
+ oi := r.put("null", true, true)
+ r.restore(oi)
+ stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(nullVersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, r.opts("")); err != nil {
+ t.Fatal(err)
+ }
+ if !r.fileInfo(nullVersionID).Deleted {
+ t.Fatal("ordinary DELETE did not replace null with a delete marker")
+ }
+ before := restoreBoundarySnapshot(t, r)
+ err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+ if !isErrVersionNotFound(err) {
+ t.Errorf("stale cleanup should reject the replacement marker: %v", err)
+ }
+ if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+ t.Error("stale cleanup changed stored versions or their data")
+ }
+ if _, _, err := r.read(""); !isErrObjectNotFound(err) {
+ t.Errorf("stale cleanup exposed the old version: %v", err)
+ }
+ // An explicit user purge must still remove that same marker.
+ if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, r.opts(nullVersionID)); err != nil {
+ t.Fatal(err)
+ }
+ if label, data, err := r.read(""); err != nil || label != "old" || !bytes.Equal(data, r.bodies["old"]) {
+ t.Fatalf("ordinary marker purge: label=%s err=%v", label, err)
+ }
+ })
+ }
+}
+
+func TestExpireRestoredMetadataReadKeepsMinorityVersion(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, operation := range []string{"cleanup", "restore", "restore-cleanup", "lookup-cleanup", "lookup-restore", "get", "delete"} {
+ t.Run(fmt.Sprintf("pools=%d/%s", pools, operation), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ oi := r.put("target", false, true)
+ r.restore(oi)
+ fi := r.fileInfo(oi.VersionID)
+ // A readable latest version lets pools lookup finish before the
+ // set reads the specifically addressed, incomplete old version.
+ r.put("newer", false, false)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ for _, disk := range disks[1:] {
+ raw, err := disk.ReadXL(t.Context(), r.bucket, r.object, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var meta xlMetaV2
+ if err := meta.Load(raw.Buf); err != nil {
+ t.Fatal(err)
+ }
+ del := FileInfo{VersionID: oi.VersionID}
+ del.SetSkipTierFreeVersion()
+ if _, err := meta.DeleteVersion(del); err != nil {
+ t.Fatal(err)
+ }
+ buf, err := meta.AppendTo(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Remove only this metadata entry; retain every real shard.
+ if err := disk.WriteAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile), buf); err != nil {
+ t.Fatal(err)
+ }
+ }
+ before := restoreBoundarySnapshot(t, r)
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+ opts := r.opts(oi.VersionID)
+ var err error
+ switch operation {
+ case "cleanup":
+ err = expireTransitionedObject(ctx, r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredVersionAction}, lcEventSrc_Scanner)
+ case "restore":
+ opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+ err = r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, opts)
+ case "restore-cleanup":
+ // The initial read succeeded before remote download failed;
+ // cleanup must not purge a now-incomplete target on reread.
+ err = er.clearRestoreStatus(ctx, r.bucket, r.object, oi.VersionID, nil)
+ case "lookup-cleanup", "lookup-restore":
+ // Version-sensitive lookup also covers native restore routing.
+ opts.MetadataChg = true
+ if operation == "lookup-cleanup" {
+ opts.Transition.ExpireRestored = true
+ } else {
+ opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+ }
+ _, _, err = r.z.getPoolInfoExistingWithOpts(ctx, r.bucket, r.object, opts)
+ case "get":
+ _, err = r.z.GetObjectInfo(ctx, r.bucket, r.object, opts)
+ case "delete":
+ _, err = r.z.DeleteObject(ctx, r.bucket, r.object, opts)
+ }
+ if ctx.Err() != nil {
+ t.Fatalf("%s timed out instead of returning its read error: %v", operation, err)
+ }
+ if err == nil {
+ t.Fatal("incomplete target unexpectedly readable")
+ }
+ if operation != "get" && operation != "delete" {
+ if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+ t.Error("failed restore read changed metadata, inline data or free versions")
+ }
+ if n := r.dataDirs(fi.DataDir); n != len(disks) {
+ t.Errorf("failed restore read removed shards: %d/%d directories remain", n, len(disks))
+ }
+ return
+ }
+ // Ordinary GET/DELETE retain their existing dangling cleanup.
+ raw, err := disks[0].ReadXL(t.Context(), r.bucket, r.object, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ versions, err := getFileInfoVersions(raw.Buf, r.bucket, r.object, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(versions.Versions) != 1 || len(versions.FreeVersions) != 1 || r.dataDirs(fi.DataDir) != len(disks)-1 {
+ t.Fatal("ordinary dangling cleanup no longer removed the minority version")
+ }
+ })
+ }
+ }
+}
+
+func TestNativeRestoreInitialMissingReadDoesNotWrite(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ r.put("live", false, false)
+ before := restoreBoundarySnapshot(t, r)
+ opts := r.opts(mustGetUUID())
+ opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, opts); !isErrVersionNotFound(err) {
+ t.Fatalf("missing restore target: %v", err)
+ }
+ if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+ t.Fatal("initial restore failure wrote metadata without a resolved version")
+ }
+}
+
+func TestExpireRestoredStorageRejectsOtherTypes(t *testing.T) {
+ for _, kind := range []string{"null-marker", "uuid-marker", "legacy", "legacy-json"} {
+ t.Run(kind, func(t *testing.T) {
+ disk, _, err := newXLStorageTestSetup(t)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := t.Context()
+ const bucket, object = "restore-types", "obj"
+ if err := disk.MakeVol(ctx, bucket); err != nil {
+ t.Fatal(err)
+ }
+ var meta xlMetaV2
+ versionID, metaName, dataDir := nullVersionID, xlStorageFormatFile, ""
+ var buf []byte
+ switch kind {
+ case "null-marker", "uuid-marker":
+ if kind == "uuid-marker" {
+ versionID = mustGetUUID()
+ }
+ err = meta.AddVersion(FileInfo{VersionID: versionID, Deleted: true, ModTime: time.Now().UTC()})
+ case "legacy", "legacy-json":
+ legacy := getSampleXLMeta(1)
+ legacy.DataDir, dataDir = "legacy", "legacy"
+ if kind == "legacy-json" {
+ metaName = xlStorageFormatFileV1
+ buf, err = json.Marshal(legacy)
+ disk.storage.Lock()
+ disk.storage.formatLegacy = true
+ disk.storage.Unlock()
+ } else {
+ err = meta.AddLegacy(&legacy)
+ }
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if buf == nil {
+ buf, err = meta.AppendTo(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := disk.WriteAll(ctx, bucket, pathJoin(object, metaName), buf); err != nil {
+ t.Fatal(err)
+ }
+ payload := []byte("sole legacy contents")
+ dataPath := filepath.Join(disk.Endpoint().Path, bucket, object, dataDir, "part.1")
+ if dataDir != "" {
+ if err := os.MkdirAll(filepath.Dir(dataPath), 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(dataPath, payload, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := disk.DeleteVersion(ctx, bucket, object, FileInfo{VersionID: versionID, ExpireRestored: true, TransitionStatus: lifecycle.TransitionComplete}, false, DeleteOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ after, err := disk.ReadAll(ctx, bucket, pathJoin(object, metaName))
+ if err != nil || !bytes.Equal(buf, after) {
+ t.Errorf("restore cleanup changed %s metadata: %v", kind, err)
+ }
+ if dataDir != "" {
+ if data, err := os.ReadFile(dataPath); err != nil || !bytes.Equal(data, payload) {
+ t.Errorf("restore cleanup removed legacy contents: %v", err)
+ }
+ }
+ if err := disk.DeleteVersion(ctx, bucket, object, FileInfo{VersionID: versionID}, false, DeleteOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := disk.ReadAll(ctx, bucket, pathJoin(object, metaName)); err != errFileNotFound {
+ t.Errorf("ordinary purge left %s: %v", kind, err)
+ }
+ })
+ }
+}
diff --git a/cmd/bucket-lifecycle-restore-safety_test.go b/cmd/bucket-lifecycle-restore-safety_test.go
new file mode 100644
index 000000000..85a4be182
--- /dev/null
+++ b/cmd/bucket-lifecycle-restore-safety_test.go
@@ -0,0 +1,284 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "reflect"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/versioning"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+const restoreSafetyRules = `tierEnabled1000WARM`
+
+// Reuse the original restore-copy fixture with a small, isolated real cluster.
+func newRestoreSafetyTest(t *testing.T, pools int, state versioning.State) *restoreTest {
+ t.Helper()
+ prevTier := globalTierConfigMgr
+ t.Cleanup(func() { globalTierConfigMgr = prevTier })
+ dirs, err := getRandomDisks(4 * pools)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { removeRoots(dirs) })
+ var endpoints EndpointServerPools
+ for i := range pools {
+ endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
+ }
+ obj, _, err := initObjectLayer(t.Context(), endpoints)
+ if err != nil {
+ t.Fatal(err)
+ }
+ z := obj.(*erasureServerPools)
+ t.Cleanup(func() { z.Shutdown(context.Background()) })
+ globalTierConfigMgr = NewTierConfigMgr()
+ globalTierConfigMgr.drivercache["WARM"] = &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
+ r := &restoreTest{t: t, z: z, bucket: getRandomBucketName(), object: "obj", labels: map[string]string{}, bodies: map[string][]byte{}}
+ if err := z.MakeBucket(t.Context(), r.bucket, MakeBucketOptions{VersioningEnabled: state != ""}); err != nil {
+ t.Fatal(err)
+ }
+ meta, err := globalBucketMetadataSys.Get(r.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if state == versioning.Suspended {
+ meta.VersioningConfigXML = []byte(`Suspended`)
+ }
+ meta.LifecycleConfigXML = []byte(restoreSafetyRules)
+ if err := meta.Save(t.Context(), z); err != nil {
+ t.Fatal(err)
+ }
+ globalBucketMetadataSys.Set(r.bucket, meta)
+ return r
+}
+
+type restoreCleanupFaultDisk struct {
+ consistencyReadFaultDisk
+ deletes *atomic.Int64
+}
+
+func (d restoreCleanupFaultDisk) DeleteVersion(ctx context.Context, bucket, object string, fi FileInfo, force bool, opts DeleteOptions) error {
+ if bucket == d.bucket && object == d.object {
+ d.deletes.Add(1)
+ }
+ return d.StorageAPI.DeleteVersion(ctx, bucket, object, fi, force, opts)
+}
+
+func TestExpireRestoredRequiresFreshMetadata(t *testing.T) {
+ for _, restored := range []bool{true, false} {
+ t.Run(fmt.Sprintf("expire-restored=%v", restored), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, "")
+ oi := r.put("v1", true, true)
+ r.restore(oi)
+ before := r.fileInfo(oi.VersionID)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ original := er.getDisks
+ disks := original()
+ var deletes atomic.Int64
+ faulty := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ faulty[i] = restoreCleanupFaultDisk{consistencyReadFaultDisk{d, r.bucket, r.object}, &deletes}
+ }
+ er.getDisks = func() []StorageAPI { return faulty }
+ // Restore the callback before any fixture reads and on a fatal assertion.
+ defer func() { er.getDisks = original }()
+ opts := ObjectOptions{VersionID: nullVersionID, Transition: TransitionOptions{ExpireRestored: restored}, Expiration: ExpirationOptions{Expire: restored}}
+ _, err := er.DeleteObject(t.Context(), r.bucket, r.object, opts)
+ er.getDisks = original
+ if restored {
+ var quorum InsufficientWriteQuorum
+ if !errors.As(err, &quorum) {
+ t.Errorf("cleanup without fresh metadata: got %v, want write-quorum error", err)
+ }
+ if n := deletes.Load(); n != 0 {
+ t.Errorf("cleanup attempted %d blind disk deletes", n)
+ }
+ if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before, after) {
+ t.Error("failed metadata read changed the stored version")
+ }
+ if n := r.dataDirs(before.DataDir); n != len(disks) {
+ t.Errorf("cleanup removed data: %d/%d directories remain", n, len(disks))
+ }
+ } else {
+ // Ordinary unversioned DELETE retains the existing tryDel behavior.
+ if err != nil || deletes.Load() != int64(len(disks)) {
+ t.Fatalf("ordinary DELETE: err=%v, deletes=%d", err, deletes.Load())
+ }
+ if _, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{}); !isErrObjectNotFound(err) {
+ t.Fatalf("ordinary DELETE left the object: %v", err)
+ }
+ }
+ })
+ }
+}
+
+func TestExpireRestoredRenewedOrMissing(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi := r.put("v1", false, true)
+ r.restore(oi)
+ stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ before := r.fileInfo(oi.VersionID)
+ renewed := completedRestoreObj(time.Now().Add(48 * time.Hour)).String()
+ opts := r.opts(oi.VersionID)
+ opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+ o.UserDefined[xhttp.AmzRestore] = renewed
+ return ReplicateDecision{}, nil
+ }
+ if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil {
+ t.Fatal(err)
+ }
+ err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+ if !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("stale task should be rejected, got %v", err)
+ }
+ if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed || got.DataDir != before.DataDir {
+ t.Error("stale task changed renewed restore")
+ }
+ if n := r.dataDirs(before.DataDir); n != 4 {
+ t.Errorf("renewed copy has %d directories, want 4", n)
+ }
+ if _, data, err := r.read(oi.VersionID); err != nil || !bytes.Equal(data, r.bodies["v1"]) {
+ t.Fatalf("renewed copy not readable: %v", err)
+ }
+
+ for _, id := range []string{mustGetUUID(), nullVersionID} {
+ opts := r.opts(id)
+ opts.Expiration.Expire = true
+ opts.Transition.ExpireRestored = true
+ if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, opts); !isErrVersionNotFound(err) {
+ t.Errorf("missing version %s: %v", id, err)
+ }
+ }
+ missing := r.opts(nullVersionID)
+ missing.Expiration.Expire = true
+ missing.Transition.ExpireRestored = true
+ if _, err := r.z.DeleteObject(t.Context(), r.bucket, "missing-object", missing); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+ t.Errorf("missing object: %v", err)
+ }
+ if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed {
+ t.Error("missing-target cleanup changed existing restore")
+ }
+}
+
+func TestExpireRestoredStorageBoundary(t *testing.T) {
+ for _, status := range []string{"", lifecycle.TransitionPending, lifecycle.TransitionComplete} {
+ for _, inline := range []bool{false, true} {
+ for _, requestStatus := range []string{"", lifecycle.TransitionComplete} {
+ t.Run(fmt.Sprintf("stored=%s/inline=%v/request=%s", status, inline, requestStatus), func(t *testing.T) {
+ disk, _, err := newXLStorageTestSetup(t)
+ if err != nil {
+ t.Fatal(err)
+ }
+ const bucket, object = "restore-storage", "obj"
+ ctx := t.Context()
+ if err := disk.MakeVol(ctx, bucket); err != nil {
+ t.Fatal(err)
+ }
+ fi := newFileInfo(object, 2, 2)
+ fi.VersionID, fi.DataDir = mustGetUUID(), mustGetUUID()
+ fi.ModTime, fi.Size = time.Now().UTC(), 1024
+ fi.TransitionStatus = status
+ fi.AddObjectPart(1, "", fi.Size, fi.Size, fi.ModTime, nil, nil)
+ fi.Metadata = map[string]string{xhttp.AmzRestore: completedRestoreObj(time.Now().Add(-time.Hour)).String(), "x-amz-meta-preserve": "keep"}
+ payload := []byte("sole local contents")
+ if inline {
+ fi.Data = payload
+ fi.SetInlineData()
+ }
+ if err := disk.WriteMetadata(ctx, "", bucket, object, fi); err != nil {
+ t.Fatal(err)
+ }
+ dir := filepath.Join(disk.Endpoint().Path, bucket, object, fi.DataDir)
+ if err := os.MkdirAll(dir, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "part.1"), payload, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ raw, err := disk.ReadXL(ctx, bucket, object, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var xl xlMetaV2
+ if err := xl.Load(raw.Buf); err != nil {
+ t.Fatal(err)
+ }
+ del := FileInfo{VersionID: fi.VersionID, ExpireRestored: true, TransitionStatus: requestStatus}
+ dataDir, err := xl.DeleteVersion(del)
+ if err != nil {
+ t.Fatal(err)
+ }
+ wantDir := ""
+ if status == lifecycle.TransitionComplete {
+ wantDir = fi.DataDir
+ }
+ if dataDir != wantDir {
+ t.Errorf("format authorized directory %q, want %q", dataDir, wantDir)
+ }
+ before, err := getFileInfo(raw.Buf, bucket, object, fi.VersionID, fileInfoOpts{Data: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := disk.DeleteVersion(ctx, bucket, object, del, false, DeleteOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ afterRaw, err := disk.ReadXL(ctx, bucket, object, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ after, err := getFileInfo(afterRaw.Buf, bucket, object, fi.VersionID, fileInfoOpts{Data: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, dirErr := os.ReadFile(filepath.Join(dir, "part.1"))
+ if status != lifecycle.TransitionComplete {
+ if !reflect.DeepEqual(before, after) {
+ t.Error("non-complete cleanup changed metadata or inline data")
+ }
+ if dirErr != nil || !bytes.Equal(data, payload) {
+ t.Errorf("sole local contents removed: %v", dirErr)
+ }
+ } else {
+ if !os.IsNotExist(dirErr) {
+ t.Errorf("completed restore directory still present: %v", dirErr)
+ }
+ if len(after.Data) != 0 || after.Metadata[xhttp.AmzRestore] != "" {
+ t.Error("completed restore kept cached data or restore headers")
+ }
+ if after.TransitionStatus != status || after.VersionID != before.VersionID || after.Metadata["x-amz-meta-preserve"] != "keep" {
+ t.Error("cleanup changed the transitioned version identity or user metadata")
+ }
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/cmd/bucket-lifecycle-restore_test.go b/cmd/bucket-lifecycle-restore_test.go
new file mode 100644
index 000000000..6917f539d
--- /dev/null
+++ b/cmd/bucket-lifecycle-restore_test.go
@@ -0,0 +1,453 @@
+// Copyright (c) 2026 Denis Babchuk
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/versioning"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// restoreTestTier is a remote tier held in memory that counts its reads.
+type restoreTestTier struct {
+ *tierExpiryTestBackend
+ gets atomic.Int64
+}
+
+func (b *restoreTestTier) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+ b.gets.Add(1)
+ return b.tierExpiryTestBackend.Get(ctx, object, rv, opts)
+}
+
+// restoreTest is a cluster with a remote tier WARM and a bucket whose
+// versions are transitioned to it and restored.
+type restoreTest struct {
+ t *testing.T
+ z *erasureServerPools
+ bucket string
+ object string
+ queue chan expiryOp
+ labels map[string]string // version ID ("null" for the null version) -> label
+ bodies map[string][]byte // label -> data
+}
+
+func restoreTestVersionKey(versionID string) string {
+ if versionID == "" {
+ return nullVersionID
+ }
+ return versionID
+}
+
+// put writes a version of the object to the first pool. A restored version
+// is 1 MiB, so that its restored copy has a data directory of its own.
+func (r *restoreTest) put(label string, null, restored bool) ObjectInfo {
+ r.t.Helper()
+ body := []byte(r.object + " " + label)
+ if restored {
+ body = bytes.Repeat(body, (1<<20)/len(body)+1)
+ }
+ opts := ObjectOptions{Versioned: !null}
+ if null {
+ opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object)
+ }
+ oi, err := r.z.serverPools[0].PutObject(r.t.Context(), r.bucket, r.object,
+ mustGetPutObjReader(r.t, bytes.NewReader(body), int64(len(body)), "", ""), opts)
+ if err != nil {
+ r.t.Fatal(err)
+ }
+ r.labels[restoreTestVersionKey(oi.VersionID)] = label
+ r.bodies[label] = body
+ return oi
+}
+
+// opts returns the options of a request on a version of the object, as the
+// S3 handlers set them.
+func (r *restoreTest) opts(versionID string) ObjectOptions {
+ return ObjectOptions{
+ VersionID: versionID,
+ Versioned: globalBucketVersioningSys.PrefixEnabled(r.bucket, r.object),
+ VersionSuspended: globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object),
+ }
+}
+
+// restore transitions a version to the tier, restores it, and moves the
+// expiry date of the restore into the past.
+func (r *restoreTest) restore(oi ObjectInfo) {
+ r.t.Helper()
+ ctx := r.t.Context()
+ vopts := r.opts(oi.VersionID)
+ topts := vopts
+ topts.MTime = oi.ModTime
+ topts.Transition = TransitionOptions{Tier: "WARM", ETag: oi.ETag}
+ if err := r.z.TransitionObject(ctx, r.bucket, r.object, topts); err != nil {
+ r.t.Fatal(err)
+ }
+ ropts := vopts
+ ropts.Transition = TransitionOptions{RestoreRequest: &RestoreObjectRequest{Days: 1}}
+ // Through the pool, as with a single pool: with more than one pool the
+ // pools layer holds the object lock that the write of the restored copy
+ // waits for.
+ if err := r.z.serverPools[0].RestoreTransitionedObject(ctx, r.bucket, r.object, ropts); err != nil {
+ r.t.Fatal(err)
+ }
+ expired := completedRestoreObj(time.Now().Add(-24 * time.Hour).UTC()).String()
+ mopts := vopts
+ mopts.MTime = oi.ModTime
+ mopts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+ o.UserDefined[xhttp.AmzRestore] = expired
+ return ReplicateDecision{}, nil
+ }
+ if _, err := r.z.PutObjectMetadata(ctx, r.bucket, r.object, mopts); err != nil {
+ r.t.Fatal(err)
+ }
+}
+
+// fileInfo returns the version of the object from a drive of the first pool.
+func (r *restoreTest) fileInfo(versionID string) FileInfo {
+ r.t.Helper()
+ infos, errs := readAllFileInfo(r.t.Context(), r.z.serverPools[0].getHashedSet(r.object).getDisks(), "", r.bucket, r.object, restoreTestVersionKey(versionID), false, false)
+ if errs[0] != nil {
+ r.t.Fatal(errs[0])
+ }
+ return infos[0]
+}
+
+// dataDirs returns how many drives of the first pool hold the data
+// directory of a version.
+func (r *restoreTest) dataDirs(dataDir string) int {
+ r.t.Helper()
+ n := 0
+ for _, disk := range r.z.serverPools[0].getHashedSet(r.object).getDisks() {
+ _, err := os.Stat(filepath.Join(disk.Endpoint().Path, r.bucket, r.object, dataDir))
+ if err == nil {
+ n++
+ } else if !os.IsNotExist(err) {
+ r.t.Fatal(err)
+ }
+ }
+ return n
+}
+
+// versions returns the versions of the object in each pool, newest first,
+// and its free versions in all pools.
+func (r *restoreTest) versions() (views []string, free int) {
+ r.t.Helper()
+ for _, pool := range r.z.serverPools {
+ rf, err := pool.getHashedSet(r.object).getDisks()[0].ReadXL(r.t.Context(), r.bucket, r.object, false)
+ if errors.Is(err, errFileNotFound) {
+ views = append(views, "")
+ continue
+ }
+ if err != nil {
+ r.t.Fatal(err)
+ }
+ fivs, err := getFileInfoVersions(rf.Buf, r.bucket, r.object, false)
+ if err != nil {
+ r.t.Fatal(err)
+ }
+ var labels []string
+ for _, fi := range fivs.Versions {
+ if fi.Deleted {
+ labels = append(labels, "DM")
+ } else {
+ labels = append(labels, r.labels[restoreTestVersionKey(fi.VersionID)])
+ }
+ }
+ views = append(views, strings.Join(labels, " "))
+ free += len(fivs.FreeVersions)
+ }
+ return views, free
+}
+
+// read returns the label and the data of a version, or of the latest version
+// when versionID is empty, or the error of the read.
+func (r *restoreTest) read(versionID string) (string, []byte, error) {
+ r.t.Helper()
+ gr, err := r.z.GetObjectNInfo(r.t.Context(), r.bucket, r.object, nil, nil, ObjectOptions{VersionID: versionID})
+ if err != nil {
+ return "", nil, err
+ }
+ defer gr.Close()
+ data, err := io.ReadAll(gr)
+ if err != nil {
+ return "", nil, err
+ }
+ return r.labels[restoreTestVersionKey(gr.ObjInfo.VersionID)], data, nil
+}
+
+// runQueue runs the queued expiries with an expiry worker.
+func (r *restoreTest) runQueue() {
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ globalExpiryState.Worker(r.queue)
+ }()
+ r.queue <- nil // the worker returns once it ran every expiry queued before
+ <-done
+}
+
+// objInfos returns the versions of the object as a drive of the
+// first pool holds them, as the scanner and the walks of decommission and
+// the drive heal see them.
+func (r *restoreTest) objInfos() []ObjectInfo {
+ r.t.Helper()
+ rf, err := r.z.serverPools[0].getHashedSet(r.object).getDisks()[0].ReadXL(r.t.Context(), r.bucket, r.object, false)
+ if err != nil {
+ r.t.Fatal(err)
+ }
+ fivs, err := getFileInfoVersions(rf.Buf, r.bucket, r.object, false)
+ if err != nil {
+ r.t.Fatal(err)
+ }
+ versioned := globalBucketVersioningSys.PrefixEnabled(r.bucket, r.object) || globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object)
+ objInfos := make([]ObjectInfo, len(fivs.Versions))
+ for i, fi := range fivs.Versions {
+ objInfos[i] = fi.ToObjectInfo(r.bucket, r.object, versioned)
+ }
+ return objInfos
+}
+
+// The restored copy of a transitioned version expires on the expiry date of
+// its restore. The expiry removes the copy and nothing else: the version
+// stays where it is in the version stack, transitioned, and is read from the
+// tier again. Every path that applies ILM runs the expiry: the scanner and a
+// GET or HEAD queue it for an expiry worker, decommission and the drive heal
+// run it at once. The expiry of the restored copy of the latest version used
+// to delete the object instead: a versioned bucket got a delete marker that
+// hid it. With versioning suspended it went to the null version, removed its
+// data if there was one and failed if not, and the restored copy stayed.
+func TestExpireRestoredCopy(t *testing.T) {
+ prevTier := globalTierConfigMgr
+ t.Cleanup(func() { globalTierConfigMgr = prevTier })
+ prevExpiry := globalExpiryState
+ t.Cleanup(func() { globalExpiryState = prevExpiry })
+
+ // Nothing here is old enough to transition; the rule makes the bucket
+ // one with a tier, as restored versions are in.
+ const rules = `tierEnabled` +
+ `1000WARM`
+
+ type version struct {
+ label string
+ null bool // the null version
+ restored bool // transitioned and restored, the restore has expired
+ }
+ testCases := []struct {
+ name string
+ versioning versioning.State // "", Enabled or Suspended
+ versions []version // oldest first
+ }{
+ {name: "versioned", versioning: versioning.Enabled, versions: []version{{label: "v1"}, {label: "v2", restored: true}}},
+ {name: "versioned-noncurrent", versioning: versioning.Enabled, versions: []version{{label: "v1", restored: true}, {label: "v2"}}},
+ {name: "versioned-null", versioning: versioning.Enabled, versions: []version{{label: "v1", null: true, restored: true}}},
+ {name: "suspended", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true}, {label: "v2", restored: true}}},
+ {name: "suspended-no-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", restored: true}}},
+ {name: "suspended-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", null: true, restored: true}}},
+ {name: "suspended-null-noncurrent", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true, restored: true}, {label: "v2"}}},
+ {name: "unversioned", versions: []version{{label: "v1", null: true, restored: true}}},
+ }
+
+ sources := []struct {
+ name string
+ expire func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event)
+ }{
+ {"scanner", func(r *restoreTest, _ ObjectInfo, _ lifecycle.Event) {
+ lc, err := globalLifecycleSys.Get(r.bucket)
+ if err != nil {
+ r.t.Fatal(err)
+ }
+ lr, err := globalBucketObjectLockSys.Get(r.bucket)
+ if err != nil {
+ r.t.Fatal(err)
+ }
+ item := scannerItem{bucket: r.bucket, objectName: r.object, lifeCycle: lc}
+ item.applyActions(r.t.Context(), r.z, r.objInfos(), lr, &sizeSummary{}, nil)
+ r.runQueue()
+ }},
+ {"head", func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) {
+ applyExpiryRule(evt, lcEventSrc_s3HeadObject, oi)
+ r.runQueue()
+ }},
+ {"list", func(r *restoreTest, _ ObjectInfo, _ lifecycle.Event) {
+ // ListObjects of the key with max-keys=1 reads its latest
+ // version without the versioning of the bucket, a listing of
+ // versions reads them all.
+ if _, err := r.z.ListObjects(r.t.Context(), r.bucket, r.object, "", "", 1); err != nil {
+ r.t.Fatal(err)
+ }
+ if _, err := r.z.ListObjectVersions(r.t.Context(), r.bucket, r.object, "", "", "", 100); err != nil {
+ r.t.Fatal(err)
+ }
+ r.runQueue()
+ }},
+ {"decom", func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) {
+ for _, o := range r.objInfos() {
+ if o.VersionID == oi.VersionID {
+ applyExpiryOnTransitionedObject(r.t.Context(), r.z, o, evt, lcEventSrc_Decom)
+ return
+ }
+ }
+ r.t.Fatal("version not found on the drive")
+ }},
+ }
+
+ for _, pools := range []int{1, 2} {
+ t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+ ctx := t.Context()
+ dirs, err := getRandomDisks(4 * pools)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { removeRoots(dirs) })
+ var endpoints EndpointServerPools
+ for i := range pools {
+ endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
+ }
+ obj, _, err := initObjectLayer(ctx, endpoints)
+ if err != nil {
+ t.Fatal(err)
+ }
+ z := obj.(*erasureServerPools)
+ t.Cleanup(func() { z.Shutdown(context.Background()) })
+ // After initObjectLayer, which sets up a new tier config.
+ tier := &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
+ globalTierConfigMgr = NewTierConfigMgr()
+ globalTierConfigMgr.drivercache["WARM"] = tier
+
+ for _, tc := range testCases {
+ for _, src := range sources {
+ t.Run(tc.name+"/"+src.name, func(t *testing.T) {
+ r := &restoreTest{
+ t: t, z: z,
+ bucket: "restore-" + tc.name + "-" + src.name, object: "obj",
+ queue: make(chan expiryOp, 100),
+ labels: map[string]string{}, bodies: map[string][]byte{},
+ }
+ if err := z.MakeBucket(ctx, r.bucket, MakeBucketOptions{VersioningEnabled: tc.versioning != ""}); err != nil {
+ t.Fatal(err)
+ }
+ meta, err := globalBucketMetadataSys.Get(r.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if tc.versioning == versioning.Suspended {
+ meta.VersioningConfigXML = []byte("Suspended")
+ }
+ meta.LifecycleConfigXML = []byte(rules)
+ if err := meta.Save(ctx, z); err != nil {
+ t.Fatal(err)
+ }
+ globalBucketMetadataSys.Set(r.bucket, meta)
+ es := &expiryState{ctx: ctx, objAPI: z}
+ workers := []chan expiryOp{r.queue}
+ es.workers.Store(&workers)
+ globalExpiryState = es
+
+ var restored ObjectInfo
+ for _, v := range tc.versions {
+ oi := r.put(v.label, v.null, v.restored)
+ if v.restored {
+ r.restore(oi)
+ restored = oi
+ }
+ }
+ rlabel := r.labels[restoreTestVersionKey(restored.VersionID)]
+
+ // The fixture: the restored copy is on the drives,
+ // its restore has expired, ILM expires it.
+ fi := r.fileInfo(restored.VersionID)
+ if fi.TransitionStatus != lifecycle.TransitionComplete || fi.DataDir == "" || r.dataDirs(fi.DataDir) == 0 {
+ t.Fatalf("fixture: %s transition status %q, restored copy on %d drives", rlabel, fi.TransitionStatus, r.dataDirs(fi.DataDir))
+ }
+ oi, err := z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(restored.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ lc, err := globalLifecycleSys.Get(r.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ lr, err := globalBucketObjectLockSys.Get(r.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ evt := evalActionFromLifecycle(ctx, *lc, lr, nil, oi)
+ if !evt.Action.DeleteRestored() || oi.RestoreExpires.IsZero() {
+ t.Fatalf("fixture: %s restore expires %v, action %v", rlabel, oi.RestoreExpires, evt.Action)
+ }
+ before, _ := r.versions()
+ for key, label := range r.labels {
+ if _, data, err := r.read(key); err != nil || !bytes.Equal(data, r.bodies[label]) {
+ t.Fatalf("fixture: GET %s: %v", label, err)
+ }
+ }
+
+ src.expire(r, oi, evt)
+
+ after, free := r.versions()
+ if fmt.Sprint(after) != fmt.Sprint(before) || free != 0 {
+ t.Errorf("versions in each pool: before %q, after %q with %d free versions", before, after, free)
+ }
+ if label, _, err := r.read(""); err != nil || label != tc.versions[len(tc.versions)-1].label {
+ t.Errorf("GET reads %q, %v; want %q", label, err, tc.versions[len(tc.versions)-1].label)
+ }
+ if n := r.dataDirs(fi.DataDir); n != 0 {
+ t.Errorf("restored copy of %s still on %d drives", rlabel, n)
+ }
+ oi, err = z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(restored.VersionID))
+ if err != nil {
+ t.Fatalf("HEAD %s: %v", rlabel, err)
+ }
+ if oi.TransitionedObject.Status != lifecycle.TransitionComplete || oi.UserDefined[xhttp.AmzRestore] != "" || !oi.RestoreExpires.IsZero() {
+ t.Errorf("HEAD %s: transition status %q, x-amz-restore %q", rlabel, oi.TransitionedObject.Status, oi.UserDefined[xhttp.AmzRestore])
+ }
+ for _, v := range tc.versions {
+ for key, label := range r.labels {
+ if label != v.label {
+ continue
+ }
+ gets := tier.gets.Load()
+ _, data, err := r.read(key)
+ switch {
+ case err != nil:
+ t.Errorf("GET %s: %v", v.label, err)
+ case !bytes.Equal(data, r.bodies[v.label]):
+ t.Errorf("GET %s reads %d bytes that differ", v.label, len(data))
+ case v.restored && tier.gets.Load() == gets:
+ t.Errorf("GET %s does not read the tier", v.label)
+ }
+ }
+ }
+ })
+ }
+ }
+ })
+ }
+}
diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go
index e7a6d99e5..54511b851 100644
--- a/cmd/bucket-lifecycle.go
+++ b/cmd/bucket-lifecycle.go
@@ -37,6 +37,7 @@ import (
"github.com/minio/minio/internal/amztime"
sse "github.com/minio/minio/internal/bucket/encryption"
"github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/replication"
"github.com/minio/minio/internal/event"
xhttp "github.com/minio/minio/internal/http"
@@ -611,8 +612,9 @@ func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) error {
traceFn := globalLifecycleSys.trace(*oi)
opts := ObjectOptions{
- Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
- Expiration: ExpirationOptions{Expire: true},
+ Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+ VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+ Expiration: ExpirationOptions{Expire: true},
}
if lcEvent.Action.DeleteVersioned() {
opts.VersionID = oi.VersionID
@@ -622,6 +624,17 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
// delete locally restored copy of object or object version
// from the source, while leaving metadata behind. The data on
// transitioned tier lies untouched and still accessible
+ //
+ // The restored copy belongs to this version, also when it is the
+ // latest one (DeleteRestoredAction). Without its version ID the
+ // delete applies to the object: a versioned bucket gets a delete
+ // marker that hides it, a suspended one strips the null version.
+ // An empty version ID is the null version, as callers that read
+ // it without the versioning of the bucket leave it.
+ opts.VersionID = oi.VersionID
+ if opts.VersionID == "" {
+ opts.VersionID = nullVersionID
+ }
opts.Transition.ExpireRestored = true
_, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
if err == nil {
@@ -636,7 +649,6 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
// touching the remote tier. If the lifecycle configuration changed after
// the scanner queued this task, DeleteObject rejects the stale action while
// the remote object is still protected.
- opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name)
keepRemote := opts.VersionID == "" &&
(opts.Versioned || (opts.VersionSuspended && oi.VersionID != "" && oi.VersionID != nullVersionID))
if lcEvent.Action == lifecycle.DeleteAction || lcEvent.Action == lifecycle.DeleteVersionAction {
@@ -976,8 +988,10 @@ func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string
}, nil
}
-// set ObjectOptions for PUT call to restore temporary copy of transitioned data
-func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) {
+// set ObjectOptions for PUT call to restore temporary copy of transitioned data.
+// opts are the restore's options, with the versioning POST captured: the
+// bucket's versioning may have changed since, and must not mint a new version.
+func putRestoreOpts(rreq *RestoreObjectRequest, objInfo ObjectInfo, opts ObjectOptions) (putOpts ObjectOptions) {
meta := make(map[string]string)
sc := rreq.OutputLocation.S3.StorageClass
if sc == "" {
@@ -1000,8 +1014,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
}
return ObjectOptions{
- Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, object),
- VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+ Versioned: opts.Versioned,
+ VersionSuspended: opts.VersionSuspended,
UserDefined: meta,
}
}
@@ -1009,19 +1023,37 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
if len(objInfo.UserTags) != 0 {
meta[xhttp.AmzObjectTagging] = objInfo.UserTags
}
- // Set restore object status
+ // Set restore object status; a completed restore has no attempt.
restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
+ delete(meta, restoreAttemptKey)
return ObjectOptions{
- Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, object),
- VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),
+ Versioned: opts.Versioned,
+ VersionSuspended: opts.VersionSuspended,
UserDefined: meta,
- VersionID: objInfo.VersionID,
+ VersionID: restoreVersionID(objInfo.VersionID),
MTime: objInfo.ModTime,
Expires: objInfo.Expires,
}
}
+// restoredCopyExpires reports whether the restored copy of oi, as read now,
+// expires by its own reason: lifecycle expires the restored copy of this
+// version, its restore deadline is valid and has passed, its transition is
+// complete and no restore is running. Another rule falling due is no reason.
+func restoredCopyExpires(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, oi ObjectInfo) bool {
+ if !evalActionFromLifecycle(ctx, lc, lr, rcfg, oi).Action.DeleteRestored() {
+ return false
+ }
+ status, err := parseRestoreObjStatus(oi.UserDefined[xhttp.AmzRestore])
+ if err != nil {
+ return false
+ }
+ expiry, completed := status.Expiry()
+ return completed && !expiry.IsZero() && UTCNow().After(expiry) &&
+ oi.TransitionedObject.Status == lifecycle.TransitionComplete && !oi.RestoreOngoing
+}
+
var errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
// IsRemote returns true if this object version's contents are in its remote
diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go
index 329772df9..77d265367 100644
--- a/cmd/data-scanner.go
+++ b/cmd/data-scanner.go
@@ -1204,7 +1204,7 @@ func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) (ok bool) {
timeILM := globalScannerMetrics.timeILM(lcEvent.Action)
if err := expireTransitionedObject(ctx, objLayer, &obj, lcEvent, src); err != nil {
- if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+ if errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
return false
}
ilmLogIf(ctx, fmt.Errorf("expireTransitionedObject(%s, %s): %w", obj.Bucket, obj.Name, err))
@@ -1271,7 +1271,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
if err != nil {
- if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+ if errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
traceFn(ILMExpiry, nil, nil)
return false
}
diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go
index 8fb371270..fad01b0cf 100644
--- a/cmd/erasure-healing.go
+++ b/cmd/erasure-healing.go
@@ -373,6 +373,13 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
if err != nil {
return result, err
}
+ // The restore fields fence native restores and restored-copy cleanup.
+ // Heal copies them only as a quorum of the drives holds them; while the
+ // drives disagree, it waits rather than create a quorum for one side.
+ latestMeta, err = restoreHealMeta(partsMetadata, latestMeta, max(len(storageDisks)/2+1, readQuorum))
+ if err != nil {
+ return result, toObjectErr(err, bucket, object, versionID)
+ }
// No modtime quorum
filterDisksByETag := quorumETag != ""
diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go
index 5c9ee5d80..8a0de02ad 100644
--- a/cmd/erasure-object.go
+++ b/cmd/erasure-object.go
@@ -45,7 +45,6 @@ import (
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/grid"
- "github.com/minio/minio/internal/hash"
xhttp "github.com/minio/minio/internal/http"
xioutil "github.com/minio/minio/internal/ioutil"
"github.com/minio/minio/internal/logger"
@@ -509,6 +508,11 @@ func joinErrs(errs []error) string {
}
func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int, opts ObjectOptions) (FileInfo, error) {
+ // Restore and restored-copy cleanup must preserve the version on read
+ // failure, including lookups and failure cleanup that reread metadata.
+ if opts.Transition.ExpireRestored || opts.Transition.RestoreRequest != nil {
+ return FileInfo{}, errErasureReadQuorum
+ }
m, ok := isObjectDangling(metaArr, errs, dataErrsByPart)
if !ok {
// We only come here if we cannot figure out if the object
@@ -2045,6 +2049,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
// expiration attempted on a bucket with no lifecycle
// rules shall be rejected.
if lc == nil && opts.Expiration.Expire {
+ if !opts.Transition.ExpireRestored {
+ return objInfo, errLifecycleNotApplicable
+ }
if opts.VersionID != "" {
return objInfo, VersionNotFound{
Bucket: bucket,
@@ -2089,6 +2096,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
isErr = true
}
if isErr {
+ if !opts.Transition.ExpireRestored {
+ return goi, errLifecycleNotApplicable
+ }
if goi.VersionID != "" {
return goi, VersionNotFound{
Bucket: bucket,
@@ -2107,6 +2117,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object)
}
+ // An ILM version expiry that the pools checked with the versions in all
+ // pools; the set still guards the protection it reads, see
+ // checkedExpiryRefused.
+ checkedExpiry := opts.Expiration.Expire && opts.Expiration.CheckedAllPools && opts.VersionID != "" && !opts.Transition.ExpireRestored
+
storageDisks := er.getDisks()
versionFound := true
objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
@@ -2114,7 +2129,10 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
tryDel := false
if gerr != nil && goi.Name == "" {
if _, ok := gerr.(InsufficientReadQuorum); ok {
- if opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
+ // A restored-copy expiry needs fresh metadata to confirm its target
+ // and deadline, a checked expiry to guard its protection; ordinary
+ // unversioned DELETE may still try a blind delete.
+ if opts.Transition.ExpireRestored || checkedExpiry || opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
// Add an MRF heal for next time.
er.addPartial(bucket, object, opts.VersionID)
return objInfo, InsufficientWriteQuorum{}
@@ -2134,6 +2152,26 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
}
}
+ // A queued null restore may now address a replacement delete marker.
+ // Such errors carry ObjectInfo, but never authorize cache cleanup.
+ if opts.Transition.ExpireRestored && (gerr != nil || goi.DeleteMarker) {
+ return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
+ }
+ // A restored copy expires only by its own reason, decided on the
+ // version's authoritative generation: a drive that missed a renewal or a
+ // later POST must not authorize removing the cache, nor drives that
+ // disagree on the restore status. Another lifecycle deletion falling due,
+ // or the all-pools check of an ordinary expiry, does not stand in for it.
+ if opts.Transition.ExpireRestored {
+ gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
+ if err != nil || gen.split || gen.fi.Deleted || lc == nil {
+ return goi, errLifecycleNotApplicable
+ }
+ if !restoredCopyExpires(ctx, *lc, rcfg, replcfg, gen.fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)) {
+ return goi, errLifecycleNotApplicable
+ }
+ }
+
if opts.EvalMetadataFn != nil {
dsc, err := opts.EvalMetadataFn(&goi, gerr)
if err != nil {
@@ -2151,7 +2189,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
}
}
- if opts.Expiration.Expire {
+ if opts.Expiration.Expire && !opts.Expiration.CheckedAllPools {
if gerr == nil {
evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
var isErr bool
@@ -2162,6 +2200,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
isErr = true
}
if isErr {
+ if !opts.Transition.ExpireRestored {
+ return goi, errLifecycleNotApplicable
+ }
if goi.VersionID != "" {
return goi, VersionNotFound{
Bucket: bucket,
@@ -2175,6 +2216,8 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
}
}
}
+ } else if checkedExpiry && checkedExpiryRefused(*lc, rcfg, replcfg, goi, opts.Expiration.ApprovedProtection) {
+ return goi, errLifecycleNotApplicable
}
// Determine whether to mark object deleted for replication
@@ -2621,107 +2664,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
return err
}
-// RestoreTransitionedObject - restore transitioned object content locally on this cluster.
-// This is similar to PostObjectRestore from AWS GLACIER
-// storage class. When PostObjectRestore API is called, a temporary copy of the object
-// is restored locally to the bucket on source cluster until the restore expiry date.
-// The copy that was transitioned continues to reside in the transitioned tier.
-func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
- return er.restoreTransitionedObject(ctx, bucket, object, opts)
-}
-
-// update restore status header in the metadata
-func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error {
- oi := objInfo.Clone()
- oi.metadataOnly = true // Perform only metadata updates.
-
- // allow retry in the case of failure to restore
- delete(oi.UserDefined, xhttp.AmzRestore)
-
- if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{
- VersionID: oi.VersionID,
- }, ObjectOptions{
- VersionID: oi.VersionID,
- }); err != nil {
- storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
- return err
- }
- return nil
-}
-
-// restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts
-// as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta
-func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error {
- setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error {
- if rerr == nil {
- return nil // nothing to do; restore object was successful
- }
- er.updateRestoreMetadata(ctx, bucket, object, oi, opts)
- return rerr
- }
- var oi ObjectInfo
- // get the file info on disk for transitioned object
- actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
- if err != nil {
- return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
- }
-
- oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
- ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi)
- if len(oi.Parts) == 1 {
- var rs *HTTPRangeSpec
- gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
- if err != nil {
- return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
- }
- defer gr.Close()
- hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size)
- if err != nil {
- return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
- }
- pReader := NewPutObjReader(hashReader)
- _, err = er.PutObject(ctx, bucket, object, pReader, ropts)
- return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
- }
-
- res, err := er.NewMultipartUpload(ctx, bucket, object, ropts)
- if err != nil {
- return setRestoreHeaderFn(oi, err)
- }
-
- var uploadedParts []CompletePart
- var rs *HTTPRangeSpec
- // get reader from the warm backend - note that even in the case of encrypted objects, this stream is still encrypted.
- gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
- if err != nil {
- return setRestoreHeaderFn(oi, err)
- }
- defer gr.Close()
-
- // rehydrate the parts back on disk as per the original xl.meta prior to transition
- for _, partInfo := range oi.Parts {
- hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size)
- if err != nil {
- return setRestoreHeaderFn(oi, err)
- }
- pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{})
- if err != nil {
- return setRestoreHeaderFn(oi, err)
- }
- if pInfo.Size != partInfo.Size {
- return setRestoreHeaderFn(oi, InvalidObjectState{Bucket: bucket, Object: object})
- }
- uploadedParts = append(uploadedParts, CompletePart{
- PartNumber: pInfo.PartNumber,
- ETag: pInfo.ETag,
- })
- }
- _, err = er.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{
- MTime: oi.ModTime,
- })
- return setRestoreHeaderFn(oi, err)
-}
-
// DecomTieredObject - moves tiered object to another pool during decommissioning.
func (er erasureObjects) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error {
if opts.UserDefined == nil {
diff --git a/cmd/erasure-server-pool-consistency.go b/cmd/erasure-server-pool-consistency.go
index ad3e24d38..7d449b60a 100644
--- a/cmd/erasure-server-pool-consistency.go
+++ b/cmd/erasure-server-pool-consistency.go
@@ -19,6 +19,7 @@ package cmd
import (
"context"
+ "errors"
"maps"
"sort"
"strings"
@@ -670,7 +671,9 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
// deleteObjectsReconciled removes data while retaining each item's authoritative
// copy until every other pool has completed its cleanup. The caller holds the
// pools-layer write locks and separates marker creation from this batch.
-func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
+// approved holds the Object Lock state each checked ILM expiry was approved
+// with, see ExpirationOptions.ApprovedProtection.
+func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket string, objects []ObjectToDelete, approved []objectLockState, opts ObjectOptions) ([]DeletedObject, []error) {
deleted := make([]DeletedObject, len(objects))
errs := make([]error, len(objects))
primary := make([]int, len(objects))
@@ -760,11 +763,23 @@ func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket
}
deleteOpts := opts
deleteOpts.VersionID, deleteOpts.NoLock = obj.VersionID, true
+ // DeleteObjects already checked ordinary ILM UUID/explicit null
+ // versions against all pools. Empty IDs and user deletes have no such
+ // authorization; restored-copy expiry uses the single-object path.
+ deleteOpts.Expiration.CheckedAllPools = opts.Expiration.Expire && obj.VersionID != ""
+ if deleteOpts.Expiration.CheckedAllPools {
+ deleteOpts.Expiration.ApprovedProtection = approved[i]
+ }
deleteOpts.DeleteReplication = obj.ReplicationState()
if opts.PrefixEnabledFn != nil {
deleteOpts.Versioned = opts.PrefixEnabledFn(obj.ObjectName)
}
_, err := z.deleteObjectReconciled(ctx, bucket, obj.ObjectName, deleteOpts)
+ if errors.Is(err, errLifecycleNotApplicable) {
+ // A later lifecycle rejection did not commit a local purge. Do
+ // not let the worker replicate the request's pending decision.
+ deleted[i].ReplicationState = ReplicationState{}
+ }
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
errs[i] = err
} else {
diff --git a/cmd/erasure-server-pool-consistency_test.go b/cmd/erasure-server-pool-consistency_test.go
index 0dec71fd5..8c94fb49b 100644
--- a/cmd/erasure-server-pool-consistency_test.go
+++ b/cmd/erasure-server-pool-consistency_test.go
@@ -522,10 +522,20 @@ func TestPoolsDeleteVersionSpecialCalls(t *testing.T) {
opts := ObjectOptions{Versioned: true, VersionID: oi.VersionID, InclFreeVersions: !expiration, Expiration: ExpirationOptions{Expire: expiration}}
_, err := z.DeleteObject(t.Context(), bucket, object, opts)
if expiration {
- // No lifecycle rule authorizes expiration. Preserve its existing
- // version-not-found result, even with an unrelated unreadable pool.
- if !isErrVersionNotFound(err) {
- t.Errorf("expiration changed its scanner contract: %v", err)
+ // No rule authorizes expiration. Distinguish refusal from real
+ // absence, even with an unrelated unreadable pool, so a worker
+ // cannot acknowledge or replicate a purge that never happened.
+ if !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("expiration did not preserve its rejection: %v", err)
+ }
+ gr, err := z.serverPools[0].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: oi.VersionID})
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, err := io.ReadAll(gr)
+ gr.Close()
+ if err != nil || string(data) != "payload" || gr.ObjInfo.VersionID != oi.VersionID {
+ t.Errorf("rejected expiration changed the stored version: %v", err)
}
} else if err != nil {
t.Errorf("free-version cleanup was forced through all-pool resolution: %v", err)
diff --git a/cmd/erasure-server-pool-decom.go b/cmd/erasure-server-pool-decom.go
index bc5e1c514..19e6a01e8 100644
--- a/cmd/erasure-server-pool-decom.go
+++ b/cmd/erasure-server-pool-decom.go
@@ -819,6 +819,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
// a noncurrent version and has to be moved like any other. The
// scanner applies the expiry once it lives in another pool.
return lifecycle.NoneAction
+ case evt.Action.Delete() && !z.migrationExpiryApplies(ctx, objInfo, evt):
+ // Another pool holds versions with which the rule does not
+ // apply. The version moves like any other.
+ return lifecycle.NoneAction
case evt.Action.Delete():
return evt.Action
default:
diff --git a/cmd/erasure-server-pool-lifecycle-authority_test.go b/cmd/erasure-server-pool-lifecycle-authority_test.go
new file mode 100644
index 000000000..d75525356
--- /dev/null
+++ b/cmd/erasure-server-pool-lifecycle-authority_test.go
@@ -0,0 +1,1399 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "maps"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/replication"
+ "github.com/minio/minio/internal/bucket/versioning"
+ "github.com/minio/minio/internal/config/storageclass"
+ "github.com/minio/minio/internal/event"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/minio/minio/internal/pubsub"
+)
+
+// An all-pools ILM approval is only as sound as the history each set reports
+// and the protection it reads under the lock. These cases leave drives with
+// stale, partial or unreadable metadata, as failed or interrupted writes and
+// offline drives do, and check that no expiry deletes what they hide.
+
+const authorityNoncurrentRule = `ncvEnabledncv-1`
+
+var authorityHoldKey = strings.ToLower(xhttp.AmzObjectLockLegalHold)
+
+func newLockedSplitILM(t *testing.T, rules string) *splitILM {
+ t.Helper()
+ return &splitILM{migrationILM: newMigrationILM(t, versioning.Enabled, true, rules), ids: map[string]string{}, bodies: map[string]string{}}
+}
+
+// authorityParity pins new objects to 12+4 on the 16 drives of a set. Other
+// tests can leave another storage class configured. Without one, 4 is the
+// default; Update cannot restore an unconfigured class, so leave it then.
+func authorityParity(t *testing.T) {
+ t.Helper()
+ if globalStorageClass.GetParityForSC("") < 0 {
+ return
+ }
+ previous := globalStorageClass
+ globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
+ t.Cleanup(func() { globalStorageClass.Update(previous) })
+}
+
+// authorityDisk fails the history scan (ReadXL) or the version removal of
+// one key on a drive.
+type authorityDisk struct {
+ StorageAPI
+ bucket, key string
+ failRead, failWrite bool
+}
+
+func (d authorityDisk) ReadXL(ctx context.Context, bucket, key string, data bool) (RawFileInfo, error) {
+ if d.failRead && bucket == d.bucket && key == d.key {
+ return RawFileInfo{}, errDiskNotFound
+ }
+ return d.StorageAPI.ReadXL(ctx, bucket, key, data)
+}
+
+func (d authorityDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error {
+ if d.failWrite && volume == d.bucket && path == d.key {
+ return errDiskNotFound
+ }
+ return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
+}
+
+func (d authorityDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error {
+ errs := make([]error, len(versions))
+ var pass []FileInfoVersions
+ var idx []int
+ for i, v := range versions {
+ if d.failWrite && volume == d.bucket && v.Name == d.key {
+ errs[i] = errDiskNotFound
+ continue
+ }
+ pass = append(pass, v)
+ idx = append(idx, i)
+ }
+ if len(pass) > 0 {
+ for j, err := range d.StorageAPI.DeleteVersions(ctx, volume, pass, opts) {
+ errs[idx[j]] = err
+ }
+ }
+ return errs
+}
+
+// wrapAuthorityDisks makes the drives of pool that drive selects fail reads
+// or writes of key.
+func wrapAuthorityDisks(t *testing.T, s *splitILM, pool int, key string, failRead, failWrite bool, drive func(int) bool) {
+ t.Helper()
+ p := s.z.serverPools[pool]
+ p.erasureDisksMu.Lock()
+ prior := p.erasureDisks
+ wrapped := make([][]StorageAPI, len(prior))
+ for i, disks := range prior {
+ wrapped[i] = append([]StorageAPI(nil), disks...)
+ for j := range disks {
+ if drive(j) {
+ wrapped[i][j] = authorityDisk{StorageAPI: disks[j], bucket: s.bucket, key: key, failRead: failRead, failWrite: failWrite}
+ }
+ }
+ }
+ p.erasureDisks = wrapped
+ p.erasureDisksMu.Unlock()
+ t.Cleanup(func() {
+ p.erasureDisksMu.Lock()
+ p.erasureDisks = prior
+ p.erasureDisksMu.Unlock()
+ })
+}
+
+func authorityDrives(from, to int) func(int) bool {
+ return func(i int) bool { return i >= from && i < to }
+}
+
+// removeAuthorityCopies removes version id of key from the drives of pool
+// that drive selects, as a write that did not reach them leaves it.
+func removeAuthorityCopies(t *testing.T, s *splitILM, pool int, key, id string, drive func(int) bool) {
+ t.Helper()
+ for i, disk := range s.z.serverPools[pool].getHashedSet(key).getDisks() {
+ if !drive(i) {
+ continue
+ }
+ fi := FileInfo{VersionID: id}
+ fi.SetSkipTierFreeVersion()
+ if err := disk.DeleteVersion(t.Context(), s.bucket, key, fi, false, DeleteOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+func authorityHeader(t *testing.T, disk StorageAPI, bucket, key, id string) xlMetaV2VersionHeader {
+ t.Helper()
+ rf, err := disk.ReadXL(t.Context(), bucket, key, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var xl xlMetaV2
+ if err := xl.LoadOrConvert(rf.Buf); err != nil {
+ t.Fatal(err)
+ }
+ for _, ver := range xl.versions {
+ if uuid.UUID(ver.header.VersionID).String() == id {
+ return ver.header
+ }
+ }
+ t.Fatalf("drive has no version %s", id)
+ return xlMetaV2VersionHeader{}
+}
+
+func authorityExpireVersion(t *testing.T, s *splitILM, key, id string) error {
+ t.Helper()
+ _, err := s.z.DeleteObject(t.Context(), s.bucket, key, ObjectOptions{Versioned: true, VersionID: id, Expiration: ExpirationOptions{Expire: true}})
+ return err
+}
+
+func authorityPresent(t *testing.T, s *splitILM, pool int, key, id string) {
+ t.Helper()
+ if _, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}); err != nil {
+ t.Fatalf("pool %d lost version %s: %v", pool, id, err)
+ }
+}
+
+// minorityRevision writes the versions old and new of key to pool 0, applies
+// metadata to old, and leaves its last drive with the revision before, as a
+// drive that missed the update does. It reports whether that stale revision
+// sorts before the fifteen others.
+func minorityRevision(t *testing.T, s *splitILM, key string, metadata map[string]string) (id string, staleFirst bool) {
+ t.Helper()
+ ctx := t.Context()
+ s.write(key, splitAt("old", 5, 0))
+ s.write(key, splitAt("new", 3, 0))
+ id = s.ids[key+" old"]
+ disks := s.z.serverPools[0].getHashedSet(key).getDisks()
+ last := disks[len(disks)-1]
+ stale, err := last.ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ setLifecycleCopyMetadata(t, s, key, id, 0, metadata, false)
+ if err := last.WriteMetadata(ctx, "", s.bucket, key, stale); err != nil {
+ t.Fatal(err)
+ }
+ return id, authorityHeader(t, last, s.bucket, key, id).sortsBefore(authorityHeader(t, disks[0], s.bucket, key, id))
+}
+
+// One drive missed a legal hold that the other fifteen committed. Its stale
+// revision must not be selected, and must not keep the fifteen from
+// deciding, whichever signature sorts first.
+func TestPoolsLifecycleMinorityLockRevision(t *testing.T) {
+ authorityParity(t)
+ s := newLockedSplitILM(t, authorityNoncurrentRule)
+ ctx := t.Context()
+ for attempt := range 32 {
+ key := fmt.Sprintf("ncv-lock-%d", attempt)
+ stamp := UTCNow().Format(time.RFC3339Nano)
+ id, staleFirst := minorityRevision(t, s, key, poolLockMetadata("", "ON", stamp, stamp))
+ versions, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key)
+ switch {
+ case err != nil:
+ t.Errorf("attempt %d (stale sorts first: %t): history: %v", attempt, staleFirst, err)
+ case len(versions) != 2:
+ t.Fatalf("attempt %d: merged %d versions", attempt, len(versions))
+ case versions[1].UserDefined[authorityHoldKey] != "ON" || events[1].Action != lifecycle.NoneAction:
+ t.Errorf("attempt %d (stale sorts first: %t): merged hold=%q event=%s", attempt, staleFirst, versions[1].UserDefined[authorityHoldKey], events[1].Action)
+ }
+ if err := authorityExpireVersion(t, s, key, id); !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("attempt %d (stale sorts first: %t): expiry of a held version: %v", attempt, staleFirst, err)
+ }
+ oi, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id})
+ if err != nil || oi.UserDefined[authorityHoldKey] != "ON" {
+ t.Fatalf("held version changed: hold=%q err=%v", oi.UserDefined[authorityHoldKey], err)
+ }
+ if staleFirst {
+ return
+ }
+ }
+ t.Fatal("no attempt where the stale revision sorts first")
+}
+
+// The same layout without protection: a drive missed an ordinary metadata
+// update of a version that expires. The fifteen still decide its expiry.
+func TestPoolsLifecycleMinorityStaleRevision(t *testing.T) {
+ authorityParity(t)
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ for attempt := range 32 {
+ key := fmt.Sprintf("ncv-stale-%d", attempt)
+ id, staleFirst := minorityRevision(t, s, key, map[string]string{"x-amz-meta-note": fmt.Sprint(attempt)})
+ _, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key)
+ if err != nil || len(events) != 2 || events[1].Action != lifecycle.DeleteVersionAction {
+ t.Errorf("attempt %d (stale sorts first: %t): events=%v err=%v", attempt, staleFirst, events, err)
+ }
+ if err := authorityExpireVersion(t, s, key, id); err != nil {
+ t.Fatalf("attempt %d (stale sorts first: %t): expiry: %v", attempt, staleFirst, err)
+ }
+ for i, disk := range s.z.serverPools[0].getHashedSet(key).getDisks() {
+ if _, err := disk.ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{}); !errors.Is(err, errFileVersionNotFound) {
+ t.Fatalf("attempt %d: drive %d still holds the expired version: %v", attempt, i, err)
+ }
+ }
+ if staleFirst {
+ return
+ }
+ }
+ t.Fatal("no attempt where the stale revision sorts first")
+}
+
+// authorityRevision encodes a 12+4 object version as drive index holds it,
+// with the legal hold status hold.
+func authorityRevision(t *testing.T, id [16]byte, modTime int64, index int, hold string) xlMetaV2ShallowVersion {
+ t.Helper()
+ v := xlMetaV2Version{Type: ObjectType, ObjectV2: &xlMetaV2Object{
+ VersionID: id, DataDir: [16]byte{1}, ErasureAlgorithm: ReedSolomon, ErasureM: 12, ErasureN: 4,
+ ErasureBlockSize: blockSizeV2, ErasureIndex: index, ErasureDist: make([]uint8, 16), BitrotChecksumAlgo: HighwayHash,
+ PartNumbers: []int{1}, PartSizes: []int64{5}, Size: 5, ModTime: modTime,
+ MetaUser: map[string]string{authorityHoldKey: hold, "etag": "e", "content-type": "text/plain"},
+ MetaSys: map[string][]byte{"x-minio-internal-a": {1}, "x-minio-internal-b": {2}, "x-minio-internal-c": {3}},
+ }}
+ for i := range v.ObjectV2.ErasureDist {
+ v.ObjectV2.ErasureDist[i] = uint8(i + 1)
+ }
+ meta, err := v.MarshalMsg(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return xlMetaV2ShallowVersion{header: v.header(), meta: meta}
+}
+
+// The header Signature is a 32-bit fold of weak hashes, so two revisions can
+// share a header. Revisions are told apart by their metadata: drives that
+// hold byte-for-byte different metadata under one header are not one
+// revision, and the merged history only carries the accepted metadata.
+func TestPoolsLifecycleRevisionIdentity(t *testing.T) {
+ newID, oldID := [16]byte{2}, [16]byte{1}
+ history := func(off func(drive int) bool) [][]xlMetaV2ShallowVersion {
+ metas := make([][]xlMetaV2ShallowVersion, 16)
+ for i := range metas {
+ old := authorityRevision(t, oldID, 1, i+1, "ON")
+ if off(i) {
+ // The stale revision under the header of the held one.
+ header := old.header
+ old = authorityRevision(t, oldID, 1, i+1, "OFF")
+ old.header = header
+ }
+ metas[i] = []xlMetaV2ShallowVersion{authorityRevision(t, newID, 2, i+1, ""), old}
+ }
+ return metas
+ }
+ metaVs := slices.Repeat([]uint8{xlMetaVersion}, 16)
+ readQuorum := func(xlMetaV2ShallowVersion, uint8) int { return 12 }
+
+ t.Run("revision-key", func(t *testing.T) {
+ on, onOther := authorityRevision(t, oldID, 1, 1, "ON"), authorityRevision(t, oldID, 1, 7, "ON")
+ off := authorityRevision(t, oldID, 1, 1, "OFF")
+ off.header = on.header
+ if versionRevision(on, xlMetaVersion) != versionRevision(onOther, xlMetaVersion) {
+ t.Error("one revision on two drives has two keys")
+ }
+ if versionRevision(on, xlMetaVersion) == versionRevision(off, xlMetaVersion) {
+ t.Error("different metadata under one header has one key")
+ }
+ })
+ t.Run("stale-minority-first", func(t *testing.T) {
+ merged, err := authoritativeHistory(t.Context(), history(func(i int) bool { return i == 0 }), metaVs, xlMetaVersion, 9, readQuorum)
+ if err != nil || len(merged) != 2 {
+ t.Fatalf("history: err=%v versions=%d", err, len(merged))
+ }
+ var v xlMetaV2Version
+ if _, err := v.unmarshalV(xlMetaVersion, merged[1].meta); err != nil {
+ t.Fatal(err)
+ }
+ if hold := v.ObjectV2.MetaUser[authorityHoldKey]; hold != "ON" {
+ t.Fatalf("merged history carries hold=%q", hold)
+ }
+ })
+ t.Run("split-8-8", func(t *testing.T) {
+ if merged, err := authoritativeHistory(t.Context(), history(func(i int) bool { return i%2 == 0 }), metaVs, xlMetaVersion, 9, readQuorum); !errors.Is(err, errErasureReadQuorum) {
+ t.Fatalf("8/8 metadata split: %d versions, err=%v", len(merged), err)
+ }
+ })
+ // The user and the system map are separate sections of the digest: an
+ // entry that moves between them changes what the evaluator reads.
+ t.Run("map-boundary", func(t *testing.T) {
+ for _, entry := range [][2]string{
+ {authorityHoldKey, "ON"},
+ {strings.ToLower(xhttp.AmzObjectLockRetainUntilDate), "2099-01-01T00:00:00Z"},
+ {xhttp.AmzObjectTagging, "expire=yes"},
+ } {
+ ver := authorityRevision(t, oldID, 1, 1, "")
+ var v xlMetaV2Version
+ if _, err := v.unmarshalV(xlMetaVersion, ver.meta); err != nil {
+ t.Fatal(err)
+ }
+ v.ObjectV2.MetaUser, v.ObjectV2.MetaSys = map[string]string{entry[0]: entry[1]}, nil
+ user, err := v.MarshalMsg(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ v.ObjectV2.MetaUser, v.ObjectV2.MetaSys = nil, map[string][]byte{entry[0]: []byte(entry[1])}
+ sys, err := v.MarshalMsg(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if versionRevision(xlMetaV2ShallowVersion{header: ver.header, meta: user}, xlMetaVersion) ==
+ versionRevision(xlMetaV2ShallowVersion{header: ver.header, meta: sys}, xlMetaVersion) {
+ t.Errorf("%s moved between the maps keeps the revision", entry[0])
+ }
+ }
+ })
+ t.Run("canceled", func(t *testing.T) {
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ if _, err := authoritativeHistory(ctx, history(func(int) bool { return false }), metaVs, xlMetaVersion, 9, readQuorum); !errors.Is(err, context.Canceled) {
+ t.Fatalf("canceled history: %v", err)
+ }
+ })
+}
+
+// authorityMovedHold serves a drive whose metadata of version id carries the
+// legal hold in the system map instead of the user map, under the header it
+// had: the evaluator no longer reads the hold there.
+type authorityMovedHold struct {
+ StorageAPI
+ bucket, key, id string
+}
+
+func (d authorityMovedHold) ReadXL(ctx context.Context, bucket, key string, data bool) (RawFileInfo, error) {
+ rf, err := d.StorageAPI.ReadXL(ctx, bucket, key, data)
+ if err != nil || bucket != d.bucket || key != d.key {
+ return rf, err
+ }
+ var xl xlMetaV2
+ if err = xl.LoadOrConvert(rf.Buf); err != nil {
+ return rf, err
+ }
+ for i := range xl.versions {
+ if uuid.UUID(xl.versions[i].header.VersionID).String() != d.id {
+ continue
+ }
+ var v xlMetaV2Version
+ if _, err = v.unmarshalV(xl.metaV, xl.versions[i].meta); err != nil {
+ return rf, err
+ }
+ v.ObjectV2.MetaSys[authorityHoldKey] = []byte(v.ObjectV2.MetaUser[authorityHoldKey])
+ delete(v.ObjectV2.MetaUser, authorityHoldKey)
+ if xl.versions[i].meta, err = v.MarshalMsg(nil); err != nil {
+ return rf, err
+ }
+ }
+ rf.Buf, err = xl.AppendTo(nil)
+ return rf, err
+}
+
+// One drive's metadata moves the hold of a held version to the system map
+// under the same header. Its revision is not the held one, and the bulk
+// expiry keeps the version.
+func TestPoolsLifecycleMovedHoldKeepsVersion(t *testing.T) {
+ s := newLockedSplitILM(t, authorityNoncurrentRule)
+ ctx := t.Context()
+ const key = "ncv-moved-hold"
+ s.write(key, splitAt("old", 5, 0))
+ s.write(key, splitAt("new", 3, 0))
+ id := s.ids[key+" old"]
+ stamp := UTCNow().Format(time.RFC3339Nano)
+ setLifecycleCopyMetadata(t, s, key, id, 0, map[string]string{authorityHoldKey: "ON", ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: stamp}, false)
+ p := s.z.serverPools[0]
+ p.erasureDisksMu.Lock()
+ prior := p.erasureDisks
+ wrapped := make([][]StorageAPI, len(prior))
+ for i, disks := range prior {
+ wrapped[i] = append([]StorageAPI(nil), disks...)
+ wrapped[i][0] = authorityMovedHold{StorageAPI: disks[0], bucket: s.bucket, key: key, id: id}
+ }
+ p.erasureDisks = wrapped
+ p.erasureDisksMu.Unlock()
+ t.Cleanup(func() { p.erasureDisksMu.Lock(); p.erasureDisks = prior; p.erasureDisksMu.Unlock() })
+
+ versions, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key)
+ if err != nil || versions[1].UserDefined[authorityHoldKey] != "ON" || events[1].Action != lifecycle.NoneAction {
+ t.Errorf("history: events=%v err=%v", events, err)
+ }
+ if err := authorityDeletePath(t, s, key, id, "bulk"); !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("bulk expiry of a held version: %v", err)
+ }
+ authorityPresent(t, s, 0, key, id)
+}
+
+// A legacy version can carry several content checksums for each part. The
+// map is encoded in iteration order, and must not change the revision of
+// identical metadata; a different checksum does change it (control).
+func TestPoolsLifecycleLegacyPartChecksums(t *testing.T) {
+ legacy := func(index int, sha256Value string) []byte {
+ m := getSampleXLMeta(1)
+ m.Stat.ModTime, m.Stat.Size, m.Parts[0].Size = time.Unix(1700000000, 0).UTC(), 20, 20
+ m.Erasure.DataBlocks, m.Erasure.ParityBlocks, m.Erasure.Index = 12, 4, index
+ m.Erasure.Distribution = []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
+ m.Parts[0].Checksums = map[string]string{"CRC32": "AAAAAA==", "CRC32C": "AAAAAQ==", "SHA1": "AAAAAg==", "SHA256": sha256Value}
+ buf, err := json.Marshal(m)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return buf
+ }
+ revision := func(buf []byte) revisionKey {
+ var xl xlMetaV2
+ if err := xl.LoadOrConvert(buf); err != nil {
+ t.Fatal(err)
+ }
+ return versionRevision(xl.versions[0], xl.metaV)
+ }
+ keys := make(map[revisionKey]bool)
+ for i := range 32 {
+ keys[revision(legacy(i%16+1, "AAAAAw=="))] = true
+ }
+ if len(keys) != 1 {
+ t.Fatalf("identical legacy metadata has %d revisions", len(keys))
+ }
+ if revision(legacy(1, "AAAABA==")) == revision(legacy(1, "AAAAAw==")) {
+ t.Fatal("a different part checksum keeps the revision")
+ }
+ disks := make([]StorageAPI, 16)
+ for i := range disks {
+ disks[i] = authorityRawDisk{buf: legacy(i+1, "AAAAAw==")}
+ }
+ er := erasureObjects{getDisks: func() []StorageAPI { return disks }, defaultParityCount: 4}
+ if versions, err := er.objectVersions(t.Context(), "bucket", "key"); err != nil || len(versions) != 1 {
+ t.Fatalf("healthy legacy history: %d versions, err=%v", len(versions), err)
+ }
+}
+
+// authorityRawDisk serves the same xl.meta for every key.
+type authorityRawDisk struct {
+ StorageAPI
+ buf []byte
+}
+
+func (d authorityRawDisk) ReadXL(context.Context, string, string, bool) (RawFileInfo, error) {
+ return RawFileInfo{Buf: d.buf}, nil
+}
+
+// A legacy (xl.json) version stores the bitrot checksum of each drive's own
+// shard. Healthy drives differ there and still hold one revision.
+func TestPoolsLifecycleLegacyHistory(t *testing.T) {
+ for _, distinct := range []bool{false, true} {
+ name := "same-checksums" // control
+ if distinct {
+ name = "per-shard-checksums"
+ }
+ t.Run(name, func(t *testing.T) {
+ disks := make([]StorageAPI, 16)
+ for i := range disks {
+ m := getSampleXLMeta(1)
+ m.Stat.ModTime, m.Stat.Size, m.Parts[0].Size = time.Unix(1700000000, 0).UTC(), 20, 20
+ m.Erasure.DataBlocks, m.Erasure.ParityBlocks, m.Erasure.Index = 12, 4, i+1
+ m.Erasure.Distribution = []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
+ shard := []byte{1, 1}
+ if distinct {
+ shard[0] = byte(i + 1)
+ }
+ sum := sha256.Sum256(shard)
+ m.Erasure.Checksums[0] = ChecksumInfo{PartNumber: 1, Algorithm: SHA256, Hash: sum[:]}
+ buf, err := json.Marshal(m)
+ if err != nil {
+ t.Fatal(err)
+ }
+ disks[i] = authorityRawDisk{buf: buf}
+ }
+ er := erasureObjects{getDisks: func() []StorageAPI { return disks }, defaultParityCount: 4}
+ if versions, err := er.objectVersions(t.Context(), "bucket", "key"); err != nil || len(versions) != 1 {
+ t.Fatalf("healthy legacy history: %d versions, err=%v", len(versions), err)
+ }
+ })
+ }
+}
+
+// A fresh version committed to 15 of 16 drives, with eight other drives
+// unreadable: only seven answering drives show it.
+func TestPoolsLifecyclePartialHistory(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "eav-partial-history"
+ s.write(key, splitAt("old", 5, 0, 1))
+ s.write(key, splitAt("fresh", 0, 1))
+ id := s.ids[key+" fresh"]
+ removeAuthorityCopies(t, s, 1, key, id, authorityDrives(0, 1))
+ wrapAuthorityDisks(t, s, 1, key, true, false, authorityDrives(1, 9))
+
+ if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) {
+ t.Errorf("partial history: %v", err)
+ }
+ _, err := s.z.DeleteObject(ctx, s.bucket, key, ObjectOptions{Versioned: true, DeletePrefix: true, DeletePrefixObject: true, Expiration: ExpirationOptions{Expire: true}})
+ if !isErrWriteQuorum(err) {
+ t.Errorf("DeleteAll on a partial history: %v", err)
+ }
+ authorityPresent(t, s, 1, key, id)
+}
+
+// Eight drives hold one revision of a version and eight another. Neither is
+// authoritative, so neither may decide the expiry.
+func TestPoolsLifecycleConflictingRevisions(t *testing.T) {
+ s := newLockedSplitILM(t, authorityNoncurrentRule)
+ ctx := t.Context()
+ const key = "ncv-split-revision"
+ s.write(key, splitAt("old", 5, 0))
+ s.write(key, splitAt("new", 3, 0))
+ id := s.ids[key+" old"]
+ disks := s.z.serverPools[0].getHashedSet(key).getDisks()
+ half := len(disks) / 2
+ stale := make([]FileInfo, len(disks))
+ for i := half; i < len(disks); i++ {
+ fi, err := disks[i].ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ stale[i] = fi
+ }
+ stamp := UTCNow().Format(time.RFC3339Nano)
+ setLifecycleCopyMetadata(t, s, key, id, 0, poolLockMetadata("", "ON", stamp, stamp), false)
+ for i := half; i < len(disks); i++ {
+ if err := disks[i].WriteMetadata(ctx, "", s.bucket, key, stale[i]); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) {
+ t.Errorf("8/8 revisions: %v", err)
+ }
+ if err := authorityExpireVersion(t, s, key, id); !isErrWriteQuorum(err) {
+ t.Errorf("expiry on 8/8 revisions: %v", err)
+ }
+ for i, disk := range disks {
+ if _, err := disk.ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{}); err != nil {
+ t.Fatalf("drive %d lost the version: %v", i, err)
+ }
+ }
+}
+
+// Every drive answers and the old version is complete, but a fresh version
+// that some drives show reached too few of them to be authoritative.
+func TestPoolsLifecycleMissingVersion(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "eav-missing-version"
+ s.write(key, splitAt("old", 5, 0, 1))
+ s.write(key, splitAt("fresh", 0, 1))
+ id := s.ids[key+" fresh"]
+ removeAuthorityCopies(t, s, 1, key, id, authorityDrives(0, 9))
+
+ if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) {
+ t.Errorf("history without an observed version: %v", err)
+ }
+ _, err := s.z.DeleteObject(ctx, s.bucket, key, ObjectOptions{Versioned: true, DeletePrefix: true, DeletePrefixObject: true, Expiration: ExpirationOptions{Expire: true}})
+ if !isErrWriteQuorum(err) {
+ t.Errorf("DeleteAll: %v", err)
+ }
+ disks := s.z.serverPools[1].getHashedSet(key).getDisks()
+ for i := 9; i < len(disks); i++ {
+ if _, err := disks[i].ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{}); err != nil {
+ t.Fatalf("drive %d lost the fresh version: %v", i, err)
+ }
+ }
+}
+
+// The completeness checks also hold where a set has fewer answering drives
+// than a read quorum or no version reaches one.
+func TestPoolsLifecycleHistoryEarlyReturns(t *testing.T) {
+ deleteAll := ObjectOptions{Versioned: true, DeletePrefix: true, DeletePrefixObject: true, Expiration: ExpirationOptions{Expire: true}}
+
+ // Seven drives hold the key, the other nine do not have it at all.
+ t.Run("minority-copy-only", func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "eav-minority-only"
+ s.write(key, splitAt("old", 5, 0))
+ s.write(key, splitAt("fresh", 0, 1))
+ id := s.ids[key+" fresh"]
+ removeAuthorityCopies(t, s, 1, key, id, authorityDrives(0, 9))
+ disks := s.z.serverPools[1].getHashedSet(key).getDisks()
+ if _, err := disks[0].ReadXL(ctx, s.bucket, key, false); !errors.Is(err, errFileNotFound) {
+ t.Fatalf("fixture: drive 0 still holds the key: %v", err)
+ }
+ if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) {
+ t.Errorf("minority copy: %v", err)
+ }
+ if _, err := s.z.DeleteObject(ctx, s.bucket, key, deleteAll); !isErrWriteQuorum(err) {
+ t.Errorf("DeleteAll: %v", err)
+ }
+ for i := 9; i < len(disks); i++ {
+ if _, err := disks[i].ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{}); err != nil {
+ t.Fatalf("drive %d lost the fresh version: %v", i, err)
+ }
+ }
+ })
+
+ // Control: two versions on seven drives each, none reaches a quorum.
+ t.Run("disjoint-minorities", func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "eav-disjoint"
+ s.write(key, splitAt("old", 5, 0))
+ s.write(key, splitAt("a", 2, 1))
+ s.write(key, splitAt("b", 1, 1))
+ removeAuthorityCopies(t, s, 1, key, s.ids[key+" a"], authorityDrives(7, 16))
+ removeAuthorityCopies(t, s, 1, key, s.ids[key+" b"], func(i int) bool { return i < 7 || i >= 14 })
+ if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) {
+ t.Fatalf("disjoint minorities: %v", err)
+ }
+ if _, err := s.z.DeleteObject(ctx, s.bucket, key, deleteAll); !isErrWriteQuorum(err) {
+ t.Fatalf("DeleteAll: %v", err)
+ }
+ })
+}
+
+// A set without an unknown drive less than half of its drives still tells
+// its history; half of them unknown could hide a version.
+func TestPoolsLifecycleUnknownDriveThreshold(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ pool int // the pool with unreadable drives
+ unknown int
+ refused bool
+ }{
+ {"history/7", 0, 7, false},
+ {"history/8", 0, 8, true},
+ {"absent/7", 1, 7, false},
+ {"absent/8", 1, 8, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "ncv-threshold"
+ // Empty versions need only a simple majority of the drives, so
+ // seven unknown drives leave enough copies.
+ var ids []string
+ for _, days := range []int{5, 3} {
+ id := mustGetUUID()
+ putConsistencyObject(t, s.z, s.bucket, key, 0, "", ObjectOptions{Versioned: true, VersionID: id, MTime: UTCNow().AddDate(0, 0, -days)})
+ ids = append(ids, id)
+ }
+ wrapAuthorityDisks(t, s, tc.pool, key, true, false, authorityDrives(0, tc.unknown))
+ err := authorityExpireVersion(t, s, key, ids[0])
+ if tc.refused {
+ if !isErrWriteQuorum(err) {
+ t.Errorf("%d unknown drives: %v", tc.unknown, err)
+ }
+ authorityPresent(t, s, 0, key, ids[0])
+ return
+ }
+ if err != nil {
+ t.Fatalf("%d unknown drives: %v", tc.unknown, err)
+ }
+ if _, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: ids[0]}); !isErrVersionNotFound(err) {
+ t.Fatalf("expired version remains: %v", err)
+ }
+ })
+ }
+}
+
+// Control: free versions are tier GC records. Residue of one on a few drives
+// must not block an expiry.
+func TestPoolsLifecycleFreeVersionResidue(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ residue int // the pool with the residue
+ }{
+ {"beside-history", 0},
+ {"alone", 1},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "ncv-free-residue"
+ s.write(key, splitTiered("gone", 6, tc.residue))
+ s.write(key, splitAt("old", 5, 0))
+ s.write(key, splitAt("new", 3, 0))
+ gone := s.ids[key+" gone"]
+ disks := s.z.serverPools[tc.residue].getHashedSet(key).getDisks()
+ fvID := mustGetUUID()
+ for i, disk := range disks {
+ fi := FileInfo{VersionID: gone}
+ if i < 3 {
+ fi.SetTierFreeVersionID(fvID)
+ } else {
+ fi.SetSkipTierFreeVersion()
+ }
+ if err := disk.DeleteVersion(ctx, s.bucket, key, fi, false, DeleteOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if h := authorityHeader(t, disks[0], s.bucket, key, fvID); !h.FreeVersion() {
+ t.Fatalf("fixture lacks a free version: %v", h)
+ }
+ versions, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key)
+ if err != nil || len(versions) != 2 || events[1].Action != lifecycle.DeleteVersionAction {
+ t.Fatalf("history with free-version residue: %d versions, events=%v, err=%v", len(versions), events, err)
+ }
+ if err := authorityExpireVersion(t, s, key, s.ids[key+" old"]); err != nil {
+ t.Fatalf("expiry beside free-version residue: %v", err)
+ }
+ })
+ }
+}
+
+// An interrupted 12+4 write left nine matching copies of a newer version:
+// a majority of the drives, but less than its read quorum. It must not make
+// the last committed version noncurrent.
+func TestPoolsLifecycleReducedParityPartialVersion(t *testing.T) {
+ for _, path := range []string{"single", "bulk"} {
+ t.Run(path, func(t *testing.T) {
+ authorityParity(t)
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "ncv-partial-write"
+ s.write(key, splitAt("committed", 5, 0))
+ s.write(key, splitAt("partial", 3, 0))
+ committed, partial := s.ids[key+" committed"], s.ids[key+" partial"]
+ oi, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: partial})
+ if err != nil || oi.DataBlocks != 12 || oi.ParityBlocks != 4 {
+ t.Fatalf("fixture is not 12+4: data=%d parity=%d err=%v", oi.DataBlocks, oi.ParityBlocks, err)
+ }
+ removeAuthorityCopies(t, s, 0, key, partial, authorityDrives(9, 16))
+
+ if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) {
+ t.Errorf("history with a partial version: %v", err)
+ }
+ if err := authorityDeletePath(t, s, key, committed, path); !isErrWriteQuorum(err) {
+ t.Errorf("expiry of the last committed version: %v", err)
+ }
+ authorityPresent(t, s, 0, key, committed)
+ })
+ }
+}
+
+// Control: two drives do not answer, every version still has its quorum.
+func TestPoolsLifecycleDegradedHistory(t *testing.T) {
+ authorityParity(t)
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "ncv-degraded"
+ s.write(key, splitAt("old", 5, 0))
+ s.write(key, splitAt("new", 3, 0))
+ id := s.ids[key+" old"]
+ wrapAuthorityDisks(t, s, 0, key, true, false, authorityDrives(0, 2))
+ if err := authorityExpireVersion(t, s, key, id); err != nil {
+ t.Fatalf("degraded expiry: %v", err)
+ }
+ if _, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) {
+ t.Fatalf("expired version remains: %v", err)
+ }
+}
+
+// authorityPoolLock sets the Object Lock metadata of the copy of version id
+// in pool.
+func authorityPoolLock(t *testing.T, s *splitILM, key, id string, pool int, metadata map[string]string) {
+ t.Helper()
+ metadata = maps.Clone(metadata)
+ if metadata[strings.ToLower(xhttp.AmzObjectLockMode)] != "" {
+ metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = UTCNow().AddDate(1, 0, 0).Format(time.RFC3339)
+ } else {
+ metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = ""
+ }
+ setLifecycleCopyMetadata(t, s, key, id, pool, metadata, false)
+}
+
+// authorityDeletePath expires version id by the single path or by the bulk
+// path, which takes the ordinary or the tiered route as the version is.
+func authorityDeletePath(t *testing.T, s *splitILM, key, id, path string) error {
+ t.Helper()
+ if path == "single" {
+ return authorityExpireVersion(t, s, key, id)
+ }
+ _, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: id}}}, ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}})
+ return errs[0]
+}
+
+var authorityPaths = []string{"single", "bulk", "tiered-bulk"}
+
+// The copies of a version in two pools carry protection revisions that do not
+// tell which is the newest. Refuse instead of guessing.
+func TestPoolsLifecycleAmbiguousProtection(t *testing.T) {
+ stamp := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+ old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano)
+ for _, tc := range []struct {
+ name string
+ copies [2]map[string]string
+ }{
+ {"hold-equal-stamps", [2]map[string]string{poolLockMetadata("", "OFF", old, stamp), poolLockMetadata("", "ON", old, stamp)}},
+ {"hold-missing-stamp", [2]map[string]string{poolLockMetadata("", "ON", old, ""), poolLockMetadata("", "OFF", old, stamp)}},
+ {"hold-malformed-stamp", [2]map[string]string{poolLockMetadata("", "ON", old, "yesterday"), poolLockMetadata("", "OFF", old, stamp)}},
+ {"retention-equal-stamps", [2]map[string]string{poolLockMetadata("", "OFF", stamp, old), poolLockMetadata("GOVERNANCE", "OFF", stamp, old)}},
+ } {
+ for _, path := range authorityPaths {
+ t.Run(tc.name+"/"+path, func(t *testing.T) {
+ s := newLockedSplitILM(t, authorityNoncurrentRule)
+ const key = "ncv-ambiguous"
+ v := splitAt("old", 5, 0, 1)
+ v.tiered = path == "tiered-bulk"
+ s.write(key, v)
+ s.write(key, splitAt("new", 3, 0, 1))
+ id := s.ids[key+" old"]
+ for pool, metadata := range tc.copies {
+ authorityPoolLock(t, s, key, id, pool, metadata)
+ }
+ if _, _, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key); !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("ambiguous protection: %v", err)
+ }
+ if err := authorityDeletePath(t, s, key, id, path); !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("expiry with ambiguous protection: %v", err)
+ }
+ for pool := range s.z.serverPools {
+ authorityPresent(t, s, pool, key, id)
+ }
+ })
+ }
+ }
+}
+
+// Protection stored under canonical-case keys counts as the evaluator reads
+// it, in the comparison of copies, in the merged version and in the set.
+func TestPoolsLifecycleCanonicalProtection(t *testing.T) {
+ older := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano)
+ recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+ until := UTCNow().AddDate(1, 0, 0).Format(time.RFC3339)
+ hold := func(status, stamp string) map[string]string {
+ return map[string]string{xhttp.AmzObjectLockLegalHold: status, ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: stamp}
+ }
+ retention := func(mode, stamp string) map[string]string {
+ m := map[string]string{ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp: stamp}
+ if mode != "" {
+ m[xhttp.AmzObjectLockMode], m[xhttp.AmzObjectLockRetainUntilDate] = mode, until
+ }
+ return m
+ }
+ for _, tc := range []struct {
+ name string
+ copies [2]map[string]string
+ expires bool
+ }{
+ {"hold-equal-stamps", [2]map[string]string{hold("OFF", recent), hold("ON", recent)}, false},
+ {"hold-newer", [2]map[string]string{hold("OFF", older), hold("ON", recent)}, false},
+ {"retention-equal-stamps", [2]map[string]string{retention("", recent), retention("COMPLIANCE", recent)}, false},
+ {"retention-newer", [2]map[string]string{retention("", older), retention("GOVERNANCE", recent)}, false},
+ {"hold-released", [2]map[string]string{hold("ON", older), hold("OFF", recent)}, true}, // control
+ } {
+ for _, path := range []string{"single", "bulk"} {
+ t.Run(tc.name+"/"+path, func(t *testing.T) {
+ s := newLockedSplitILM(t, authorityNoncurrentRule)
+ const key = "ncv-canonical"
+ id, mtime := mustGetUUID(), UTCNow().AddDate(0, 0, -5)
+ for pool, metadata := range tc.copies {
+ putConsistencyObject(t, s.z, s.bucket, key, pool, "old", ObjectOptions{Versioned: true, VersionID: id, MTime: mtime, UserDefined: maps.Clone(metadata)})
+ }
+ s.write(key, splitAt("new", 3, 0, 1))
+ _, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key)
+ if !tc.expires && err == nil && events[1].Action != lifecycle.NoneAction {
+ t.Errorf("protected version approved: %s", events[1].Action)
+ }
+ err = authorityDeletePath(t, s, key, id, path)
+ if tc.expires {
+ if err != nil {
+ t.Fatalf("released version: %v", err)
+ }
+ if _, err := s.z.serverPools[0].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) {
+ t.Fatalf("released version remains: %v", err)
+ }
+ return
+ }
+ if !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("expiry of a protected version: %v", err)
+ }
+ for pool := range s.z.serverPools {
+ authorityPresent(t, s, pool, key, id)
+ }
+ })
+ }
+ }
+}
+
+// Control: a strictly newer release in another pool still expires the
+// version by the ordinary bulk path too, beside the single and tiered bulk
+// paths of TestPoolsLifecycleDuplicateLockMetadata.
+func TestPoolsLifecycleProtectionReleaseBulk(t *testing.T) {
+ old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano)
+ recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+ for _, tc := range []struct {
+ name string
+ copies [2]map[string]string
+ }{
+ {"hold-released", [2]map[string]string{poolLockMetadata("", "ON", old, old), poolLockMetadata("", "OFF", old, recent)}},
+ {"retention-released", [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", old, old), poolLockMetadata("", "OFF", recent, old)}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s := newLockedSplitILM(t, authorityNoncurrentRule)
+ const key = "ncv-released"
+ s.write(key, splitAt("old", 5, 0, 1))
+ s.write(key, splitAt("new", 3, 0, 1))
+ id := s.ids[key+" old"]
+ for pool, metadata := range tc.copies {
+ authorityPoolLock(t, s, key, id, pool, metadata)
+ }
+ if err := authorityDeletePath(t, s, key, id, "bulk"); err != nil {
+ t.Fatalf("released version: %v", err)
+ }
+ for pool := range s.z.serverPools {
+ if _, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) {
+ t.Errorf("pool %d still holds the released version: %v", pool, err)
+ }
+ }
+ })
+ }
+}
+
+// A set takes an all-pools approval only with the protection it was approved
+// with. Protection that the set reads and the approval does not strictly
+// supersede keeps the version.
+func TestPoolsLifecycleCheckedExpirySafeguard(t *testing.T) {
+ old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano)
+ recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+ for _, tc := range []struct {
+ name string
+ local map[string]string
+ approved objectLockState
+ deleted bool
+ }{
+ {"hold-newer-locally", poolLockMetadata("", "ON", old, recent), objectLockState{legalHold: "OFF", legalHoldTimestamp: old}, false},
+ {"hold-missing-local-stamp", poolLockMetadata("", "ON", old, ""), objectLockState{legalHold: "OFF", legalHoldTimestamp: recent}, false},
+ {"hold-equal-stamps", poolLockMetadata("", "ON", old, recent), objectLockState{legalHold: "OFF", legalHoldTimestamp: recent}, false},
+ {"hold-missing-approved-stamp", poolLockMetadata("", "ON", old, old), objectLockState{legalHold: "OFF"}, false},
+ {"retention-newer-locally", poolLockMetadata("GOVERNANCE", "OFF", recent, old), objectLockState{retentionTimestamp: old, legalHold: "OFF", legalHoldTimestamp: old}, false},
+ // Controls: the approval strictly supersedes the local protection.
+ {"hold-released", poolLockMetadata("", "ON", old, old), objectLockState{legalHold: "OFF", legalHoldTimestamp: recent}, true},
+ {"retention-released", poolLockMetadata("GOVERNANCE", "OFF", old, old), objectLockState{retentionTimestamp: recent, legalHold: "OFF", legalHoldTimestamp: old}, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s := newLockedSplitILM(t, authorityNoncurrentRule)
+ ctx := t.Context()
+ const key = "ncv-safeguard"
+ s.write(key, splitAt("old", 5, 0))
+ s.write(key, splitAt("new", 3, 0))
+ id := s.ids[key+" old"]
+ authorityPoolLock(t, s, key, id, 0, tc.local)
+ opts := ObjectOptions{Versioned: true, VersionID: id, Expiration: ExpirationOptions{Expire: true, CheckedAllPools: true}}
+ opts.Expiration.ApprovedProtection = tc.approved
+ _, err := s.z.serverPools[0].DeleteObject(ctx, s.bucket, key, opts)
+ if !tc.deleted {
+ if !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("protected version: %v", err)
+ }
+ authorityPresent(t, s, 0, key, id)
+ return
+ }
+ if err != nil {
+ t.Fatalf("released version: %v", err)
+ }
+ if _, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) {
+ t.Fatalf("released version remains: %v", err)
+ }
+ })
+ }
+}
+
+// A pending purge vetoes an approved expiry in the set, also when the read
+// returns it with MethodNotAllowed, as for a delete marker.
+func TestPoolsLifecycleCheckedExpiryPendingPurge(t *testing.T) {
+ for _, marker := range []bool{false, true} {
+ name := "version"
+ if marker {
+ name = "delete-marker"
+ }
+ t.Run(name, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "ncv-pending-purge"
+ label := "old"
+ s.write(key, splitAt("first", 6, 0))
+ if marker {
+ label = "dm"
+ s.write(key, splitMarker(label, 5, 0))
+ } else {
+ s.write(key, splitAt(label, 5, 0))
+ }
+ s.write(key, splitAt("new", 3, 0))
+ id := s.ids[key+" "+label]
+ target := newILMTestTarget(t)
+ setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket)
+ oi, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id})
+ if oi.DeleteMarker != marker || (err != nil) != marker || marker && !isErrMethodNotAllowed(err) {
+ t.Fatalf("fixture: marker=%t err=%v", oi.DeleteMarker, err)
+ }
+ _, err = s.z.serverPools[0].DeleteObject(ctx, s.bucket, key, ObjectOptions{
+ VersionID: id, Versioned: true, MTime: oi.ModTime,
+ DeleteReplication: ReplicationState{
+ VersionPurgeStatusInternal: target.arn + "=PENDING;",
+ PurgeTargets: map[string]VersionPurgeStatusType{target.arn: replication.VersionPurgePending},
+ },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ pending, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id})
+ if !isErrMethodNotAllowed(err) || pending.VersionPurgeStatus != replication.VersionPurgePending {
+ t.Fatalf("fixture: purge=%q err=%v", pending.VersionPurgeStatus, err)
+ }
+ _, err = s.z.serverPools[0].DeleteObject(ctx, s.bucket, key, ObjectOptions{Versioned: true, VersionID: id, Expiration: ExpirationOptions{Expire: true, CheckedAllPools: true}})
+ if !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("approved expiry of a pending purge: %v", err)
+ }
+ after, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id})
+ if !isErrMethodNotAllowed(err) || after.VersionPurgeStatus != replication.VersionPurgePending {
+ t.Fatalf("pending purge changed: purge=%q err=%v", after.VersionPurgeStatus, err)
+ }
+ })
+ }
+}
+
+// The copy of a version in another pool carries a pending purge that the
+// merge would drop with the replication state of the first copy.
+func TestPoolsLifecycleDuplicatePendingPurge(t *testing.T) {
+ for _, path := range []string{"single", "bulk"} {
+ t.Run(path, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "ncv-pending-copy"
+ s.write(key, splitAt("old", 5, 0, 1))
+ s.write(key, splitAt("new", 3, 0, 1))
+ id := s.ids[key+" old"]
+ target := newILMTestTarget(t)
+ setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket)
+ oi, err := s.z.serverPools[1].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id})
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = s.z.serverPools[1].DeleteObject(ctx, s.bucket, key, ObjectOptions{
+ VersionID: id, Versioned: true, MTime: oi.ModTime,
+ DeleteReplication: ReplicationState{
+ VersionPurgeStatusInternal: target.arn + "=PENDING;",
+ PurgeTargets: map[string]VersionPurgeStatusType{target.arn: replication.VersionPurgePending},
+ },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ files, err := s.z.serverPools[1].getHashedSet(key).objectVersions(ctx, s.bucket, key)
+ if err != nil || len(files) != 2 || files[1].ToObjectInfo(s.bucket, key, true).VersionPurgeStatus != replication.VersionPurgePending {
+ t.Fatalf("fixture lacks the pending copy: %d versions, err=%v", len(files), err)
+ }
+ if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("copies with a different purge state: %v", err)
+ }
+ if err := authorityDeletePath(t, s, key, id, path); !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("expiry beside a pending purge: %v", err)
+ }
+ stored, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id})
+ if err != nil || stored.VersionPurgeStatus != "" {
+ t.Fatalf("first copy changed: purge=%q err=%v", stored.VersionPurgeStatus, err)
+ }
+ if _, err := s.z.serverPools[1].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}); !isErrMethodNotAllowed(err) {
+ t.Fatalf("pending copy changed: %v", err)
+ }
+ })
+ }
+}
+
+// Copies of a version compare their per-target replication state as parsed:
+// the order of the serialized targets does not count, a target does.
+func TestPoolsLifecycleDuplicateReplicationTargets(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statuses [2]string
+ refused bool
+ }{
+ {"reordered", [2]string{"arn:a=COMPLETED;arn:b=COMPLETED;", "arn:b=COMPLETED;arn:a=COMPLETED;"}, false}, // control
+ {"other-targets", [2]string{"arn:a=COMPLETED;", "arn:a=COMPLETED;arn:b=COMPLETED;"}, true},
+ {"other-status", [2]string{"arn:a=COMPLETED;arn:b=COMPLETED;", "arn:a=COMPLETED;arn:b=FAILED;"}, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "ncv-replicated-copies"
+ id, mtime := mustGetUUID(), UTCNow().AddDate(0, 0, -5)
+ for pool, status := range tc.statuses {
+ putConsistencyObject(t, s.z, s.bucket, key, pool, "old", ObjectOptions{
+ Versioned: true, VersionID: id, MTime: mtime,
+ UserDefined: map[string]string{ReservedMetadataPrefixLower + ReplicationStatus: status},
+ })
+ }
+ s.write(key, splitAt("new", 3, 0, 1))
+ for pool, status := range tc.statuses {
+ oi, err := s.z.serverPools[pool].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id})
+ if err != nil || oi.ReplicationStatusInternal != status {
+ t.Fatalf("fixture pool %d: status=%q err=%v", pool, oi.ReplicationStatusInternal, err)
+ }
+ }
+ _, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key)
+ if tc.refused {
+ if !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("copies with other replication targets: %v", err)
+ }
+ if err := authorityExpireVersion(t, s, key, id); !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("expiry: %v", err)
+ }
+ authorityPresent(t, s, 0, key, id)
+ return
+ }
+ if err != nil || events[1].Action != lifecycle.DeleteVersionAction {
+ t.Fatalf("equivalent copies: events=%v err=%v", events, err)
+ }
+ if err := authorityExpireVersion(t, s, key, id); err != nil {
+ t.Fatalf("expiry of equivalent copies: %v", err)
+ }
+ })
+ }
+}
+
+// Null versions written in two pools of a suspended bucket share a version
+// ID without being one write. Copies with another ModTime are older writes:
+// the newest decides, unless an older one is still protected. Copies with
+// the same ModTime are duplicates and must still agree.
+func TestPoolsLifecycleNullGenerations(t *testing.T) {
+ hold := func(status, stamp string) map[string]string {
+ return map[string]string{authorityHoldKey: status, ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: stamp}
+ }
+ older := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano)
+ recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+ for _, tc := range []struct {
+ name string
+ lock bool
+ old, new map[string]string
+ sameModTime bool
+ wantNewest bool // evaluated on the newer write, else refused
+ replicateOld bool
+ }{
+ {"older-unprotected", false, hold("OFF", ""), nil, false, true, true},
+ {"older-held", true, hold("ON", older), hold("OFF", recent), false, false, false},
+ {"same-modtime-duplicates", false, hold("OFF", ""), nil, true, false, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s := &splitILM{migrationILM: newMigrationILM(t, versioning.Suspended, tc.lock, splitILMRules), ids: map[string]string{}, bodies: map[string]string{}}
+ ctx := t.Context()
+ const key = "exp-null-generations"
+ oldTime, newTime := UTCNow().AddDate(0, 0, -5), UTCNow().AddDate(0, 0, -3)
+ if tc.sameModTime {
+ newTime = oldTime
+ }
+ oldMeta := maps.Clone(tc.old)
+ if tc.replicateOld {
+ oldMeta[ReservedMetadataPrefixLower+ReplicationStatus] = "arn:a=COMPLETED;"
+ }
+ putConsistencyObject(t, s.z, s.bucket, key, 0, "old", ObjectOptions{VersionSuspended: true, MTime: oldTime, UserDefined: oldMeta})
+ newest := putConsistencyObject(t, s.z, s.bucket, key, 1, "new", ObjectOptions{VersionSuspended: true, MTime: newTime, UserDefined: maps.Clone(tc.new)})
+
+ versions, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key)
+ opts := ObjectOptions{VersionSuspended: true, Expiration: ExpirationOptions{Expire: true}}
+ if !tc.wantNewest {
+ if !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("history: %d versions, events=%v, err=%v", len(versions), events, err)
+ }
+ if _, err := s.z.DeleteObject(ctx, s.bucket, key, opts); !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("expiry: %v", err)
+ }
+ for pool := range s.z.serverPools {
+ oi, err := s.z.serverPools[pool].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: nullVersionID})
+ if err != nil || oi.DeleteMarker {
+ t.Fatalf("pool %d lost its null version: marker=%t err=%v", pool, oi.DeleteMarker, err)
+ }
+ }
+ return
+ }
+ if err != nil || len(versions) != 1 || !versions[0].ModTime.Equal(newest.ModTime) || events[0].Action != lifecycle.DeleteAction {
+ t.Fatalf("history: %d versions, events=%v, err=%v", len(versions), events, err)
+ }
+ if _, err := s.z.DeleteObject(ctx, s.bucket, key, opts); err != nil {
+ t.Fatalf("expiry of the newest null write: %v", err)
+ }
+ })
+ }
+}
+
+// authoritySideEffects collects the version IDs that removal notifications
+// and ILM expiry audit traces report.
+func authoritySideEffects(t *testing.T, run func()) (notified, traced []string) {
+ t.Helper()
+ done := make(chan struct{})
+ defer close(done)
+ events := make(chan event.Event, 16)
+ if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedDelete), events, done, nil); err != nil {
+ t.Fatal(err)
+ }
+ traces := make(chan madmin.TraceInfo, 16)
+ if err := globalTrace.Subscribe(madmin.TraceILM, traces, done, nil); err != nil {
+ t.Fatal(err)
+ }
+ run()
+ // Both publishers deliver synchronously, so everything sent is buffered.
+ for {
+ select {
+ case ev := <-events:
+ notified = append(notified, ev.S3.Object.VersionID)
+ case tr := <-traces:
+ if tr.FuncName == ILMExpiry {
+ traced = append(traced, tr.Custom["version-id"])
+ }
+ default:
+ slices.Sort(notified)
+ slices.Sort(traced)
+ return notified, traced
+ }
+ }
+}
+
+// A batch expiry that another pool cannot read is not a removal.
+func TestPoolsLifecycleBatchQuorumNotAnnounced(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ const key = "ncv-quorum-event"
+ s.write(key, splitAt("old", 5, 0))
+ s.write(key, splitAt("new", 3, 0))
+ id := s.ids[key+" old"]
+ lifecycleReadDisks(t, s, key, 1, errDiskNotFound)
+ result := &lifecycleBatchResult{ObjectLayer: s.z}
+ notified, traced := authoritySideEffects(t, func() {
+ deleteObjectVersions(t.Context(), result, s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: id}}}, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}})
+ })
+ if !isErrWriteQuorum(result.errs[0]) {
+ t.Fatalf("fixture result: %v", result.errs[0])
+ }
+ authorityPresent(t, s, 0, key, id)
+ if len(notified) != 0 || len(traced) != 0 {
+ t.Fatalf("unapplied expiry announced: notifications=%v traces=%v", notified, traced)
+ }
+}
+
+// Each entry of a batch gets the side effects of its own result: a completed
+// or already absent delete is announced and audited, a refused or failed one
+// is not.
+func TestPoolsLifecycleBatchSideEffects(t *testing.T) {
+ for _, path := range []string{"single-pool", "bulk", "tiered-bulk"} {
+ t.Run(path, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ const key, failing = "ncv-side-effects", "ncv-side-effects-failing"
+ for _, k := range []string{key, failing} {
+ v := splitAt("old", 5, 0)
+ v.tiered = path == "tiered-bulk"
+ s.write(k, v)
+ s.write(k, splitAt("new", 3, 0))
+ }
+ if path == "single-pool" {
+ prior := s.z.serverPools
+ t.Cleanup(func() { s.z.serverPools = prior })
+ s.z.serverPools = s.z.serverPools[:1]
+ }
+ wrapAuthorityDisks(t, s, 0, failing, false, true, authorityDrives(0, 9))
+ success, absent := s.ids[key+" old"], mustGetUUID()
+ entries := []ObjectToDelete{
+ {ObjectV: ObjectV{ObjectName: key, VersionID: success}},
+ {ObjectV: ObjectV{ObjectName: failing, VersionID: s.ids[failing+" old"]}},
+ {ObjectV: ObjectV{ObjectName: key, VersionID: absent}},
+ }
+ if path != "single-pool" {
+ // A single pool has no all-pools check to refuse a bulk entry.
+ entries = append(entries, ObjectToDelete{ObjectV: ObjectV{ObjectName: key, VersionID: s.ids[key+" new"]}})
+ }
+ evts := make([]lifecycle.Event, len(entries))
+ for i := range evts {
+ evts[i] = lifecycle.Event{Action: lifecycle.DeleteVersionAction}
+ }
+ result := &lifecycleBatchResult{ObjectLayer: s.z}
+ notified, traced := authoritySideEffects(t, func() {
+ deleteObjectVersions(t.Context(), result, s.bucket, entries, evts)
+ })
+ if result.errs[0] != nil || !isErrWriteQuorum(result.errs[1]) ||
+ result.errs[2] != nil && !isErrVersionNotFound(result.errs[2]) ||
+ len(entries) > 3 && !errors.Is(result.errs[3], errLifecycleNotApplicable) {
+ t.Fatalf("fixture results: %v", result.errs)
+ }
+ want := []string{success, absent}
+ slices.Sort(want)
+ if !slices.Equal(notified, want) {
+ t.Errorf("removal notifications for %v, want %v", notified, want)
+ }
+ if !slices.Equal(traced, want) {
+ t.Errorf("ILM expiry audit for %v, want %v", traced, want)
+ }
+ })
+ }
+}
+
+// A single pool keeps the explicit refusal rather than a not-found result.
+func TestPoolsLifecycleSinglePoolRefusal(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ s.write("exp-new", splitAt("fresh", 0, 0))
+ prior := s.z.serverPools
+ t.Cleanup(func() { s.z.serverPools = prior })
+ s.z.serverPools = s.z.serverPools[:1]
+ _, err := s.z.DeleteObject(t.Context(), s.bucket, "exp-new", ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}})
+ if !errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+ t.Fatalf("single-pool stale expiry: %v", err)
+ }
+}
+
+// Migration leaves a version to the logical expiry. When that expiry is
+// refused at retirement while the source still holds the version, the source
+// is kept: a refusal is never absence. Decommission and rebalance both
+// retire through removeMovedVersions.
+func TestPoolsMigrationLifecycleRefusedExpiryKeepsSource(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ ctx := t.Context()
+ const key = "exp-moved"
+ s.write(key, splitAt("old", 5, 0, 1))
+ s.write(key, splitAt("new", 0.01, 1))
+ id := s.ids[key+" old"]
+ source, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id})
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, updatedAt, err := globalBucketMetadataSys.GetLifecycleConfig(s.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ version := FileInfo{Name: key, VersionID: id, ModTime: source.ModTime}
+ err = s.z.removeMovedVersions(ctx, s.bucket, key, 0, []FileInfo{version}, []migrationExpiredVersion{{
+ version: version,
+ event: lifecycle.Event{Action: lifecycle.DeleteAction},
+ lifecycleUpdatedAt: updatedAt,
+ source: lcEventSrc_Decom,
+ }})
+ if _, ok := err.(PreConditionFailed); !ok {
+ t.Fatalf("retirement after a refused expiry: %v", err)
+ }
+ authorityPresent(t, s, 0, key, id)
+ if views := s.views(); !slices.Equal(views[0][key], []string{"old"}) || !slices.Equal(views[1][key], []string{"new", "old"}) {
+ t.Fatalf("refused expiry changed the pools: %v", views)
+ }
+}
diff --git a/cmd/erasure-server-pool-lifecycle-batch_test.go b/cmd/erasure-server-pool-lifecycle-batch_test.go
new file mode 100644
index 000000000..363a8768a
--- /dev/null
+++ b/cmd/erasure-server-pool-lifecycle-batch_test.go
@@ -0,0 +1,270 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "testing"
+
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/replication"
+ "github.com/minio/minio/internal/bucket/versioning"
+ "github.com/minio/minio/internal/event"
+ "github.com/minio/minio/internal/pubsub"
+)
+
+// Reuse the author's scanner/worker fixture. A second configured pool is
+// sufficient to trigger the tiered NNV regression; no versions need move there.
+func TestPoolsLifecycleTieredNoncurrentBatch(t *testing.T) {
+ for _, tiered := range []bool{false, true} {
+ name := "plain"
+ if tiered {
+ name = "tiered"
+ }
+ t.Run(name, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ old := func(label string, days float64) splitVersion {
+ v := splitAt(label, days, 0)
+ v.tiered = tiered
+ return v
+ }
+ s.run([]splitKey{
+ {"nnv-all-in-0", []splitVersion{old("v1", 5), old("v2", 4), splitAt("v3", 3, 0), splitAt("v4", 2, 0)}, "v4", "v4", [][]string{{"v4", "v3"}, nil}},
+ {"ncv-all-in-0", []splitVersion{old("v1", 5), old("v2", 4), splitAt("v3", 0.01, 0)}, "v3", "v3", [][]string{{"v3", "v2"}, nil}},
+ })
+ })
+ }
+}
+
+// Observe the actual pool results consumed by the expiry worker, without
+// replacing the storage deletion or the replication enqueue path.
+type lifecycleBatchResult struct {
+ ObjectLayer
+ deleted []DeletedObject
+ errs []error
+}
+
+func (o *lifecycleBatchResult) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
+ o.deleted, o.errs = o.ObjectLayer.DeleteObjects(ctx, bucket, objects, opts)
+ return o.deleted, o.errs
+}
+
+func TestPoolsLifecycleTieredBatchReplication(t *testing.T) {
+ for _, identity := range []string{"uuid", "null"} {
+ t.Run(identity, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ const key = "nnv-replicated"
+ oldest := splitTiered("v1", 5, 0)
+ oldest.null = identity == "null"
+ for _, v := range []splitVersion{oldest, splitTiered("v2", 4, 0), splitTiered("v3", 3, 0), splitAt("v4", 2, 0)} {
+ s.write(key, v)
+ }
+ target := newILMTestTarget(t)
+ registerILMTestTarget(t, s.z, s.bucket, target, func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodDelete {
+ t.Errorf("unexpected target method: %s", r.Method)
+ w.WriteHeader(500)
+ return
+ }
+ target.deletes.Add(1)
+ opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true, ReplicationRequest: true}
+ opts.SetReplicaStatus(replication.Replica)
+ if _, err := s.z.DeleteObject(r.Context(), target.bucket, key, opts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+ t.Errorf("target purge: %v", err)
+ w.WriteHeader(500)
+ return
+ }
+ w.WriteHeader(http.StatusNoContent)
+ })
+ setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket)
+ stats := NewReplicationStats(t.Context(), nil)
+ previous := globalReplicationStats.Swap(stats)
+ t.Cleanup(func() { globalReplicationStats.Store(previous) })
+ restore, _ := newILMTestReplicationPool(t.Context(), s.z, stats)
+ t.Cleanup(restore)
+ var entries []ObjectToDelete
+ for _, label := range []string{"v1", "v3"} {
+ vid := s.ids[key+" "+label]
+ if vid == "" {
+ vid = nullVersionID
+ }
+ oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ entry := ilmPurgeToDelete(t.Context(), s.bucket, oi, true)
+ if entry.VersionPurgeStatus != replication.VersionPurgePending {
+ t.Fatalf("fixture lacks purge decision: %+v", entry)
+ }
+ entries = append(entries, entry)
+ putConsistencyObject(t, s.z, target.bucket, key, 0, label, ObjectOptions{Versioned: identity != "null" || label != "v1", VersionID: vid})
+ }
+ result := &lifecycleBatchResult{ObjectLayer: s.z}
+ deleteObjectVersions(t.Context(), result, s.bucket, entries, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}})
+ if result.errs[0] != nil {
+ t.Errorf("eligible delete: %v", result.errs[0])
+ }
+ if !errors.Is(result.errs[1], errLifecycleNotApplicable) {
+ t.Errorf("retained version result: %v", result.errs[1])
+ }
+ if result.deleted[1].VersionPurgeStatus() != "" || result.deleted[1].ReplicationState.ReplicateDecisionStr != "" {
+ t.Errorf("rejected entry retained replication state: %+v", result.deleted[1])
+ }
+ source, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: entries[0].VersionID, Versioned: true})
+ if !isErrMethodNotAllowed(err) || source.VersionPurgeStatus != replication.VersionPurgePending {
+ t.Errorf("source must be a pending purge tombstone before enqueue: status=%q err=%v", source.VersionPurgeStatus, err)
+ }
+ retained, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: entries[1].VersionID, Versioned: true})
+ if err != nil || retained.VersionPurgeStatus != "" {
+ t.Errorf("rejected source changed: %+v, %v", retained, err)
+ }
+ queue := globalReplicationPool.Get().workers[0]
+ if len(queue) != 1 {
+ t.Fatalf("queued %d purges, want only eligible v1", len(queue))
+ }
+ deletion, ok := (<-queue).(DeletedObjectReplicationInfo)
+ if !ok || deletion.VersionID != entries[0].VersionID || deletion.EventType != ReplicateILMDelete || deletion.VersionPurgeStatus() != replication.VersionPurgePending {
+ t.Fatalf("wrong purge queued: %+v", deletion)
+ }
+ if result := replicateDelete(t.Context(), deletion, s.z); result.VersionPurgeStatus() != replication.VersionPurgeComplete {
+ t.Fatalf("replicate purge: %+v", result)
+ }
+ if target.deletes.Load() != 1 {
+ t.Errorf("target delete count=%d", target.deletes.Load())
+ }
+ for _, bucket := range []string{s.bucket, target.bucket} {
+ if _, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: entries[0].VersionID}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+ t.Errorf("purged v1 remains in %s: %v", bucket, err)
+ }
+ if _, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: entries[1].VersionID}); err != nil {
+ t.Errorf("retained v3 missing from %s: %v", bucket, err)
+ }
+ }
+ })
+ }
+}
+
+func TestPoolsLifecycleTieredBatchIdentity(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ versioned bool
+ id string
+ expire bool
+ wantGone bool
+ }{
+ {"uuid-ilm", true, "uuid", true, true},
+ {"null-ilm", true, "null", true, true},
+ {"empty-unversioned-ilm", false, "", true, false},
+ {"uuid-user-delete", true, "uuid", false, true},
+ {"empty-user-delete", false, "", false, true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ state := versioning.State("")
+ if tc.versioned {
+ state = versioning.Enabled
+ }
+ s := newSplitILM(t, state, splitILMRules)
+ const key = "nnv-identity"
+ v := splitTiered("v1", 5, 0)
+ v.null = tc.id != "uuid"
+ s.write(key, v)
+ if tc.versioned {
+ s.write(key, splitAt("v2", 4, 0))
+ s.write(key, splitAt("v3", 3, 0))
+ }
+ vid := tc.id
+ if vid == "uuid" {
+ vid = s.ids[key+" v1"]
+ }
+ vc, _ := globalBucketVersioningSys.Get(s.bucket)
+ _, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: vid}}}, ObjectOptions{PrefixEnabledFn: vc.PrefixEnabled, Expiration: ExpirationOptions{Expire: tc.expire}})
+ if tc.expire && !tc.wantGone {
+ if !errors.Is(errs[0], errLifecycleNotApplicable) {
+ t.Fatalf("ineligible expiry: %v", errs[0])
+ }
+ } else if errs[0] != nil {
+ t.Fatalf("delete: %v", errs[0])
+ }
+ query := vid
+ if query == "" {
+ query = nullVersionID
+ }
+ _, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: query})
+ gone := isErrVersionNotFound(err) || isErrObjectNotFound(err)
+ if gone != tc.wantGone || (!gone && err != nil) {
+ t.Fatalf("gone=%t err=%v, want gone=%t", gone, err, tc.wantGone)
+ }
+ })
+ }
+}
+
+// A batch entry whose expiry the all-pools recheck rejects is kept. The worker
+// must not report it as removed, by event, ILM trace or audit, while eligible
+// entries of the same batch are still reported.
+func TestPoolsLifecycleBatchRejectionNotAnnounced(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ const key = "nnv-announce"
+ for _, v := range []splitVersion{splitTiered("v1", 5, 0), splitTiered("v2", 4, 0), splitTiered("v3", 3, 0), splitAt("v4", 2, 0)} {
+ s.write(key, v)
+ }
+ eligible, retained := s.ids[key+" v1"], s.ids[key+" v3"]
+ done := make(chan struct{})
+ defer close(done)
+ events := make(chan event.Event, 8)
+ if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedDelete), events, done, nil); err != nil {
+ t.Fatal(err)
+ }
+ traces := make(chan madmin.TraceInfo, 8)
+ if err := globalTrace.Subscribe(madmin.TraceILM, traces, done, nil); err != nil {
+ t.Fatal(err)
+ }
+ result := &lifecycleBatchResult{ObjectLayer: s.z}
+ deleteObjectVersions(t.Context(), result, s.bucket, []ObjectToDelete{
+ {ObjectV: ObjectV{ObjectName: key, VersionID: eligible}},
+ {ObjectV: ObjectV{ObjectName: key, VersionID: retained}},
+ }, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}})
+ if result.errs[0] != nil || !errors.Is(result.errs[1], errLifecycleNotApplicable) {
+ t.Fatalf("fixture results: eligible=%v retained=%v", result.errs[0], result.errs[1])
+ }
+ if _, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: retained, Versioned: true}); err != nil {
+ t.Fatalf("rejected version must remain: %v", err)
+ }
+ // Both publishers deliver synchronously, so everything sent is buffered.
+ var notified, traced []string
+ for drained := false; !drained; {
+ select {
+ case ev := <-events:
+ notified = append(notified, ev.S3.Object.VersionID)
+ case tr := <-traces:
+ if tr.FuncName == ILMExpiry {
+ traced = append(traced, tr.Custom["version-id"])
+ }
+ default:
+ drained = true
+ }
+ }
+ if len(notified) != 1 || notified[0] != eligible {
+ t.Errorf("removal events for %v, want only eligible %s (retained %s)", notified, eligible, retained)
+ }
+ if len(traced) != 1 || traced[0] != eligible {
+ t.Errorf("ILM expiry traces for %v, want only eligible %s (retained %s)", traced, eligible, retained)
+ }
+}
diff --git a/cmd/erasure-server-pool-lifecycle-safety_test.go b/cmd/erasure-server-pool-lifecycle-safety_test.go
new file mode 100644
index 000000000..0ee319350
--- /dev/null
+++ b/cmd/erasure-server-pool-lifecycle-safety_test.go
@@ -0,0 +1,361 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "maps"
+ "net/http"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/replication"
+ "github.com/minio/minio/internal/bucket/versioning"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// A partial metadata update can leave the newest lock/tag revision on a copy
+// other than the one selected by content ModTime. Keep the data identity intact.
+func setLifecycleCopyMetadata(t *testing.T, s *splitILM, key, id string, pool int, metadata map[string]string, advance bool) {
+ t.Helper()
+ oi, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id})
+ if err != nil {
+ t.Fatal(err)
+ }
+ opts := ObjectOptions{VersionID: id, Versioned: true, MTime: oi.ModTime}
+ if advance {
+ opts.MTime = opts.MTime.Add(time.Hour)
+ }
+ opts.EvalMetadataFn = func(oi *ObjectInfo, _ error) (ReplicateDecision, error) {
+ maps.Copy(oi.UserDefined, metadata)
+ return ReplicateDecision{}, nil
+ }
+ if _, err := s.z.serverPools[pool].PutObjectMetadata(t.Context(), s.bucket, key, opts); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestPoolsLifecycleDuplicateLockMetadata(t *testing.T) {
+ old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano)
+ recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+ for _, tc := range []struct {
+ name string
+ copies [2]map[string]string
+ mode, hold string
+ delete bool
+ newerPool1 bool
+ }{
+ {name: "hold-on", copies: [2]map[string]string{poolLockMetadata("", "OFF", old, old), poolLockMetadata("", "ON", old, recent)}, hold: "ON"},
+ {name: "held-earlier-pool", copies: [2]map[string]string{poolLockMetadata("", "ON", old, recent), poolLockMetadata("", "OFF", old, old)}, hold: "ON", newerPool1: true},
+ {name: "retention-on", copies: [2]map[string]string{poolLockMetadata("", "OFF", old, old), poolLockMetadata("GOVERNANCE", "OFF", recent, old)}, mode: "GOVERNANCE", hold: "OFF"},
+ {name: "independent-fields", copies: [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", recent, old), poolLockMetadata("", "ON", old, recent)}, mode: "GOVERNANCE", hold: "ON"},
+ {name: "hold-released", copies: [2]map[string]string{poolLockMetadata("", "ON", old, old), poolLockMetadata("", "OFF", old, recent)}, hold: "OFF", delete: true},
+ {name: "retention-released", copies: [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", old, old), poolLockMetadata("", "OFF", recent, old)}, hold: "OFF", delete: true},
+ } {
+ for _, batch := range []bool{false, true} {
+ path := "single"
+ if batch {
+ path = "batch"
+ }
+ t.Run(tc.name+"/"+path, func(t *testing.T) {
+ s := &splitILM{migrationILM: newMigrationILM(t, versioning.Enabled, true, splitILMRules), ids: map[string]string{}, bodies: map[string]string{}}
+ const key = "ncv-copy-lock"
+ v := splitAt("old", 5, 0, 1)
+ v.tiered = batch
+ s.write(key, v)
+ s.write(key, splitAt("new", 3, 0, 1))
+ id := s.ids[key+" old"]
+ for pool, metadata := range tc.copies {
+ metadata = maps.Clone(metadata)
+ if metadata["x-amz-object-lock-mode"] != "" {
+ metadata["x-amz-object-lock-retain-until-date"] = UTCNow().AddDate(1, 0, 0).Format(time.RFC3339)
+ } else {
+ metadata["x-amz-object-lock-retain-until-date"] = ""
+ }
+ setLifecycleCopyMetadata(t, s, key, id, pool, metadata, tc.newerPool1 && pool == 1)
+ }
+ versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key)
+ switch {
+ case tc.newerPool1:
+ // Another ModTime makes the copies separate writes. The
+ // held older one refuses the object instead of merging.
+ if !errors.Is(err, errLifecycleNotApplicable) {
+ t.Fatalf("held older write: %v", err)
+ }
+ case err != nil:
+ t.Fatal(err)
+ default:
+ if len(versions) != 2 || versions[1].VersionID != id || versions[1].NumVersions != 2 {
+ t.Fatalf("wrong version history: %+v", versions)
+ }
+ state := storedObjectLockState(versions[1].UserDefined)
+ if state.mode != tc.mode || state.legalHold != tc.hold {
+ t.Errorf("merged state=%+v, want mode=%s hold=%s", state, tc.mode, tc.hold)
+ }
+ want := lifecycle.NoneAction
+ if tc.delete {
+ want = lifecycle.DeleteVersionAction
+ }
+ // Read-side expiry rechecks consume the same event producer.
+ if events[1].Action != want {
+ t.Errorf("expiry event=%s want=%s", events[1].Action, want)
+ }
+ }
+ opts := ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}}
+ if batch {
+ _, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: id}}}, opts)
+ err = errs[0]
+ } else {
+ opts.VersionID = id
+ _, err = s.z.DeleteObject(t.Context(), s.bucket, key, opts)
+ }
+ if tc.delete && err != nil || !tc.delete && !errors.Is(err, errLifecycleNotApplicable) {
+ t.Errorf("delete error=%v, wantDelete=%t", err, tc.delete)
+ }
+ for pool := range s.z.serverPools {
+ _, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id})
+ // Single DeleteObject retains its existing first-copy dispatch.
+ gone := tc.delete && (batch || pool == 0)
+ if gone && !isErrVersionNotFound(err) || !gone && err != nil {
+ t.Errorf("pool=%d err=%v wantGone=%t", pool, err, gone)
+ }
+ }
+ })
+ }
+ }
+}
+
+func TestPoolsLifecycleDuplicateTagMetadata(t *testing.T) {
+ const rule = `taggedEnabledexpireyes1`
+ for _, tc := range []struct {
+ name, stale, fresh string
+ expire bool
+ }{
+ {"removed", "expire=yes", "", false},
+ {"changed", "expire=yes", "expire=no", false},
+ {"enabled", "expire=no", "expire=yes", true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, rule)
+ const key = "tagged-copy"
+ s.write(key, splitTiered("old", 5, 0, 1))
+ s.write(key, splitAt("new", 3, 0, 1))
+ id := s.ids[key+" old"]
+ old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano)
+ recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano)
+ for pool, tags := range []string{tc.stale, tc.fresh} {
+ stamp := old
+ if pool == 1 {
+ stamp = recent
+ }
+ setLifecycleCopyMetadata(t, s, key, id, pool, map[string]string{xhttp.AmzObjectTagging: tags, ReservedMetadataPrefixLower + TaggingTimestamp: stamp}, false)
+ }
+ versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(versions) != 2 || versions[1].UserTags != tc.fresh || versions[1].UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] != recent {
+ t.Fatalf("lost newest tag revision: %+v", versions)
+ }
+ want := lifecycle.NoneAction
+ if tc.expire {
+ want = lifecycle.DeleteVersionAction
+ }
+ if events[1].Action != want {
+ t.Errorf("shared read/delete expiry event=%s want=%s", events[1].Action, want)
+ }
+ _, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: id}}}, ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}})
+ if tc.expire && errs[0] != nil || !tc.expire && !errors.Is(errs[0], errLifecycleNotApplicable) {
+ t.Errorf("delete error=%v", errs[0])
+ }
+ for pool := range s.z.serverPools {
+ _, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id})
+ if tc.expire && !isErrVersionNotFound(err) || !tc.expire && err != nil {
+ t.Errorf("pool=%d err=%v wantGone=%t", pool, err, tc.expire)
+ }
+ }
+ })
+ }
+}
+
+// All-pool validation uses ReadXL. Run once at the first addressed read in
+// batch dispatch, after validation and before the set's lifecycle check.
+type lifecycleDispatchDisk struct {
+ StorageAPI
+ bucket, key string
+ once *sync.Once
+ scans *atomic.Int64
+ beforeRead func()
+}
+
+func (d lifecycleDispatchDisk) ReadXL(ctx context.Context, bucket, key string, data bool) (RawFileInfo, error) {
+ if bucket == d.bucket && key == d.key {
+ d.scans.Add(1)
+ }
+ return d.StorageAPI.ReadXL(ctx, bucket, key, data)
+}
+
+func (d lifecycleDispatchDisk) ReadVersion(ctx context.Context, orig, bucket, key, id string, opts ReadOptions) (FileInfo, error) {
+ if bucket == d.bucket && key == d.key {
+ d.once.Do(d.beforeRead)
+ }
+ return d.StorageAPI.ReadVersion(ctx, orig, bucket, key, id, opts)
+}
+
+func TestPoolsLifecycleBatchRejectionReplication(t *testing.T) {
+ for _, mode := range []string{"withdraw-before-validation", "withdraw-after-validation", "already-absent", "normal"} {
+ t.Run(mode, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ const key = "ncv-dispatch-recheck"
+ s.write(key, splitTiered("old", 5, 0))
+ s.write(key, splitAt("new", 3, 0))
+ id := s.ids[key+" old"]
+ target := newILMTestTarget(t)
+ registerILMTestTarget(t, s.z, s.bucket, target, func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodDelete || r.URL.Query().Get("versionId") != id {
+ t.Errorf("unexpected purge: %s %s", r.Method, r.URL)
+ w.WriteHeader(500)
+ return
+ }
+ target.deletes.Add(1)
+ opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true, ReplicationRequest: true}
+ opts.SetReplicaStatus(replication.Replica)
+ if _, err := s.z.DeleteObject(r.Context(), target.bucket, key, opts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+ t.Error(err)
+ w.WriteHeader(500)
+ return
+ }
+ w.WriteHeader(http.StatusNoContent)
+ })
+ setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket)
+ putConsistencyObject(t, s.z, target.bucket, key, 0, "replica", ObjectOptions{Versioned: true, VersionID: id})
+ stats := NewReplicationStats(t.Context(), nil)
+ previous := globalReplicationStats.Swap(stats)
+ t.Cleanup(func() { globalReplicationStats.Store(previous) })
+ restore, _ := newILMTestReplicationPool(t.Context(), s.z, stats)
+ t.Cleanup(restore)
+ oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id})
+ if err != nil {
+ t.Fatal(err)
+ }
+ entry := ilmPurgeToDelete(t.Context(), s.bucket, oi, true)
+ if entry.VersionPurgeStatus != replication.VersionPurgePending {
+ t.Fatal("fixture lacks pending purge")
+ }
+ withdraw := func() {
+ meta, err := globalBucketMetadataSys.Get(s.bucket)
+ if err != nil {
+ t.Error(err)
+ return
+ }
+ meta.lifecycleConfig = nil
+ meta.LifecycleConfigXML = nil
+ globalBucketMetadataSys.Set(s.bucket, meta)
+ }
+ if mode == "withdraw-before-validation" {
+ withdraw()
+ }
+ originals := s.z.serverPools[0].getHashedSet(key).getDisks()
+ var once sync.Once
+ var scans atomic.Int64
+ var dispatched atomic.Bool
+ for _, pool := range s.z.serverPools {
+ pool.erasureDisksMu.Lock()
+ prior := pool.erasureDisks
+ wrapped := make([][]StorageAPI, len(prior))
+ for i, disks := range prior {
+ wrapped[i] = append([]StorageAPI(nil), disks...)
+ for j, disk := range disks {
+ wrapped[i][j] = lifecycleDispatchDisk{StorageAPI: disk, bucket: s.bucket, key: key, once: &once, scans: &scans, beforeRead: func() {
+ dispatched.Store(true)
+ if mode != "withdraw-before-validation" && scans.Load() == 0 {
+ t.Error("dispatch injection ran before all-pool validation")
+ }
+ switch mode {
+ case "withdraw-after-validation":
+ withdraw()
+ case "already-absent":
+ fi := FileInfo{VersionID: id}
+ fi.SetTierFreeVersionID(mustGetUUID())
+ for _, disk := range originals {
+ if err := disk.DeleteVersion(t.Context(), s.bucket, key, fi, false, DeleteOptions{}); err != nil {
+ t.Error(err)
+ }
+ }
+ }
+ }}
+ }
+ }
+ pool.erasureDisks = wrapped
+ pool.erasureDisksMu.Unlock()
+ t.Cleanup(func() { pool.erasureDisksMu.Lock(); pool.erasureDisks = prior; pool.erasureDisksMu.Unlock() })
+ }
+ result := &lifecycleBatchResult{ObjectLayer: s.z}
+ deleteObjectVersions(t.Context(), result, s.bucket, []ObjectToDelete{entry}, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}})
+ if mode != "withdraw-before-validation" && !dispatched.Load() {
+ t.Fatal("dispatch injection did not run")
+ }
+ queue := globalReplicationPool.Get().workers[0]
+ reject := mode == "withdraw-before-validation" || mode == "withdraw-after-validation"
+ if reject {
+ if !errors.Is(result.errs[0], errLifecycleNotApplicable) || result.deleted[0].found || result.deleted[0].VersionPurgeStatus() != "" || result.deleted[0].ReplicationState.ReplicateDecisionStr != "" {
+ t.Errorf("rejection acknowledged a purge: %+v, %v", result.deleted[0], result.errs[0])
+ }
+ if len(queue) != 0 {
+ t.Fatalf("rejected delete queued %d remote purges", len(queue))
+ }
+ for _, bucket := range []string{s.bucket, target.bucket} {
+ oi, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: id})
+ if err != nil || oi.VersionPurgeStatus != "" {
+ t.Errorf("rejection changed %s: purge=%s err=%v", bucket, oi.VersionPurgeStatus, err)
+ }
+ }
+ if target.deletes.Load() != 0 {
+ t.Fatal("rejected delete reached remote target")
+ }
+ return
+ }
+ if result.errs[0] != nil || len(queue) != 1 {
+ t.Fatalf("valid/absent purge: error=%v queued=%d", result.errs[0], len(queue))
+ }
+ source, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id})
+ if mode == "normal" && (!isErrMethodNotAllowed(err) || source.VersionPurgeStatus != replication.VersionPurgePending) {
+ t.Fatalf("purge not committed locally: %+v %v", source, err)
+ }
+ if mode == "already-absent" && !isErrVersionNotFound(err) {
+ t.Fatalf("source must actually be absent: %v", err)
+ }
+ deletion := (<-queue).(DeletedObjectReplicationInfo)
+ if result := replicateDelete(t.Context(), deletion, s.z); result.VersionPurgeStatus() != replication.VersionPurgeComplete {
+ t.Fatalf("purge failed: %+v", result)
+ }
+ if target.deletes.Load() != 1 {
+ t.Fatalf("remote deletes=%d", target.deletes.Load())
+ }
+ for _, bucket := range []string{s.bucket, target.bucket} {
+ if _, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+ t.Errorf("purged version remains in %s: %v", bucket, err)
+ }
+ }
+ })
+ }
+}
diff --git a/cmd/erasure-server-pool-lifecycle.go b/cmd/erasure-server-pool-lifecycle.go
new file mode 100644
index 000000000..597cdf249
--- /dev/null
+++ b/cmd/erasure-server-pool-lifecycle.go
@@ -0,0 +1,785 @@
+// Copyright (c) 2026 Denis Babchuk
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/binary"
+ "errors"
+ "maps"
+ "slices"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ objectlock "github.com/minio/minio/internal/bucket/object/lock"
+ "github.com/minio/minio/internal/bucket/replication"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// An ordinary ILM rejection is not evidence that the version is absent. In
+// particular it must not acknowledge a pending replication purge.
+var errLifecycleNotApplicable = errors.New("lifecycle expiry no longer applies")
+
+// ILM decides on the versions of an object that one pool holds: the scanner
+// and the drive heal read the drives of one set, decommission and rebalance
+// walk one pool, and a GET or HEAD of a version reads the pool that holds it.
+// IsLatest, NumVersions and SuccessorModTime count only these. An object can
+// have versions in several pools: from two concurrent first writes, from
+// overwrites that went to another pool, from data movement. The expiries
+// these decisions run delete in every pool, so a version that is the latest
+// in its own pool but not of the object had a newer version hidden under a
+// delete marker or deleted with all versions, and a delete marker without
+// noncurrent versions in its pool was removed over the versions of another
+// pool. Deciding on fewer versions only delays the expiry of a noncurrent
+// version: its successor in the pool is not older than the one of the object,
+// and fewer newer versions count against NewerNoncurrentVersions.
+//
+// Before an expiry deletes in any pool, it is evaluated again with the
+// versions of the object in all pools, as the scanner would evaluate it if
+// one pool held them all. An expiry that no longer applies is not run, as the
+// recheck of a set does it: the scanner queues it again once the rule
+// applies to the object.
+
+// objectVersions returns the versions of object that the drives of the set
+// hold authoritatively, newest first, without free versions. It returns no
+// versions when the set does not hold the object, and an error when the
+// drives cannot tell its complete history.
+//
+// Expiry deletes by this history, so a revision of a version, its header
+// and its metadata (versionRevision), counts only when more than half of
+// the configured drives and the read quorum of its metadata hold it. A
+// version that some drive shows without such a revision, or too many
+// unreadable drives, could hide what an expiry must not delete. Free
+// versions are tier GC records and do not count.
+func (er erasureObjects) objectVersions(ctx context.Context, bucket, object string) ([]FileInfo, error) {
+ disks := er.getDisks()
+ rawFileInfos, errs := readAllRawFileInfo(ctx, disks, bucket, object, false)
+ majority := len(disks)/2 + 1
+
+ var (
+ metas [][]xlMetaV2ShallowVersion
+ metaVs []uint8
+ metaV uint8
+ unknown int
+ )
+ for i, rf := range rawFileInfos {
+ if errs[i] != nil {
+ if !errors.Is(errs[i], errFileNotFound) && !errors.Is(errs[i], errVolumeNotFound) {
+ unknown++
+ }
+ continue
+ }
+ var xl xlMetaV2
+ if err := xl.LoadOrConvert(rf.Buf); err != nil {
+ unknown++
+ continue
+ }
+ metas = append(metas, xl.versions)
+ metaVs = append(metaVs, xl.metaV)
+ metaV = xl.metaV
+ }
+ // Without an unknown drive the history is complete, also on one drive.
+ if unknown > 0 && unknown >= len(disks)/2 {
+ return nil, toObjectErr(errErasureReadQuorum, bucket, object)
+ }
+
+ quorumMetas, quorumErrs := make([]FileInfo, len(disks)), make([]error, len(disks))
+ merged, err := authoritativeHistory(ctx, metas, metaVs, metaV, majority, func(ver xlMetaV2ShallowVersion, metaV uint8) int {
+ return er.versionReadQuorum(ctx, bucket, object, ver, metaV, quorumMetas, quorumErrs)
+ })
+ if err != nil {
+ return nil, toObjectErr(err, bucket, object)
+ }
+ if merged == nil {
+ return nil, nil // not in this set
+ }
+ all, err := xlMetaV2{versions: merged, metaV: metaV}.ListVersions(bucket, object, false)
+ if err != nil {
+ return nil, toObjectErr(err, bucket, object)
+ }
+ versions := all[:0]
+ for _, fi := range all {
+ if !fi.TierFreeVersion() {
+ versions = append(versions, fi)
+ }
+ }
+ return versions, nil
+}
+
+// revisionKey identifies a revision of a version: its header and a strong
+// digest of its metadata. The header Signature is a 32-bit fold of weak
+// hashes; two revisions can share it.
+type revisionKey struct {
+ header xlMetaV2VersionHeader
+ digest [sha256.Size]byte
+}
+
+// versionRevision returns the revision of ver, whose metadata is encoded as
+// metaV describes. As Signature does, it leaves out the fields that differ
+// between the drives of one write: the erasure index, empty part lists and,
+// for legacy versions, the release and the per-shard checksums. The digest
+// input is canonical: each map, including the part checksums of a legacy
+// version, is its own section with its entry count and sorted,
+// length-prefixed entries, since msgp encodes maps in iteration order and an
+// entry must not move between maps unnoticed. The header of a
+// legacy version carries a signature over its per-shard checksums, so the
+// key replaces it with one from the digest.
+func versionRevision(ver xlMetaV2ShallowVersion, metaV uint8) revisionKey {
+ h := sha256.New()
+ writeUint := func(n int) {
+ var b [8]byte
+ binary.LittleEndian.PutUint64(b[:], uint64(n))
+ h.Write(b[:])
+ }
+ writeField := func(b []byte) {
+ writeUint(len(b))
+ h.Write(b)
+ }
+ section := func(name string, entries int) {
+ writeField([]byte(name))
+ writeUint(entries)
+ }
+ writeStrings := func(name string, m map[string]string) {
+ section(name, len(m))
+ for _, k := range slices.Sorted(maps.Keys(m)) {
+ writeField([]byte(k))
+ writeField([]byte(m[k]))
+ }
+ }
+ writeBytes := func(name string, m map[string][]byte) {
+ section(name, len(m))
+ for _, k := range slices.Sorted(maps.Keys(m)) {
+ writeField([]byte(k))
+ writeField(m[k])
+ }
+ }
+ key := revisionKey{header: ver.header}
+ var v xlMetaV2Version
+ if _, err := v.unmarshalV(metaV, ver.meta); err == nil {
+ var (
+ bts []byte
+ err error = errFileCorrupt
+ )
+ section("type", 1)
+ writeField([]byte{byte(v.Type)})
+ switch {
+ case v.Type == ObjectType && v.ObjectV2 != nil:
+ c := *v.ObjectV2
+ c.ErasureIndex = 0
+ if !slices.ContainsFunc(c.PartETags, func(tag string) bool { return tag != "" }) {
+ c.PartETags = nil
+ }
+ if len(c.PartActualSizes) == 0 {
+ c.PartActualSizes = nil
+ }
+ writeStrings("user", c.MetaUser)
+ writeBytes("sys", c.MetaSys)
+ c.MetaUser, c.MetaSys = nil, nil
+ bts, err = c.MarshalMsg(nil)
+ case v.Type == DeleteType && v.DeleteMarker != nil:
+ c := *v.DeleteMarker
+ writeBytes("sys", c.MetaSys)
+ c.MetaSys = nil
+ bts, err = c.MarshalMsg(nil)
+ case v.Type == LegacyType && v.ObjectV1 != nil:
+ c := *v.ObjectV1
+ c.Erasure.Index = 0
+ c.Erasure.Checksums = nil
+ c.Minio.Release = ""
+ writeStrings("meta", c.Meta)
+ c.Meta = nil
+ // Part checksums are the only other map: one section per part.
+ section("parts", len(c.Parts))
+ c.Parts = slices.Clone(c.Parts)
+ for i := range c.Parts {
+ writeStrings("part-checksums", c.Parts[i].Checksums)
+ c.Parts[i].Checksums = nil
+ }
+ bts, err = c.MarshalMsg(nil)
+ }
+ if err == nil {
+ section("fields", 1)
+ writeField(bts)
+ h.Sum(key.digest[:0])
+ if v.Type == LegacyType {
+ copy(key.header.Signature[:], key.digest[:])
+ }
+ return key
+ }
+ h.Reset()
+ }
+ // Undecodable metadata only matches the same bytes.
+ section("raw", 1)
+ writeField(ver.meta)
+ h.Sum(key.digest[:0])
+ return key
+}
+
+// authoritativeHistory returns the versions of metas, the version lists of
+// the drives that answered, as merged newest first, when every version
+// that is not a free version has a revision that at least majority drives
+// and its readQuorum hold. It returns no versions when none is live,
+// errErasureReadQuorum when the drives cannot tell the history, and the
+// error of ctx once it ends. metaVs are the metadata versions of metas,
+// metaV the one the merged result is decoded with.
+func authoritativeHistory(ctx context.Context, metas [][]xlMetaV2ShallowVersion, metaVs []uint8, metaV uint8, majority int, readQuorum func(xlMetaV2ShallowVersion, uint8) int) ([]xlMetaV2ShallowVersion, error) {
+ // Count the drives that hold each revision; a drive counts once for each
+ // version.
+ type revision struct {
+ ver xlMetaV2ShallowVersion
+ metaV uint8
+ count int
+ }
+ keys := make([][]revisionKey, len(metas))
+ revisions := make(map[[16]byte]map[revisionKey]*revision)
+ live := make(map[[16]byte]bool)
+ for i, versions := range metas {
+ keys[i] = make([]revisionKey, len(versions))
+ seen := make(map[[16]byte]bool, len(versions))
+ for j, ver := range versions {
+ // Decoding and hashing every history has a cost; stop with the
+ // request.
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ id := ver.header.VersionID
+ if !ver.header.FreeVersion() {
+ live[id] = true
+ }
+ if seen[id] {
+ continue
+ }
+ seen[id] = true
+ key := versionRevision(ver, metaVs[i])
+ keys[i][j] = key
+ if revisions[id] == nil {
+ revisions[id] = make(map[revisionKey]*revision)
+ }
+ r := revisions[id][key]
+ if r == nil {
+ r = &revision{ver: ver, metaV: metaVs[i]}
+ revisions[id][key] = r
+ }
+ r.count++
+ }
+ }
+ accepted := make(map[[16]byte]revisionKey, len(live))
+ for id := range live {
+ var best *revision
+ var bestKey revisionKey
+ for key, r := range revisions[id] {
+ if !key.header.FreeVersion() && (best == nil || r.count > best.count) {
+ best, bestKey = r, key
+ }
+ }
+ // Only one revision can hold more than half of the drives.
+ if best == nil || best.count < majority || best.count < readQuorum(best.ver, best.metaV) {
+ return nil, errErasureReadQuorum
+ }
+ accepted[id] = bestKey
+ }
+
+ // The strict merge orders the history. It has to hold exactly the
+ // accepted revisions. A minority revision that sorts first would reset
+ // its count, and one that shares the header could stand in for the
+ // accepted one, so the merge only sees the accepted revision of each
+ // version and free versions as they are. It sees the headers of the
+ // keys, which do not differ between the drives of a legacy version.
+ filtered := make([][]xlMetaV2ShallowVersion, len(metas))
+ for i, versions := range metas {
+ for j, ver := range versions {
+ key, ok := accepted[ver.header.VersionID]
+ switch {
+ case ver.header.FreeVersion():
+ case ok && keys[i][j] == key:
+ ver.header = key.header
+ default:
+ continue
+ }
+ filtered[i] = append(filtered[i], ver)
+ }
+ }
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ merged := mergeXLV2Versions(majority, true, 0, filtered...)
+ emitted := make(map[[16]byte]bool, len(accepted))
+ for _, ver := range merged {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
+ if ver.header.FreeVersion() {
+ continue
+ }
+ id := ver.header.VersionID
+ if key, ok := accepted[id]; !ok || emitted[id] || versionRevision(ver, metaV) != key {
+ return nil, errErasureReadQuorum
+ }
+ emitted[id] = true
+ }
+ if len(emitted) != len(accepted) {
+ return nil, errErasureReadQuorum
+ }
+ if len(accepted) == 0 {
+ return nil, nil
+ }
+ return merged, nil
+}
+
+// versionReadQuorum returns the read quorum of the metadata of ver, as
+// objectQuorumFromMeta derives it when all drives of the set hold it, or
+// more than the drives when its metadata cannot be read. metaArr and errs
+// have one entry for each drive; errs stays nil.
+func (er erasureObjects) versionReadQuorum(ctx context.Context, bucket, object string, ver xlMetaV2ShallowVersion, metaV uint8, metaArr []FileInfo, errs []error) int {
+ var v xlMetaV2Version
+ if _, err := v.unmarshalV(metaV, ver.meta); err != nil {
+ return len(metaArr) + 1
+ }
+ fi, err := v.ToFileInfo(bucket, object, false)
+ if err != nil {
+ return len(metaArr) + 1
+ }
+ for i := range metaArr {
+ metaArr[i] = fi
+ }
+ readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
+ if err != nil {
+ return len(metaArr) + 1
+ }
+ return readQuorum
+}
+
+// lifecycleEventsAllPools returns the versions of object in all pools,
+// newest first, and the lifecycle event of each as the scanner evaluates it
+// on them. A version with copies in several pools, as an interrupted move
+// leaves it, counts once, and so do other writes of its version ID.
+func (z *erasureServerPools) lifecycleEventsAllPools(ctx context.Context, bucket, object string) ([]ObjectInfo, []lifecycle.Event, error) {
+ lc, err := globalLifecycleSys.Get(bucket)
+ if err != nil {
+ if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
+ return nil, nil, errLifecycleNotApplicable
+ }
+ return nil, nil, err
+ }
+ lr, err := globalBucketObjectLockSys.Get(bucket)
+ if err != nil {
+ return nil, nil, err
+ }
+ rcfg, err := getReplicationConfig(ctx, bucket)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ poolVersions := make([][]FileInfo, len(z.serverPools))
+ errs := make([]error, len(z.serverPools))
+ var wg sync.WaitGroup
+ for i, pool := range z.serverPools {
+ wg.Add(1)
+ go func(i int, set *erasureObjects) {
+ defer wg.Done()
+ poolVersions[i], errs[i] = set.objectVersions(ctx, bucket, object)
+ }(i, pool.getHashedSet(object))
+ }
+ wg.Wait()
+ for _, err := range errs {
+ if err != nil {
+ return nil, nil, err
+ }
+ }
+
+ vc, _ := globalBucketVersioningSys.Get(bucket)
+ versioned := vc != nil && vc.Versioned(decodeDirObject(object))
+ var all []PoolObjInfo
+ for pool, versions := range poolVersions {
+ for _, fi := range versions {
+ all = append(all, PoolObjInfo{Index: pool, ObjInfo: fi.ToObjectInfo(bucket, object, versioned)})
+ }
+ }
+ // Each pool lists its versions newest first. On equal times the lower
+ // pool comes first, as reads choose it.
+ sort.SliceStable(all, func(i, j int) bool { return all[i].ObjInfo.ModTime.After(all[j].ObjInfo.ModTime) })
+ seen := make(map[string]int, len(all))
+ var versions [][]PoolObjInfo
+ for _, copy := range all {
+ if i, ok := seen[copy.ObjInfo.VersionID]; ok {
+ versions[i] = append(versions[i], copy)
+ continue
+ }
+ seen[copy.ObjInfo.VersionID] = len(versions)
+ versions = append(versions, []PoolObjInfo{copy})
+ }
+
+ e := lifecycle.NewEvaluator(*lc).WithLockRetention(&lr).WithReplicationConfig(rcfg)
+ objInfos := make([]ObjectInfo, len(versions))
+ objOpts := make([]lifecycle.ObjectOpts, len(versions))
+ for i, copies := range versions {
+ // Copies with another ModTime are other writes of the version ID,
+ // such as null versions written in several pools, not duplicates of
+ // one write. The newest is the version. Data movement writes a new
+ // data dir, so only ModTime tells the duplicates of a write apart.
+ // An older write that is still protected keeps the whole object.
+ newest := 1
+ for newest < len(copies) && copies[newest].ObjInfo.ModTime.Equal(copies[0].ObjInfo.ModTime) {
+ newest++
+ }
+ for _, older := range copies[newest:] {
+ if opts := older.ObjInfo.ToLifecycleOpts(); e.IsObjectLocked(opts) || e.IsPendingReplication(opts) {
+ return nil, nil, errLifecycleNotApplicable
+ }
+ }
+ copies = copies[:newest]
+ oi := copies[0].ObjInfo
+ if len(copies) > 1 {
+ // The copies must tell which protection is the newest, and no
+ // copy may carry replication state that the merge would drop.
+ if !lockRevisionsOrdered(copies) || !replicationStatesAgree(copies) {
+ return nil, nil, errLifecycleNotApplicable
+ }
+ // Count the version once, but retain independently ordered lock
+ // and tag revisions, including explicit removals, from every copy.
+ oi = mergedPoolObjectInfo(copies)
+ setLifecycleLockState(oi.UserDefined, newestLockState(copies))
+ }
+ oi.IsLatest = i == 0
+ oi.NumVersions = len(versions)
+ oi.SuccessorModTime = time.Time{}
+ if i > 0 {
+ oi.SuccessorModTime = objInfos[i-1].ModTime
+ }
+ objInfos[i] = oi
+ objOpts[i] = oi.ToLifecycleOpts()
+ }
+ events, err := e.Eval(objOpts)
+ if err != nil {
+ return nil, nil, err
+ }
+ return objInfos, events, nil
+}
+
+// lockRevisionTime parses the timestamp of an Object Lock revision. Unlike
+// olderThan, a missing, malformed or zero timestamp orders nothing.
+func lockRevisionTime(stamp string) (time.Time, bool) {
+ t, err := time.Parse(time.RFC3339Nano, stamp)
+ return t, err == nil && !t.IsZero()
+}
+
+// lockRevisionNewer reports whether the revision stamped newer strictly
+// supersedes the one stamped older.
+func lockRevisionNewer(newer, older string) bool {
+ n, ok := lockRevisionTime(newer)
+ o, ok2 := lockRevisionTime(older)
+ return ok && ok2 && n.After(o)
+}
+
+// lifecycleLockState returns the Object Lock state of metadata as the
+// lifecycle evaluator reads it (objectlock.GetObjectRetentionMeta and
+// GetObjectLegalHoldMeta): a lowercase key before its canonical form. The
+// stored state (storedObjectLockState) only has lowercase keys.
+func lifecycleLockState(metadata map[string]string) objectLockState {
+ get := func(canonical string) string {
+ if v, ok := metadata[strings.ToLower(canonical)]; ok {
+ return v
+ }
+ return metadata[canonical]
+ }
+ return objectLockState{
+ mode: get(objectlock.AmzObjectLockMode),
+ retainUntil: get(objectlock.AmzObjectLockRetainUntilDate),
+ retentionTimestamp: get(ReservedMetadataPrefix + ObjectLockRetentionTimestamp),
+ legalHold: get(objectlock.AmzObjectLockLegalHold),
+ legalHoldTimestamp: get(ReservedMetadataPrefix + ObjectLockLegalHoldTimestamp),
+ }
+}
+
+// setLifecycleLockState sets state in metadata so that the evaluator reads
+// it: lowercase keys take precedence over canonical ones.
+func setLifecycleLockState(metadata map[string]string, state objectLockState) {
+ set := func(canonical, value string) {
+ lower := strings.ToLower(canonical)
+ _, hasLower := metadata[lower]
+ _, hasCanonical := metadata[canonical]
+ if value != "" || hasLower || hasCanonical {
+ metadata[lower] = value
+ }
+ }
+ set(objectlock.AmzObjectLockMode, state.mode)
+ set(objectlock.AmzObjectLockRetainUntilDate, state.retainUntil)
+ set(ReservedMetadataPrefix+ObjectLockRetentionTimestamp, state.retentionTimestamp)
+ set(objectlock.AmzObjectLockLegalHold, state.legalHold)
+ set(ReservedMetadataPrefix+ObjectLockLegalHoldTimestamp, state.legalHoldTimestamp)
+}
+
+// newestLockState returns the newest hold and the newest retention of the
+// copies of a version, which lockRevisionsOrdered found equal or ordered.
+func newestLockState(copies []PoolObjInfo) objectLockState {
+ state := lifecycleLockState(copies[0].ObjInfo.UserDefined)
+ for _, copy := range copies[1:] {
+ next := lifecycleLockState(copy.ObjInfo.UserDefined)
+ if lockRevisionNewer(next.legalHoldTimestamp, state.legalHoldTimestamp) {
+ state.legalHold, state.legalHoldTimestamp = next.legalHold, next.legalHoldTimestamp
+ }
+ if lockRevisionNewer(next.retentionTimestamp, state.retentionTimestamp) {
+ state.mode, state.retainUntil, state.retentionTimestamp = next.mode, next.retainUntil, next.retentionTimestamp
+ }
+ }
+ return state
+}
+
+// checkedExpiryRefused reports whether a set must keep a version whose ILM
+// expiry the pools approved with the Object Lock state approved, by the
+// metadata goi the set reads under the same lock. A pending purge always
+// keeps it: its status orders nothing. The approved hold or retention only
+// replaces the one the set reads when it is strictly newer; otherwise the
+// protection the set reads stands, also when it is newer than the approval.
+func checkedExpiryRefused(lc lifecycle.Lifecycle, lr objectlock.Retention, rcfg *replication.Config, goi ObjectInfo, approved objectLockState) bool {
+ e := lifecycle.NewEvaluator(lc).WithLockRetention(&lr).WithReplicationConfig(rcfg)
+ obj := goi.ToLifecycleOpts()
+ if e.IsPendingReplication(obj) {
+ return true
+ }
+ local := lifecycleLockState(goi.UserDefined)
+ obj.UserDefined = maps.Clone(goi.UserDefined)
+ if obj.UserDefined == nil {
+ obj.UserDefined = make(map[string]string)
+ }
+ if lockRevisionNewer(approved.legalHoldTimestamp, local.legalHoldTimestamp) {
+ obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = approved.legalHold
+ }
+ if lockRevisionNewer(approved.retentionTimestamp, local.retentionTimestamp) {
+ obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = approved.mode
+ obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = approved.retainUntil
+ }
+ return e.IsObjectLocked(obj)
+}
+
+// lockRevisionsOrdered reports whether the copies of a version agree on the
+// legal hold and on the retention, each on its own, or carry revisions with
+// distinct valid timestamps, so that the newest one is known.
+func lockRevisionsOrdered(copies []PoolObjInfo) bool {
+ ordered := func(revision func(objectLockState) (value, stamp string)) bool {
+ stamps := make(map[[2]string]string, len(copies))
+ for _, copy := range copies {
+ value, stamp := revision(lifecycleLockState(copy.ObjInfo.UserDefined))
+ stamps[[2]string{value, stamp}] = stamp
+ }
+ if len(stamps) == 1 {
+ return true
+ }
+ seen := make([]time.Time, 0, len(stamps))
+ for _, stamp := range stamps {
+ t, ok := lockRevisionTime(stamp)
+ if !ok {
+ return false
+ }
+ for _, other := range seen {
+ if t.Equal(other) {
+ return false
+ }
+ }
+ seen = append(seen, t)
+ }
+ return true
+ }
+ return ordered(func(s objectLockState) (string, string) {
+ return s.legalHold, s.legalHoldTimestamp
+ }) && ordered(func(s objectLockState) (string, string) {
+ return s.mode + "\x00" + s.retainUntil, s.retentionTimestamp
+ })
+}
+
+// replicationStatesAgree reports whether the copies of a version carry the
+// same replication and purge state. The per-target states are compared as
+// parsed, regardless of their order.
+func replicationStatesAgree(copies []PoolObjInfo) bool {
+ first := copies[0].ObjInfo
+ for _, copy := range copies[1:] {
+ oi := copy.ObjInfo
+ if oi.ReplicationStatus != first.ReplicationStatus || oi.VersionPurgeStatus != first.VersionPurgeStatus ||
+ !maps.Equal(replicationStatusesMap(oi.ReplicationStatusInternal), replicationStatusesMap(first.ReplicationStatusInternal)) ||
+ !maps.Equal(versionPurgeStatusesMap(oi.VersionPurgeStatusInternal), versionPurgeStatusesMap(first.VersionPurgeStatusInternal)) {
+ return false
+ }
+ }
+ return true
+}
+
+// lifecycleAllPools holds the versions of an object in all pools and their
+// lifecycle events, see lifecycleEventsAllPools.
+type lifecycleAllPools struct {
+ versions []ObjectInfo
+ events []lifecycle.Event
+ err error
+}
+
+// checkExpiryAllPools returns nil when the ILM expiry that opts describe
+// still applies to object with the versions it has in all pools, see
+// lifecycleAllPools.expiryApplies, and the Object Lock state with which it
+// approved the expiry of a version.
+func (z *erasureServerPools) checkExpiryAllPools(ctx context.Context, bucket, object string, opts ObjectOptions) (objectLockState, error) {
+ var all lifecycleAllPools
+ all.versions, all.events, all.err = z.lifecycleEventsAllPools(ctx, bucket, object)
+ if err := all.expiryApplies(bucket, object, opts); err != nil {
+ return objectLockState{}, err
+ }
+ return all.protection(opts.VersionID), nil
+}
+
+// version returns the index of the version versionID, or -1.
+func (all lifecycleAllPools) version(versionID string) int {
+ for i, oi := range all.versions {
+ if oi.VersionID == versionID || oi.VersionID == "" && versionID == nullVersionID {
+ return i
+ }
+ }
+ return -1
+}
+
+// protection returns the Object Lock state of the version versionID that
+// the evaluator decided on, to be checked against the state the set reads.
+func (all lifecycleAllPools) protection(versionID string) objectLockState {
+ if versionID == "" {
+ return objectLockState{}
+ }
+ if i := all.version(versionID); i >= 0 {
+ return lifecycleLockState(all.versions[i].UserDefined)
+ }
+ return objectLockState{}
+}
+
+// expiryApplies returns nil when the ILM expiry that opts describe applies
+// to the versions. A missing version and a rejected expiry remain distinct:
+// - an expiry of all versions (DeletePrefix) needs the latest version to
+// expire with all versions;
+// - a delete marker (no VersionID) hides the latest version, which has to
+// expire;
+// - a version (VersionID) has to expire by itself: a noncurrent version,
+// or a delete marker without noncurrent versions.
+func (all lifecycleAllPools) expiryApplies(bucket, object string, opts ObjectOptions) error {
+ if all.err != nil {
+ if _, ok := all.err.(InsufficientReadQuorum); ok {
+ return InsufficientWriteQuorum{}
+ }
+ return all.err
+ }
+ name := decodeDirObject(object)
+ switch {
+ case opts.DeletePrefix:
+ if len(all.events) > 0 && all.events[0].Action.DeleteAll() {
+ return nil
+ }
+ case opts.VersionID == "":
+ if len(all.events) > 0 && (all.events[0].Action == lifecycle.DeleteAction || all.events[0].Action == lifecycle.DeleteAllVersionsAction) {
+ return nil
+ }
+ default:
+ if i := all.version(opts.VersionID); i >= 0 {
+ if all.events[i].Action == lifecycle.DeleteVersionAction {
+ return nil
+ }
+ return errLifecycleNotApplicable
+ }
+ return VersionNotFound{Bucket: bucket, Object: name, VersionID: opts.VersionID}
+ }
+ if len(all.versions) > 0 {
+ return errLifecycleNotApplicable
+ }
+ return ObjectNotFound{Bucket: bucket, Object: name}
+}
+
+// expiryNeedsAllPools reports whether an ILM expiry in opts deletes or hides
+// versions by a rule that depends on the other versions of the object. A
+// restored copy expires by its own restore date. Without versions each copy
+// of an object is a write of its own, and a set rechecks the rule on its
+// copy (deleteObjectFromAllPools).
+func (z *erasureServerPools) expiryNeedsAllPools(opts ObjectOptions) bool {
+ if !opts.Expiration.Expire || z.SinglePool() || opts.Transition.ExpireRestored {
+ return false
+ }
+ return opts.DeletePrefix || opts.VersionID != "" || opts.Versioned || opts.VersionSuspended
+}
+
+// recheckReadExpiry only rechecks a local ordinary-delete candidate. The
+// global latest expiring does not authorize hiding a different version that
+// GET/HEAD has already read. On an incomplete or rejected recheck, keep that
+// successful read and do not queue an expiry. Leave response metadata intact.
+func recheckReadExpiry(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, evt lifecycle.Event) (ObjectInfo, lifecycle.Event) {
+ z, ok := objectAPI.(*erasureServerPools)
+ if !ok || !evt.Action.Delete() || evt.Action.DeleteRestored() || !z.expiryNeedsAllPools(ObjectOptions{
+ Expiration: ExpirationOptions{Expire: true},
+ VersionID: oi.VersionID,
+ Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+ VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+ DeletePrefix: evt.Action.DeleteAll(),
+ }) {
+ return oi, evt
+ }
+ versions, events, err := z.lifecycleEventsAllPools(ctx, oi.Bucket, encodeDirObject(oi.Name))
+ if err == nil {
+ versionID := oi.VersionID
+ if versionID == "" {
+ versionID = nullVersionID
+ }
+ for i, version := range versions {
+ id := version.VersionID
+ if id == "" {
+ id = nullVersionID
+ }
+ if id == versionID {
+ if events[i].Action.Delete() && !events[i].Action.DeleteRestored() {
+ return version, events[i]
+ }
+ break
+ }
+ }
+ }
+ return oi, lifecycle.Event{Action: lifecycle.NoneAction}
+}
+
+// migrationExpiryApplies reports whether the ILM expiry evt, which a
+// decommission or rebalance decided for oi on the versions of the pool it
+// walks, applies with the versions of the object in all pools too. Only then
+// may the walk leave the version to the expiry instead of moving it. A
+// version that is the latest in the walked pool can be a noncurrent version
+// of the object, and a delete marker alone in that pool can hide versions in
+// another one. Such a version is moved like any other, and the scanner
+// applies the rule in the pool that holds the object then.
+func (z *erasureServerPools) migrationExpiryApplies(ctx context.Context, oi ObjectInfo, evt lifecycle.Event) bool {
+ // The options the expiry deletes with, see
+ // applyExpiryOnNonTransitionedObjects and expireTransitionedObject.
+ opts := ObjectOptions{
+ Expiration: ExpirationOptions{Expire: true},
+ Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+ VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+ }
+ if evt.Action.DeleteVersioned() {
+ opts.VersionID = oi.VersionID
+ }
+ if evt.Action.DeleteAll() {
+ opts.DeletePrefix = true
+ }
+ opts.Transition.ExpireRestored = evt.Action.DeleteRestored()
+ if !z.expiryNeedsAllPools(opts) {
+ return true
+ }
+ _, err := z.checkExpiryAllPools(ctx, oi.Bucket, encodeDirObject(oi.Name), opts)
+ return err == nil
+}
diff --git a/cmd/erasure-server-pool-lifecycle_test.go b/cmd/erasure-server-pool-lifecycle_test.go
new file mode 100644
index 000000000..ea39d2819
--- /dev/null
+++ b/cmd/erasure-server-pool-lifecycle_test.go
@@ -0,0 +1,501 @@
+// Copyright (c) 2026 Denis Babchuk
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "fmt"
+ "io"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/versioning"
+)
+
+// The scanner and the walks of decommission and rebalance evaluate ILM on the
+// versions of a key in one pool: IsLatest, NumVersions and SuccessorModTime
+// count only these. A key can have versions in several pools: two concurrent
+// first writes, overwrites that went to another pool. The expiry they decide
+// runs in every pool.
+
+// splitILM lays out the versions of keys in the two pools of migrationILM.
+type splitILM struct {
+ *migrationILM
+ ids map[string]string // object and label -> version ID
+ bodies map[string]string // body -> label
+}
+
+func newSplitILM(t *testing.T, status versioning.State, rules string) *splitILM {
+ t.Helper()
+ return &splitILM{migrationILM: newMigrationILM(t, status, false, rules), ids: map[string]string{}, bodies: map[string]string{}}
+}
+
+// splitVersion is a version of a key in a test layout.
+type splitVersion struct {
+ pools []int // the pools that hold a copy of it
+ label string
+ days float64 // its age
+ marker bool
+ tiered bool // its data is in a remote tier
+ null bool // a null version (unversioned or suspended bucket)
+ size int // its size, at least the size of its label
+}
+
+func (s *splitILM) write(object string, v splitVersion) {
+ s.t.Helper()
+ ctx := s.t.Context()
+ mtime := UTCNow().Add(-time.Duration(v.days * float64(24*time.Hour)))
+ versionID := mustGetUUID()
+ if v.null {
+ versionID = ""
+ }
+ for _, pool := range v.pools {
+ if v.marker {
+ _, err := s.z.serverPools[pool].DeleteObject(ctx, s.bucket, object,
+ ObjectOptions{Versioned: true, VersionID: versionID, DeleteMarker: true, MTime: mtime})
+ if err != nil {
+ s.t.Fatal(err)
+ }
+ continue
+ }
+ opts := ObjectOptions{Versioned: !v.null, VersionID: versionID, MTime: mtime}
+ if v.null && s.status() == versioning.Suspended {
+ opts = ObjectOptions{VersionSuspended: true, MTime: mtime}
+ }
+ body := object + " " + v.label
+ if len(body) < v.size {
+ body += strings.Repeat(".", v.size-len(body))
+ }
+ putConsistencyObject(s.t, s.z, s.bucket, object, pool, body, opts)
+ s.bodies[body] = v.label
+ if v.tiered {
+ vid := versionID
+ if vid == "" {
+ vid = nullVersionID
+ }
+ infos, errs := readAllFileInfo(ctx, s.z.serverPools[pool].getHashedSet(object).getDisks(), "", s.bucket, object, vid, false, false)
+ if errs[0] != nil {
+ s.t.Fatal(errs[0])
+ }
+ fi := infos[0]
+ fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", "remote-"+v.label
+ if err := s.z.serverPools[pool].DecomTieredObject(ctx, s.bucket, object, fi, ObjectOptions{}); err != nil {
+ s.t.Fatal(err)
+ }
+ }
+ }
+ if v.marker {
+ s.markers[s.key(object, versionID)] = v.label
+ } else {
+ s.labels[s.key(object, versionID)] = v.label
+ }
+ s.ids[object+" "+v.label] = versionID
+ s.track(object)
+}
+
+func (s *splitILM) status() versioning.State {
+ vc, _ := globalBucketVersioningSys.Get(s.bucket)
+ if vc == nil {
+ return ""
+ }
+ return vc.Status
+}
+
+// scan applies ILM to every object in a pool as the scanner of a drive of
+// that pool does, from the xl.meta of that drive, and runs the queued
+// expiries as an expiry worker does.
+func (s *splitILM) scan(pool int) {
+ s.t.Helper()
+ ctx := s.t.Context()
+ lc, err := globalLifecycleSys.Get(s.bucket)
+ if err != nil {
+ s.t.Fatal(err)
+ }
+ lr, err := globalBucketObjectLockSys.Get(s.bucket)
+ if err != nil {
+ s.t.Fatal(err)
+ }
+ vc, _ := globalBucketVersioningSys.Get(s.bucket)
+ for _, object := range s.objects {
+ disk := s.z.serverPools[pool].getHashedSet(object).getDisks()[0]
+ rf, err := disk.ReadXL(ctx, s.bucket, object, false)
+ if err != nil {
+ continue // not in this pool
+ }
+ fivs, err := getFileInfoVersions(rf.Buf, s.bucket, object, false)
+ if err != nil {
+ s.t.Fatal(err)
+ }
+ objInfos := make([]ObjectInfo, len(fivs.Versions))
+ for i, fi := range fivs.Versions {
+ objInfos[i] = fi.ToObjectInfo(s.bucket, object, vc != nil && vc.Versioned(object))
+ }
+ item := scannerItem{bucket: s.bucket, objectName: object, lifeCycle: lc}
+ item.applyActions(ctx, s.z, objInfos, lr, &sizeSummary{}, nil)
+ }
+ s.runQueue()
+}
+
+// head applies ILM to a version as a HEAD or GET with its versionId does.
+func (s *splitILM) head(object, label string) lifecycle.Action {
+ s.t.Helper()
+ ctx := s.t.Context()
+ oi, err := s.z.GetObjectInfo(ctx, s.bucket, object, ObjectOptions{VersionID: s.ids[object+" "+label]})
+ if err != nil {
+ s.t.Fatal(err)
+ }
+ lc, err := globalLifecycleSys.Get(s.bucket)
+ if err != nil {
+ s.t.Fatal(err)
+ }
+ lr, err := globalBucketObjectLockSys.Get(s.bucket)
+ if err != nil {
+ s.t.Fatal(err)
+ }
+ evt := evalActionFromLifecycle(ctx, *lc, lr, nil, oi)
+ if evt.Action.Delete() {
+ applyExpiryRule(evt, lcEventSrc_s3HeadObject, oi)
+ }
+ s.runQueue()
+ return evt.Action
+}
+
+// runQueue runs the queued expiries with an expiry worker.
+func (s *splitILM) runQueue() {
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ globalExpiryState.Worker(s.queue)
+ }()
+ s.queue <- nil // the worker returns once it ran every expiry queued before
+ <-done
+}
+
+// get returns the label of the version a GET of object reads, or "-" when it
+// reads as deleted.
+func (s *splitILM) get(object string) string {
+ s.t.Helper()
+ ctx := s.t.Context()
+ oi, err := s.z.GetObjectInfo(ctx, s.bucket, object, ObjectOptions{})
+ if err != nil {
+ if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrMethodNotAllowed(err) {
+ return "-"
+ }
+ return fmt.Sprintf("<%v>", err)
+ }
+ if oi.VersionID != "" && oi.VersionID != nullVersionID {
+ return s.label(object, oi.VersionID, false)
+ }
+ // Copies of a null version in two pools are told apart by their data.
+ gr, err := s.z.GetObjectNInfo(ctx, s.bucket, object, nil, nil, ObjectOptions{})
+ if err != nil {
+ return fmt.Sprintf("<%v>", err)
+ }
+ defer gr.Close()
+ data, err := io.ReadAll(gr)
+ if err != nil {
+ s.t.Fatal(err)
+ }
+ if l, ok := s.bodies[string(data)]; ok {
+ return l
+ }
+ return "?" + string(data)
+}
+
+// views returns the versions of each object in each pool, newest first.
+func (s *splitILM) views() []map[string][]string {
+ views := make([]map[string][]string, len(s.z.serverPools))
+ for pool := range s.z.serverPools {
+ views[pool] = s.view(pool)
+ }
+ return views
+}
+
+type splitKey struct {
+ name string
+ versions []splitVersion
+ before string // what a GET reads before ILM ran
+ after string // what a GET reads once ILM ran
+ want [][]string // versions in each pool once ILM ran, nil to skip
+}
+
+// run lays out keys, runs two scanner cycles over all pools and checks that
+// a GET of each key never reads anything but before or after, and the
+// versions each pool holds at the end. The second cycle must not change
+// anything.
+func (s *splitILM) run(keys []splitKey) {
+ s.t.Helper()
+ for _, k := range keys {
+ for _, v := range k.versions {
+ s.write(k.name, v)
+ }
+ if got := s.get(k.name); got != k.before {
+ s.t.Fatalf("%s: fixture reads %s, want %s", k.name, got, k.before)
+ }
+ }
+ var first []map[string][]string
+ for cycle := 1; cycle <= 2; cycle++ {
+ for pool := range s.z.serverPools {
+ s.scan(pool)
+ for _, k := range keys {
+ if got := s.get(k.name); got != k.before && got != k.after {
+ s.t.Errorf("%s: reads %s after the scanner of pool %d (cycle %d), want %s", k.name, got, pool, cycle, k.after)
+ }
+ }
+ }
+ if cycle == 1 {
+ first = s.views()
+ }
+ }
+ last := s.views()
+ for _, k := range keys {
+ if got := s.get(k.name); got != k.after {
+ s.t.Errorf("%s: reads %s once ILM ran, want %s", k.name, got, k.after)
+ }
+ for pool := range last {
+ if !slices.Equal(first[pool][k.name], last[pool][k.name]) {
+ s.t.Errorf("%s: the second scanner cycle changed pool %d from %v to %v", k.name, pool, first[pool][k.name], last[pool][k.name])
+ }
+ if k.want != nil && !slices.Equal(last[pool][k.name], k.want[pool]) {
+ s.t.Errorf("%s: pool %d holds %v, want %v", k.name, pool, last[pool][k.name], k.want[pool])
+ }
+ }
+ }
+}
+
+const splitILMRules = "expEnabledexp-" +
+ "1" +
+ "eavEnabledeav-" +
+ "1true" +
+ "edmEnablededm-" +
+ "true" +
+ "dmeEnableddme-" +
+ "1" +
+ "ncvEnabledncv-" +
+ "1" +
+ "exsEnabledexs-100" +
+ "1" +
+ "nnvEnablednnv-" +
+ "11"
+
+func splitAt(label string, days float64, pools ...int) splitVersion {
+ return splitVersion{pools: pools, label: label, days: days}
+}
+
+func splitMarker(label string, days float64, pools ...int) splitVersion {
+ return splitVersion{pools: pools, label: label, days: days, marker: true}
+}
+
+func splitTiered(label string, days float64, pools ...int) splitVersion {
+ return splitVersion{pools: pools, label: label, days: days, tiered: true}
+}
+
+func splitNull(label string, days float64, pools ...int) splitVersion {
+ return splitVersion{pools: pools, label: label, days: days, null: true}
+}
+
+// splitILMOnePool are keys in one pool. ILM applies to them as usual.
+func splitILMOnePool(pool int) []splitKey {
+ in := func(vs ...string) [][]string {
+ want := [][]string{nil, nil}
+ want[pool] = vs
+ return want
+ }
+ return []splitKey{
+ {"exp-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool)}, "v2", "-", in("DM", "v2", "v1")},
+ {"exp-one-tiered", []splitVersion{splitAt("v1", 5, pool), splitTiered("v2", 4, pool)}, "v2", "-", in("DM", "v2", "v1")},
+ {"eav-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool)}, "v2", "-", in()},
+ {"edm-one", []splitVersion{splitMarker("dm", 3, pool)}, "-", "-", in()},
+ {"dme-one", []splitVersion{splitAt("v1", 5, pool), splitMarker("dm", 3, pool)}, "-", "-", in()},
+ {"ncv-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool), splitAt("v3", 0.01, pool)}, "v3", "v3", in("v3", "v2")},
+ {"nnv-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool), splitAt("v3", 3, pool), splitAt("v4", 2, pool)}, "v4", "v4", in("v4", "v3")},
+ }
+}
+
+func TestPoolsLifecycleSplitObject(t *testing.T) {
+ t.Run("versioned", func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ s.run([]splitKey{
+ // Expiration with Days applies to the current version v2 only.
+ {"exp-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+ {"exp-old-in-1", []splitVersion{splitAt("v2", 0.01, 0), splitAt("v1", 5, 1)}, "v2", "v2", [][]string{{"v2"}, {"v1"}}},
+ {"exp-tiered-old-in-0", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+ // v2 is old enough: one delete marker hides the key.
+ {"exp-both-old", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1)}, "v2", "-", [][]string{{"DM", "v1"}, {"v2"}}},
+ // The rule applies to v2 only by its size.
+ {"exs-big-in-1", []splitVersion{splitAt("v1", 5, 0), {pools: []int{1}, label: "v2", days: 4, size: 200}}, "v2", "-", [][]string{{"DM", "v1"}, {"v2"}}},
+ // The marker hides v1 and is not a marker without noncurrent
+ // versions; v1 is not the current version.
+ {"exp-marker-in-0", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", [][]string{{"dm"}, {"v1"}}},
+ // ExpiredObjectAllVersions applies to the current version v2 only.
+ {"eav-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+ {"eav-old-in-1", []splitVersion{splitAt("v2", 0.01, 0), splitAt("v1", 5, 1)}, "v2", "v2", [][]string{{"v2"}, {"v1"}}},
+ {"eav-tiered-old-in-0", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+ {"eav-both-old", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1)}, "v2", "-", [][]string{nil, nil}},
+ // A copy of v1 in both pools, as an interrupted move leaves it.
+ {"eav-copies", []splitVersion{splitAt("v1", 5, 0, 1)}, "v1", "-", [][]string{nil, nil}},
+ // The marker hides v1 in the other pool.
+ {"edm-marker-in-0", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", [][]string{{"dm"}, {"v1"}}},
+ {"edm-marker-in-1", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 1)}, "-", "-", [][]string{{"v1"}, {"dm"}}},
+ {"edm-copies", []splitVersion{splitMarker("dm", 3, 0, 1)}, "-", "-", [][]string{nil, nil}},
+ // v2 in the other pool is the current version, the marker does
+ // not expire.
+ {"dme-new-in-1", []splitVersion{splitMarker("dm", 3, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"dm"}, {"v2"}}},
+ {"dme-new-in-0", []splitVersion{splitAt("v2", 0.01, 0), splitMarker("dm", 3, 1)}, "v2", "v2", [][]string{{"v2"}, {"dm"}}},
+ // The marker is the current version and expires with v1.
+ {"dme-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 1)}, "-", "-", [][]string{nil, nil}},
+ // v1 is noncurrent since v2 was written, v2 since v3 was.
+ {"ncv-split", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1), splitAt("v3", 2, 0)}, "v3", "v3", [][]string{{"v3"}, {"v2"}}},
+ })
+ })
+ t.Run("versioned/one-pool", func(t *testing.T) {
+ for pool := range 2 {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ s.run(splitILMOnePool(pool))
+ }
+ })
+ t.Run("versioned/head-version", func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ keys := []splitKey{
+ {"exp-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+ {"eav-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+ }
+ for _, k := range keys {
+ for _, v := range k.versions {
+ s.write(k.name, v)
+ }
+ act := s.head(k.name, "v1")
+ t.Logf("%s: HEAD of v1 evaluates %s", k.name, act)
+ if got := s.get(k.name); got != k.after {
+ t.Errorf("%s: reads %s after a HEAD of v1, want %s", k.name, got, k.after)
+ }
+ views := s.views()
+ for pool := range views {
+ if !slices.Equal(views[pool][k.name], k.want[pool]) {
+ t.Errorf("%s: pool %d holds %v, want %v", k.name, pool, views[pool][k.name], k.want[pool])
+ }
+ }
+ }
+ })
+ t.Run("suspended", func(t *testing.T) {
+ s := newSplitILM(t, versioning.Suspended, splitILMRules)
+ s.run([]splitKey{
+ {"exp-null-old-in-0", []splitVersion{splitNull("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}},
+ })
+ })
+ t.Run("unversioned", func(t *testing.T) {
+ s := newSplitILM(t, "", splitILMRules)
+ s.run([]splitKey{
+ {"eav-old-in-0", []splitVersion{splitNull("old", 5, 0), splitNull("new", 0.01, 1)}, "new", "new", nil},
+ {"exp-old-in-0", []splitVersion{splitNull("old", 5, 0), splitNull("new", 0.01, 1)}, "new", "new", nil},
+ })
+ views := s.views()
+ for object, want := range map[string]int{"eav-old-in-0": 1, "exp-old-in-0": 0} {
+ if len(views[0][object]) != want || len(views[1][object]) != 1 {
+ t.Errorf("%s: pools hold %v and %v, want %d and 1 copies", object, views[0][object], views[1][object], want)
+ }
+ }
+ })
+}
+
+// walk moves pool 0 out with a decommission or a rebalance pass, as move
+// does, but reports a decommission that did not complete instead of stopping
+// the test, so the reads are still checked.
+func (s *splitILM) walk(kind string) {
+ s.t.Helper()
+ ctx := s.t.Context()
+ if kind == "rebalance" {
+ migrationTestWalk(s.t, s.z, s.bucket, 0, kind)
+ s.z.rebalMu.Lock()
+ s.z.rebalMeta.PoolStats[0].Info.Status = rebalCompleted
+ s.z.rebalMu.Unlock()
+ return
+ }
+ if err := s.z.StartDecommission(ctx, 0); err != nil {
+ s.t.Fatal(err)
+ }
+ s.z.doDecommissionInRoutine(ctx, 0)
+ s.z.poolMetaMutex.RLock()
+ info := *s.z.poolMeta.Pools[0].Decommission
+ s.z.poolMetaMutex.RUnlock()
+ if !info.Complete || info.Failed || info.ItemsDecommissionFailed > 0 {
+ s.t.Errorf("decommission did not complete: complete=%t failed=%t itemsFailed=%d; pool 0 holds %v",
+ info.Complete, info.Failed, info.ItemsDecommissionFailed, s.view(0))
+ }
+}
+
+// Rules that expire a whole object (DelMarkerExpiration,
+// ExpiredObjectAllVersions) or a delete marker without noncurrent versions
+// (ExpiredObjectDeleteMarker) depend on every version of the object. A walk
+// sees the versions in its own pool only.
+func TestPoolsMigrationLifecycleWholeObject(t *testing.T) {
+ recent := 1.0 / 24
+ objects := []struct {
+ name string
+ versions []splitVersion // oldest first
+ before string // what a GET reads before the walk
+ after string // what a GET reads once ILM caught up
+ want []string // versions in pool 1 once ILM caught up, newest first
+ }{
+ // The marker expires and takes v1 with it.
+ {"dme-one-pool", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 0)}, "-", "-", nil},
+ {"dme-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", nil},
+ // v2 is the current version, the marker does not expire.
+ {"dme-newer-in-pool-1", []splitVersion{splitMarker("dm", 3, 0), splitAt("v2", recent, 1)}, "v2", "v2", []string{"v2", "dm"}},
+ // v2 expires and takes v1 with it.
+ {"eav-one-pool", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 0)}, "v2", "-", nil},
+ {"eav-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitAt("v2", 4, 0)}, "v2", "-", nil},
+ // v2 is the current version and does not expire.
+ {"eav-newer-in-pool-1", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", recent, 1)}, "v2", "v2", []string{"v2", "v1"}},
+ // The marker hides v1, it is not a marker without noncurrent versions.
+ {"edm-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", []string{"dm", "v1"}},
+ {"keep-marker", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 0)}, "-", "-", []string{"dm", "v1"}},
+ {"keep-versions", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 0)}, "v2", "v2", []string{"v2", "v1"}},
+ }
+ for _, kind := range []string{"decommission", "rebalance"} {
+ t.Run(kind, func(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ for _, o := range objects {
+ for _, v := range o.versions {
+ s.write(o.name, v)
+ }
+ if got := s.get(o.name); got != o.before {
+ t.Fatalf("%s: fixture reads %s, want %s", o.name, got, o.before)
+ }
+ }
+ s.walk(kind)
+ for _, o := range objects {
+ if got := s.get(o.name); got != o.before && got != o.after {
+ t.Errorf("%s: reads %s after the %s, before ILM caught up; want %s or %s", o.name, got, kind, o.before, o.after)
+ }
+ }
+ s.runQueue()
+ s.scan(1)
+ want := map[string][]string{}
+ for _, o := range objects {
+ want[o.name] = o.want
+ if got := s.get(o.name); got != o.after {
+ t.Errorf("%s: reads %s once ILM caught up, want %s", o.name, got, o.after)
+ }
+ }
+ s.check(want)
+ })
+ }
+}
diff --git a/cmd/erasure-server-pool-lock-domain_test.go b/cmd/erasure-server-pool-lock-domain_test.go
new file mode 100644
index 000000000..13249b213
--- /dev/null
+++ b/cmd/erasure-server-pool-lock-domain_test.go
@@ -0,0 +1,1185 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "crypto/md5"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "slices"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio/internal/amztime"
+ objectlock "github.com/minio/minio/internal/bucket/object/lock"
+ "github.com/minio/minio/internal/bucket/versioning"
+ "github.com/minio/minio/internal/crypto"
+ "github.com/minio/minio/internal/dsync"
+ "github.com/minio/minio/internal/hash"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// A single pool whose two erasure sets share the test drives, but lock as the
+// sets of a distributed pool do whose sets use different hosts. With one
+// shared local table the sets are one lock domain, as on a single node.
+const (
+ ldShared = "shared" // one local lock table for both sets
+ ldLocal = "local-split" // a local lock table per set
+ ldDsync = "dsync-disjoint" // in-process dsync, no locker in common
+ ldOverlap = "dsync-overlap" // in-process dsync, one of three lockers in common
+)
+
+var ldModes = []string{ldShared, ldLocal, ldDsync, ldOverlap}
+
+// ldWait is how long an operation must stay blocked behind a held commit.
+const ldWait = 400 * time.Millisecond
+
+type ldGateKey struct{}
+
+// ldGate holds the storage commit of the operation whose context carries it.
+type ldGate struct {
+ entered, release chan struct{}
+ once, open sync.Once
+}
+
+func newLDGate() *ldGate {
+ return &ldGate{entered: make(chan struct{}), release: make(chan struct{})}
+}
+
+func (g *ldGate) let() { g.open.Do(func() { close(g.release) }) }
+
+func ldHold(ctx context.Context) {
+ g, _ := ctx.Value(ldGateKey{}).(*ldGate)
+ if g == nil {
+ return
+ }
+ g.once.Do(func() { close(g.entered) })
+ select {
+ case <-g.release:
+ case <-ctx.Done():
+ }
+}
+
+// ldGateDisk holds every metadata commit of a gated operation.
+type ldGateDisk struct{ StorageAPI }
+
+func (d ldGateDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) {
+ ldHold(ctx)
+ return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
+}
+
+func (d ldGateDisk) RenamePart(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string, meta []byte, skipParent string) error {
+ ldHold(ctx)
+ return d.StorageAPI.RenamePart(ctx, srcVolume, srcPath, dstVolume, dstPath, meta, skipParent)
+}
+
+func (d ldGateDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error {
+ ldHold(ctx)
+ return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
+}
+
+func (d ldGateDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error {
+ ldHold(ctx)
+ return d.StorageAPI.DeleteVersions(ctx, volume, versions, opts)
+}
+
+func (d ldGateDisk) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error {
+ ldHold(ctx)
+ return d.StorageAPI.WriteMetadata(ctx, origvolume, volume, path, fi)
+}
+
+func (d ldGateDisk) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error {
+ ldHold(ctx)
+ return d.StorageAPI.UpdateMetadata(ctx, volume, path, fi, opts)
+}
+
+// ldSplit turns the single pool of z into two sets that share its drives and
+// lock as mode says. Their drives hold gated commits.
+func ldSplit(t *testing.T, z *erasureServerPools, mode string) {
+ t.Helper()
+ pool := z.serverPools[0]
+ sets, count := pool.sets, pool.setCount
+ t.Cleanup(func() { pool.sets, pool.setCount = sets, count })
+ first, second := *sets[0], *sets[0]
+ second.setIndex = 1
+ switch mode {
+ case ldShared:
+ case ldLocal:
+ first.nsMutex, second.nsMutex = newNSLock(false), newNSLock(false)
+ case ldDsync, ldOverlap:
+ l := make([]dsync.NetLocker, 8)
+ for i := range l {
+ l[i] = newLocker()
+ }
+ a, b := l[:4], l[4:]
+ if mode == ldOverlap {
+ // Each set can grant its quorum of two without the shared one.
+ a, b = l[:3], l[2:5]
+ }
+ first.nsMutex, second.nsMutex = newNSLock(true), newNSLock(true)
+ first.getLockers = func() ([]dsync.NetLocker, string) { return slices.Clone(a), "lock-domain-test" }
+ second.getLockers = func() ([]dsync.NetLocker, string) { return slices.Clone(b), "lock-domain-test" }
+ default:
+ t.Fatalf("unknown mode %q", mode)
+ }
+ disks := sets[0].getDisks
+ gated := func() []StorageAPI {
+ ds := disks()
+ for i, d := range ds {
+ if d != nil {
+ ds[i] = ldGateDisk{d}
+ }
+ }
+ return ds
+ }
+ first.getDisks, second.getDisks = gated, gated
+ pool.sets, pool.setCount = []*erasureObjects{&first, &second}, 2
+}
+
+// ldPool returns a split single pool and its unversioned test bucket.
+func ldPool(t *testing.T, mode string) (*erasureServerPools, string) {
+ t.Helper()
+ z, bucket := bulkDeletePools(t, 1)
+ ldSplit(t, z, mode)
+ return z, bucket
+}
+
+// ldKey returns a name with prefix that hashes to set.
+func ldKey(t *testing.T, z *erasureServerPools, prefix string, set int) string {
+ t.Helper()
+ for i := range 1000 {
+ key := fmt.Sprintf("%s-%d", prefix, i)
+ if z.serverPools[0].getHashedSetIndex(key) == set {
+ return key
+ }
+ }
+ t.Fatalf("no name with prefix %q hashes to set %d", prefix, set)
+ return ""
+}
+
+// ldOverlap runs hold until it is inside its storage commit, then runs other.
+// It reports whether other was still waiting after ldWait, then lets hold
+// commit and returns both results.
+func ldOverlapRun(t *testing.T, hold, other func(context.Context) error) (waited bool, holdErr, otherErr error) {
+ t.Helper()
+ g := newLDGate()
+ defer g.let()
+ ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute)
+ defer cancel()
+ held := make(chan error, 1)
+ go func() { held <- hold(context.WithValue(ctx, ldGateKey{}, g)) }()
+ select {
+ case <-g.entered:
+ case err := <-held:
+ t.Fatalf("held operation finished before its commit: %v", err)
+ case <-ctx.Done():
+ t.Fatal("held operation never reached its commit")
+ }
+ done := make(chan error, 1)
+ go func() { done <- other(ctx) }()
+ select {
+ case otherErr = <-done:
+ case <-time.After(ldWait):
+ waited = true
+ }
+ g.let()
+ holdErr = <-held
+ if waited {
+ otherErr = <-done
+ }
+ return waited, holdErr, otherErr
+}
+
+func ldReader(r io.Reader, size int64) *PutObjReader {
+ hr, err := hash.NewReader(context.Background(), r, size, "", "", size)
+ if err != nil {
+ panic(err)
+ }
+ return NewPutObjReader(hr)
+}
+
+func ldPut(ctx context.Context, z *erasureServerPools, bucket, key, body string) error {
+ _, err := z.PutObject(ctx, bucket, key, ldReader(strings.NewReader(body), int64(len(body))), ObjectOptions{})
+ return err
+}
+
+func ldMustPut(t *testing.T, z *erasureServerPools, bucket, key, body string) {
+ t.Helper()
+ if err := ldPut(t.Context(), z, bucket, key, body); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func ldBody(t *testing.T, z *erasureServerPools, bucket, key string) (string, ObjectInfo) {
+ t.Helper()
+ gr, err := z.GetObjectNInfo(t.Context(), bucket, key, nil, nil, ObjectOptions{})
+ if err != nil {
+ t.Fatalf("read %s: %v", key, err)
+ }
+ defer gr.Close()
+ body, err := io.ReadAll(gr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(body), gr.ObjInfo
+}
+
+func ldAbsent(t *testing.T, z *erasureServerPools, bucket, key string) {
+ t.Helper()
+ if _, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{}); !isErrObjectNotFound(err) {
+ t.Errorf("%s: %v, want not found", key, err)
+ }
+}
+
+// ldMetadataCopy is a metadata-only self copy of key from a snapshot read
+// without its lock, as the COPY handler read a copy onto itself.
+func ldMetadataCopy(ctx context.Context, z *erasureServerPools, bucket, key, meta string) error {
+ oi, err := z.GetObjectInfo(ctx, bucket, key, ObjectOptions{NoLock: true})
+ if err != nil {
+ return err
+ }
+ oi.metadataOnly = true
+ oi.UserDefined = cloneMSS(oi.UserDefined)
+ oi.UserDefined["x-amz-meta-copied"] = meta
+ _, err = z.CopyObject(ctx, bucket, key, bucket, key, oi, ObjectOptions{}, ObjectOptions{})
+ return err
+}
+
+// In one pool, a DELETE, bulk DELETE, COPY or tag update of a key must wait for
+// a write of the same key that is inside its commit, and the reverse, whatever
+// the hosts of the key's set are.
+func TestLockDomainSinglePoolWritersExclude(t *testing.T) {
+ type tc struct {
+ name string
+ setup func(t *testing.T, z *erasureServerPools, bucket, key string)
+ hold, other func(ctx context.Context, z *erasureServerPools, bucket, key string) error
+ check func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error)
+ }
+ seed := func(t *testing.T, z *erasureServerPools, bucket, key string) { ldMustPut(t, z, bucket, key, "seed") }
+ put := func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ return ldPut(ctx, z, bucket, key, "written")
+ }
+ noErrs := func(t *testing.T, holdErr, otherErr error) {
+ t.Helper()
+ if holdErr != nil || otherErr != nil {
+ t.Fatalf("held: %v, other: %v", holdErr, otherErr)
+ }
+ }
+ cases := []tc{{
+ name: "put/delete",
+ setup: seed,
+ hold: put,
+ other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ _, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{})
+ return err
+ },
+ check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) {
+ noErrs(t, holdErr, otherErr)
+ ldAbsent(t, z, bucket, key)
+ },
+ }, {
+ // The If-Match check and the delete are atomic with writers: the
+ // condition is checked against the PUT's version.
+ name: "put/delete-if-match",
+ setup: seed,
+ hold: put,
+ other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ seedETag := getMD5Hash([]byte("seed"))
+ _, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{CheckPrecondFn: func(oi ObjectInfo) bool { return oi.ETag != seedETag }})
+ return err
+ },
+ check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) {
+ if holdErr != nil || !isErrPreconditionFailed(otherErr) {
+ t.Fatalf("put: %v; If-Match delete of the replaced version: %v, want precondition failed", holdErr, otherErr)
+ }
+ if body, _ := ldBody(t, z, bucket, key); body != "written" {
+ t.Errorf("body %q, want the PUT's", body)
+ }
+ },
+ }, {
+ // ILM's delete-all deletes one key through prefix deletion.
+ name: "put/delete-all",
+ setup: seed,
+ hold: put,
+ other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ _, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{DeletePrefix: true, DeletePrefixObject: true})
+ return err
+ },
+ check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) {
+ noErrs(t, holdErr, otherErr)
+ ldAbsent(t, z, bucket, key)
+ },
+ }, {
+ name: "put/bulk-delete",
+ setup: seed,
+ hold: put,
+ other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ _, errs := z.DeleteObjects(ctx, bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key}}}, ObjectOptions{})
+ return errs[0]
+ },
+ check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) {
+ noErrs(t, holdErr, otherErr)
+ ldAbsent(t, z, bucket, key)
+ },
+ }, {
+ name: "put/copy-metadata",
+ setup: seed,
+ hold: put,
+ other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ return ldMetadataCopy(ctx, z, bucket, key, "1")
+ },
+ check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) {
+ noErrs(t, holdErr, otherErr)
+ if body, _ := ldBody(t, z, bucket, key); body != "written" {
+ t.Errorf("body %q after copy, want the PUT's", body)
+ }
+ },
+ }, {
+ name: "put/copy-rewrite",
+ setup: func(t *testing.T, z *erasureServerPools, bucket, key string) {
+ ldMustPut(t, z, bucket, key, "seed")
+ ldMustPut(t, z, bucket, key+"-src", "copied")
+ },
+ hold: put,
+ other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ gr, err := z.GetObjectNInfo(ctx, bucket, key+"-src", nil, nil, ObjectOptions{})
+ if err != nil {
+ return err
+ }
+ defer gr.Close()
+ src := gr.ObjInfo
+ src.PutObjReader = ldReader(gr, src.Size)
+ _, err = z.CopyObject(ctx, bucket, key+"-src", bucket, key, src, ObjectOptions{}, ObjectOptions{})
+ return err
+ },
+ check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) {
+ noErrs(t, holdErr, otherErr)
+ if body, _ := ldBody(t, z, bucket, key); body != "copied" {
+ t.Errorf("body %q, want the copy's, which ran after the PUT", body)
+ }
+ },
+ }, {
+ name: "delete/tags",
+ setup: seed,
+ hold: func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ _, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{})
+ return err
+ },
+ other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ _, err := z.PutObjectTags(ctx, bucket, key, "k=v", ObjectOptions{})
+ return err
+ },
+ check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) {
+ if holdErr != nil || !isErrObjectNotFound(otherErr) {
+ t.Fatalf("delete: %v; tags: %v, want not found", holdErr, otherErr)
+ }
+ ldAbsent(t, z, bucket, key)
+ },
+ }, {
+ name: "copy/tags",
+ setup: seed,
+ hold: func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ return ldMetadataCopy(ctx, z, bucket, key, "1")
+ },
+ other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error {
+ _, err := z.PutObjectTags(ctx, bucket, key, "k=v", ObjectOptions{})
+ return err
+ },
+ check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) {
+ noErrs(t, holdErr, otherErr)
+ oi, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
+ if err != nil || oi.UserTags != "k=v" || oi.UserDefined["x-amz-meta-copied"] != "1" {
+ t.Errorf("after copy then tags: tags %q, meta %q, %v", oi.UserTags, oi.UserDefined["x-amz-meta-copied"], err)
+ }
+ },
+ }}
+ for _, mode := range ldModes {
+ t.Run(mode, func(t *testing.T) {
+ z, bucket := ldPool(t, mode)
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ key := ldKey(t, z, strings.ReplaceAll(c.name, "/", "-"), 1)
+ c.setup(t, z, bucket, key)
+ waited, holdErr, otherErr := ldOverlapRun(t,
+ func(ctx context.Context) error { return c.hold(ctx, z, bucket, key) },
+ func(ctx context.Context) error { return c.other(ctx, z, bucket, key) })
+ if !waited {
+ holder, contender, _ := strings.Cut(c.name, "/")
+ t.Errorf("%s ran while %s held its commit (other: %v)", contender, holder, otherErr)
+ }
+ c.check(t, z, bucket, key, holdErr, otherErr)
+ })
+ }
+ })
+ }
+}
+
+// ldUpload starts an upload of key whose upload resource, and that of its
+// first part, hash to the set of key when sameSet is set.
+func ldUpload(t *testing.T, z *erasureServerPools, bucket, key string, sameSet bool) string {
+ t.Helper()
+ pool := z.serverPools[0]
+ for range 64 {
+ res, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ set := pool.getHashedSetIndex(key)
+ if !sameSet || (pool.getHashedSetIndex(pathJoin(key, res.UploadID)) == set &&
+ pool.getHashedSetIndex(pathJoin(key, res.UploadID, "1")) == set) {
+ return res.UploadID
+ }
+ if err := z.AbortMultipartUpload(t.Context(), bucket, key, res.UploadID, ObjectOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ t.Fatal("no upload ID with the wanted hashes")
+ return ""
+}
+
+// The upload-ID guards of complete and abort exclude a PutObjectPart of the
+// same upload. Get-info and list-parts read, as PutObjectPart's own guard
+// does, and do not wait for it.
+func TestLockDomainUploadGuards(t *testing.T) {
+ part := []byte("the only part")
+ partETag := getMD5Hash(part)
+ putPart := func(ctx context.Context, z *erasureServerPools, bucket, key, id string) error {
+ _, err := z.PutObjectPart(ctx, bucket, key, id, 1, ldReader(bytes.NewReader(part), int64(len(part))), ObjectOptions{})
+ return err
+ }
+ for _, mode := range ldModes {
+ t.Run(mode, func(t *testing.T) {
+ z, bucket := ldPool(t, mode)
+ for _, op := range []string{"complete", "abort", "info", "list"} {
+ t.Run(op, func(t *testing.T) {
+ key := ldKey(t, z, "upload-"+op, 1)
+ id := ldUpload(t, z, bucket, key, false)
+ waited, holdErr, otherErr := ldOverlapRun(t,
+ func(ctx context.Context) error { return putPart(ctx, z, bucket, key, id) },
+ func(ctx context.Context) error {
+ var err error
+ switch op {
+ case "complete":
+ _, err = z.CompleteMultipartUpload(ctx, bucket, key, id, []CompletePart{{PartNumber: 1, ETag: partETag}}, ObjectOptions{})
+ case "abort":
+ err = z.AbortMultipartUpload(ctx, bucket, key, id, ObjectOptions{})
+ case "info":
+ _, err = z.GetMultipartInfo(ctx, bucket, key, id, ObjectOptions{})
+ case "list":
+ _, err = z.ListObjectParts(ctx, bucket, key, id, 0, 10, ObjectOptions{})
+ }
+ return err
+ })
+ exclusive := op == "complete" || op == "abort"
+ if waited != exclusive {
+ t.Errorf("%s waited=%v for a part upload inside its commit, want %v (err %v)", op, waited, exclusive, otherErr)
+ }
+ if holdErr != nil || otherErr != nil {
+ t.Fatalf("part: %v, %s: %v", holdErr, op, otherErr)
+ }
+ switch op {
+ case "complete":
+ if body, _ := ldBody(t, z, bucket, key); body != string(part) {
+ t.Errorf("completed object %q", body)
+ }
+ case "abort":
+ if _, err := z.GetMultipartInfo(t.Context(), bucket, key, id, ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) {
+ t.Errorf("upload after abort: %v", err)
+ }
+ }
+ })
+ }
+ })
+ }
+}
+
+// Bulk delete groups its names by set and must still wait for a completion
+// of one of them, also when the batch names the completion's upload
+// resource itself; neither may hold the other.
+func TestLockDomainBulkDeleteAgainstUploads(t *testing.T) {
+ part := []byte("the only part")
+ partETag := getMD5Hash(part)
+ for _, mode := range ldModes {
+ t.Run(mode, func(t *testing.T) {
+ z, bucket := ldPool(t, mode)
+ for _, alias := range []bool{false, true} {
+ t.Run(fmt.Sprintf("complete/alias=%v", alias), func(t *testing.T) {
+ key := ldKey(t, z, fmt.Sprintf("bulk-complete-%v", alias), 1)
+ id := ldUpload(t, z, bucket, key, false)
+ if _, err := z.PutObjectPart(t.Context(), bucket, key, id, 1, mustGetPutObjReader(t, bytes.NewReader(part), int64(len(part)), "", ""), ObjectOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ names := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key}}}
+ if alias {
+ names = append(names, ObjectToDelete{ObjectV: ObjectV{ObjectName: pathJoin(key, id)}})
+ }
+ waited, holdErr, otherErr := ldOverlapRun(t,
+ func(ctx context.Context) error {
+ _, err := z.CompleteMultipartUpload(ctx, bucket, key, id, []CompletePart{{PartNumber: 1, ETag: partETag}}, ObjectOptions{})
+ return err
+ },
+ func(ctx context.Context) error {
+ _, errs := z.DeleteObjects(ctx, bucket, names, ObjectOptions{})
+ return errors.Join(errs...)
+ })
+ if !waited {
+ t.Errorf("bulk delete ran while the completion held its commit (%v)", otherErr)
+ }
+ if holdErr != nil || otherErr != nil {
+ t.Fatalf("complete: %v, bulk: %v", holdErr, otherErr)
+ }
+ ldAbsent(t, z, bucket, key)
+ })
+ }
+ // The names alias the locks of a part upload; they are other keys
+ // and need not wait, but neither side may block the other.
+ t.Run("put-part/alias", func(t *testing.T) {
+ key := ldKey(t, z, "bulk-part", 1)
+ id := ldUpload(t, z, bucket, key, true)
+ names := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: pathJoin(key, id)}}, {ObjectV: ObjectV{ObjectName: pathJoin(key, id, "1")}}}
+ _, holdErr, otherErr := ldOverlapRun(t,
+ func(ctx context.Context) error {
+ _, err := z.PutObjectPart(ctx, bucket, key, id, 1, ldReader(bytes.NewReader(part), int64(len(part))), ObjectOptions{})
+ return err
+ },
+ func(ctx context.Context) error {
+ _, errs := z.DeleteObjects(ctx, bucket, names, ObjectOptions{})
+ return errors.Join(errs...)
+ })
+ if holdErr != nil || otherErr != nil {
+ t.Fatalf("part: %v, bulk: %v", holdErr, otherErr)
+ }
+ if _, err := z.CompleteMultipartUpload(t.Context(), bucket, key, id, []CompletePart{{PartNumber: 1, ETag: partETag}}, ObjectOptions{}); err != nil {
+ t.Fatalf("upload after the bulk delete: %v", err)
+ }
+ })
+ })
+ }
+}
+
+// A group of a bulk delete whose lock is held by a writer of its set fails
+// alone; the other set's group commits. A canceled request fails the groups it
+// has not run, keeps the ones it has, and leaves no lock behind.
+func TestLockDomainBulkDeleteGroups(t *testing.T) {
+ for _, mode := range ldModes {
+ t.Run(mode, func(t *testing.T) {
+ z, bucket := ldPool(t, mode)
+ split := mode != ldShared
+ for _, cancelIt := range []bool{false, true} {
+ t.Run(fmt.Sprintf("cancel=%v", cancelIt), func(t *testing.T) {
+ k0 := ldKey(t, z, fmt.Sprintf("group-%v", cancelIt), 0)
+ k1 := ldKey(t, z, fmt.Sprintf("group-%v", cancelIt), 1)
+ ldMustPut(t, z, bucket, k0, "zero")
+ ldMustPut(t, z, bucket, k1, "one")
+ // A writer of k1 in the lock domain of its writes.
+ writer := z.serverPools[0].getHashedSet(k1).NewNSLock(bucket, k1)
+ wctx, err := writer.GetLock(t.Context(), globalOperationTimeout)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var once sync.Once
+ unlock := func() { once.Do(func() { writer.Unlock(wctx) }) }
+ defer unlock()
+
+ ctx, cancel := context.WithTimeout(t.Context(), 1500*time.Millisecond)
+ defer cancel()
+ names := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: k1}}, {ObjectV: ObjectV{ObjectName: k0}}}
+ type result struct {
+ objs []DeletedObject
+ errs []error
+ }
+ done := make(chan result, 1)
+ go func() {
+ objs, errs := z.DeleteObjects(ctx, bucket, names, ObjectOptions{})
+ done <- result{objs, errs}
+ }()
+ if cancelIt {
+ if split {
+ // Cancel once the first group has committed.
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ if _, err := z.GetObjectInfo(t.Context(), bucket, k0, ObjectOptions{}); isErrObjectNotFound(err) {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("set 0's group did not commit")
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ } else {
+ time.Sleep(200 * time.Millisecond)
+ }
+ cancel()
+ }
+ res := <-done
+ if res.objs[0].ObjectName != k1 || res.objs[1].ObjectName != k0 {
+ t.Fatalf("results out of order: %+v", res.objs)
+ }
+ if res.errs[0] == nil {
+ t.Errorf("k1 deleted while its writer held the lock")
+ }
+ if cancelIt && !errors.Is(res.errs[0], context.Canceled) {
+ t.Errorf("k1 after cancellation: %v", res.errs[0])
+ }
+ if split != (res.errs[1] == nil) {
+ t.Errorf("k0 (other set): %v, want committed=%v", res.errs[1], split)
+ }
+ unlock()
+ if body, _ := ldBody(t, z, bucket, k1); body != "one" {
+ t.Errorf("k1 = %q", body)
+ }
+ if split {
+ ldAbsent(t, z, bucket, k0)
+ }
+ // No lock is left behind.
+ lk := z.serverPools[0].getHashedSet(k1).NewNSLock(bucket, k1)
+ lctx, err := lk.GetLock(t.Context(), newDynamicTimeout(5*time.Second, 5*time.Second))
+ if err != nil {
+ t.Fatalf("lock left behind: %v", err)
+ }
+ lk.Unlock(lctx)
+ })
+ }
+ })
+ }
+}
+
+// ldBulkScenario runs one versioned bulk delete over names of both sets and
+// describes its results and the versions left, without generated version IDs.
+func ldBulkScenario(t *testing.T, mode string) []string {
+ t.Helper()
+ z, _ := ldPool(t, mode)
+ bucket := "lock-domain-versioned"
+ if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
+ t.Fatal(err)
+ }
+ vput := func(key, body string) ObjectInfo {
+ oi, err := z.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ return oi
+ }
+ a, b, c := ldKey(t, z, "map-a", 1), ldKey(t, z, "map-b", 0), ldKey(t, z, "map-c", 1)
+ dir := ldKey(t, z, "map-dir", 0) + "/"
+ va1, va2 := vput(a, "a1"), vput(a, "a2")
+ vb := vput(b, "b1")
+ vput(c, "c1")
+ if _, err := z.PutObject(t.Context(), bucket, dir, mustGetPutObjReader(t, strings.NewReader(""), 0, "", ""), ObjectOptions{Versioned: true}); err != nil {
+ t.Fatal(err)
+ }
+ absent := mustGetUUID()
+ invalid := strings.Repeat("x", 1100)
+ names := []ObjectToDelete{
+ {ObjectV: ObjectV{ObjectName: a}}, // marker
+ {ObjectV: ObjectV{ObjectName: b, VersionID: vb.VersionID}}, // purge
+ {ObjectV: ObjectV{ObjectName: invalid}}, // invalid
+ {ObjectV: ObjectV{ObjectName: a, VersionID: va1.VersionID}}, // purge, same key
+ {ObjectV: ObjectV{ObjectName: c}}, // marker
+ {ObjectV: ObjectV{ObjectName: dir}}, // directory marker
+ {ObjectV: ObjectV{ObjectName: "map-missing"}}, // absent
+ {ObjectV: ObjectV{ObjectName: a, VersionID: va1.VersionID}}, // duplicate
+ {ObjectV: ObjectV{ObjectName: b, VersionID: absent}}, // absent version
+ }
+ // Names hash with the deployment ID: compare them by role.
+ label := strings.NewReplacer(dir, "DIR/", a, "A", b, "B", c, "C", invalid, "INVALID",
+ va1.VersionID, "a1", va2.VersionID, "a2", vb.VersionID, "b1", absent, "absent")
+ objs, errs := z.DeleteObjects(t.Context(), bucket, names, ObjectOptions{Versioned: true})
+ var out []string
+ for i := range names {
+ o := objs[i]
+ vid := label.Replace(o.VersionID)
+ if len(vid) > 10 {
+ vid = "generated"
+ }
+ out = append(out, label.Replace(fmt.Sprintf("%d %s vid=%q marker=%v markerVid=%v found=%v err=%v",
+ i, o.ObjectName, vid, o.DeleteMarker, o.DeleteMarkerVersionID != "", o.found, errs[i])))
+ }
+ for _, key := range []string{a, b, c, dir} {
+ info, err := z.ListObjectVersions(t.Context(), bucket, key, "", "", "", 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var vs []string
+ for _, v := range info.Objects {
+ if v.Name != key {
+ continue
+ }
+ id := "other"
+ switch v.VersionID {
+ case va2.VersionID:
+ id = "a2"
+ case va1.VersionID:
+ id = "a1"
+ case vb.VersionID:
+ id = "b1"
+ }
+ vs = append(vs, fmt.Sprintf("%s:marker=%v:latest=%v", id, v.DeleteMarker, v.IsLatest))
+ }
+ out = append(out, label.Replace(key)+" "+strings.Join(vs, ","))
+ }
+ return out
+}
+
+// Grouping by set keeps every result where its name was and gives the same
+// results and the same versions as the one bulk lock of a single node.
+func TestLockDomainBulkDeleteMapping(t *testing.T) {
+ want := ldBulkScenario(t, ldShared)
+ for _, mode := range ldModes[1:] {
+ t.Run(mode, func(t *testing.T) {
+ got := ldBulkScenario(t, mode)
+ if !slices.Equal(got, want) {
+ t.Errorf("results differ from the single bulk lock:\n got %q\nwant %q", got, want)
+ }
+ })
+ }
+}
+
+// A restore takes the object lock of its set only to commit, not for its
+// download. DELETE and COPY of the object wait for that commit, also where
+// the sets of the pool lock with lockers of their own, and the restore still
+// completes.
+func TestLockDomainRestoreExcludesDeleteAndCopy(t *testing.T) {
+ for _, mode := range ldModes {
+ for _, op := range []string{"delete", "copy"} {
+ t.Run(mode+"/"+op, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ ldSplit(t, r.z, mode)
+ r.object = ldKey(t, r.z, "restore-"+op, 1)
+ oi, body := nativeRestoreSeed(t, r, 0, false, false)
+ waited, restoreErr, opErr := ldOverlapRun(t, func(ctx context.Context) error {
+ return r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ }, func(ctx context.Context) error {
+ switch op {
+ case "delete":
+ _, err := r.z.DeleteObject(ctx, r.bucket, r.object, r.opts(""))
+ return err
+ default:
+ ropts := r.opts(oi.VersionID)
+ ropts.NoLock = true
+ cur, err := r.z.GetObjectInfo(ctx, r.bucket, r.object, ropts)
+ if err != nil {
+ return err
+ }
+ cur.metadataOnly = true
+ cur.UserDefined = cloneMSS(cur.UserDefined)
+ cur.UserDefined["x-amz-meta-copied"] = "1"
+ vopts := r.opts(oi.VersionID)
+ _, err = r.z.CopyObject(ctx, r.bucket, r.object, r.bucket, r.object, cur, vopts, vopts)
+ return err
+ }
+ })
+ if !waited {
+ t.Errorf("%s ran during the restore's commit (%v)", op, opErr)
+ }
+ if restoreErr != nil {
+ t.Fatalf("restore: %v", restoreErr)
+ }
+ if opErr != nil {
+ t.Fatalf("%s after the restore: %v", op, opErr)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ })
+ }
+ }
+}
+
+// ldAPI is a split single pool served by the S3 API router.
+func ldAPI(t *testing.T, mode string) (*erasureServerPools, string, http.Handler) {
+ t.Helper()
+ z, bucket := ldPool(t, mode)
+ if err := newTestConfig(globalMinioDefaultRegion, z); err != nil {
+ t.Fatal(err)
+ }
+ return z, bucket, initTestAPIEndPoints(z, nil)
+}
+
+func ldRequest(ctx context.Context, t *testing.T, router http.Handler, method, path, body string, headers map[string]string) *httptest.ResponseRecorder {
+ t.Helper()
+ req, err := newTestSignedRequestV4(method, path, int64(len(body)), strings.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, headers)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rec := httptest.NewRecorder()
+ router.ServeHTTP(rec, req.WithContext(ctx))
+ return rec
+}
+
+func ldCopyRequest(ctx context.Context, t *testing.T, router http.Handler, bucket, src, dst string, headers map[string]string) *httptest.ResponseRecorder {
+ t.Helper()
+ h := map[string]string{xhttp.AmzCopySource: "/" + bucket + "/" + src}
+ for k, v := range headers {
+ h[k] = v
+ }
+ return ldRequest(ctx, t, router, http.MethodPut, "/"+bucket+"/"+dst, "", h)
+}
+
+// The COPY handler locks its destination, and a copy onto itself reads its
+// source, in the lock domain of the object's writes. A copy onto itself must
+// read the version that the PUT before it committed.
+func TestLockDomainCopyHandler(t *testing.T) {
+ for _, mode := range ldModes {
+ t.Run(mode, func(t *testing.T) {
+ z, bucket, router := ldAPI(t, mode)
+ for _, self := range []bool{false, true} {
+ t.Run(fmt.Sprintf("self=%v", self), func(t *testing.T) {
+ key := ldKey(t, z, fmt.Sprintf("copy-handler-%v", self), 1)
+ src := key + "-src"
+ ldMustPut(t, z, bucket, key, "seed")
+ ldMustPut(t, z, bucket, src, "source")
+ var rec *httptest.ResponseRecorder
+ waited, holdErr, _ := ldOverlapRun(t,
+ func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written") },
+ func(ctx context.Context) error {
+ if self {
+ rec = ldCopyRequest(ctx, t, router, bucket, key, key, map[string]string{
+ xhttp.AmzMetadataDirective: "REPLACE", "x-amz-meta-copied": "1",
+ })
+ } else {
+ rec = ldCopyRequest(ctx, t, router, bucket, src, key, nil)
+ }
+ return nil
+ })
+ if !waited {
+ t.Errorf("COPY ran while a PUT of its destination held its commit")
+ }
+ if holdErr != nil || rec.Code != http.StatusOK {
+ t.Fatalf("put: %v; copy: %d %s", holdErr, rec.Code, rec.Body.String())
+ }
+ body, oi := ldBody(t, z, bucket, key)
+ want := "source"
+ if self {
+ want = "written"
+ if oi.UserDefined["X-Amz-Meta-Copied"] != "1" {
+ t.Errorf("copied metadata missing: %v", oi.UserDefined)
+ }
+ }
+ if body != want || oi.ETag != getMD5Hash([]byte(want)) {
+ t.Errorf("after the copy: body %q etag %q, want %q with its own ETag", body, oi.ETag, want)
+ }
+ })
+ }
+ })
+ }
+}
+
+// Copies in opposite directions always make progress: between keys of
+// different sets, and where both locks share one domain, as on a single node
+// or with several pools.
+func TestLockDomainCopyHandlerReciprocal(t *testing.T) {
+ for _, mode := range append(slices.Clone(ldModes), "pools") {
+ t.Run(mode, func(t *testing.T) {
+ var z *erasureServerPools
+ var bucket string
+ var router http.Handler
+ keys := [2]string{"reciprocal-a", "reciprocal-b"}
+ if mode == "pools" {
+ z, bucket = bulkDeletePools(t, 2)
+ if err := newTestConfig(globalMinioDefaultRegion, z); err != nil {
+ t.Fatal(err)
+ }
+ router = initTestAPIEndPoints(z, nil)
+ } else {
+ z, bucket, router = ldAPI(t, mode)
+ keys = [2]string{ldKey(t, z, "reciprocal", 0), ldKey(t, z, "reciprocal", 1)}
+ }
+ // Not inlined: the source is read while the destination is written.
+ body := strings.Repeat("reciprocal copy source ", 1<<14)
+ for _, k := range keys {
+ ldMustPut(t, z, bucket, k, body)
+ }
+ ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
+ defer cancel()
+ for round := range 8 {
+ var wg sync.WaitGroup
+ codes := make([]int, 2)
+ start := make(chan struct{})
+ for i := range keys {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ <-start
+ codes[i] = ldCopyRequest(ctx, t, router, bucket, keys[i], keys[1-i], nil).Code
+ }()
+ }
+ close(start)
+ wg.Wait()
+ if codes[0] != http.StatusOK || codes[1] != http.StatusOK {
+ t.Fatalf("round %d: reciprocal copies returned %v", round, codes)
+ }
+ }
+ })
+ }
+}
+
+// The COPY handler looks up its destination while it holds the destination's
+// lock: Object Lock's retention check and the tier sweeper's lookup must not
+// take that lock again.
+func TestLockDomainCopyHandlerDestinationLookups(t *testing.T) {
+ prevTier := globalTierConfigMgr
+ t.Cleanup(func() { globalTierConfigMgr = prevTier })
+ for _, mode := range ldModes {
+ t.Run(mode, func(t *testing.T) {
+ z, _, router := ldAPI(t, mode)
+ globalTierConfigMgr = NewTierConfigMgr()
+ globalTierConfigMgr.Tiers["WARM"] = madmin.TierConfig{Version: madmin.TierConfigVer, Type: madmin.S3, Name: "WARM", S3: &madmin.TierS3{Bucket: "warm"}}
+ bucket := "lock-domain-worm"
+ if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{LockEnabled: true, VersioningEnabled: true}); err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+ defer cancel()
+ src := ldKey(t, z, "lookup-src", 0)
+ if _, err := z.PutObject(t.Context(), bucket, src, mustGetPutObjReader(t, strings.NewReader(""), 0, "", ""), ObjectOptions{Versioned: true}); err != nil {
+ t.Fatal(err)
+ }
+ for _, dst := range []string{ldKey(t, z, "lookup-dst", 1), ldKey(t, z, "lookup-dir", 1) + "/"} {
+ if rec := ldCopyRequest(ctx, t, router, bucket, src, dst, nil); rec.Code != http.StatusOK {
+ t.Fatalf("copy to %s: %d %s", dst, rec.Code, rec.Body.String())
+ }
+ // Self copy, metadata only.
+ if rec := ldCopyRequest(ctx, t, router, bucket, dst, dst, map[string]string{
+ xhttp.AmzMetadataDirective: "REPLACE", "x-amz-meta-copied": "1",
+ }); rec.Code != http.StatusOK {
+ t.Fatalf("self copy of %s: %d %s", dst, rec.Code, rec.Body.String())
+ }
+ }
+ })
+ }
+}
+
+// SelectObjectContent reads under the lock of the object's writes.
+func TestLockDomainSelectWaitsForWriter(t *testing.T) {
+ const request = `SELECT * FROM S3ObjectSQL` +
+ `NONE` +
+ ``
+ for _, mode := range ldModes {
+ t.Run(mode, func(t *testing.T) {
+ z, bucket, router := ldAPI(t, mode)
+ key := ldKey(t, z, "select", 1)
+ ldMustPut(t, z, bucket, key, "seed,1\n")
+ var rec *httptest.ResponseRecorder
+ waited, holdErr, _ := ldOverlapRun(t,
+ func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written,2\n") },
+ func(ctx context.Context) error {
+ rec = ldRequest(ctx, t, router, http.MethodPost, "/"+bucket+"/"+key+"?select&select-type=2", request, nil)
+ return nil
+ })
+ if !waited {
+ t.Errorf("SELECT ran while a PUT held its commit")
+ }
+ if holdErr != nil || rec.Code != http.StatusOK || !bytes.Contains(rec.Body.Bytes(), []byte("written,2")) {
+ t.Fatalf("put: %v; select: %d %q", holdErr, rec.Code, rec.Body.String())
+ }
+ })
+ }
+}
+
+// Batch key rotation locks the object in the lock domain of its writes.
+func TestLockDomainKeyRotationWaitsForWriter(t *testing.T) {
+ for _, mode := range ldModes {
+ t.Run(mode, func(t *testing.T) {
+ z, bucket := ldPool(t, mode)
+ key := ldKey(t, z, "rotate", 1)
+ ldMustPut(t, z, bucket, key, "seed")
+ job := &BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}}
+ // The rotation's listing found an SSE-S3 object; it reads it again
+ // under the lock.
+ listed := ObjectInfo{Bucket: bucket, Name: key, UserDefined: map[string]string{crypto.MetaSealedKeyS3: "listed"}}
+ waited, _, _ := ldOverlapRun(t,
+ func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written") },
+ func(ctx context.Context) error { return job.KeyRotate(ctx, z, listed) })
+ if !waited {
+ t.Errorf("key rotation ran while a PUT held its commit")
+ }
+ })
+ }
+}
+
+// ldCopyGate holds a COPY of one destination inside the object layer's
+// CopyObject, after the handler's Object Lock check, and reports when a
+// metadata update of that destination reaches the object layer.
+type ldCopyGate struct {
+ ObjectLayer
+ bucket, object string
+ copying, updating chan struct{}
+ release chan struct{}
+ copyOnce, updatingOnce sync.Once
+}
+
+func (g *ldCopyGate) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, info ObjectInfo, srcOpts, dstOpts ObjectOptions) (ObjectInfo, error) {
+ if dstBucket == g.bucket && dstObject == g.object {
+ g.copyOnce.Do(func() { close(g.copying) })
+ select {
+ case <-g.release:
+ case <-ctx.Done():
+ return ObjectInfo{}, ctx.Err()
+ }
+ }
+ return g.ObjectLayer.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, info, srcOpts, dstOpts)
+}
+
+func (g *ldCopyGate) PutObjectMetadata(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+ if bucket == g.bucket && object == g.object {
+ g.updatingOnce.Do(func() { close(g.updating) })
+ }
+ return g.ObjectLayer.PutObjectMetadata(ctx, bucket, object, opts)
+}
+
+// A retention or legal-hold update of a COPY destination cannot land between
+// the handler's Object Lock check and the copy: it waits for the copy, which
+// then succeeds, and is applied after it.
+func TestLockDomainCopyExcludesObjectLockUpdates(t *testing.T) {
+ for _, mode := range append(slices.Clone(ldModes), "pools") {
+ t.Run(mode, func(t *testing.T) {
+ var z *erasureServerPools
+ if mode == "pools" {
+ z, _ = bulkDeletePools(t, 2)
+ } else {
+ z, _ = ldPool(t, mode)
+ }
+ if err := newTestConfig(globalMinioDefaultRegion, z); err != nil {
+ t.Fatal(err)
+ }
+ bucket := "lock-domain-worm-copy"
+ if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{LockEnabled: true, VersioningEnabled: true}); err != nil {
+ t.Fatal(err)
+ }
+ plain := initTestAPIEndPoints(z, nil)
+ put := func(key string) {
+ t.Helper()
+ if rec := ldRequest(t.Context(), t, plain, http.MethodPut, "/"+bucket+"/"+key, "", nil); rec.Code != http.StatusOK {
+ t.Fatalf("PUT %s: %d %s", key, rec.Code, rec.Body.String())
+ }
+ }
+ put("copy-source/")
+ for _, self := range []bool{false, true} {
+ for _, state := range []string{"GOVERNANCE", "COMPLIANCE", "LEGAL-HOLD"} {
+ t.Run(fmt.Sprintf("self=%v/%s", self, state), func(t *testing.T) {
+ // Directory objects: Object Lock checks their null version.
+ key := fmt.Sprintf("copy-destination-%v-%s/", self, state)
+ if mode != "pools" {
+ for z.serverPools[0].getHashedSetIndex(encodeDirObject(key)) != 1 {
+ key = "x" + key
+ }
+ }
+ put(key)
+ g := &ldCopyGate{
+ ObjectLayer: z, bucket: bucket, object: key,
+ copying: make(chan struct{}), updating: make(chan struct{}), release: make(chan struct{}),
+ }
+ setObjectLayer(g)
+ defer setObjectLayer(z)
+ router := initTestAPIEndPoints(g, nil)
+ ctx, cancel := context.WithTimeout(t.Context(), time.Minute)
+ release := sync.OnceFunc(func() { close(g.release) })
+ var wg sync.WaitGroup
+ defer func() { release(); cancel(); wg.Wait() }()
+
+ src := "copy-source/"
+ if self {
+ src = key
+ }
+ copied, updated := make(chan *httptest.ResponseRecorder, 1), make(chan *httptest.ResponseRecorder, 1)
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ copied <- ldCopyRequest(ctx, t, router, bucket, src, key, map[string]string{
+ xhttp.AmzMetadataDirective: "REPLACE", "x-amz-meta-copied": "1",
+ })
+ }()
+ select {
+ case <-g.copying:
+ case rec := <-copied:
+ t.Fatalf("COPY returned before its write: %d %s", rec.Code, rec.Body.String())
+ case <-ctx.Done():
+ t.Fatal("COPY never reached its write")
+ }
+
+ query := url.Values{"versionId": {nullVersionID}}
+ body := `ON`
+ if state == "LEGAL-HOLD" {
+ query.Set("legal-hold", "")
+ } else {
+ query.Set("retention", "")
+ body = fmt.Sprintf(`%s%s`,
+ state, amztime.ISO8601Format(UTCNow().Add(365*24*time.Hour)))
+ }
+ sum := md5.Sum([]byte(body))
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ updated <- ldRequest(ctx, t, router, http.MethodPut, makeTestTargetURL("", bucket, key, query), body,
+ map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])})
+ }()
+ select {
+ case <-g.updating:
+ case rec := <-updated:
+ t.Fatalf("update returned before reaching the object layer: %d %s", rec.Code, rec.Body.String())
+ case <-ctx.Done():
+ t.Fatal("update never reached the object layer")
+ }
+ select {
+ case rec := <-updated:
+ t.Errorf("%s update ran between the COPY's Object Lock check and its write: %d", state, rec.Code)
+ updated <- rec
+ case <-time.After(ldWait):
+ }
+ release()
+ for name, result := range map[string]chan *httptest.ResponseRecorder{"COPY": copied, "update": updated} {
+ select {
+ case rec := <-result:
+ if rec.Code != http.StatusOK {
+ t.Errorf("%s: %d %s", name, rec.Code, rec.Body.String())
+ }
+ case <-ctx.Done():
+ t.Fatalf("%s did not finish", name)
+ }
+ }
+ after, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after.UserDefined["X-Amz-Meta-Copied"] != "1" {
+ t.Errorf("copied metadata missing: %v", after.UserDefined)
+ }
+ if state == "LEGAL-HOLD" {
+ if objectlock.GetObjectLegalHoldMeta(after.UserDefined).Status != objectlock.LegalHoldOn {
+ t.Error("legal hold applied before the copy was lost")
+ }
+ } else if string(objectlock.GetObjectRetentionMeta(after.UserDefined).Mode) != state {
+ t.Error("retention applied before the copy was lost")
+ }
+ })
+ }
+ }
+ })
+ }
+}
diff --git a/cmd/erasure-server-pool-locks.go b/cmd/erasure-server-pool-locks.go
new file mode 100644
index 000000000..b577bf566
--- /dev/null
+++ b/cmd/erasure-server-pool-locks.go
@@ -0,0 +1,322 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "math/rand/v2"
+ "slices"
+ "strings"
+ "time"
+
+ "github.com/minio/minio-go/v7/pkg/set"
+)
+
+// Object lock domains.
+//
+// Every erasure set has its own lockers, one per host among its drives, and a
+// distributed lock is granted by a majority of them. Two locks of one name
+// exclude each other only when they ask the same lockers. In a single pool the
+// writers (PutObject, multipart commit, PutObjectPart, tags, retention, legal
+// hold, metadata, restore, transition, heal) lock in the set an object hashes
+// to. The pools layer locks an object, and its upload IDs, in the same set, so
+// that a pool whose sets use different hosts still has one lock per object.
+// With several pools every object write takes the pools lock instead, which
+// these helpers then return unchanged.
+//
+// Generic NewNSLock keeps one stable domain for names that are not objects:
+// bucket and IAM metadata locks, format and leader coordination, and the
+// /[replicate]/ serializer.
+
+// NewObjectNSLock returns the lock of one object in the lock domain that its
+// writes use: in a single pool the object's own erasure set, otherwise the
+// pools lock. Directory objects are named as storage names them.
+func (z *erasureServerPools) NewObjectNSLock(bucket, object string) RWLocker {
+ if z.SinglePool() {
+ object = encodeDirObject(object)
+ }
+ return z.objectNSLock(bucket, object, object)
+}
+
+// uploadIDNSLock returns the guard of one upload ID. It is chosen by the
+// upload's object, as PutObjectPart locks it in that object's set; the name
+// locked stays object/uploadID.
+func (z *erasureServerPools) uploadIDNSLock(bucket, object, uploadID string) RWLocker {
+ return z.objectNSLock(bucket, object, pathJoin(object, uploadID))
+}
+
+// objectNSLock returns the lock of resource, a name owned by object, in the
+// lock domain of object.
+func (z *erasureServerPools) objectNSLock(bucket, object, resource string) RWLocker {
+ if z.SinglePool() {
+ return z.serverPools[0].getHashedSet(encodeDirObject(object)).NewNSLock(bucket, resource)
+ }
+ return z.NewNSLock(bucket, resource)
+}
+
+// objectLockRequest is one object lock that lockObjects takes.
+type objectLockRequest struct {
+ bucket, object string
+ write bool
+}
+
+// lockObjects takes the locks of several objects, such as the source and
+// destination of a copy, before any of them is read. A name that appears twice
+// gets one lock, a write lock if either asks for one. The locks are taken in
+// one order, by lock domain and then by bucket and object, and transactionally
+// (see getNSLocks), so that two copies in opposite directions, or an operation
+// that orders its locks differently, cannot hold each other forever. The
+// returned context ends when any of the locks is lost; the returned function
+// releases them all.
+func (z *erasureServerPools) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) {
+ type entry struct {
+ objectLockRequest
+ domain int
+ }
+ locks := make([]entry, 0, len(reqs))
+ for _, r := range reqs {
+ r.object = encodeDirObject(r.object)
+ i := slices.IndexFunc(locks, func(e entry) bool { return e.bucket == r.bucket && e.object == r.object })
+ if i >= 0 {
+ locks[i].write = locks[i].write || r.write
+ continue
+ }
+ domain := 0
+ if z.SinglePool() {
+ domain = z.serverPools[0].getHashedSetIndex(r.object)
+ }
+ locks = append(locks, entry{objectLockRequest: r, domain: domain})
+ }
+ slices.SortFunc(locks, func(a, b entry) int {
+ if a.domain != b.domain {
+ return a.domain - b.domain
+ }
+ return strings.Compare(pathJoin(a.bucket, a.object), pathJoin(b.bucket, b.object))
+ })
+ nsLocks := make([]nsLockRequest, len(locks))
+ for i, l := range locks {
+ nsLocks[i] = nsLockRequest{
+ newLock: func() RWLocker { return z.NewObjectNSLock(l.bucket, l.object) },
+ write: l.write,
+ atomic: true,
+ }
+ }
+ return getNSLocks(ctx, nsLocks, globalOperationTimeout)
+}
+
+// nsLockRequest is one lock of a transactional acquisition.
+type nsLockRequest struct {
+ // newLock returns a new instance of the lock for each attempt. An
+ // instance names its holder, and a distributed unlock retries in the
+ // background; a later attempt with the same instance could be released
+ // by such a retry of the attempt before.
+ newLock func() RWLocker
+ write bool
+ // atomic: the lock is granted as a whole or not at all. A distributed
+ // lock is, also over several names. The local backend takes several
+ // names one at a time, holding the first while it waits for the next;
+ // its lock context is the context it was given.
+ atomic bool
+}
+
+const (
+ // nsLockTryTimeout bounds each wait for a lock while holding another.
+ nsLockTryTimeout = 2 * time.Second
+ // nsLockRetryWait is the longest random pause before a new attempt.
+ nsLockRetryWait = 250 * time.Millisecond
+)
+
+// getNSLocks acquires every lock of locks, in their order, or none. While it
+// holds a lock it waits for the next one only up to nsLockTryTimeout; then it
+// releases what it holds in reverse order, pauses, and tries again with a
+// fresh context, until the timeout of all attempts. A wait that holds nothing
+// can last for the rest of the timeout. Each lock is taken in the context of
+// the one before, so the returned context ends when any lease is lost.
+func getNSLocks(ctx context.Context, locks []nsLockRequest, timeout *dynamicTimeout) (LockContext, func(), error) {
+ type held struct {
+ lock RWLocker
+ write bool
+ lkctx LockContext
+ }
+ release := func(hs []held) {
+ for i := len(hs) - 1; i >= 0; i-- {
+ if hs[i].write {
+ hs[i].lock.Unlock(hs[i].lkctx)
+ } else {
+ hs[i].lock.RUnlock(hs[i].lkctx)
+ }
+ }
+ }
+ // Fail as a single lock does: a canceled request with its own error,
+ // anything else as a timeout.
+ failed := func() error {
+ if err := ctx.Err(); errors.Is(err, context.Canceled) {
+ return err
+ }
+ return OperationTimedOut{}
+ }
+ start := time.Now()
+ deadline := start.Add(timeout.Timeout())
+ for {
+ hs := make([]held, 0, len(locks))
+ lctx := ctx
+ var err error
+ for _, l := range locks {
+ wait := time.Until(deadline)
+ if len(hs) > 0 || !l.atomic {
+ wait = min(wait, nsLockTryTimeout)
+ }
+ if wait <= 0 {
+ err = OperationTimedOut{}
+ break
+ }
+ lk := l.newLock()
+ try := newDynamicTimeout(wait, wait)
+ // The local backend waits up to the timeout for each name: bound
+ // the whole attempt by its context instead.
+ tctx, cancel := lctx, context.CancelFunc(func() {})
+ if !l.atomic {
+ tctx, cancel = context.WithTimeout(lctx, wait)
+ }
+ var lkctx LockContext
+ if l.write {
+ lkctx, err = lk.GetLock(tctx, try)
+ } else {
+ lkctx, err = lk.GetRLock(tctx, try)
+ }
+ cancel()
+ if err != nil {
+ break
+ }
+ if !l.atomic {
+ lkctx.ctx = lctx
+ }
+ hs = append(hs, held{lock: lk, write: l.write, lkctx: lkctx})
+ lctx = lkctx.Context()
+ }
+ if err == nil {
+ timeout.LogSuccess(time.Since(start))
+ if len(hs) == 0 {
+ return LockContext{ctx: ctx, cancel: func() {}}, func() {}, nil
+ }
+ return LockContext{ctx: lctx, cancel: func() {}}, func() { release(hs) }, nil
+ }
+ release(hs)
+ // A lost lease cancels the context of a later wait; only the
+ // request's own context ends the attempts.
+ if ctx.Err() != nil || (!errors.As(err, &OperationTimedOut{}) && !errors.Is(err, context.Canceled)) {
+ return LockContext{}, nil, err
+ }
+ pause := rand.N(nsLockRetryWait) + time.Millisecond
+ if time.Until(deadline) <= pause {
+ timeout.LogFailure()
+ return LockContext{}, nil, OperationTimedOut{}
+ }
+ t := time.NewTimer(pause)
+ select {
+ case <-ctx.Done():
+ t.Stop()
+ return LockContext{}, nil, failed()
+ case <-t.C:
+ }
+ }
+}
+
+// sharesLocalNSLock reports whether every set of the pool uses one local lock
+// table, as a single node does: there all sets are one lock domain.
+func (s *erasureSets) sharesLocalNSLock() bool {
+ first := s.sets[0].nsMutex
+ if first.isDistErasure {
+ return false
+ }
+ for _, set := range s.sets[1:] {
+ if set.nsMutex != first {
+ return false
+ }
+ }
+ return true
+}
+
+// deleteObjectsInSets is the bulk delete of a single pool whose sets do not
+// share one lock table. The names are grouped by the set they hash to, the
+// lock domain of their writers. The groups run one after another in ascending
+// set index; each takes its set's lock over all of its names in one
+// acquisition, deletes its sub-batch, and releases the lock before the next
+// group starts. No two sets' locks are held at once.
+//
+// Each group's lock is taken in the request's context. A group whose lock
+// cannot be taken fails as a whole and the next group still runs; once the
+// request is canceled the remaining groups fail without being attempted. The
+// results keep their positions in objects. Replication, events and auditing
+// stay with the callers, on the merged results.
+func (z *erasureServerPools) deleteObjectsInSets(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions, dobjects []DeletedObject, derrs []error, timeout *dynamicTimeout) {
+ pool := z.serverPools[0]
+ groups := make(map[int][]int)
+ for i, obj := range objects {
+ idx := pool.getHashedSetIndex(obj.ObjectName)
+ groups[idx] = append(groups[idx], i)
+ }
+ order := make([]int, 0, len(groups))
+ for idx := range groups {
+ order = append(order, idx)
+ }
+ slices.Sort(order)
+
+ for _, idx := range order {
+ members := groups[idx]
+ fail := func(err error) {
+ for _, i := range members {
+ derrs[i] = err
+ }
+ }
+ if err := ctx.Err(); err != nil {
+ fail(err)
+ continue
+ }
+ er := pool.sets[idx]
+ names := set.NewStringSet()
+ batch := make([]ObjectToDelete, len(members))
+ for j, i := range members {
+ names.Add(objects[i].ObjectName)
+ batch[j] = objects[i]
+ }
+ resources := names.ToSlice()
+ lkctx, unlock, err := getNSLocks(ctx, []nsLockRequest{{
+ newLock: func() RWLocker { return er.NewNSLock(bucket, resources...) },
+ write: true,
+ atomic: er.nsMutex.isDistErasure || len(resources) == 1,
+ }}, timeout)
+ if err != nil {
+ fail(err)
+ continue
+ }
+ deleted, errs := er.DeleteObjects(lkctx.Context(), bucket, batch, opts)
+ unlock()
+ // Merge as the pools loop of DeleteObjects does for one pool.
+ for j, i := range members {
+ if errs[j] == nil && deleted[j].found {
+ dobjects[i], derrs[i] = deleted[j], nil
+ continue
+ }
+ if derrs[i] == nil {
+ dobjects[i], derrs[i] = deleted[j], errs[j]
+ }
+ }
+ }
+}
diff --git a/cmd/erasure-server-pool-locks_test.go b/cmd/erasure-server-pool-locks_test.go
new file mode 100644
index 000000000..696c22771
--- /dev/null
+++ b/cmd/erasure-server-pool-locks_test.go
@@ -0,0 +1,555 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio/internal/dsync"
+)
+
+// ldTry reports whether lk can be taken now, and releases it again.
+func ldTry(t *testing.T, lk RWLocker, write bool) bool {
+ t.Helper()
+ dt := newDynamicTimeout(100*time.Millisecond, 100*time.Millisecond)
+ if write {
+ lc, err := lk.GetLock(t.Context(), dt)
+ if err == nil {
+ lk.Unlock(lc)
+ }
+ return err == nil
+ }
+ lc, err := lk.GetRLock(t.Context(), dt)
+ if err == nil {
+ lk.RUnlock(lc)
+ }
+ return err == nil
+}
+
+// ldFree reports whether lk can be taken within a few seconds: a distributed
+// unlock completes in the background.
+func ldFree(t *testing.T, lk RWLocker, write bool) bool {
+ t.Helper()
+ deadline := time.Now().Add(5 * time.Second)
+ for !ldTry(t, lk, write) {
+ if time.Now().After(deadline) {
+ return false
+ }
+ }
+ return true
+}
+
+// ldTaken waits until someone else holds lk.
+func ldTaken(t *testing.T, lk RWLocker) {
+ t.Helper()
+ deadline := time.Now().Add(5 * time.Second)
+ for ldTry(t, lk, false) {
+ if time.Now().After(deadline) {
+ t.Fatal("lock never taken")
+ }
+ time.Sleep(time.Millisecond)
+ }
+}
+
+func ldHoldLock(t *testing.T, lk RWLocker) func() {
+ t.Helper()
+ lc, err := lk.GetLock(t.Context(), globalOperationTimeout)
+ if err != nil {
+ t.Fatal(err)
+ }
+ done := false
+ return func() {
+ if !done {
+ done = true
+ lk.Unlock(lc)
+ }
+ }
+}
+
+// Object locks, and upload guards by their object, are in the set the
+// object's writes lock in; with several pools they stay the pools lock.
+func TestObjectNSLockRouting(t *testing.T) {
+ for _, mode := range ldModes {
+ t.Run(mode, func(t *testing.T) {
+ z, bucket := ldPool(t, mode)
+ pool := z.serverPools[0]
+ key := ldKey(t, z, "route", 1)
+ dir := ldKey(t, z, "route-dir", 0) + "/"
+ for _, name := range []string{key, dir} {
+ release := ldHoldLock(t, z.NewObjectNSLock(bucket, name))
+ encoded := encodeDirObject(name)
+ if ldTry(t, pool.getHashedSet(encoded).NewNSLock(bucket, encoded), false) {
+ t.Errorf("%s: a writer of the object's set ignores the object lock", name)
+ }
+ release()
+ }
+ id := mustGetUUID()
+ release := ldHoldLock(t, z.uploadIDNSLock(bucket, key, id))
+ if ldTry(t, pool.getHashedSet(key).NewNSLock(bucket, pathJoin(key, id)), false) {
+ t.Error("PutObjectPart's guard ignores the upload guard")
+ }
+ release()
+ if mode != ldShared && pool.sharesLocalNSLock() {
+ t.Error("split sets reported as one local table")
+ }
+ if mode == ldShared && !pool.sharesLocalNSLock() {
+ t.Error("one local table not recognized")
+ }
+ })
+ }
+ t.Run("pools", func(t *testing.T) {
+ z, bucket := bulkDeletePools(t, 2)
+ release := ldHoldLock(t, z.NewNSLock(bucket, "pooled"))
+ defer release()
+ if ldTry(t, z.NewObjectNSLock(bucket, "pooled"), false) || ldTry(t, z.uploadIDNSLock(bucket, "pooled", ""), false) {
+ t.Error("multi-pool object locks left the pools lock")
+ }
+ })
+}
+
+// lockObjects merges a name asked twice into one write lock and takes source
+// and destination in their own sets.
+func TestLockObjects(t *testing.T) {
+ for _, mode := range ldModes {
+ t.Run(mode, func(t *testing.T) {
+ z, bucket := ldPool(t, mode)
+ pool := z.serverPools[0]
+ src, dst := ldKey(t, z, "lock-src", 1), ldKey(t, z, "lock-dst", 0)
+ lkctx, unlock, err := z.lockObjects(t.Context(), objectLockRequest{bucket: bucket, object: src}, objectLockRequest{bucket: bucket, object: dst, write: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if lkctx.Context().Err() != nil {
+ t.Fatal("lock context ended")
+ }
+ if !ldTry(t, pool.getHashedSet(src).NewNSLock(bucket, src), false) {
+ t.Error("source not shared with readers")
+ }
+ if ldTry(t, pool.getHashedSet(src).NewNSLock(bucket, src), true) {
+ t.Error("source not locked against writers")
+ }
+ if ldTry(t, pool.getHashedSet(dst).NewNSLock(bucket, dst), false) {
+ t.Error("destination not locked")
+ }
+ unlock()
+ if !ldFree(t, pool.getHashedSet(dst).NewNSLock(bucket, dst), true) || !ldFree(t, pool.getHashedSet(src).NewNSLock(bucket, src), true) {
+ t.Error("locks left behind")
+ }
+
+ // A copy onto itself takes one write lock; a directory object is
+ // locked by its stored name.
+ self := ldKey(t, z, "lock-self", 1) + "/"
+ _, unlock, err = z.lockObjects(t.Context(), objectLockRequest{bucket: bucket, object: self}, objectLockRequest{bucket: bucket, object: self, write: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ encoded := encodeDirObject(self)
+ if ldTry(t, pool.getHashedSet(encoded).NewNSLock(bucket, encoded), false) {
+ t.Error("self copy not write locked")
+ }
+ unlock()
+ })
+ }
+}
+
+// ldLeaseLock is a lock whose lease the test ends.
+type ldLeaseLock struct {
+ RWLocker
+ lost chan context.CancelFunc
+}
+
+func (l ldLeaseLock) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
+ lc, err := l.RWLocker.GetLock(ctx, timeout)
+ if err != nil {
+ return lc, err
+ }
+ lease, cancel := context.WithCancel(lc.Context())
+ select {
+ case l.lost <- cancel:
+ default:
+ panic("too many lock attempts")
+ }
+ return LockContext{ctx: lease, cancel: cancel}, nil
+}
+
+// getNSLocks never holds one lock while it waits for another for long: it
+// releases what it holds and retries, also after a lost lease, until its
+// timeout or the request ends, and leaves nothing behind.
+func TestGetNSLocksTransactional(t *testing.T) {
+ ns := newNSLock(false)
+ lock := func(name string) RWLocker { return ns.NewNSLock(nil, "bucket", name) }
+ newLock := func(name string) func() RWLocker { return func() RWLocker { return lock(name) } }
+ reqs := func() []nsLockRequest {
+ return []nsLockRequest{{newLock: newLock("a"), write: true, atomic: true}, {newLock: newLock("b"), write: true, atomic: true}}
+ }
+
+ t.Run("partial", func(t *testing.T) {
+ releaseB := ldHoldLock(t, lock("b"))
+ defer releaseB()
+ type result struct {
+ unlock func()
+ err error
+ }
+ done := make(chan result, 1)
+ go func() {
+ _, unlock, err := getNSLocks(t.Context(), reqs(), globalOperationTimeout)
+ done <- result{unlock, err}
+ }()
+ // "a" is taken while "b" is waited for, and free again between
+ // attempts.
+ ldTaken(t, lock("a"))
+ deadline := time.Now().Add(3 * nsLockTryTimeout)
+ for !ldTry(t, lock("a"), true) {
+ if time.Now().After(deadline) {
+ t.Fatal("a stays held while b is waited for")
+ }
+ }
+ releaseB()
+ res := <-done
+ if res.err != nil {
+ t.Fatal(res.err)
+ }
+ if ldTry(t, lock("a"), false) || ldTry(t, lock("b"), false) {
+ t.Error("not both locks held")
+ }
+ res.unlock()
+ if !ldTry(t, lock("a"), true) || !ldTry(t, lock("b"), true) {
+ t.Error("locks left behind")
+ }
+ })
+
+ t.Run("timeout", func(t *testing.T) {
+ releaseB := ldHoldLock(t, lock("b"))
+ defer releaseB()
+ start := time.Now()
+ _, _, err := getNSLocks(t.Context(), reqs(), newDynamicTimeout(3*time.Second, 3*time.Second))
+ if !errors.As(err, &OperationTimedOut{}) {
+ t.Fatalf("got %v, want a timeout", err)
+ }
+ if took := time.Since(start); took > 6*time.Second {
+ t.Errorf("timed out after %v", took)
+ }
+ if !ldTry(t, lock("a"), true) {
+ t.Error("a left behind")
+ }
+ })
+
+ t.Run("cancel", func(t *testing.T) {
+ releaseB := ldHoldLock(t, lock("b"))
+ defer releaseB()
+ ctx, cancel := context.WithCancel(t.Context())
+ time.AfterFunc(300*time.Millisecond, cancel)
+ _, _, err := getNSLocks(ctx, reqs(), globalOperationTimeout)
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("got %v, want canceled", err)
+ }
+ if !ldTry(t, lock("a"), true) {
+ t.Error("a left behind")
+ }
+ })
+
+ t.Run("lease", func(t *testing.T) {
+ lost := make(chan context.CancelFunc, 64)
+ leased := []nsLockRequest{{newLock: func() RWLocker { return ldLeaseLock{lock("a"), lost} }, write: true, atomic: true}, {newLock: newLock("b"), write: true, atomic: true}}
+ // Losing the first lease while the second is waited for retries
+ // with a fresh context.
+ releaseB := ldHoldLock(t, lock("b"))
+ type result struct {
+ lkctx LockContext
+ unlock func()
+ err error
+ }
+ done := make(chan result, 1)
+ go func() {
+ lkctx, unlock, err := getNSLocks(t.Context(), leased, globalOperationTimeout)
+ done <- result{lkctx, unlock, err}
+ }()
+ (<-lost)()
+ releaseB()
+ res := <-done
+ if res.err != nil {
+ t.Fatal(res.err)
+ }
+ if res.lkctx.Context().Err() != nil {
+ t.Fatal("returned context ended before any lease was lost")
+ }
+ // Losing a held lease ends the returned context.
+ var cancel context.CancelFunc
+ for len(lost) > 0 {
+ cancel = <-lost
+ }
+ cancel()
+ select {
+ case <-res.lkctx.Context().Done():
+ case <-time.After(time.Second):
+ t.Error("lost lease did not end the locks' context")
+ }
+ res.unlock()
+ if !ldTry(t, lock("a"), true) || !ldTry(t, lock("b"), true) {
+ t.Error("locks left behind")
+ }
+ })
+}
+
+// A group whose lock times out fails alone and the later groups still run.
+func TestDeleteObjectsInSetsLaterGroupsRun(t *testing.T) {
+ for _, mode := range ldModes[1:] {
+ t.Run(mode, func(t *testing.T) {
+ z, bucket := ldPool(t, mode)
+ k0, k1 := ldKey(t, z, "later", 0), ldKey(t, z, "later", 1)
+ ldMustPut(t, z, bucket, k0, "zero")
+ ldMustPut(t, z, bucket, k1, "one")
+ release := ldHoldLock(t, z.serverPools[0].getHashedSet(k0).NewNSLock(bucket, k0))
+ defer release()
+ objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: k0}}, {ObjectV: ObjectV{ObjectName: k1}}}
+ dobjects := []DeletedObject{{ObjectName: k0}, {ObjectName: k1}}
+ derrs := make([]error, 2)
+ z.deleteObjectsInSets(t.Context(), bucket, objects, ObjectOptions{}, dobjects, derrs, newDynamicTimeout(time.Second, time.Second))
+ if !errors.As(derrs[0], &OperationTimedOut{}) || derrs[1] != nil {
+ t.Fatalf("results %v, want a timeout for k0 only", derrs)
+ }
+ release()
+ if body, _ := ldBody(t, z, bucket, k0); body != "zero" {
+ t.Errorf("k0 = %q", body)
+ }
+ ldAbsent(t, z, bucket, k1)
+ })
+ }
+}
+
+// The local lock takes the names of a group one by one. A group that waits
+// for a name must not keep the names it holds: a part upload holds its part
+// lock and then reads its upload ID, which the batch names too.
+func TestDeleteObjectsInSetsLocalGroupReleases(t *testing.T) {
+ z, bucket := ldPool(t, ldLocal)
+ pool := z.serverPools[0]
+ var upload, part string
+ for i := range 1000 {
+ upload = ldKey(t, z, "local-upload", 1) + fmt.Sprintf("/u%d", i)
+ part = upload + "/1"
+ if pool.getHashedSetIndex(upload) == 1 && pool.getHashedSetIndex(part) == 1 {
+ break
+ }
+ }
+ er := pool.getHashedSet(upload)
+ releasePart := ldHoldLock(t, er.NewNSLock(bucket, part))
+ defer releasePart()
+ done := make(chan []error, 1)
+ go func() {
+ _, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: upload}}, {ObjectV: ObjectV{ObjectName: part}}}, ObjectOptions{})
+ done <- errs
+ }()
+ // The batch takes the upload name, waits for the part, and lets the
+ // part upload's read of its upload ID through.
+ ldTaken(t, er.NewNSLock(bucket, upload))
+ read := er.NewNSLock(bucket, upload)
+ rctx, err := read.GetRLock(t.Context(), newDynamicTimeout(4*nsLockTryTimeout, 4*nsLockTryTimeout))
+ if err != nil {
+ t.Fatalf("batch kept a name while waiting: %v", err)
+ }
+ read.RUnlock(rctx)
+ releasePart()
+ select {
+ case errs := <-done:
+ if errors.Join(errs...) != nil {
+ t.Fatal(errs)
+ }
+ case <-time.After(30 * time.Second):
+ t.Fatal("bulk delete did not finish")
+ }
+}
+
+// ldLostUnlockReply applies its first unlock but loses the reply until gate
+// opens, so dsync retries that unlock in the background.
+type ldLostUnlockReply struct {
+ dsync.NetLocker
+ gate <-chan struct{}
+ applied, retried chan<- struct{}
+ calls atomic.Int32
+}
+
+func (l *ldLostUnlockReply) Unlock(ctx context.Context, args dsync.LockArgs) (bool, error) {
+ n := l.calls.Add(1)
+ ok, err := l.NetLocker.Unlock(ctx, args)
+ switch n {
+ case 1:
+ l.applied <- struct{}{}
+ <-l.gate
+ return false, errors.New("unlock reply lost")
+ case 2:
+ l.retried <- struct{}{}
+ }
+ return ok, err
+}
+
+// ldFailFirst fails the first acquisition of any of its instances.
+type ldFailFirst struct {
+ RWLocker
+ calls *atomic.Int32
+}
+
+func (l ldFailFirst) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
+ if l.calls.Add(1) == 1 {
+ return LockContext{}, OperationTimedOut{}
+ }
+ return l.RWLocker.GetLock(ctx, timeout)
+}
+
+// A distributed unlock of a failed attempt that is retried late must not
+// release the lock the next attempt took.
+func TestGetNSLocksFreshLockPerAttempt(t *testing.T) {
+ gate := make(chan struct{})
+ applied, retried := make(chan struct{}, 3), make(chan struct{}, 3)
+ lockers := make([]dsync.NetLocker, 3)
+ for i := range lockers {
+ lockers[i] = &ldLostUnlockReply{NetLocker: newLocker(), gate: gate, applied: applied, retried: retried}
+ }
+ ns := newNSLock(true)
+ dst := func() RWLocker {
+ return ns.NewNSLock(func() ([]dsync.NetLocker, string) { return lockers, "test" }, "bucket", "destination")
+ }
+ local := newNSLock(false)
+ var calls atomic.Int32
+ src := func() RWLocker { return ldFailFirst{local.NewNSLock(nil, "bucket", "source"), &calls} }
+ ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second)
+ defer cancel()
+ opened := false
+ open := func() {
+ if !opened {
+ opened = true
+ close(gate)
+ }
+ }
+ defer open()
+ _, unlock, err := getNSLocks(ctx, []nsLockRequest{{newLock: dst, write: true, atomic: true}, {newLock: src, write: true, atomic: true}}, newDynamicTimeout(10*time.Second, 10*time.Second))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer unlock()
+ for range lockers {
+ select {
+ case <-applied:
+ case <-ctx.Done():
+ t.Fatal("the first attempt's unlock never reached the lockers")
+ }
+ }
+ open()
+ for range lockers {
+ select {
+ case <-retried:
+ case <-ctx.Done():
+ t.Fatal("the lost unlock was not retried")
+ }
+ }
+ competitor := dst()
+ lc, err := competitor.GetLock(ctx, newDynamicTimeout(300*time.Millisecond, 300*time.Millisecond))
+ if err == nil {
+ competitor.Unlock(lc)
+ t.Fatal("a late unlock retry released the held lock")
+ }
+}
+
+// One attempt at a local lock over several names is bounded as a whole, and so
+// is the acquisition.
+func TestGetNSLocksLocalAttemptBound(t *testing.T) {
+ ns := newNSLock(false)
+ a, b := ns.NewNSLock(nil, "bucket", "a"), ns.NewNSLock(nil, "bucket", "b")
+ releaseA, releaseB := ldHoldLock(t, a), ldHoldLock(t, b)
+ defer releaseA()
+ defer releaseB()
+ time.AfterFunc(900*time.Millisecond, releaseA)
+ time.AfterFunc(1800*time.Millisecond, releaseB)
+ start := time.Now()
+ _, unlock, err := getNSLocks(t.Context(), []nsLockRequest{{
+ newLock: func() RWLocker { return ns.NewNSLock(nil, "bucket", "a", "b") }, write: true,
+ }}, newDynamicTimeout(1500*time.Millisecond, 1500*time.Millisecond))
+ if err == nil {
+ unlock()
+ t.Fatalf("acquired after %v with a deadline of 1.5s", time.Since(start))
+ }
+ if !errors.As(err, &OperationTimedOut{}) {
+ t.Fatalf("got %v, want a timeout", err)
+ }
+}
+
+// ldLeaseLayer gives the COPY handler locks whose lease the test ends, and
+// holds its destination lookup until the lookup's context ends.
+type ldLeaseLayer struct {
+ ObjectLayer
+ destination string
+ lost chan context.CancelFunc
+ lookup chan context.Context
+}
+
+func (l *ldLeaseLayer) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) {
+ lc, unlock, err := l.ObjectLayer.lockObjects(ctx, reqs...)
+ if err != nil {
+ return lc, unlock, err
+ }
+ lease, cancel := context.WithCancel(lc.Context())
+ l.lost <- cancel
+ return LockContext{ctx: lease, cancel: cancel}, func() { cancel(); unlock() }, nil
+}
+
+func (l *ldLeaseLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+ if object == l.destination && opts.NoLock {
+ l.lookup <- ctx
+ <-ctx.Done()
+ return ObjectInfo{}, ctx.Err()
+ }
+ return l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+}
+
+// The COPY handler's work under its locks ends when a lease is lost, the
+// destination lookups included.
+func TestLockDomainCopyLookupEndsWithLease(t *testing.T) {
+ z, bucket, _ := ldAPI(t, ldShared)
+ ldMustPut(t, z, bucket, "source", "")
+ prevTier := globalTierConfigMgr
+ t.Cleanup(func() { globalTierConfigMgr = prevTier })
+ globalTierConfigMgr = NewTierConfigMgr()
+ globalTierConfigMgr.Tiers["WARM"] = madmin.TierConfig{Version: madmin.TierConfigVer, Type: madmin.S3, Name: "WARM", S3: &madmin.TierS3{Bucket: "warm"}}
+ layer := &ldLeaseLayer{ObjectLayer: z, destination: "destination", lost: make(chan context.CancelFunc, 1), lookup: make(chan context.Context, 1)}
+ router := initTestAPIEndPoints(layer, []string{"CopyObject"})
+ ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+ defer cancel()
+ code := make(chan int, 1)
+ go func() { code <- ldCopyRequest(ctx, t, router, bucket, "source", "destination", nil).Code }()
+ var lookup context.Context
+ select {
+ case lookup = <-layer.lookup:
+ case <-ctx.Done():
+ t.Fatal("COPY did not look up its destination")
+ }
+ (<-layer.lost)()
+ select {
+ case <-lookup.Done():
+ case <-time.After(5 * time.Second):
+ t.Fatal("the destination lookup outlived the lost lease")
+ }
+ if c := <-code; c == http.StatusOK {
+ t.Error("COPY succeeded after losing its lock")
+ }
+}
diff --git a/cmd/erasure-server-pool-rebalance.go b/cmd/erasure-server-pool-rebalance.go
index 8626dd965..da1dd6a8e 100644
--- a/cmd/erasure-server-pool-rebalance.go
+++ b/cmd/erasure-server-pool-rebalance.go
@@ -605,6 +605,10 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
// a noncurrent version and has to be moved like any other. The
// scanner applies the expiry once it lives in another pool.
return lifecycle.NoneAction
+ case evt.Action.Delete() && !z.migrationExpiryApplies(ctx, objInfo, evt):
+ // Another pool holds versions with which the rule does not
+ // apply. The version moves like any other.
+ return lifecycle.NoneAction
case evt.Action.Delete():
return evt.Action
}
diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go
index d1e83bb71..0811edebc 100644
--- a/cmd/erasure-server-pool.go
+++ b/cmd/erasure-server-pool.go
@@ -1239,8 +1239,26 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
object = encodeDirObject(object)
}
- // Serialize logical deletion with pooled writes and metadata updates.
- lk := z.NewNSLock(bucket, object)
+ // Serialize logical deletion with the writes and metadata updates of the
+ // object: take the lock they take (see NewObjectNSLock). With one pool
+ // that is the object's own erasure set, which is where PutObject,
+ // multipart commit and the metadata updates lock, so a pool whose sets
+ // use different hosts still excludes them. With several pools it is the
+ // pools lock, as before. Exact-key deletes that use prefix deletion
+ // (DeletePrefixObject) are deletes of one object. A true prefix deletion
+ // keeps the pools lock on the prefix name, which never covered the
+ // objects below it.
+ //
+ // Nodes that still run a release before this routing lock single-pool
+ // deletes and copies in the first set. In a pool whose sets use different
+ // hosts they do not exclude the deletes and copies of updated nodes, nor
+ // their writes, so such a pool must not run mixed versions.
+ var lk RWLocker
+ if opts.DeletePrefix && !opts.DeletePrefixObject {
+ lk = z.NewNSLock(bucket, object)
+ } else {
+ lk = z.NewObjectNSLock(bucket, object)
+ }
lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
if err != nil {
return ObjectInfo{}, err
@@ -1248,6 +1266,19 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
ctx = lkctx.Context()
defer lk.Unlock(lkctx)
+ // An ILM expiry is decided on the versions in one pool. Under the lock,
+ // check it with the versions in all pools; the sets then take it as
+ // decided, without a recheck by the versions each one holds. They only
+ // keep a version whose protection they read differently.
+ if z.expiryNeedsAllPools(opts) {
+ var approved objectLockState
+ if approved, err = z.checkExpiryAllPools(ctx, bucket, object, opts); err != nil {
+ return ObjectInfo{Name: decodeDirObject(object), VersionID: opts.VersionID}, err
+ }
+ opts.Expiration.CheckedAllPools = true
+ opts.Expiration.ApprovedProtection = approved
+ }
+
if opts.DeletePrefix {
return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
}
@@ -1309,10 +1340,10 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
// Evaluate the conditional (If-Match) precondition while the write lock
// acquired above is held, before the delete-marker short-circuit and before
- // any version is removed, so the object cannot change between the check and
- // the delete. This is scoped to a single erasure set (see the note at the
- // lock above): only there do the delete lock and the write path share the
- // same lock namespace, making the check-then-delete atomic.
+ // any version is removed. The object's writers take the same lock (see the
+ // note at the lock above), so the object cannot change between the check
+ // and the delete. Multi-pool conditional deletes return above, through
+ // deleteObjectReconciled, under the same pools lock.
if opts.CheckPrecondFn != nil {
// pinfo.ObjInfo is the current latest version. getPoolInfoExistingWithOpts
// intentionally strips VersionID, so for a version-scoped delete read the
@@ -1496,6 +1527,18 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
objSets.Add(objects[i].ObjectName)
}
+ // A single pool locks each name in the set its writers lock it in, one
+ // set at a time. A single node keeps one lock table for all sets, so it
+ // keeps the one bulk lock below: its local lock takes the names one by
+ // one and holds the earlier names while it waits.
+ if z.SinglePool() && !z.serverPools[0].sharesLocalNSLock() {
+ z.deleteObjectsInSets(ctx, bucket, objects, opts, dobjects, derrs, globalOperationTimeout)
+ for i := range dobjects {
+ dobjects[i].ObjectName = decodeDirObject(dobjects[i].ObjectName)
+ }
+ return dobjects, derrs
+ }
+
// Acquire a bulk write lock across 'objects'
multiDeleteLock := z.NewNSLock(bucket, objSets.ToSlice()...)
lkctx, err := multiDeleteLock.GetLock(ctx, globalOperationTimeout)
@@ -1517,10 +1560,31 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
batch = make([]ObjectToDelete, 0, len(objects))
batchIdx = make([]int, 0, len(objects))
data, dataIdx := make([]ObjectToDelete, 0, len(objects)), make([]int, 0, len(objects))
+ approved := make([]objectLockState, 0, len(objects))
+ ilm := make(map[string]lifecycleAllPools)
for i, obj := range objects {
if derrs[i] != nil {
continue
}
+ // A version that ILM expires is checked with the versions of
+ // its object in all pools, see checkExpiryAllPools. When the
+ // expiry no longer applies, nothing is deleted and nothing is
+ // left to replicate.
+ var protection objectLockState
+ vopts := ObjectOptions{Expiration: opts.Expiration, VersionID: obj.VersionID}
+ if obj.VersionID != "" && z.expiryNeedsAllPools(vopts) {
+ all, ok := ilm[obj.ObjectName]
+ if !ok {
+ all.versions, all.events, all.err = z.lifecycleEventsAllPools(ctx, bucket, obj.ObjectName)
+ ilm[obj.ObjectName] = all
+ }
+ if err := all.expiryApplies(bucket, obj.ObjectName, vopts); err != nil {
+ dobjects[i] = DeletedObject{ObjectName: obj.ObjectName, VersionID: obj.VersionID}
+ derrs[i] = err
+ continue
+ }
+ protection = all.protection(obj.VersionID)
+ }
if deletesData(obj, opts) {
if z.copyWithoutDeleteQuorum(ctx, bucket, obj.ObjectName, obj.VersionID) {
derrs[i] = InsufficientWriteQuorum{}
@@ -1528,13 +1592,14 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
}
data = append(data, obj)
dataIdx = append(dataIdx, i)
+ approved = append(approved, protection)
continue
}
batch = append(batch, obj)
batchIdx = append(batchIdx, i)
}
if len(data) > 0 {
- deleted, errs := z.deleteObjectsReconciled(ctx, bucket, data, opts)
+ deleted, errs := z.deleteObjectsReconciled(ctx, bucket, data, approved, opts)
for j, i := range dataIdx {
dobjects[i], derrs[i] = deleted[j], errs[j]
}
@@ -1598,8 +1663,10 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
+ // A caller that also locks the source takes both with lockObjects, before
+ // it reads the source, and passes NoLock.
if !dstOpts.NoLock {
- ns := z.NewNSLock(dstBucket, dstObject)
+ ns := z.NewObjectNSLock(dstBucket, dstObject)
lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
if err != nil {
return ObjectInfo{}, err
@@ -1625,7 +1692,21 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
return oi, err
}
- poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
+ var poolIdx int
+ if dstOpts.MetadataChg && cpSrcDstSame && srcInfo.metadataOnly && srcOpts.VersionID == dstOpts.VersionID {
+ // Restore status updates address an existing version. Keep ordinary
+ // copy/write allocation unchanged, including its latest-version lookup.
+ poolIdx, err = z.getPoolIdxExistingWithOpts(ctx, dstBucket, dstObject, ObjectOptions{
+ VersionID: dstOpts.VersionID,
+ MetadataChg: true,
+ NoLock: true,
+ Transition: dstOpts.Transition,
+ SkipDecommissioned: true,
+ SkipRebalancing: true,
+ })
+ } else {
+ poolIdx, err = z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
+ }
if err != nil {
return objInfo, err
}
@@ -2182,7 +2263,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
return MultipartInfo{}, err
}
- uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+ uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID)
lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
if err != nil {
return MultipartInfo{}, err
@@ -2223,7 +2304,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
return ListPartsInfo{}, err
}
- uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
+ uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID)
lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout)
if err != nil {
return ListPartsInfo{}, err
@@ -2274,7 +2355,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
}
}(ctx)
- lk := z.NewNSLock(bucket, pathJoin(object, uploadID))
+ lk := z.uploadIDNSLock(bucket, object, uploadID)
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
if err != nil {
return err
@@ -2312,9 +2393,12 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
}
}()
+ // Several pools take the object lock first, then the upload lock below.
+ // A single pool takes the upload lock here and the set takes the object
+ // lock, in the object's own erasure set, only right before it commits.
if !z.SinglePool() {
if !opts.NoLock {
- lk := z.NewNSLock(bucket, encodeDirObject(object))
+ lk := z.NewObjectNSLock(bucket, encodeDirObject(object))
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
if err != nil {
return objInfo, err
@@ -2375,13 +2459,18 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
// Hold write locks to verify uploaded parts, also disallows any
// parallel PutObjectPart() requests.
- uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID))
- wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
- if err != nil {
- return objInfo, err
+ // An empty upload ID would name the object lock itself, which this
+ // completion holds or takes later: let the storage layer return its
+ // normal missing bucket/upload error instead.
+ if uploadID != "" {
+ uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID)
+ wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout)
+ if err != nil {
+ return objInfo, err
+ }
+ ctx = wlkctx.Context()
+ defer uploadIDLock.Unlock(wlkctx)
}
- ctx = wlkctx.Context()
- defer uploadIDLock.Unlock(wlkctx)
if z.SinglePool() {
return z.serverPools[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
@@ -3309,39 +3398,25 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec
return z.serverPools[idx].TransitionObject(ctx, bucket, object, opts)
}
-// RestoreTransitionedObject - restore transitioned object content locally on this cluster.
-func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+func (z *erasureServerPools) CheckAbandonedParts(ctx context.Context, bucket, object string, opts madmin.HealOpts) error {
object = encodeDirObject(object)
if z.SinglePool() {
- return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts)
+ return z.serverPools[0].CheckAbandonedParts(ctx, bucket, object, opts)
}
-
- if !opts.NoLock {
- // Acquire write lock before restoring transitioned object
- lk := z.NewNSLock(bucket, object)
- lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
+ // Several pools: remove abandoned data under the pools lock, which every
+ // write of the object takes, as healObjectInPool does. The lock of one
+ // pool's set would not exclude a write that has moved its data directory
+ // in place but not yet published the metadata that references it, such
+ // as a native restore's commit.
+ if opts.Remove && !opts.DryRun && !opts.NoLock {
+ lk := z.NewObjectNSLock(bucket, object)
+ lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
if err != nil {
return err
}
ctx = lkctx.Context()
defer lk.Unlock(lkctx)
- }
-
- // Avoid restoring object from a pool being decommissioned.
- opts.SkipDecommissioned = true
- opts.NoLock = true
- idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
- if err != nil {
- return err
- }
-
- return z.serverPools[idx].RestoreTransitionedObject(ctx, bucket, object, opts)
-}
-
-func (z *erasureServerPools) CheckAbandonedParts(ctx context.Context, bucket, object string, opts madmin.HealOpts) error {
- object = encodeDirObject(object)
- if z.SinglePool() {
- return z.serverPools[0].CheckAbandonedParts(ctx, bucket, object, opts)
+ opts.NoLock = true
}
errs := make([]error, len(z.serverPools))
var wg sync.WaitGroup
diff --git a/cmd/globals.go b/cmd/globals.go
index e71294508..607a2ecf5 100644
--- a/cmd/globals.go
+++ b/cmd/globals.go
@@ -319,6 +319,8 @@ var (
globalOperationTimeout = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops
globalDeleteOperationTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute) // default time for delete ops
+ // Queued restores retry this wait; keep their failures out of delete timeouts.
+ globalRestoreLockTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute)
// Listing recovery has its own failure history and a five-second caller budget.
globalListingOperationTimeout = newDynamicTimeout(5*time.Second, 5*time.Second)
diff --git a/cmd/metacache-server-pool.go b/cmd/metacache-server-pool.go
index 2889e9a96..276cef0fb 100644
--- a/cmd/metacache-server-pool.go
+++ b/cmd/metacache-server-pool.go
@@ -265,9 +265,12 @@ func (z *erasureServerPools) listPathShortcutEOF(ctx context.Context, bucket str
}
// listingRLock covers the lock domains used by normal same-key mutations.
-// Multi-pool writes use the pools lock. In a single pool, DELETE/COPY use the
-// first set while PUT and metadata updates use the destination set, so recovery
-// takes both in that order. No writer routing or wire protocol changes here.
+// Multi-pool writes use the pools lock. In a single pool every object write,
+// DELETE and COPY included, locks the object's own set (see NewObjectNSLock),
+// which is the source set here. Releases before that routing locked
+// single-pool DELETE and COPY in the first set, so recovery still takes the
+// first set too, in that order, while nodes of those releases may run. It can
+// drop that lock only behind a version boundary that keeps such nodes out.
func (z *erasureServerPools) listingRLock(ctx context.Context, bucket, object string, source *erasureObjects) (LockContext, func(), error) {
firstSet := z.serverPools[0].sets[0]
first := z.NewNSLock(bucket, object)
diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go
index 964a01149..307268adb 100644
--- a/cmd/object-api-interface.go
+++ b/cmd/object-api-interface.go
@@ -159,6 +159,14 @@ type WalkOptions struct {
// ExpirationOptions represents object options for object expiration at objectLayer.
type ExpirationOptions struct {
Expire bool
+ // CheckedAllPools: the expiry was checked with the versions of the
+ // object in all pools (checkExpiryAllPools). A set does not check it
+ // again with the versions it holds alone.
+ CheckedAllPools bool
+ // ApprovedProtection: the Object Lock state of the version with which
+ // the check approved its expiry. The set compares it with the state it
+ // reads (checkedExpiryRefused). In memory only.
+ ApprovedProtection objectLockState `msg:"-"`
}
// TransitionOptions represents object options for transition ObjectLayer operation
@@ -169,6 +177,15 @@ type TransitionOptions struct {
RestoreRequest *RestoreObjectRequest
RestoreExpiry time.Time
ExpireRestored bool
+
+ // attempt is the native restore a call belongs to: the attempt that
+ // POST marked on the version, see restoreRequest.
+ attempt *restoreAttempt
+ // owner is the restore lock of the version, which POST hands to the
+ // worker that runs attempt, see erasureServerPools.RestoreTransitionedObject.
+ owner *restoreOwner
+ // restoreDate is the request date POST recorded with attempt.
+ restoreDate string
}
// MakeBucketOptions represents bucket options for ObjectLayer bucket operations
@@ -250,7 +267,13 @@ func (o *ObjectOptions) SetEvalRetentionBypassFn(f EvalRetentionBypassFn) {
// ObjectLayer implements primitives for object API layer.
type ObjectLayer interface {
// Locking operations on object.
+ // NewNSLock locks names in one stable domain. It is for names that are
+ // not objects, such as bucket and IAM metadata locks.
NewNSLock(bucket string, objects ...string) RWLocker
+ // NewObjectNSLock locks one object in the lock domain of its writes.
+ NewObjectNSLock(bucket, object string) RWLocker
+ // lockObjects locks several objects together, see erasureServerPools.
+ lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error)
// Storage operations.
Shutdown(context.Context) error
@@ -287,6 +310,8 @@ type ObjectLayer interface {
DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error)
TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error
RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error
+ // restoreRequest records a POST restore on a version, see erasureServerPools.
+ restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error)
// Multipart operations.
ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)
diff --git a/cmd/object-api-interface_gen.go b/cmd/object-api-interface_gen.go
index 919cdb174..8637f5498 100644
--- a/cmd/object-api-interface_gen.go
+++ b/cmd/object-api-interface_gen.go
@@ -79,10 +79,13 @@ func (z BucketOptions) Msgsize() (s int) {
// MarshalMsg implements msgp.Marshaler
func (z ExpirationOptions) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
- // map header, size 1
+ // map header, size 2
// string "Expire"
- o = append(o, 0x81, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65)
+ o = append(o, 0x82, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65)
o = msgp.AppendBool(o, z.Expire)
+ // string "CheckedAllPools"
+ o = append(o, 0xaf, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x41, 0x6c, 0x6c, 0x50, 0x6f, 0x6f, 0x6c, 0x73)
+ o = msgp.AppendBool(o, z.CheckedAllPools)
return
}
@@ -110,6 +113,12 @@ func (z *ExpirationOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "Expire")
return
}
+ case "CheckedAllPools":
+ z.CheckedAllPools, bts, err = msgp.ReadBoolBytes(bts)
+ if err != nil {
+ err = msgp.WrapError(err, "CheckedAllPools")
+ return
+ }
default:
bts, err = msgp.Skip(bts)
if err != nil {
@@ -124,7 +133,7 @@ func (z *ExpirationOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z ExpirationOptions) Msgsize() (s int) {
- s = 1 + 7 + msgp.BoolSize
+ s = 1 + 7 + msgp.BoolSize + 16 + msgp.BoolSize
return
}
diff --git a/cmd/object-copy-metadata_test.go b/cmd/object-copy-metadata_test.go
index 2cb05f3eb..72c729599 100644
--- a/cmd/object-copy-metadata_test.go
+++ b/cmd/object-copy-metadata_test.go
@@ -340,9 +340,9 @@ func TestCopyRewritesObjectData(t *testing.T) {
name: "data copy always rewrites",
want: true,
},
- // PostRestoreObjectHandler, updateRestoreMetadata and batchKeyRotate all
- // address the same version on both sides and never set Versioned, so they
- // only ever reach these two cases.
+ // In-place metadata callers such as batchKeyRotate address the same
+ // version on both sides and never set Versioned, so they only ever
+ // reach these two cases.
{
name: "unversioned in-place metadata update",
metadataOnly: true,
diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go
index ddfc8dde3..41441882a 100644
--- a/cmd/object-handlers-common.go
+++ b/cmd/object-handlers-common.go
@@ -467,9 +467,17 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
deletedObjs, errs := o.DeleteObjects(ctx, bucket, toDel, ObjectOptions{
PrefixEnabledFn: vc.PrefixEnabled,
VersionSuspended: vc.Suspended(),
+ Expiration: ExpirationOptions{Expire: true},
})
for i, dobj := range deletedObjs {
+ if errs[i] != nil && !isErrObjectNotFound(errs[i]) && !isErrVersionNotFound(errs[i]) {
+ // Only a completed delete, or one confirmed already absent, is
+ // a removal. After a refusal, a quorum, configuration or any
+ // other error the version may remain; do not audit or announce
+ // it. The next scan re-evaluates it.
+ continue
+ }
oi := ObjectInfo{
Bucket: bucket,
Name: dobj.ObjectName,
diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go
index 26784ac16..4f9100df2 100644
--- a/cmd/object-handlers.go
+++ b/cmd/object-handlers.go
@@ -34,6 +34,7 @@ import (
"sort"
"strconv"
"strings"
+ "sync"
"sync/atomic"
"time"
@@ -183,7 +184,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
// Take read lock on object, here so subsequent lower-level
// calls do not need to.
- lock := objectAPI.NewNSLock(bucket, object)
+ lock := objectAPI.NewObjectNSLock(bucket, object)
lkctx, err := lock.GetRLock(ctx, globalOperationTimeout)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -492,9 +493,10 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
return
}
event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo)
+ expiryInfo, event := recheckReadExpiry(ctx, objectAPI, objInfo, event)
if event.Action.Delete() {
// apply whatever the expiry rule is.
- applyExpiryRule(event, lcEventSrc_s3GetObject, objInfo)
+ applyExpiryRule(event, lcEventSrc_s3GetObject, expiryInfo)
if !event.Action.DeleteRestored() {
// If the ILM action is not on restored object return error.
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchKey), r.URL)
@@ -972,9 +974,10 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
return
}
event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo)
+ expiryInfo, event := recheckReadExpiry(ctx, objectAPI, objInfo, event)
if event.Action.Delete() {
// apply whatever the expiry rule is.
- applyExpiryRule(event, lcEventSrc_s3HeadObject, objInfo)
+ applyExpiryRule(event, lcEventSrc_s3HeadObject, expiryInfo)
if !event.Action.DeleteRestored() {
// If the ILM action is not on restored object return error.
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
@@ -1468,12 +1471,35 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
return checkCopyObjectPreconditions(ctx, w, r, o)
}
getOpts.CheckPrecondFn = checkCopyPrecondFn
- if cpSrcDstSame {
- getOpts.NoLock = true
+
+ // Lock the source for reading and a local destination for writing before
+ // the source is opened, both in the lock domain of their writers. Taking
+ // them together in one order keeps copies in opposite directions from
+ // holding each other; a copy onto itself takes one write lock. The source
+ // is then read, preconditions and inline data included, and the
+ // destination looked up and written, all under these locks and without
+ // taking them again. That work runs in lockCtx, which ends when either
+ // lock is lost and so aborts the copy; errors are still reported in the
+ // request's context. Both locks are released once the copy has been
+ // written: a synchronous replication below reads the destination with its
+ // own lock.
+ locks := []objectLockRequest{{bucket: srcBucket, object: srcObject}}
+ if !remoteCallRequired {
+ locks = append(locks, objectLockRequest{bucket: dstBucket, object: dstObject, write: true})
}
+ lkctx, unlock, err := objectAPI.lockObjects(ctx, locks...)
+ if err != nil {
+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+ return
+ }
+ unlockCopy := sync.OnceFunc(unlock)
+ defer unlockCopy()
+ lockCtx := lkctx.Context()
+ getOpts.NoLock = true
+ dstOpts.NoLock = true
var rs *HTTPRangeSpec
- gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, getOpts)
+ gr, err := getObjectNInfo(lockCtx, srcBucket, srcObject, rs, r.Header, getOpts)
if err != nil {
if isErrPreconditionFailed(err) {
return
@@ -1540,7 +1566,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
length := actualSize
if !cpSrcDstSame {
- if err := enforceBucketQuotaHard(ctx, dstBucket, actualSize); err != nil {
+ if err := enforceBucketQuotaHard(lockCtx, dstBucket, actualSize); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -1558,12 +1584,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
- reader = etag.NewReader(ctx, reader, nil, nil)
+ reader = etag.NewReader(lockCtx, reader, nil, nil)
} else {
reader = gr
}
- srcInfo.Reader, err = hash.NewReader(ctx, reader, length, "", "", actualSize)
+ srcInfo.Reader, err = hash.NewReader(lockCtx, reader, length, "", "", actualSize)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -1658,7 +1684,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
}
- if err = rotateKey(ctx, oldKey, newKeyID, newKey, srcBucket, srcObject, encMetadata, kmsCtx); err != nil {
+ if err = rotateKey(lockCtx, oldKey, newKeyID, newKey, srcBucket, srcObject, encMetadata, kmsCtx); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -1743,7 +1769,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
dstOpts.IndexCB = cb
defer s2c.Close()
reader = etag.Wrap(s2c, checksumReader)
- srcInfo.Reader, err = hash.NewReader(ctx, reader, -1, "", "", actualSize)
+ srcInfo.Reader, err = hash.NewReader(lockCtx, reader, -1, "", "", actualSize)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -1756,7 +1782,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
if isTargetEncrypted && !remoteCallRequired {
var encReader io.Reader
kind, _ := crypto.IsRequested(r.Header)
- encReader, objEncKey, err = newEncryptReader(ctx, srcInfo.Reader, kind, newKeyID, newKey, dstBucket, dstObject, encMetadata, kmsCtx)
+ encReader, objEncKey, err = newEncryptReader(lockCtx, srcInfo.Reader, kind, newKeyID, newKey, dstBucket, dstObject, encMetadata, kmsCtx)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -1771,7 +1797,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
// do not try to verify encrypted content
- srcInfo.Reader, err = hash.NewReader(ctx, reader, targetSize, "", "", actualSize)
+ srcInfo.Reader, err = hash.NewReader(lockCtx, reader, targetSize, "", "", actualSize)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -1843,7 +1869,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectRetentionAction)
holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectLegalHoldAction)
- getObjectInfo := objectAPI.GetObjectInfo
+ // The destination is locked above: look it up without its lock, in the
+ // locks' context.
+ getObjectInfo := func(_ context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+ opts.NoLock = true
+ return objectAPI.GetObjectInfo(lockCtx, bucket, object, opts)
+ }
// apply default bucket configuration/governance headers for dest side.
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms, replicaTrusted)
@@ -2029,9 +2060,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
// For an ordinary federated copy, actualSize is the logical plaintext
// length to declare. srcInfo.Size is not a reliable wire length because
// the read path may already have adjusted it.
- writeCtx, modified := withFederatedWriteTime(ctx)
+ writeCtx, modified := withFederatedWriteTime(lockCtx)
remoteObjInfo, rerr := core.PutObject(writeCtx, dstBucket, dstObject, srcInfo.Reader,
actualSize, "", "", opts)
+ gr.Close()
+ unlockCopy()
if rerr != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL)
return
@@ -2078,7 +2111,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
// Copy source object to destination, if source and destination
// object is same then only metadata is updated.
- objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts)
+ objInfo, err = copyObjectFn(lockCtx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts)
+ gr.Close()
+ unlockCopy()
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -3951,7 +3986,13 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
return
}
- objInfo, err := getObjectInfo(ctx, bucket, object, opts)
+ // Preserve a failed restore read even before the request body is parsed.
+ opts.Transition.RestoreRequest = &RestoreObjectRequest{}
+ // Resolve the target without the object lock; the status step below
+ // takes the restore lock and then the object lock.
+ preOpts := opts
+ preOpts.NoLock = true
+ objInfo, err := getObjectInfo(ctx, bucket, object, preOpts)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -3976,46 +4017,56 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
writeErrorResponse(ctx, w, apiErr, r.URL)
return
}
+ opts.Transition.RestoreRequest = rreq
statusCode := http.StatusOK
- alreadyRestored := false
- if err == nil {
- if objInfo.RestoreOngoing && rreq.Type != SelectRestoreRequest {
+ // set or upgrade restore expiry
+ restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
+
+ // update self with restore metadata
+ var (
+ attempt *restoreAttempt
+ owner *restoreOwner
+ restoreDate string
+ )
+ if rreq.Type != SelectRestoreRequest {
+ // Native restore needs every server to follow the restore lock and
+ // the staged commit; refuse it, writing nothing, while that is not
+ // established.
+ if err := nativeRestoreAvailable(ctx); err != nil {
+ apiErr := errorCodes.ToAPIErr(ErrBusy)
+ apiErr.Code, apiErr.Description = "ServiceUnavailable", err.Error()
+ writeErrorResponse(ctx, w, apiErr, r.URL)
+ return
+ }
+ // Decide on a fresh read under the object lock: renew a restored copy
+ // that is still there, or mark the version ongoing for a new attempt.
+ // Only the version resolved above, with its identity, is restored.
+ statusOpts := opts
+ statusOpts.Transition.RestoreExpiry = restoreExpiry
+ status, err := objectAPI.restoreRequest(GlobalContext, bucket, object, statusOpts, objInfo)
+ if errors.Is(err, errRestoreInProgress) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectRestoreAlreadyInProgress), r.URL)
return
}
- if !objInfo.RestoreOngoing && !objInfo.RestoreExpires.IsZero() {
- statusCode = http.StatusAccepted
- alreadyRestored = true
- }
- }
- // set or upgrade restore expiry
- restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
- metadata := cloneMSS(objInfo.UserDefined)
-
- // update self with restore metadata
- if rreq.Type != SelectRestoreRequest {
- objInfo.metadataOnly = true // Perform only metadata updates.
- metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days)
- metadata[xhttp.AmzRestoreRequestDate] = time.Now().UTC().Format(http.TimeFormat)
- if alreadyRestored {
- metadata[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
- } else {
- metadata[xhttp.AmzRestore] = ongoingRestoreObj().String()
- }
- objInfo.UserDefined = metadata
- if _, err := objectAPI.CopyObject(GlobalContext, bucket, object, bucket, object, objInfo, ObjectOptions{
- VersionID: objInfo.VersionID,
- }, ObjectOptions{
- VersionID: objInfo.VersionID,
- MTime: objInfo.ModTime,
- }); err != nil {
+ if errors.As(err, &InvalidObjectState{}) || errors.As(err, &MethodNotAllowed{}) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL)
return
}
- // for previously restored object, just update the restore expiry
- if alreadyRestored {
+ if err != nil {
+ // Such as a read quorum error while telling whether it restores.
+ writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
+ // Events keep the version ID as this request resolved it.
+ status.objInfo.VersionID = objInfo.VersionID
+ objInfo = status.objInfo
+ // for previously restored object, just update the restore expiry
+ if status.attempt == nil {
+ return
+ }
+ attempt = status.attempt
+ // The restore lock goes to the worker below, which releases it.
+ owner, restoreDate = status.owner, status.date
}
restoreObject := mustGetUUID()
@@ -4035,6 +4086,9 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
// now process the restore in background
go func() {
rctx := GlobalContext
+ // The restore releases the lock it was handed; this covers a path
+ // that does not reach it. Releasing twice is a no-op.
+ defer owner.release()
if !rreq.SelectParameters.IsEmpty() {
actualSize, err := objInfo.GetActualSize()
if err != nil {
@@ -4080,14 +4134,20 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
rreq.SelectParameters.Close()
return
}
- opts := ObjectOptions{
+ // The versioning context and the explicit version POST resolved.
+ restoreOpts := ObjectOptions{
Transition: TransitionOptions{
RestoreRequest: rreq,
RestoreExpiry: restoreExpiry,
+ attempt: attempt,
+ owner: owner,
+ restoreDate: restoreDate,
},
- VersionID: objInfo.VersionID,
+ VersionID: attempt.VersionID,
+ Versioned: opts.Versioned,
+ VersionSuspended: opts.VersionSuspended,
}
- if err := objectAPI.RestoreTransitionedObject(rctx, bucket, object, opts); err != nil {
+ if err := objectAPI.RestoreTransitionedObject(rctx, bucket, object, restoreOpts); err != nil {
s3LogIf(ctx, fmt.Errorf("Unable to restore transitioned bucket/object %s/%s: %w", bucket, object, err))
return
}
diff --git a/cmd/object-lifecycle-read_test.go b/cmd/object-lifecycle-read_test.go
new file mode 100644
index 000000000..449e3d47a
--- /dev/null
+++ b/cmd/object-lifecycle-read_test.go
@@ -0,0 +1,380 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/versioning"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// An explicit-version read uses ReadVersion. Only the added lifecycle scan
+// uses ReadXL, so its errors/absence can be injected without failing the read.
+type lifecycleReadDisk struct {
+ StorageAPI
+ bucket, object string
+ scans *atomic.Int64
+ err error
+}
+
+func (d lifecycleReadDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
+ if bucket == d.bucket && object == d.object {
+ d.scans.Add(1)
+ if d.err != nil {
+ return RawFileInfo{}, d.err
+ }
+ }
+ return d.StorageAPI.ReadXL(ctx, bucket, object, data)
+}
+
+func lifecycleReadDisks(t *testing.T, s *splitILM, object string, failedPool int, err error) *atomic.Int64 {
+ t.Helper()
+ var scans atomic.Int64
+ for p, pool := range s.z.serverPools {
+ pool.erasureDisksMu.Lock()
+ previous := pool.erasureDisks
+ wrapped := make([][]StorageAPI, len(previous))
+ for i, disks := range previous {
+ wrapped[i] = append([]StorageAPI(nil), disks...)
+ for j, disk := range disks {
+ var injected error
+ if p == failedPool {
+ injected = err
+ }
+ wrapped[i][j] = lifecycleReadDisk{StorageAPI: disk, bucket: s.bucket, object: object, scans: &scans, err: injected}
+ }
+ }
+ pool.erasureDisks = wrapped
+ pool.erasureDisksMu.Unlock()
+ t.Cleanup(func() {
+ pool.erasureDisksMu.Lock()
+ pool.erasureDisks = previous
+ pool.erasureDisksMu.Unlock()
+ })
+ }
+ return &scans
+}
+
+// Real signed GET and HEAD requests use the author's two-pool storage fixture.
+// Checking the queue as well as the response catches both false hiding and
+// accidentally authorizing the latest version instead of the version read.
+func TestPoolsLifecycleReadHTTP(t *testing.T) {
+ const bothRules = `bothEnabledboth-` +
+ `11`
+ for _, state := range []versioning.State{versioning.Enabled, versioning.Suspended} {
+ t.Run(string(state), func(t *testing.T) {
+ s := newSplitILM(t, state, splitILMRules+bothRules)
+ router := initTestAPIEndPoints(s.z, nil)
+ for _, tc := range []struct {
+ name, prefix string
+ latestAge float64
+ readLatest bool
+ implicit bool
+ action lifecycle.Action
+ local lifecycle.Action
+ fault error
+ wantScan bool
+ }{
+ {name: "old-version-new-latest", prefix: "exp-", latestAge: 0.01, local: lifecycle.DeleteAction, wantScan: true},
+ {name: "old-version-expired-latest", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, wantScan: true},
+ {name: "old-version-delete-all-latest", prefix: "eav-", latestAge: 3, local: lifecycle.DeleteAllVersionsAction, wantScan: true},
+ {name: "expired-current", prefix: "exp-", latestAge: 3, readLatest: true, local: lifecycle.DeleteAction, action: lifecycle.DeleteAction, wantScan: true},
+ {name: "expired-current-implicit", prefix: "exp-", latestAge: 3, readLatest: true, implicit: true, local: lifecycle.DeleteAction, action: lifecycle.DeleteAction, wantScan: true},
+ {name: "expired-noncurrent", prefix: "both-", latestAge: 3, local: lifecycle.DeleteAction, action: lifecycle.DeleteVersionAction, wantScan: true},
+ {name: "extra-read-quorum-error", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, fault: errDiskNotFound, wantScan: true},
+ {name: "read-version-missing-from-recheck", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, fault: errFileNotFound, wantScan: true},
+ {name: "local-none-global-expired", prefix: "ncv-", latestAge: 3, local: lifecycle.NoneAction},
+ } {
+ for _, null := range []bool{false, true} {
+ for oldPool := range 2 {
+ t.Run(fmt.Sprintf("%s/null=%t/old-pool=%d", tc.name, null, oldPool), func(t *testing.T) {
+ key := fmt.Sprintf("%s%s-%t-%d", tc.prefix, tc.name, null, oldPool)
+ old := splitAt("v1", 5, oldPool)
+ old.null = null
+ s.write(key, old)
+ s.write(key, splitAt("v2", tc.latestAge, 1-oldPool))
+ label := "v1"
+ if tc.readLatest {
+ label = "v2"
+ }
+ vid := s.ids[key+" "+label]
+ if vid == "" {
+ vid = nullVersionID
+ }
+ oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: state == versioning.Enabled, VersionSuspended: state == versioning.Suspended})
+ if err != nil {
+ t.Fatal(err)
+ }
+ lc, err := globalLifecycleSys.Get(s.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if local := lc.Eval(oi.ToLifecycleOpts()); local.Action != tc.local {
+ t.Fatalf("fixture local event=%v, want %v", local.Action, tc.local)
+ }
+ versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ index := 1
+ if tc.readLatest {
+ index = 0
+ }
+ if tc.local == lifecycle.NoneAction && events[index].Action != lifecycle.DeleteVersionAction {
+ t.Fatal("fixture must be globally expired but locally NoneAction")
+ }
+ scans := lifecycleReadDisks(t, s, key, oldPool, tc.fault)
+ for _, method := range []string{http.MethodGet, http.MethodHead} {
+ t.Run(method, func(t *testing.T) {
+ scans.Store(0)
+ requestID := vid
+ if tc.implicit {
+ requestID = ""
+ }
+ rec := consistencyRequest(t, router, method, s.bucket, key, requestID)
+ want := http.StatusOK
+ if tc.action.Delete() {
+ want = http.StatusNotFound
+ }
+ if rec.Code != want {
+ t.Errorf("status=%d want=%d body=%s", rec.Code, want, rec.Body.String())
+ }
+ if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != key+" "+label {
+ t.Errorf("body=%q, want requested version %s", rec.Body.String(), label)
+ }
+ if !tc.implicit && (scans.Load() > 0) != tc.wantScan {
+ t.Errorf("ReadXL calls=%d, want scan=%t", scans.Load(), tc.wantScan)
+ }
+ if tc.action.Delete() {
+ select {
+ case op := <-s.queue:
+ task, ok := op.(expiryTask)
+ src := lcEventSrc_s3GetObject
+ if method == http.MethodHead {
+ src = lcEventSrc_s3HeadObject
+ }
+ global := versions[index]
+ if !ok || task.src != src || task.event != events[index] || task.event.Action != tc.action ||
+ task.objInfo.VersionID != global.VersionID || task.objInfo.IsLatest != global.IsLatest ||
+ task.objInfo.NumVersions != global.NumVersions || !task.objInfo.SuccessorModTime.Equal(global.SuccessorModTime) {
+ t.Errorf("wrong queued expiry: %+v; want %s with global event %+v", task, vid, events[index])
+ }
+ default:
+ t.Error("missing expiry task")
+ }
+ }
+ if n := len(s.queue); n != 0 {
+ t.Errorf("unexpected expiry tasks: %d", n)
+ for len(s.queue) > 0 {
+ <-s.queue
+ }
+ }
+ })
+ }
+ })
+ }
+ }
+ }
+ })
+ }
+}
+
+func TestPoolsLifecycleReadAuthorizationAndClose(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ initConfigSubsystem(t.Context(), s.z)
+ globalIAMSys.Init(t.Context(), s.z, globalEtcdClient, 2*time.Second)
+ denied := newObjectAttributesAuthzUser(t, "lifecycle-read", s.bucket, `"s3:PutObject"`)
+ router := initTestAPIEndPoints(s.z, nil)
+ for _, expire := range []bool{false, true} {
+ key := fmt.Sprintf("exp-reader-%t", expire)
+ v := splitAt("v1", 5, 0)
+ v.size = 1 << 20 // a non-inline reader retains the object read lock
+ s.write(key, v)
+ if !expire {
+ s.write(key, splitAt("v2", 0.01, 1))
+ }
+ vid := s.ids[key+" v1"]
+ oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: true})
+ if err != nil || oi.Inlined {
+ t.Fatalf("fixture must retain a reader lock: inlined=%t err=%v", oi.Inlined, err)
+ }
+ scans := lifecycleReadDisks(t, s, key, -1, nil)
+ for _, method := range []string{http.MethodGet, http.MethodHead} {
+ req, err := newTestSignedRequestV4(method, getGetObjectURL("", s.bucket, key)+"?versionId="+vid, 0, nil, denied.AccessKey, denied.SecretKey, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rec := httptest.NewRecorder()
+ router.ServeHTTP(rec, req)
+ if rec.Code != http.StatusForbidden || scans.Load() != 0 || len(s.queue) != 0 {
+ t.Fatalf("denied %s: status=%d scans=%d queued=%d", method, rec.Code, scans.Load(), len(s.queue))
+ }
+ }
+ rec := consistencyRequest(t, router, http.MethodGet, s.bucket, key, vid)
+ want := http.StatusOK
+ if expire {
+ want = http.StatusNotFound
+ }
+ if rec.Code != want {
+ t.Errorf("authorized GET status=%d want=%d", rec.Code, want)
+ }
+ // The reader owns the multi-pool read lock until Close, including the
+ // early 404 path. A writer must acquire it immediately after the handler.
+ lk := s.z.NewNSLock(s.bucket, key)
+ lkctx, err := lk.GetLock(t.Context(), newDynamicTimeout(time.Second, time.Second))
+ if err != nil {
+ t.Fatalf("GET leaked its reader/lock: %v", err)
+ }
+ lk.Unlock(lkctx)
+ for len(s.queue) > 0 {
+ <-s.queue
+ }
+ }
+}
+
+func TestPoolsLifecycleReadBoundaries(t *testing.T) {
+ const transitionRules = `transitionEnabledtransition-` +
+ `1WARM`
+ for _, tc := range []struct {
+ name string
+ state versioning.State
+ null bool
+ singlePool bool
+ restored bool
+ transition bool
+ action lifecycle.Action
+ wantScan bool
+ }{
+ {name: "single-pool", state: versioning.Enabled, singlePool: true, action: lifecycle.DeleteAction},
+ {name: "single-pool-null", state: versioning.Enabled, singlePool: true, null: true, action: lifecycle.DeleteAction},
+ {name: "unversioned", null: true, action: lifecycle.DeleteAction},
+ {name: "current-null-enabled", state: versioning.Enabled, null: true, action: lifecycle.DeleteAction, wantScan: true},
+ {name: "current-null-suspended", state: versioning.Suspended, null: true, action: lifecycle.DeleteAction, wantScan: true},
+ {name: "transition", state: versioning.Enabled, transition: true, action: lifecycle.TransitionAction},
+ {name: "restored", state: versioning.Enabled, restored: true, action: lifecycle.DeleteRestoredAction},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s := newSplitILM(t, tc.state, splitILMRules+transitionRules)
+ key := "exp-" + tc.name
+ if tc.restored || tc.transition {
+ key = "transition-" + tc.name
+ }
+ v := splitAt("v1", 5, 0)
+ v.null, v.tiered = tc.null, tc.restored
+ s.write(key, v)
+ vid := s.ids[key+" v1"]
+ if vid == "" {
+ vid = nullVersionID
+ }
+ if tc.restored {
+ globalTierConfigMgr.drivercache["WARM"] = &tierExpiryTestBackend{objects: map[string][]byte{"remote-v1": []byte(key + " v1")}}
+ opts := ObjectOptions{VersionID: vid, Versioned: true}
+ opts.EvalMetadataFn = func(oi *ObjectInfo, _ error) (ReplicateDecision, error) {
+ oi.UserDefined[xhttp.AmzRestore] = completedRestoreObj(UTCNow().Add(-time.Hour)).String()
+ return ReplicateDecision{}, nil
+ }
+ if _, err := s.z.PutObjectMetadata(t.Context(), s.bucket, key, opts); err != nil {
+ t.Fatal(err)
+ }
+ }
+ z := s.z
+ if tc.singlePool {
+ // Exercise the single-pool dispatch with the same storage fixture.
+ z = &erasureServerPools{serverPools: s.z.serverPools[:1]}
+ }
+ router := initTestAPIEndPoints(z, []string{"GetObject", "HeadObject"})
+ scans := lifecycleReadDisks(t, s, key, -1, nil)
+ for _, method := range []string{http.MethodGet, http.MethodHead} {
+ scans.Store(0)
+ rec := consistencyRequest(t, router, method, s.bucket, key, vid)
+ want := http.StatusOK
+ if tc.action.Delete() && !tc.action.DeleteRestored() {
+ want = http.StatusNotFound
+ }
+ if rec.Code != want || (scans.Load() > 0) != tc.wantScan {
+ t.Errorf("%s status=%d want=%d scans=%d wantScan=%t body=%s", method, rec.Code, want, scans.Load(), tc.wantScan, rec.Body.String())
+ }
+ if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != key+" v1" {
+ t.Errorf("%s lost object contents: %q", method, rec.Body.String())
+ }
+ if tc.action.Delete() {
+ select {
+ case op := <-s.queue:
+ task, ok := op.(expiryTask)
+ if !ok || task.event.Action != tc.action || s.key(key, task.objInfo.VersionID) != s.key(key, vid) {
+ t.Errorf("%s wrong expiry: %+v", method, task)
+ }
+ default:
+ t.Errorf("%s missing expiry", method)
+ }
+ }
+ if n := len(s.queue); n != 0 {
+ t.Fatalf("%s queued %d unexpected expiries", method, n)
+ }
+ }
+ })
+ }
+}
+
+func TestPoolsLifecycleReadProxy(t *testing.T) {
+ s := newSplitILM(t, versioning.Enabled, splitILMRules)
+ const key, body = "exp-proxied", "remote object remains readable"
+ vid := mustGetUUID()
+ target := newILMTestTarget(t)
+ var remoteRequests atomic.Int64
+ registerILMTestTarget(t, s.z, s.bucket, target, func(w http.ResponseWriter, r *http.Request) {
+ remoteRequests.Add(1)
+ w.Header().Set(xhttp.ContentLength, fmt.Sprint(len(body)))
+ w.Header().Set(xhttp.LastModified, UTCNow().Add(-5*24*time.Hour).Format(http.TimeFormat))
+ w.Header().Set(xhttp.ETag, `"da39a3ee5e6b4b0d3255bfef95601890"`)
+ w.Header().Set(xhttp.AmzVersionID, vid)
+ if r.Method == http.MethodGet {
+ _, _ = w.Write([]byte(body))
+ }
+ })
+ setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket)
+ // The delete-only target fixture omits credentials; proxy discovery clones
+ // the target descriptor and requires its credentials field to be present.
+ globalBucketTargetSys.Lock()
+ globalBucketTargetSys.targetsMap[s.bucket][0].Credentials = &madmin.Credentials{AccessKey: "proxy-test"}
+ globalBucketTargetSys.Unlock()
+ stats := NewReplicationStats(t.Context(), nil)
+ previous := globalReplicationStats.Swap(stats)
+ t.Cleanup(func() { globalReplicationStats.Store(previous) })
+ scans := lifecycleReadDisks(t, s, key, -1, nil)
+ router := initTestAPIEndPoints(s.z, nil)
+ for _, method := range []string{http.MethodGet, http.MethodHead} {
+ before := remoteRequests.Load()
+ rec := consistencyRequest(t, router, method, s.bucket, key, vid)
+ if rec.Code != http.StatusOK || remoteRequests.Load() == before || scans.Load() != 0 || len(s.queue) != 0 {
+ t.Errorf("proxy %s: status=%d remote=%d scans=%d queued=%d body=%s", method, rec.Code, remoteRequests.Load()-before, scans.Load(), len(s.queue), rec.Body.String())
+ }
+ if method == http.MethodGet && rec.Body.String() != body {
+ t.Errorf("proxy body=%q", rec.Body.String())
+ }
+ }
+}
diff --git a/cmd/object-retention-authz_test.go b/cmd/object-retention-authz_test.go
index af44aac37..c62477dc0 100644
--- a/cmd/object-retention-authz_test.go
+++ b/cmd/object-retention-authz_test.go
@@ -544,6 +544,18 @@ func (p *hookBeforeWrite) CompleteMultipartUpload(ctx context.Context, bucket, k
return p.ObjectLayer.CompleteMultipartUpload(ctx, bucket, key, uploadID, parts, opts)
}
+// The COPY handler checks Object Lock under its destination's write lock, so
+// no writer can land between that check and the copy. The latest a writer can
+// land is just before the handler takes its locks.
+func (p *hookBeforeWrite) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) {
+ for _, r := range reqs {
+ if r.write {
+ p.fire(r.object)
+ }
+ }
+ return p.ObjectLayer.lockObjects(ctx, reqs...)
+}
+
func TestDirectoryObjectLockAfterPreflight(t *testing.T) {
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go
index b28c8db44..6661b5fba 100644
--- a/cmd/peer-rest-server.go
+++ b/cmd/peer-rest-server.go
@@ -116,6 +116,7 @@ var (
stopRebalanceRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerStopRebalance, grid.NewMSS, grid.NewNoPayload)
updateMetacacheListingRPC = grid.NewSingleHandler[*metacache, *metacache](grid.HandlerUpdateMetacacheListing, func() *metacache { return &metacache{} }, func() *metacache { return &metacache{} })
cleanupUploadIDCacheMetaRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerClearUploadID, grid.NewMSS, grid.NewNoPayload)
+ restoreCapabilityRPC = grid.NewSingleHandler[*grid.MSS, *grid.MSS](grid.HandlerRestoreCapability, grid.NewMSS, grid.NewMSS)
// STREAMS
// Set an output capacity of 100 for consoleLog and listenRPC
@@ -1440,6 +1441,7 @@ func registerPeerRESTHandlers(router *mux.Router, gm *grid.Manager) {
logger.FatalIf(stopRebalanceRPC.Register(gm, server.StopRebalanceHandler), "unable to register handler")
logger.FatalIf(updateMetacacheListingRPC.Register(gm, server.UpdateMetacacheListingHandler), "unable to register handler")
logger.FatalIf(cleanupUploadIDCacheMetaRPC.Register(gm, server.HandlerClearUploadID), "unable to register handler")
+ logger.FatalIf(restoreCapabilityRPC.Register(gm, restoreCapabilityHandler), "unable to register handler")
logger.FatalIf(gm.RegisterStreamingHandler(grid.HandlerTrace, grid.StreamHandler{
Handle: server.TraceHandler,
diff --git a/cmd/restore-attempt.go b/cmd/restore-attempt.go
new file mode 100644
index 000000000..ec2d01ecb
--- /dev/null
+++ b/cmd/restore-attempt.go
@@ -0,0 +1,662 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "maps"
+ "net/http"
+ "slices"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// The version that POST marks ongoing names the native restore attempt that
+// owns the marker. Clients cannot set it: requests with reserved metadata are
+// refused, and it is never extracted from headers.
+const restoreAttemptKey = ReservedMetadataPrefixLower + "restore-attempt"
+
+const (
+ // restoreTakeoverWait bounds how long a POST waits for the restore lock
+ // of its version. A running restore holds it for its whole run.
+ restoreTakeoverWait = 2 * time.Second
+ // restoreCleanupTimeout bounds each status reset and convergence pass.
+ restoreCleanupTimeout = time.Minute
+)
+
+// restoreRecoveryDelays are the in-memory retries that clear the ongoing
+// marker of a restore that failed before it could clear the marker itself.
+// A restart loses them; a later POST then takes the marker over.
+var restoreRecoveryDelays = []time.Duration{0, 100 * time.Millisecond, time.Second, 5 * time.Second, 30 * time.Second, 2 * time.Minute}
+
+var (
+ // errRestoreSuperseded: the version no longer belongs to this attempt.
+ // The attempt must not rehydrate, complete or reset anything.
+ errRestoreSuperseded = errors.New("restore attempt superseded")
+ // errRestoreInProgress: a restore holds the restore lock of the version.
+ errRestoreInProgress = errors.New("restore already in progress")
+ // errRestoreUnchanged: a restore status update found nothing to write.
+ errRestoreUnchanged = errors.New("restore status unchanged")
+)
+
+// restoreResetError marks a failed restore that may have left its ongoing
+// marker behind. The pools layer hands it to recoverRestoreStatus.
+type restoreResetError struct{ error }
+
+func (e restoreResetError) Unwrap() error { return e.error }
+
+// restoreVersionID is the explicit version ID of a resolved restore target.
+func restoreVersionID(versionID string) string {
+ if versionID == "" {
+ return nullVersionID
+ }
+ return versionID
+}
+
+// restoreAttempt identifies one native restore of one stored version.
+type restoreAttempt struct {
+ ID string `json:"id"`
+ VersionID string `json:"v"` // "null" for the null version
+ ETag string `json:"e"`
+ Size int64 `json:"s"`
+ ModTime time.Time `json:"m"`
+}
+
+func newRestoreAttempt(oi ObjectInfo) *restoreAttempt {
+ return &restoreAttempt{
+ ID: mustGetUUID(),
+ VersionID: restoreVersionID(oi.VersionID),
+ ETag: oi.ETag,
+ Size: oi.Size,
+ ModTime: oi.ModTime.UTC(),
+ }
+}
+
+func (a *restoreAttempt) String() string {
+ b, _ := json.Marshal(a)
+ return string(b)
+}
+
+// targets reports whether oi is still the version the attempt was made for.
+// A replacement under the same version ID differs in ETag, size or ModTime.
+func (a *restoreAttempt) targets(oi ObjectInfo) bool {
+ return a != nil && !oi.DeleteMarker && restoreVersionID(oi.VersionID) == a.VersionID &&
+ oi.ETag == a.ETag && oi.Size == a.Size && oi.ModTime.Equal(a.ModTime) &&
+ oi.TransitionedObject.Status == lifecycle.TransitionComplete
+}
+
+// owns reports whether the version is ongoing under this attempt.
+func (a *restoreAttempt) owns(oi ObjectInfo) bool {
+ return a.targets(oi) && oi.RestoreOngoing && oi.UserDefined[restoreAttemptKey] == a.ID
+}
+
+// ownsGeneration reports whether the attempt owns the authoritative
+// generation of the version. Where the drives disagree only in their restore
+// fields, see restoreGeneration, it owns the version only when its marker is
+// the only ongoing marker there: holding one of several is not enough.
+func (a *restoreAttempt) ownsGeneration(gen restoreGeneration, bucket, object string) bool {
+ oi := gen.fi.ToObjectInfo(bucket, object, false)
+ if !gen.split {
+ return a.owns(oi)
+ }
+ return a.targets(oi) && len(gen.ongoing) == 1 && gen.ongoing[a.ID]
+}
+
+// restoreGeneration is the metadata generation of one version that the drives
+// of a set hold authoritatively: the revision (header and normalized metadata
+// digest, versionRevision) that at least max(n/2+1, its read quorum) drives
+// hold. drives has each drive's own copy of it, with that drive's erasure
+// index and inline data, and an invalid entry where a drive holds another or
+// no revision; disks has nil there.
+//
+// A failed restore status write can leave drives that differ only in their
+// restore fields, with no revision on a quorum. Then the generation is the
+// restore-neutral revision that a quorum holds once those fields are left
+// out: fi has no restore fields, and ongoing names the attempts whose
+// ongoing marker some of its drives hold ("" for one without an attempt).
+// Every other field, protection included, still needs the quorum.
+//
+// revs has, for every drive that holds the version, the full revision of the
+// entry it holds, whether or not that revision is the generation; nil where a
+// drive does not hold the version or cannot be read. A native restore's
+// commit compares the drives against it later, see restorePublication.
+type restoreGeneration struct {
+ fi FileInfo
+ drives []FileInfo
+ disks []StorageAPI
+ split bool
+ ongoing map[string]bool
+ revs []*revisionKey
+}
+
+// restoreFieldKeys are the metadata keys a restore status write sets.
+var restoreFieldKeys = []string{xhttp.AmzRestore, xhttp.AmzRestoreExpiryDays, xhttp.AmzRestoreRequestDate, restoreAttemptKey}
+
+// restoreHealMeta returns latest, the metadata heal copies to drives that
+// lack the version, with the restore fields that at least quorum drives of
+// that version hold. When some drive holds restore fields and no set of
+// them has the quorum, healing the version waits.
+func restoreHealMeta(metas []FileInfo, latest FileInfo, quorum int) (FileInfo, error) {
+ counts := make(map[string]int)
+ fields := make(map[string]map[string]string)
+ restoring := false
+ for _, m := range metas {
+ if !m.IsValid() || m.Deleted || !m.ModTime.Equal(latest.ModTime) {
+ continue
+ }
+ f := make(map[string]string)
+ var key strings.Builder
+ for _, k := range restoreFieldKeys {
+ if v, ok := m.Metadata[k]; ok {
+ f[k] = v
+ fmt.Fprintf(&key, "%d:%s=%d:%s;", len(k), k, len(v), v)
+ restoring = true
+ }
+ }
+ counts[key.String()]++
+ fields[key.String()] = f
+ }
+ if !restoring {
+ return latest, nil
+ }
+ best, bestCount := "", 0
+ for key, c := range counts {
+ if c > bestCount {
+ best, bestCount = key, c
+ }
+ }
+ if bestCount < quorum {
+ return latest, errErasureReadQuorum
+ }
+ latest.Metadata = maps.Clone(latest.Metadata)
+ for _, k := range restoreFieldKeys {
+ delete(latest.Metadata, k)
+ }
+ maps.Copy(latest.Metadata, fields[best])
+ return latest, nil
+}
+
+// restoreNeutralRevision is the revision of ver without its restore fields.
+func restoreNeutralRevision(ver xlMetaV2ShallowVersion, metaV uint8) (revisionKey, bool) {
+ var v xlMetaV2Version
+ if _, err := v.unmarshalV(metaV, ver.meta); err != nil || v.Type != ObjectType || v.ObjectV2 == nil {
+ return revisionKey{}, false
+ }
+ o := *v.ObjectV2
+ o.MetaUser, o.MetaSys = maps.Clone(o.MetaUser), maps.Clone(o.MetaSys)
+ for _, k := range restoreFieldKeys {
+ delete(o.MetaUser, k)
+ delete(o.MetaSys, k)
+ }
+ v.ObjectV2 = &o
+ meta, err := v.MarshalMsg(nil)
+ if err != nil {
+ return revisionKey{}, false
+ }
+ return versionRevision(xlMetaV2ShallowVersion{header: v.header(), meta: meta}, xlMetaVersion), true
+}
+
+// readRestoreGeneration selects the authoritative generation of a version,
+// or fails closed: a revision that only some drives hold, such as one a
+// failed metadata update left behind, is never selected. An empty version ID
+// addresses the latest version, as a direct restore may.
+func (er erasureObjects) readRestoreGeneration(ctx context.Context, bucket, object, versionID string) (restoreGeneration, error) {
+ if versionID == "" {
+ fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, ObjectOptions{
+ NoLock: true,
+ Transition: TransitionOptions{RestoreRequest: &RestoreObjectRequest{}},
+ }, false)
+ if err != nil {
+ return restoreGeneration{}, err
+ }
+ versionID = restoreVersionID(fi.VersionID)
+ }
+ var uv [16]byte
+ if versionID != nullVersionID {
+ id, err := uuid.Parse(versionID)
+ if err != nil {
+ return restoreGeneration{}, errFileVersionNotFound
+ }
+ uv = id
+ }
+ disks := er.getDisks()
+ n := len(disks)
+ majority := n/2 + 1
+ raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, true)
+ type revision struct {
+ ver xlMetaV2ShallowVersion
+ metaV uint8
+ count int
+ }
+ keys := make([]*revisionKey, n)
+ neutral := make([]*revisionKey, n)
+ revisions := make(map[revisionKey]*revision)
+ neutrals := make(map[revisionKey]*revision)
+ absent := 0
+ for i := range disks {
+ if errs[i] != nil {
+ if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errVolumeNotFound) {
+ absent++
+ }
+ continue
+ }
+ var xl xlMetaV2
+ if xl.LoadOrConvert(raws[i].Buf) != nil {
+ continue
+ }
+ found := false
+ for _, ver := range xl.versions {
+ if ver.header.VersionID != uv || ver.header.FreeVersion() {
+ continue
+ }
+ key := versionRevision(ver, xl.metaV)
+ keys[i], found = &key, true
+ r := revisions[key]
+ if r == nil {
+ r = &revision{ver: ver, metaV: xl.metaV}
+ revisions[key] = r
+ }
+ r.count++
+ if nkey, ok := restoreNeutralRevision(ver, xl.metaV); ok {
+ neutral[i] = &nkey
+ if neutrals[nkey] == nil {
+ neutrals[nkey] = &revision{ver: ver, metaV: xl.metaV}
+ }
+ neutrals[nkey].count++
+ }
+ break
+ }
+ if !found {
+ absent++
+ }
+ }
+ accepted := func(revs map[revisionKey]*revision) (revisionKey, bool) {
+ var best *revision
+ var bestKey revisionKey
+ for key, r := range revs {
+ if best == nil || r.count > best.count {
+ best, bestKey = r, key
+ }
+ }
+ return bestKey, best != nil && best.count >= majority &&
+ best.count >= er.versionReadQuorum(ctx, bucket, object, best.ver, best.metaV, make([]FileInfo, n), make([]error, n))
+ }
+ gen := restoreGeneration{drives: make([]FileInfo, n), disks: make([]StorageAPI, n), revs: slices.Clone(keys)}
+ bestKey, ok := accepted(revisions)
+ if !ok {
+ if bestKey, ok = accepted(neutrals); !ok {
+ if absent >= majority {
+ return restoreGeneration{}, errFileVersionNotFound
+ }
+ return restoreGeneration{}, errErasureReadQuorum
+ }
+ keys, gen.split, gen.ongoing = neutral, true, make(map[string]bool)
+ }
+ for i := range disks {
+ if keys[i] == nil || *keys[i] != bestKey {
+ continue
+ }
+ fi, err := getFileInfo(raws[i].Buf, bucket, object, versionID, fileInfoOpts{Data: true})
+ if err != nil {
+ continue
+ }
+ gen.drives[i], gen.disks[i] = fi, disks[i]
+ if gen.split {
+ if st, err := parseRestoreObjStatus(fi.Metadata[xhttp.AmzRestore]); err == nil && st.Ongoing() {
+ gen.ongoing[fi.Metadata[restoreAttemptKey]] = true
+ }
+ }
+ if !gen.fi.IsValid() {
+ gen.fi = fi
+ }
+ }
+ if !gen.fi.IsValid() && !gen.fi.Deleted {
+ return restoreGeneration{}, errErasureReadQuorum
+ }
+ if gen.split {
+ gen.fi.Metadata = maps.Clone(gen.fi.Metadata)
+ for _, k := range restoreFieldKeys {
+ delete(gen.fi.Metadata, k)
+ }
+ }
+ return gen, nil
+}
+
+// updateRestoreStatus rewrites the restore fields of one stored version. It
+// selects the authoritative generation of the version, see
+// readRestoreGeneration, patches only the restore fields onto exactly that
+// generation, keeping every other field and its ModTime, and writes it back
+// only to the drives that hold it. The caller holds the object lock. update
+// edits the metadata in place, or returns an error, errRestoreUnchanged to
+// write nothing; ongoing names the attempts whose ongoing marker the
+// generation holds. A failed read never removes the version, and a failed
+// write puts the generation it read back.
+func (er erasureObjects) updateRestoreStatus(ctx context.Context, bucket, object, versionID string, update func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error) (ObjectInfo, error) {
+ gen, err := er.readRestoreGeneration(ctx, bucket, object, versionID)
+ if err != nil {
+ return ObjectInfo{}, toObjectErr(err, bucket, object, versionID)
+ }
+ fi := gen.fi
+ oi := fi.ToObjectInfo(bucket, object, false)
+ if fi.Deleted {
+ return oi, toObjectErr(errMethodNotAllowed, bucket, object)
+ }
+ ongoing := gen.ongoing
+ if !gen.split && oi.RestoreOngoing {
+ ongoing = map[string]bool{oi.UserDefined[restoreAttemptKey]: true}
+ }
+ meta := maps.Clone(fi.Metadata)
+ if err := update(oi, meta, ongoing); err != nil {
+ return oi, err
+ }
+ write := func(meta func(FileInfo) map[string]string) error {
+ metaArr := make([]FileInfo, len(gen.drives))
+ for i, d := range gen.drives {
+ if gen.disks[i] == nil {
+ continue
+ }
+ if !d.InlineData() {
+ d.Data = nil
+ }
+ d.Metadata = meta(d)
+ metaArr[i] = d
+ }
+ // The write assigns each drive its slot as erasure index: order the
+ // drives by the version's distribution first.
+ disks, metaArr := shuffleDisksAndPartsMetadata(gen.disks, metaArr, fi)
+ _, err := writeUniqueFileInfo(ctx, disks, "", bucket, object, metaArr, fi.WriteQuorum(er.defaultWQuorum()))
+ return err
+ }
+ if err := write(func(FileInfo) map[string]string { return meta }); err != nil {
+ // Without the read generation back, its drives could hold two
+ // revisions and no quorum for either. A split that stays differs
+ // only in restore fields, which the next write or reset resolves.
+ if rerr := write(func(d FileInfo) map[string]string { return d.Metadata }); rerr != nil {
+ storageLogIf(ctx, fmt.Errorf("Unable to put back the restore status of %s/%s(%s): %w", bucket, object, versionID, rerr))
+ }
+ return oi, toObjectErr(err, bucket, object)
+ }
+ fi.Metadata = meta
+ return fi.ToObjectInfo(bucket, object, false), nil
+}
+
+// clearRestoreStatus makes a version retryable again while attempt a still
+// owns its ongoing marker. It never brings back an earlier completed status:
+// POST only starts a restore when that restored copy is gone.
+func (er erasureObjects) clearRestoreStatus(ctx context.Context, bucket, object, versionID string, a *restoreAttempt) error {
+ _, err := er.updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error {
+ id := ""
+ if a != nil {
+ id = a.ID
+ if !a.targets(oi) {
+ return errRestoreUnchanged
+ }
+ }
+ // Clear only this attempt's marker. Where drives disagree, another
+ // attempt's marker may be the acknowledged one: leave that to it,
+ // or to a later POST, which takes it over under the lock.
+ if !ongoing[id] || len(ongoing) != 1 {
+ return errRestoreUnchanged
+ }
+ delete(meta, xhttp.AmzRestore)
+ delete(meta, xhttp.AmzRestoreExpiryDays)
+ delete(meta, xhttp.AmzRestoreRequestDate)
+ delete(meta, restoreAttemptKey)
+ return nil
+ })
+ if errors.Is(err, errRestoreUnchanged) {
+ return nil
+ }
+ return err
+}
+
+// restoreLock is the object lock that restores and their status updates
+// share: the lock every write of the object takes.
+func (z *erasureServerPools) restoreLock(bucket, object string) RWLocker {
+ return z.NewObjectNSLock(bucket, object)
+}
+
+// restoreRequestStatus is what a POST restore recorded on its version.
+type restoreRequestStatus struct {
+ objInfo ObjectInfo
+ // attempt is the restore that must rehydrate the version; nil when a
+ // restored copy was still there and its expiry was renewed.
+ attempt *restoreAttempt
+ // owner is the restore lock of the version, held for attempt. The
+ // caller hands it to the worker that runs attempt, or releases it.
+ owner *restoreOwner
+ // date is the request date recorded with attempt.
+ date string
+}
+
+// restoreRequest records a POST restore on target, the version that POST
+// resolved, on a fresh read under the object lock. The version must still
+// be target: a replacement under the same version ID is refused. A copy that
+// still holds its restored data is renewed. Otherwise the version is marked
+// ongoing for a new attempt, which also takes over a marker that no running
+// restore holds. Only the copy that is to be restored is written, so no copy
+// claims a restored copy it does not hold.
+//
+// It takes the restore lock R of the version before the object lock O. A
+// running restore holds R for its whole run, so a POST that cannot take R
+// within the takeover wait answers from the drives: in progress when they
+// show the version ongoing, and a retryable timeout otherwise, such as while
+// another POST of the version registers. ctx must outlive the request: a new
+// attempt keeps R in the returned status.
+func (z *erasureServerPools) restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (status restoreRequestStatus, err error) {
+ object = encodeDirObject(object)
+ versionID := restoreVersionID(target.VersionID)
+ owner, err := z.takeRestoreOwner(ctx, bucket, object, versionID, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait))
+ if err != nil {
+ if errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil {
+ running, rerr := z.restoreRunning(ctx, bucket, object, versionID)
+ switch {
+ case rerr != nil:
+ return restoreRequestStatus{}, rerr
+ case running:
+ return restoreRequestStatus{}, errRestoreInProgress
+ }
+ }
+ return restoreRequestStatus{}, err
+ }
+ defer func() {
+ if status.attempt == nil {
+ owner.release()
+ }
+ }()
+ lk := z.restoreLock(bucket, object)
+ lkctx, err := lk.GetLock(owner.ctx, globalOperationTimeout)
+ if err != nil {
+ return restoreRequestStatus{}, err
+ }
+ ctx = lkctx.Context()
+ defer lk.Unlock(lkctx)
+
+ expected := newRestoreAttempt(target)
+ copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{
+ VersionID: versionID,
+ Versioned: opts.Versioned,
+ VersionSuspended: opts.VersionSuspended,
+ Transition: opts.Transition,
+ })
+ if err != nil {
+ return restoreRequestStatus{}, err
+ }
+ // Restore the copy reads serve first, outside pools being decommissioned,
+ // as RestoreTransitionedObject does.
+ pool := -1
+ for _, c := range copies {
+ if !z.IsSuspended(c.Index) {
+ pool = c.Index
+ break
+ }
+ }
+ if pool < 0 {
+ return restoreRequestStatus{}, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
+ }
+
+ rreq := opts.Transition.RestoreRequest
+ date := UTCNow().Format(http.TimeFormat)
+ var attempt *restoreAttempt
+ oi, err := z.serverPools[pool].getHashedSet(object).updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, _ map[string]bool) error {
+ if !expected.targets(oi) {
+ // Not the version POST resolved, or no longer transitioned.
+ return InvalidObjectState{Bucket: bucket, Object: decodeDirObject(object)}
+ }
+ attempt = nil
+ meta[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days)
+ meta[xhttp.AmzRestoreRequestDate] = date
+ if !oi.RestoreOngoing && !oi.RestoreExpires.IsZero() {
+ // Cleanup removes a restored copy together with its header.
+ meta[xhttp.AmzRestore] = completedRestoreObj(opts.Transition.RestoreExpiry).String()
+ delete(meta, restoreAttemptKey)
+ return nil
+ }
+ attempt = newRestoreAttempt(oi)
+ meta[xhttp.AmzRestore] = ongoingRestoreObj().String()
+ meta[restoreAttemptKey] = attempt.ID
+ return nil
+ })
+ if err != nil {
+ if attempt != nil {
+ // The marker may have reached some drives, or all of them.
+ z.recoverRestoreStatus(bucket, object, ObjectOptions{
+ VersionID: attempt.VersionID,
+ Transition: TransitionOptions{attempt: attempt},
+ })
+ }
+ return restoreRequestStatus{}, err
+ }
+ if attempt == nil {
+ return restoreRequestStatus{objInfo: oi}, nil
+ }
+ return restoreRequestStatus{objInfo: oi, attempt: attempt, owner: owner, date: date}, nil
+}
+
+// restoreRunning reports, without the object lock and within the takeover
+// wait, whether the drives show a restore of the version ongoing in a pool.
+// An error means that this cannot be told.
+func (z *erasureServerPools) restoreRunning(ctx context.Context, bucket, object, versionID string) (bool, error) {
+ ctx, cancel := context.WithTimeout(ctx, restoreTakeoverWait)
+ defer cancel()
+ for _, pool := range z.serverPools {
+ gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, versionID)
+ if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) {
+ continue
+ }
+ if err == nil && ctx.Err() != nil {
+ err = ctx.Err()
+ }
+ if err != nil {
+ return false, toObjectErr(err, bucket, object, versionID)
+ }
+ if len(gen.ongoing) > 0 || !gen.split && gen.fi.ToObjectInfo(bucket, object, false).RestoreOngoing {
+ return true, nil
+ }
+ }
+ return false, nil
+}
+
+// restorePoolIdx returns the pool whose copy the restore rehydrates: the one
+// POST marked for its attempt, or for a direct caller the existing version.
+func (z *erasureServerPools) restorePoolIdx(ctx context.Context, bucket, object string, opts ObjectOptions) (int, error) {
+ a := opts.Transition.attempt
+ if a == nil {
+ // Restore an existing version, not the latest version's write location.
+ lookupOpts := opts
+ lookupOpts.MetadataChg = true
+ idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts)
+ if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+ return idx, restoreResetError{err}
+ }
+ return idx, err
+ }
+ found := false
+ for idx, pool := range z.serverPools {
+ gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, a.VersionID)
+ if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) {
+ continue
+ }
+ if err != nil {
+ return -1, restoreResetError{toObjectErr(err, bucket, object, a.VersionID)}
+ }
+ found = true
+ if !z.IsSuspended(idx) && a.ownsGeneration(gen, bucket, object) {
+ return idx, nil
+ }
+ }
+ if !found {
+ return -1, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: a.VersionID}
+ }
+ return -1, errRestoreSuperseded
+}
+
+// recoverRestoreStatus retries, in memory and for a bounded time, to clear the
+// ongoing marker that a failed restore left behind, so a later POST is not
+// refused. Each try takes the object lock afresh and clears the marker only
+// while it still belongs to this attempt.
+func (z *erasureServerPools) recoverRestoreStatus(bucket, object string, opts ObjectOptions) {
+ go func() {
+ var err error
+ for _, delay := range restoreRecoveryDelays {
+ select {
+ case <-GlobalContext.Done():
+ return
+ case <-time.After(delay):
+ }
+ if err = z.resetRestoreStatus(bucket, object, opts); err == nil {
+ return
+ }
+ }
+ storageLogIf(GlobalContext, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, err))
+ }()
+}
+
+// resetRestoreStatus is one try of recoverRestoreStatus.
+func (z *erasureServerPools) resetRestoreStatus(bucket, object string, opts ObjectOptions) error {
+ ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
+ defer cancel()
+ lk := z.restoreLock(bucket, object)
+ lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+ if err != nil {
+ return err
+ }
+ defer lk.Unlock(lkctx)
+ ctx = lkctx.Context()
+
+ a := opts.Transition.attempt
+ versionID := opts.VersionID
+ if a != nil {
+ versionID = a.VersionID
+ }
+ for _, pool := range z.serverPools {
+ err := pool.getHashedSet(object).clearRestoreStatus(ctx, bucket, object, versionID, a)
+ if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !isErrMethodNotAllowed(err) && !isErrBucketNotFound(err) {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/cmd/restore-capability.go b/cmd/restore-capability.go
new file mode 100644
index 000000000..311c1c6b3
--- /dev/null
+++ b/cmd/restore-capability.go
@@ -0,0 +1,184 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "slices"
+ "sync"
+ "time"
+
+ "github.com/minio/minio/internal/grid"
+)
+
+// A native restore keeps its multipart upload private and marks the version
+// it rehydrates (restore-attempt.go). A server that predates this ignores
+// the marker: it would list, change, abort or complete the upload. Native
+// restore therefore runs only while every server of the cluster positively
+// reports that it understands the marker. A server that does not answer,
+// or does not know the question, as an older server does not, blocks it.
+
+// nativeRestoreCapability is the native restore protocol this server speaks.
+const nativeRestoreCapability = "2"
+
+const (
+ peerRESTRestoreCapability = "restore"
+ // restoreCapabilityTTL bounds how long a positive answer is reused while
+ // no server reconnected.
+ restoreCapabilityTTL = 10 * time.Second
+ // restoreCapabilityTimeout bounds the question to each server.
+ restoreCapabilityTimeout = 5 * time.Second
+)
+
+var errNativeRestoreUnavailable = errors.New("native restore is unavailable during a rolling upgrade: not every server of the cluster supports it yet")
+
+// restoreCapabilityHandler answers the question on this server.
+func restoreCapabilityHandler(*grid.MSS) (*grid.MSS, *grid.RemoteErr) {
+ return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: nativeRestoreCapability}), nil
+}
+
+// restoreCapabilityPeer is another server of the cluster.
+type restoreCapabilityPeer interface {
+ // restoreConnection identifies the current connection to the server; it
+ // changes when the server reconnects, as a restarted one does. ok is
+ // false when the server is not connected.
+ restoreConnection() (key string, ok bool)
+ restoreCapability(ctx context.Context) (string, error)
+}
+
+type restorePeerClient struct{ *peerRESTClient }
+
+func (c restorePeerClient) restoreConnection() (string, bool) {
+ if c.peerRESTClient == nil || c.gridConn == nil {
+ return "", false
+ }
+ conn := c.gridConn()
+ if conn == nil {
+ return "", false
+ }
+ st := conn.Stats()
+ if st.Connected == 0 {
+ return "", false
+ }
+ // Keys are compared by the position of their server.
+ return fmt.Sprintf("%d/%d", st.LastConnectTime.UnixNano(), st.ReconnectCount), true
+}
+
+func (c restorePeerClient) restoreCapability(ctx context.Context) (string, error) {
+ conn := c.gridConn()
+ if conn == nil {
+ return "", errPeerOffline
+ }
+ resp, err := restoreCapabilityRPC.Call(ctx, conn, grid.NewMSS())
+ if err != nil {
+ return "", err
+ }
+ return resp.Get(peerRESTRestoreCapability), nil
+}
+
+// restoreCapabilityGate decides whether native restore may run, caching a
+// positive answer for restoreCapabilityTTL while the connections to all
+// servers stay the same. A negative answer is not cached.
+type restoreCapabilityGate struct {
+ peers func() []restoreCapabilityPeer
+
+ once sync.Once
+ turn chan struct{} // one check at a time
+ checked time.Time
+ keys []string
+}
+
+var globalRestoreCapability = &restoreCapabilityGate{peers: notificationRestorePeers}
+
+func notificationRestorePeers() []restoreCapabilityPeer {
+ if globalNotificationSys == nil {
+ return nil
+ }
+ peers := make([]restoreCapabilityPeer, 0, len(globalNotificationSys.peerClients))
+ for _, client := range globalNotificationSys.peerClients {
+ peers = append(peers, restorePeerClient{client})
+ }
+ return peers
+}
+
+// nativeRestoreAvailable returns errNativeRestoreUnavailable unless every
+// other server of the cluster supports native restore.
+func nativeRestoreAvailable(ctx context.Context) error {
+ return globalRestoreCapability.check(ctx)
+}
+
+func (g *restoreCapabilityGate) check(ctx context.Context) error {
+ g.once.Do(func() { g.turn = make(chan struct{}, 1) })
+ // One deadline covers the wait for another check and the questions.
+ ctx, cancel := context.WithTimeout(ctx, restoreCapabilityTimeout)
+ defer cancel()
+ select {
+ case g.turn <- struct{}{}:
+ case <-ctx.Done():
+ return errNativeRestoreUnavailable
+ }
+ defer func() { <-g.turn }()
+
+ // Read the connections only now: one may have changed while this check
+ // waited for another.
+ peers := g.peers()
+ connections := func() ([]string, bool) {
+ keys := make([]string, len(peers))
+ for i, p := range peers {
+ key, ok := p.restoreConnection()
+ if !ok {
+ return nil, false
+ }
+ keys[i] = key
+ }
+ return keys, true
+ }
+ keys, ok := connections()
+ if !ok {
+ g.checked, g.keys = time.Time{}, nil
+ return errNativeRestoreUnavailable
+ }
+ if ctx.Err() != nil {
+ return errNativeRestoreUnavailable
+ }
+ if !g.checked.IsZero() && time.Since(g.checked) < restoreCapabilityTTL && slices.Equal(keys, g.keys) {
+ return nil
+ }
+ g.checked, g.keys = time.Time{}, nil
+ answers := make([]bool, len(peers))
+ var wg sync.WaitGroup
+ for i, p := range peers {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ version, err := p.restoreCapability(ctx)
+ answers[i] = err == nil && version == nativeRestoreCapability
+ }()
+ }
+ wg.Wait()
+ // An answer counts only for the connection it was asked on: a server
+ // that reconnected meanwhile may be another binary.
+ after, ok := connections()
+ if !ok || !slices.Equal(keys, after) || slices.Contains(answers, false) || ctx.Err() != nil {
+ return errNativeRestoreUnavailable
+ }
+ g.checked, g.keys = time.Now(), keys
+ return nil
+}
diff --git a/cmd/restore-capability_test.go b/cmd/restore-capability_test.go
new file mode 100644
index 000000000..306764d12
--- /dev/null
+++ b/cmd/restore-capability_test.go
@@ -0,0 +1,271 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "reflect"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/versioning"
+ "github.com/minio/minio/internal/grid"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// A server answers the capability question over the grid. One that does not
+// know it, as an older server, or answers something else, blocks native
+// restore, and so does one that is not connected.
+func TestNativeRestoreCapabilityGrid(t *testing.T) {
+ tg, err := grid.SetupTestGrid(4)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(tg.Cleanup)
+ if err := restoreCapabilityRPC.Register(tg.Managers[1], restoreCapabilityHandler); err != nil {
+ t.Fatal(err)
+ }
+ // Managers[2] never registers the handler.
+ if err := restoreCapabilityRPC.Register(tg.Managers[3], func(*grid.MSS) (*grid.MSS, *grid.RemoteErr) {
+ return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: "0"}), nil
+ }); err != nil {
+ t.Fatal(err)
+ }
+ peer := func(i int) restoreCapabilityPeer {
+ return restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[i]) }}}
+ }
+ offline := restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return nil }}}
+ for _, tc := range []struct {
+ name string
+ peers []restoreCapabilityPeer
+ ok bool
+ }{
+ {"single-server", nil, true},
+ {"all-capable", []restoreCapabilityPeer{peer(1)}, true},
+ {"older-server", []restoreCapabilityPeer{peer(1), peer(2)}, false},
+ {"other-answer", []restoreCapabilityPeer{peer(1), peer(3)}, false},
+ {"offline", []restoreCapabilityPeer{peer(1), offline}, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return tc.peers }}
+ if err := g.check(t.Context()); (err == nil) != tc.ok {
+ t.Errorf("capability: %v, want available=%v", err, tc.ok)
+ }
+ })
+ }
+}
+
+// restoreCapabilityFake is a server whose connection and answer a test sets.
+type restoreCapabilityFake struct {
+ capable atomic.Bool
+ connected atomic.Bool
+ conn atomic.Int64
+ asked atomic.Int64
+}
+
+func newRestoreCapabilityFake() *restoreCapabilityFake {
+ p := &restoreCapabilityFake{}
+ p.capable.Store(true)
+ p.connected.Store(true)
+ return p
+}
+
+func (p *restoreCapabilityFake) restoreConnection() (string, bool) {
+ return strings.Repeat("c", int(p.conn.Load())), p.connected.Load()
+}
+
+func (p *restoreCapabilityFake) restoreCapability(context.Context) (string, error) {
+ p.asked.Add(1)
+ if !p.capable.Load() {
+ return "", errors.New("Invalid Handler for type: RestoreCapability")
+ }
+ return nativeRestoreCapability, nil
+}
+
+// A positive answer is reused while the connections stay the same; a
+// reconnect asks again, a disconnected server blocks without being asked,
+// and a negative answer is not reused.
+func TestNativeRestoreCapabilityCache(t *testing.T) {
+ p := newRestoreCapabilityFake()
+ g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+ for range 3 {
+ if err := g.check(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if n := p.asked.Load(); n != 1 {
+ t.Errorf("asked %d times while connected and capable, want 1", n)
+ }
+ // The server restarts with an older binary and reconnects.
+ p.capable.Store(false)
+ p.conn.Add(1)
+ if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) {
+ t.Errorf("after a reconnect to an older server: %v", err)
+ }
+ p.capable.Store(true)
+ if err := g.check(t.Context()); err != nil {
+ t.Errorf("a negative answer was reused: %v", err)
+ }
+ asked := p.asked.Load()
+ p.connected.Store(false)
+ if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) || p.asked.Load() != asked {
+ t.Errorf("disconnected server: %v, asked %d -> %d", err, asked, p.asked.Load())
+ }
+}
+
+// During a rolling upgrade POST restore answers 503 and writes nothing, and
+// a restore queued before it creates no upload and leaves the version
+// retryable.
+func TestNativeRestoreRollingUpgradeGate(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := restoreIsolationSeed(t, r, 0, false, true)
+ p := newRestoreCapabilityFake()
+ previous := globalRestoreCapability
+ globalRestoreCapability = &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+ t.Cleanup(func() { globalRestoreCapability = previous })
+
+ p.capable.Store(false)
+ before := r.fileInfo(oi.VersionID)
+ layer := newRestoreIsolationLayer(r)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "rolling upgrade") {
+ t.Errorf("POST during a rolling upgrade: %d %s", rec.Code, rec.Body.String())
+ }
+ if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before.Metadata, after.Metadata) {
+ t.Error("refused POST changed the version")
+ }
+ select {
+ case err := <-layer.done:
+ t.Fatalf("refused POST started a restore: %v", err)
+ default:
+ }
+
+ p.capable.Store(true)
+ release := make(chan struct{})
+ layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ <-release
+ return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ close(release)
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ // A server is replaced by an older binary while the restore is queued.
+ p.capable.Store(false)
+ p.conn.Add(1)
+ close(release)
+ if err := layer.wait(t); !errors.Is(err, errNativeRestoreUnavailable) {
+ t.Fatalf("queued restore during a rolling upgrade: %v", err)
+ }
+ if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+ t.Errorf("queued restore created an upload during a rolling upgrade: %v", uploads)
+ }
+ info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil || info.RestoreOngoing || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" {
+ t.Errorf("version after a refused queued restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+ }
+}
+
+// restoreCapabilityHeld is a fake server whose answer a test holds.
+type restoreCapabilityHeld struct {
+ restoreCapabilityFake
+ asking chan struct{}
+ release chan struct{}
+}
+
+func (p *restoreCapabilityHeld) restoreCapability(ctx context.Context) (string, error) {
+ v, err := p.restoreCapabilityFake.restoreCapability(ctx)
+ p.asking <- struct{}{}
+ select {
+ case <-p.release:
+ return v, err
+ case <-ctx.Done():
+ return "", ctx.Err()
+ }
+}
+
+// A check that waits for another reads the connections once its turn
+// comes, an answer counts only if its server did not reconnect meanwhile,
+// and a waiting check ends with its context.
+func TestNativeRestoreCapabilityRevalidates(t *testing.T) {
+ t.Run("reconnect-during-check", func(t *testing.T) {
+ held := func() *restoreCapabilityHeld {
+ p := &restoreCapabilityHeld{asking: make(chan struct{}, 4), release: make(chan struct{})}
+ p.capable.Store(true)
+ p.connected.Store(true)
+ return p
+ }
+ fast, slow := held(), held()
+ g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{fast, slow} }}
+ first := make(chan error, 1)
+ go func() { first <- g.check(t.Context()) }()
+ <-fast.asking
+ <-slow.asking
+ waiting := make(chan error, 1)
+ go func() { waiting <- g.check(t.Context()) }()
+ // Both answered "capable"; then fast reconnects as an older server.
+ fast.capable.Store(false)
+ fast.conn.Add(1)
+ close(fast.release)
+ close(slow.release)
+ if err := <-first; !errors.Is(err, errNativeRestoreUnavailable) {
+ t.Errorf("check whose server reconnected while it asked: %v", err)
+ }
+ if err := <-waiting; !errors.Is(err, errNativeRestoreUnavailable) {
+ t.Errorf("waiting check accepted a reconnected older server: %v", err)
+ }
+ })
+ t.Run("canceled-cached", func(t *testing.T) {
+ p := newRestoreCapabilityFake()
+ g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }}
+ if err := g.check(t.Context()); err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithCancel(t.Context())
+ cancel()
+ if err := g.check(ctx); err == nil {
+ t.Error("a cached answer accepted an ended context")
+ }
+ })
+ t.Run("canceled-wait", func(t *testing.T) {
+ slow := &restoreCapabilityHeld{restoreCapabilityFake: restoreCapabilityFake{}, asking: make(chan struct{}, 4), release: make(chan struct{})}
+ slow.capable.Store(true)
+ slow.connected.Store(true)
+ defer close(slow.release)
+ g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{slow} }}
+ go g.check(t.Context())
+ <-slow.asking
+ ctx, cancel := context.WithCancel(t.Context())
+ waiting := make(chan error, 1)
+ go func() { waiting <- g.check(ctx) }()
+ cancel()
+ select {
+ case err := <-waiting:
+ if err == nil {
+ t.Error("canceled check succeeded")
+ }
+ case <-time.After(time.Second):
+ t.Error("canceled check kept waiting for another check")
+ }
+ })
+}
diff --git a/cmd/restore-native-encoding_test.go b/cmd/restore-native-encoding_test.go
new file mode 100644
index 000000000..6a4fa3ba9
--- /dev/null
+++ b/cmd/restore-native-encoding_test.go
@@ -0,0 +1,248 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/versioning"
+ "github.com/minio/minio/internal/event"
+ "github.com/minio/minio/internal/hash"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/minio/minio/internal/kms"
+ "github.com/minio/minio/internal/pubsub"
+ "github.com/minio/mux"
+)
+
+// Exercise the stored representation produced by real S3 uploads. Native
+// restore copies encoded bytes, but must retain logical sizes and identity.
+func TestNativeRestoreEncodedObjects(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, encoding := range []string{"plain", "compressed", "ssec", "s3-compressed"} {
+ for _, layout := range []string{"single", "multipart", "sparse-one-part"} {
+ t.Run(fmt.Sprintf("pools=%d/%s/%s", pools, encoding, layout), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ r.object = "restore-encoded.txt"
+ previousKMS, previousTLS := GlobalKMS, globalIsTLS
+ t.Cleanup(func() { GlobalKMS, globalIsTLS = previousKMS, previousTLS })
+ globalIsTLS = true
+ headers := make(map[string]string)
+ readHeaders := make(map[string]string)
+ if encoding == "compressed" || encoding == "s3-compressed" {
+ enableAttributesPartsCompression(t)
+ }
+ if encoding == "ssec" {
+ headers = ssecTestHeaders(0x53)
+ readHeaders = ssecTestHeaders(0x53)
+ }
+ if encoding == "s3-compressed" {
+ var err error
+ GlobalKMS, err = kms.NewBuiltin("restore-test-key", bytes.Repeat([]byte{0x59}, 32))
+ if err != nil {
+ t.Fatal(err)
+ }
+ headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+ }
+ previousObjectAPI := newObjectLayerFn()
+ t.Cleanup(func() { setObjectLayer(previousObjectAPI) })
+ router := initTestAPIEndPoints(r.z, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "PutObject", "GetObject"})
+ bodies := [][]byte{bytes.Repeat([]byte("encoded-restore-"), compMinIndexSize/16+1)}
+ partNumbers := []int{1}
+ switch layout {
+ case "multipart":
+ bodies = append(bodies, []byte("last encoded part"))
+ partNumbers = []int{1, 2}
+ case "sparse-one-part":
+ partNumbers = []int{5}
+ }
+ want := bytes.Join(bodies, nil)
+ if layout == "single" {
+ headers[xhttp.AmzChecksumCRC32] = mustChecksum(t, hash.ChecksumCRC32, want)
+ putCopyChecksumSource(t, router, globalActiveCred, r.bucket, r.object, want, headers)
+ } else {
+ attributesPartsUpload(t, router, globalActiveCred, r.bucket, r.object, headers, bodies, partNumbers)
+ }
+ before, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if before.IsCompressed() != (encoding == "compressed" || encoding == "s3-compressed") {
+ t.Fatalf("unexpected compression state: %v", before.IsCompressed())
+ }
+ if before.IsCompressed() && len(before.Parts[0].Index) == 0 {
+ t.Fatal("fixture must have a compression index")
+ }
+ if layout == "single" && len(before.Checksum) == 0 {
+ t.Fatal("fixture must have a stored checksum")
+ }
+ objectURL := getGetObjectURL("", r.bucket, r.object) + "?versionId=" + url.QueryEscape(before.VersionID)
+ beforeGet := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, objectURL, nil, readHeaders)
+ if beforeGet.Code != http.StatusOK || !bytes.Equal(beforeGet.Body.Bytes(), want) {
+ t.Fatalf("fixture GET: %d bytes=%d want=%d", beforeGet.Code, beforeGet.Body.Len(), len(want))
+ }
+ beforeETags := beforeGet.Header()[xhttp.ETag]
+ if len(beforeETags) != 1 || beforeETags[0] == "" {
+ t.Fatal("fixture GET must expose a nonempty ETag")
+ }
+ beforeRaw := nativeRestoreSnapshot(t, r)
+ topts := r.opts(before.VersionID)
+ topts.MTime = before.ModTime
+ topts.Transition = TransitionOptions{Tier: "WARM", ETag: before.ETag}
+ if err := r.z.TransitionObject(t.Context(), r.bucket, r.object, topts); err != nil {
+ t.Fatal(err)
+ }
+ observed := nativeRestoreHTTPObserver{r.z, make(chan nativeRestoreHTTPResult, 1)}
+ api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return observed }}
+ events, done := make(chan event.Event, 1), make(chan struct{})
+ defer close(done)
+ if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRestoreCompleted), events, done, nil); err != nil {
+ t.Fatal(err)
+ }
+ restoreXML := []byte(`1`)
+ req, err := newTestSignedRequestV4(http.MethodPost, objectURL+"&restore", int64(len(restoreXML)), bytes.NewReader(restoreXML), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := req.ParseForm(); err != nil {
+ t.Fatal(err)
+ }
+ req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object})
+ rec := httptest.NewRecorder()
+ api.PostRestoreObjectHandler(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("restore POST: %d %s", rec.Code, rec.Body.String())
+ }
+ select {
+ case result := <-observed.done:
+ if result.err != nil {
+ t.Fatalf("background restore: %v", result.err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("restore did not finish")
+ }
+ select {
+ case <-events:
+ case <-time.After(5 * time.Second):
+ t.Fatal("restore completion event missing")
+ }
+ after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(before.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after.ETag != before.ETag || after.Size != before.Size || !after.ModTime.Equal(before.ModTime) {
+ t.Error("restore changed content identity")
+ }
+ afterRaw := nativeRestoreSnapshot(t, r)
+ for pool := range beforeRaw {
+ if len(afterRaw[pool]) != len(beforeRaw[pool]) {
+ t.Fatalf("pool %d version count changed", pool)
+ }
+ for i, fi := range beforeRaw[pool] {
+ if !bytes.Equal(fi.Checksum, afterRaw[pool][i].Checksum) {
+ t.Error("restore lost stored checksum")
+ }
+ }
+ }
+ if len(after.Parts) != len(before.Parts) {
+ t.Fatal("restore changed part count")
+ }
+ for i, part := range after.Parts {
+ old := before.Parts[i]
+ if part.Number != old.Number || part.Size != old.Size || part.ActualSize != old.ActualSize || !bytes.Equal(part.Index, old.Index) {
+ t.Errorf("part %d layout or compression index changed", i)
+ }
+ }
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ reads := tier.gets.Load()
+ got := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, objectURL, nil, readHeaders)
+ if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), want) {
+ t.Errorf("restored GET: %d bytes=%d want=%d", got.Code, got.Body.Len(), len(want))
+ }
+ afterETags := got.Header()[xhttp.ETag]
+ if len(afterETags) != 1 || afterETags[0] == "" || afterETags[0] != beforeETags[0] {
+ t.Error("restore changed client ETag")
+ }
+ for i, number := range partNumbers {
+ part := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, objectURL+fmt.Sprintf("&partNumber=%d", number), nil, readHeaders)
+ if part.Code != http.StatusPartialContent || !bytes.Equal(part.Body.Bytes(), bodies[i]) {
+ t.Errorf("restored part %d: %d bytes=%d want=%d", number, part.Code, part.Body.Len(), len(bodies[i]))
+ }
+ }
+ if tier.gets.Load() != reads {
+ t.Error("restored GET still used remote tier")
+ }
+ })
+ }
+ }
+ }
+}
+
+// XL metadata predating compression has no ActualSize. Conversion and healing
+// preserve those zero values; a plain multipart restore must derive the size.
+func TestNativeRestoreLegacyPartSizes(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, "")
+ oi, body := nativeRestoreSeed(t, r, 0, true, true)
+ for _, disk := range r.z.serverPools[0].getHashedSet(r.object).getDisks() {
+ fi, err := disk.ReadVersion(t.Context(), r.bucket, r.bucket, r.object, oi.VersionID, ReadOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ for i := range fi.Parts {
+ fi.Parts[i].ActualSize = 0
+ }
+ delete(fi.Metadata, ReservedMetadataPrefix+"actual-size")
+ if err := disk.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ before, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, part := range before.Parts {
+ if part.ActualSize != 0 {
+ t.Fatal("legacy fixture was normalized")
+ }
+ }
+ _, got, err := r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("legacy fixture read: %v", err)
+ }
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+ if err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ reads := tier.gets.Load()
+ _, got, err = r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("legacy restored read: %v", err)
+ }
+ if tier.gets.Load() != reads {
+ t.Error("legacy restored read used the remote tier")
+ }
+}
diff --git a/cmd/restore-native-generation_test.go b/cmd/restore-native-generation_test.go
new file mode 100644
index 000000000..b7383ef8f
--- /dev/null
+++ b/cmd/restore-native-generation_test.go
@@ -0,0 +1,724 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "crypto/rand"
+ "errors"
+ "fmt"
+ "maps"
+ "net/http"
+ "net/http/httptest"
+ "reflect"
+ "runtime"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/versioning"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// restoreGenerationDisk fails metadata writes or updates of the object, or
+// metadata reads of uploads, while its toggles are set.
+type restoreGenerationDisk struct {
+ StorageAPI
+ bucket, object string
+ write, update, uploadRead *atomic.Bool
+}
+
+func (d restoreGenerationDisk) WriteMetadata(ctx context.Context, orig, vol, path string, fi FileInfo) error {
+ if d.write != nil && d.write.Load() && vol == d.bucket && path == d.object {
+ return errFaultyDisk
+ }
+ return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi)
+}
+
+func (d restoreGenerationDisk) UpdateMetadata(ctx context.Context, vol, path string, fi FileInfo, opts UpdateMetadataOpts) error {
+ if d.update != nil && d.update.Load() && vol == d.bucket && path == d.object {
+ return errFaultyDisk
+ }
+ return d.StorageAPI.UpdateMetadata(ctx, vol, path, fi, opts)
+}
+
+func (d restoreGenerationDisk) ReadVersion(ctx context.Context, orig, vol, path, version string, opts ReadOptions) (FileInfo, error) {
+ if d.uploadRead != nil && d.uploadRead.Load() && vol == minioMetaMultipartBucket {
+ return FileInfo{}, errFaultyDisk
+ }
+ return d.StorageAPI.ReadVersion(ctx, orig, vol, path, version, opts)
+}
+
+// restoreGenerationOffline makes a drive's object metadata unreadable and
+// unwritable while set, its uploads still readable.
+type restoreGenerationOffline struct {
+ StorageAPI
+ bucket, object string
+ fail *atomic.Bool
+}
+
+func (d restoreGenerationOffline) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
+ if d.fail.Load() && bucket == d.bucket && object == d.object {
+ return RawFileInfo{}, errFaultyDisk
+ }
+ return d.StorageAPI.ReadXL(ctx, bucket, object, data)
+}
+
+func (d restoreGenerationOffline) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
+ if d.fail.Load() && bucket == d.bucket && object == d.object {
+ return FileInfo{}, errFaultyDisk
+ }
+ return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
+}
+
+func (d restoreGenerationOffline) WriteMetadata(ctx context.Context, orig, bucket, object string, fi FileInfo) error {
+ if d.fail.Load() && bucket == d.bucket && object == d.object {
+ return errFaultyDisk
+ }
+ return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi)
+}
+
+// restoreGenerationMetadata returns the version's metadata on each drive.
+func restoreGenerationMetadata(t *testing.T, disks []StorageAPI, r *restoreTest, versionID string) []map[string]string {
+ t.Helper()
+ metas := make([]map[string]string, len(disks))
+ for i, d := range disks {
+ fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, versionID, ReadOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ metas[i] = fi.Metadata
+ }
+ return metas
+}
+
+// An Object Lock update acknowledged on a quorum, which one drive missed,
+// survives POST restore: POST patches the restore fields onto the quorum's
+// generation and never writes the stale drive's metadata anywhere.
+func TestRestoreStatusUsesQuorumGeneration(t *testing.T) {
+ for _, kind := range []string{"legal-hold", "retention"} {
+ t.Run(kind, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ enableBucketObjectLock(t, r.bucket)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ initial := []byte(`OFF`)
+ updated := []byte(`ON`)
+ key := strings.ToLower(xhttp.AmzObjectLockLegalHold)
+ if kind == "retention" {
+ initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour))
+ updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))
+ key = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
+ }
+ if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ var fail atomic.Bool
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, update: &fail}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ fail.Store(true)
+ if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK {
+ t.Fatalf("protection update: %d %s", rec.Code, rec.Body.String())
+ }
+ fail.Store(false)
+ before := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+ acknowledged := before[1][key]
+ if before[0][key] == acknowledged {
+ t.Fatal("fixture: drive 0 did not miss the update")
+ }
+ for i := 1; i < len(disks); i++ {
+ if before[i][key] != acknowledged {
+ t.Fatal("fixture: the update did not reach the other drives")
+ }
+ }
+ if rec := restoreIsolationPOST(t, r, r.z, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ after := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+ for i := 1; i < len(disks); i++ {
+ if after[i][key] != acknowledged {
+ t.Errorf("drive %d: POST wrote %q over the acknowledged %q", i, after[i][key], acknowledged)
+ }
+ if after[i][xhttp.AmzRestoreRequestDate] == before[i][xhttp.AmzRestoreRequestDate] {
+ t.Errorf("drive %d: POST did not record its request", i)
+ }
+ }
+ // The stale drive is left to heal; reads that pick it are not
+ // changed here.
+ if after[0][key] != before[0][key] || after[0][xhttp.AmzRestoreRequestDate] != before[0][xhttp.AmzRestoreRequestDate] {
+ t.Error("POST wrote the stale drive instead of leaving it to heal")
+ }
+ })
+ }
+}
+
+// The attempt is part of a version's metadata generation. A POST whose
+// marker reached all drives but one is acknowledged: its restore owns the
+// version, and the reset of an older attempt leaves the newer marker alone.
+func TestRestoreAttemptGenerationFencing(t *testing.T) {
+ for _, mode := range []string{"worker", "old-reset", "degraded-old-reset"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ var fail, offline atomic.Bool
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, write: &fail}
+ wrapped[1] = restoreGenerationOffline{StorageAPI: disks[1], bucket: r.bucket, object: r.object, fail: &offline}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ captured := make(chan ObjectOptions, 2)
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error {
+ captured <- opts
+ return nil
+ }
+ var older ObjectOptions
+ if mode != "worker" {
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ older = <-captured
+ layer.wait(t)
+ }
+ fail.Store(true)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ fail.Store(false)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("POST on all drives but one: %d %s", rec.Code, rec.Body.String())
+ }
+ newer := <-captured
+ layer.wait(t)
+ id := newer.Transition.attempt.ID
+ if mode == "worker" {
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, newer); err != nil {
+ t.Fatalf("restore of the acknowledged attempt: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ return
+ }
+ if mode == "degraded-old-reset" {
+ // One drive of the newer attempt is unreadable: the drives
+ // that answer no longer show which attempt is acknowledged.
+ offline.Store(true)
+ }
+ if err := r.z.resetRestoreStatus(r.bucket, r.object, older); err != nil {
+ t.Fatal(err)
+ }
+ offline.Store(false)
+ after := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+ for i := 1; i < len(disks); i++ {
+ if after[i][restoreAttemptKey] != id || !strings.Contains(after[i][xhttp.AmzRestore], "true") {
+ t.Errorf("drive %d: the older attempt's reset cleared the newer marker: %q %q", i, after[i][restoreAttemptKey], after[i][xhttp.AmzRestore])
+ }
+ }
+ if after[0][restoreAttemptKey] != older.Transition.attempt.ID {
+ t.Error("the reset wrote the stale drive")
+ }
+ // The newer attempt still owns the version and restores it.
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, newer); err != nil {
+ t.Fatalf("restore of the newer attempt after the older reset: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ })
+ }
+}
+
+// POST restores the version it resolved with that version's identity. A
+// replacement under the same version ID between its read and its status
+// write is refused, and nothing restores it.
+func TestRestorePOSTRejectsReplacedTarget(t *testing.T) {
+ for _, state := range []versioning.State{"", versioning.Suspended} {
+ t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, state)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := restoreIsolationSeed(t, r, 0, true, false)
+ layer := newRestoreIsolationLayer(r)
+ var replacement ObjectInfo
+ layer.afterRead = func() {
+ data := []byte("replacement after POST resolved its target")
+ var err error
+ replacement, err = r.z.PutObject(t.Context(), r.bucket, r.object,
+ mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: state == versioning.Suspended})
+ if err != nil {
+ t.Fatal(err)
+ }
+ opts := r.opts(nullVersionID)
+ opts.MTime = replacement.ModTime
+ opts.Transition = TransitionOptions{Tier: "WARM", ETag: replacement.ETag}
+ if err := r.z.serverPools[0].TransitionObject(t.Context(), r.bucket, r.object, opts); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code < 400 {
+ layer.wait(t)
+ t.Errorf("POST accepted a replacement of its target: %d", rec.Code)
+ }
+ after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(nullVersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after.ETag != replacement.ETag || after.UserDefined[xhttp.AmzRestore] != "" || after.ETag == oi.ETag {
+ t.Errorf("replacement %q changed by POST: etag %q restore %q", replacement.ETag, after.ETag, after.UserDefined[xhttp.AmzRestore])
+ }
+ })
+ }
+}
+
+// A POST on a version whose restore holds the object lock answers within
+// the two-second takeover bound: its first read does not wait for the lock.
+func TestRestorePOSTOngoingBound(t *testing.T) {
+ for _, mode := range []string{"pools=1", "pools=2", "stale-minority", "failed-activity-read"} {
+ t.Run(mode, func(t *testing.T) {
+ pools := 1
+ if mode == "pools=2" {
+ pools = 2
+ }
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+ tier := newRestoreIsolationGateTier(0, nil)
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ defer tier.unblock()
+ var fail atomic.Bool
+ if mode == "stale-minority" {
+ // The first read may pick a drive that missed the marker.
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, write: &fail}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ fail.Store(true)
+ }
+ layer := newRestoreIsolationLayer(r)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ fail.Store(false)
+ if rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ restoreIsolationEntered(t, tier)
+ var unreadable atomic.Bool
+ second := newRestoreIsolationLayer(r)
+ if mode == "failed-activity-read" {
+ // Once POST resolved its target, nothing tells whether it
+ // is restoring.
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ wrapped[i] = restoreGenerationOffline{StorageAPI: d, bucket: r.bucket, object: r.object, fail: &unreadable}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ second.afterRead = func() { unreadable.Store(true) }
+ }
+ done := make(chan *httptest.ResponseRecorder, 1)
+ start := time.Now()
+ go func() { done <- restoreIsolationPOST(t, r, second, oi.VersionID) }()
+ select {
+ case rec := <-done:
+ want := rec.Code == http.StatusConflict
+ if mode == "failed-activity-read" {
+ want = rec.Code >= 500
+ }
+ if !want || time.Since(start) > restoreTakeoverWait+time.Second {
+ t.Errorf("POST during a running restore: %d after %s, want a bounded refusal", rec.Code, time.Since(start))
+ }
+ case <-time.After(restoreTakeoverWait + 2*time.Second):
+ t.Error("POST during a running restore waited for the object lock")
+ unreadable.Store(false)
+ tier.unblock()
+ <-done
+ }
+ unreadable.Store(false)
+ tier.unblock()
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+// A POST whose status write fails on drives that stay unwritten puts the
+// generation it read back, and a split that a failed put-back leaves
+// differs only in restore fields: the attempt's reset and a later POST both
+// resolve it, while every other field still needs the quorum.
+func TestRestorePOSTStatusWriteSplit(t *testing.T) {
+ for _, mode := range []string{"put-back", "split-reset", "split-post"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ var fail atomic.Bool
+ wrapped := append([]StorageAPI(nil), disks...)
+ for i := range 2 {
+ wrapped[i] = restoreGenerationDisk{StorageAPI: disks[i], bucket: r.bucket, object: r.object, write: &fail}
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ before := restoreGenerationMetadata(t, disks, r, oi.VersionID)
+ var attempt *restoreAttempt
+ if mode == "put-back" {
+ fail.Store(true)
+ rec := restoreIsolationPOST(t, r, r.z, oi.VersionID)
+ fail.Store(false)
+ if rec.Code < 400 {
+ t.Fatalf("POST acknowledged a status write on two of four drives: %d", rec.Code)
+ }
+ if after := restoreGenerationMetadata(t, disks, r, oi.VersionID); !reflect.DeepEqual(before, after) {
+ t.Error("failed status write was not put back")
+ }
+ } else {
+ // A status write and its put-back both failed on two drives.
+ attempt = newRestoreAttempt(oi)
+ for _, d := range disks[2:] {
+ fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ fi.Metadata[xhttp.AmzRestore] = ongoingRestoreObj().String()
+ fi.Metadata[restoreAttemptKey] = attempt.ID
+ if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if _, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); err != nil {
+ t.Fatalf("a split in restore fields only has no generation: %v", err)
+ }
+ }
+ if mode == "split-reset" {
+ opts := nativeRestoreOptions(r, oi.VersionID)
+ opts.Transition.attempt = attempt
+ if err := r.z.resetRestoreStatus(r.bucket, r.object, opts); err != nil {
+ t.Fatalf("reset of the split attempt: %v", err)
+ }
+ for i, meta := range restoreGenerationMetadata(t, disks, r, oi.VersionID) {
+ if meta[xhttp.AmzRestore] != "" || meta[restoreAttemptKey] != "" {
+ t.Errorf("drive %d keeps the split attempt: %q %q", i, meta[xhttp.AmzRestore], meta[restoreAttemptKey])
+ }
+ }
+ }
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST after the failed status write: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restore contents: %v", err)
+ }
+ })
+ }
+}
+
+// A split in any other field is not resolved: the generation fails closed.
+func TestRestoreGenerationSplitNeedsQuorum(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ for i, d := range er.getDisks()[2:] {
+ fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ fi.Metadata[xhttp.AmzRestore] = ongoingRestoreObj().String()
+ fi.Metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = fmt.Sprint("ON", i)
+ if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if _, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); !errors.Is(err, errErasureReadQuorum) {
+ t.Fatalf("split outside the restore fields: %v, want read quorum", err)
+ }
+ if rec := restoreIsolationPOST(t, r, r.z, oi.VersionID); rec.Code < 400 {
+ t.Errorf("POST over a split outside the restore fields: %d", rec.Code)
+ }
+}
+
+// restoreGenerationCleanupDisk misses restored-copy cleanups and metadata
+// writes of the object while its toggles are set.
+type restoreGenerationCleanupDisk struct {
+ StorageAPI
+ bucket, object string
+ cleanup, metaWrites *atomic.Bool
+}
+
+func (d restoreGenerationCleanupDisk) DeleteVersion(ctx context.Context, bucket, object string, fi FileInfo, force bool, opts DeleteOptions) error {
+ if bucket == d.bucket && object == d.object && fi.ExpireRestored && d.cleanup.Load() {
+ return errFaultyDisk
+ }
+ return d.StorageAPI.DeleteVersion(ctx, bucket, object, fi, force, opts)
+}
+
+func (d restoreGenerationCleanupDisk) WriteMetadata(ctx context.Context, orig, bucket, object string, fi FileInfo) error {
+ if bucket == d.bucket && object == d.object && d.metaWrites.Load() {
+ return errFaultyDisk
+ }
+ return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi)
+}
+
+// Restored-copy expiry decides on the version's authoritative generation. A
+// drive that missed a cleanup and the POST after it keeps an expired header,
+// which must not clear the quorum's ongoing restore; drives split between
+// an expired and a renewed header authorize nothing.
+func TestExpireRestoredUsesQuorumGeneration(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, mode := range []string{"after-partial-cleanup", "split-renewal"} {
+ t.Run(fmt.Sprintf("pools=%d/%s", pools, mode), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi := r.put("cache", false, true)
+ r.restore(oi)
+ stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ expire := func() error {
+ return expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+ }
+ if mode == "split-renewal" {
+ // A renewal reached two drives; the other two keep the
+ // expired header.
+ dataDir := r.fileInfo(oi.VersionID).DataDir
+ for _, d := range disks[2:] {
+ fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ fi.Metadata[xhttp.AmzRestore] = completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()
+ if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := expire(); err == nil || r.dataDirs(dataDir) != len(disks) {
+ t.Errorf("expiry on drives split between expired and renewed: %v, %d/%d cache dirs", err, r.dataDirs(dataDir), len(disks))
+ }
+ return
+ }
+ var cleanup, metaWrites atomic.Bool
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[0] = restoreGenerationCleanupDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, cleanup: &cleanup, metaWrites: &metaWrites}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ cleanup.Store(true)
+ err = expire()
+ cleanup.Store(false)
+ if err != nil {
+ t.Fatalf("cleanup on all drives but one: %v", err)
+ }
+ captured := make(chan ObjectOptions, 1)
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error {
+ captured <- opts
+ return nil
+ }
+ metaWrites.Store(true)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ metaWrites.Store(false)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("POST after the cleanup: %d %s", rec.Code, rec.Body.String())
+ }
+ opts := <-captured
+ layer.wait(t)
+ // The same cleanup, queued twice, runs again.
+ if err := expire(); err == nil {
+ t.Error("a drive's expired header cleared the quorum's ongoing restore")
+ }
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, opts); err != nil {
+ t.Fatalf("restore of the acknowledged attempt: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, r.bodies["cache"]) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ })
+ }
+ }
+}
+
+// restoreGenerationHidden hides a drive's copy of the object version while
+// set, as a drive that lost it and that heal writes again.
+type restoreGenerationHidden struct {
+ StorageAPI
+ bucket, object string
+ hidden *atomic.Bool
+}
+
+func (d restoreGenerationHidden) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
+ if d.hidden.Load() && bucket == d.bucket && object == d.object {
+ return FileInfo{}, errFileNotFound
+ }
+ return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
+}
+
+// Heal copies the restore fields to a drive only as a quorum of the drives
+// holds them. While they disagree, healing the version waits: it must not
+// create a quorum for an expired header over a renewal, nor for a
+// superseded attempt. Fields a quorum agrees on are copied.
+func TestRestoreHealNeedsRestoreQuorum(t *testing.T) {
+ for _, mode := range []string{"expired-over-renewal", "superseded-attempt", "agreed"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ setRestore := func(versionID string, drives []StorageAPI, fields map[string]string) {
+ for _, d := range drives {
+ fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, versionID, ReadOptions{ReadData: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ maps.Copy(fi.Metadata, fields)
+ if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ var versionID string
+ var stale ObjectInfo
+ var older ObjectOptions
+ hide := 3
+ switch mode {
+ case "expired-over-renewal", "agreed":
+ oi := r.put("cache", false, true)
+ r.restore(oi)
+ versionID = oi.VersionID
+ var err error
+ if stale, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(versionID)); err != nil {
+ t.Fatal(err)
+ }
+ renewed := map[string]string{xhttp.AmzRestore: completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()}
+ if mode == "agreed" {
+ // Three drives renewed, the hidden one expired.
+ setRestore(versionID, disks[:3], renewed)
+ } else {
+ // No quorum: two drives expired an hour ago, one renewed,
+ // and the hidden one expired a day ago.
+ setRestore(versionID, disks[:2], map[string]string{xhttp.AmzRestore: completedRestoreObj(UTCNow().Add(-time.Hour)).String()})
+ setRestore(versionID, disks[2:3], renewed)
+ }
+ case "superseded-attempt":
+ oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+ versionID = oi.VersionID
+ captured := make(chan ObjectOptions, 1)
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error {
+ captured <- opts
+ return nil
+ }
+ if rec := restoreIsolationPOST(t, r, layer, versionID); rec.Code != http.StatusOK {
+ t.Fatal(rec.Body.String())
+ }
+ older = <-captured
+ layer.wait(t)
+ // A newer attempt reached two drives.
+ setRestore(versionID, disks[2:], map[string]string{xhttp.AmzRestore: ongoingRestoreObj().String(), restoreAttemptKey: newRestoreAttempt(oi).ID})
+ hide = 2
+ }
+ before := restoreGenerationMetadata(t, disks, r, versionID)
+ var hidden atomic.Bool
+ wrapped := append([]StorageAPI(nil), disks...)
+ wrapped[hide] = restoreGenerationHidden{StorageAPI: disks[hide], bucket: r.bucket, object: r.object, hidden: &hidden}
+ er.getDisks = func() []StorageAPI { return wrapped }
+ hidden.Store(true)
+ _, healErr := er.HealObject(t.Context(), r.bucket, r.object, versionID, madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+ hidden.Store(false)
+ er.getDisks = func() []StorageAPI { return disks }
+ after := restoreGenerationMetadata(t, disks, r, versionID)
+ if mode == "agreed" {
+ if healErr != nil || after[hide][xhttp.AmzRestore] != after[0][xhttp.AmzRestore] {
+ t.Errorf("heal did not copy the restore fields a quorum holds: %v, %q", healErr, after[hide][xhttp.AmzRestore])
+ }
+ return
+ }
+ if healErr == nil {
+ t.Error("heal proceeded on drives that disagree on the restore fields")
+ }
+ if after[hide][xhttp.AmzRestore] != before[hide][xhttp.AmzRestore] || after[hide][restoreAttemptKey] != before[hide][restoreAttemptKey] {
+ t.Errorf("heal wrote a minority restore state: %q -> %q", before[hide][xhttp.AmzRestore], after[hide][xhttp.AmzRestore])
+ }
+ if mode == "expired-over-renewal" {
+ dataDir := r.fileInfo(versionID).DataDir
+ if err := expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err == nil || r.dataDirs(dataDir) != len(disks) {
+ t.Errorf("expiry after heal: %v, %d/%d cache dirs", err, r.dataDirs(dataDir), len(disks))
+ }
+ return
+ }
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, older); !errors.Is(err, errRestoreSuperseded) {
+ t.Errorf("superseded attempt after heal: %v", err)
+ }
+ })
+ }
+}
+
+// restoreGenerationRand draws UUIDs from crypto/rand for the whole test
+// binary, installed before any goroutine runs. While a pause is set, it holds
+// the UUID that stale cleanup draws in its deadline-bound worker, for the
+// trash path right before its rename, until the pause is released.
+var restoreGenerationRand = &restoreGenerationRandReader{}
+
+func init() { uuid.SetRand(restoreGenerationRand) }
+
+type restoreGenerationRandReader struct {
+ pause atomic.Pointer[restoreGenerationPause]
+}
+
+type restoreGenerationPause struct {
+ entered, release chan struct{}
+}
+
+func (r *restoreGenerationRandReader) Read(b []byte) (int, error) {
+ if p := r.pause.Load(); p != nil && restoreGenerationInCleanupRename() && r.pause.CompareAndSwap(p, nil) {
+ close(p.entered)
+ <-p.release
+ }
+ return rand.Read(b)
+}
+
+// restoreGenerationInCleanupRename reports whether the caller runs in stale
+// cleanup's deadline-bound worker.
+func restoreGenerationInCleanupRename() bool {
+ pcs := make([]uintptr, 32)
+ frames := runtime.CallersFrames(pcs[:runtime.Callers(3, pcs)])
+ cleanup, worker := false, false
+ for {
+ f, more := frames.Next()
+ cleanup = cleanup || strings.Contains(f.Function, "cleanupStaleUploadsOnDisk")
+ worker = worker || strings.Contains(f.Function, "ioutil.WithDeadline")
+ if !more {
+ return cleanup && worker
+ }
+ }
+}
diff --git a/cmd/restore-native-isolation_test.go b/cmd/restore-native-isolation_test.go
new file mode 100644
index 000000000..2690aefea
--- /dev/null
+++ b/cmd/restore-native-isolation_test.go
@@ -0,0 +1,930 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "reflect"
+ "runtime"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/versioning"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/minio/mux"
+)
+
+// The private attempt metadata of a native restore, as stored. A literal name
+// keeps these regressions meaningful on a tree without the key.
+const restoreIsolationAttemptKey = "x-minio-internal-restore-attempt"
+
+// restoreIsolationLayer hooks the object layer a POST restore handler uses.
+// afterRead runs once POST has read the version, before POST records its
+// status. restore, when set, replaces the background restore; every
+// background restore result is sent to done.
+type restoreIsolationLayer struct {
+ ObjectLayer
+ afterRead func()
+ once sync.Once
+ restore func(ctx context.Context, bucket, object string, opts ObjectOptions) error
+ done chan error
+}
+
+func newRestoreIsolationLayer(r *restoreTest) *restoreIsolationLayer {
+ return &restoreIsolationLayer{ObjectLayer: r.z, done: make(chan error, 4)}
+}
+
+func (l *restoreIsolationLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+ oi, err := l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+ if err == nil && l.afterRead != nil {
+ l.once.Do(l.afterRead)
+ }
+ return oi, err
+}
+
+func (l *restoreIsolationLayer) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ var err error
+ if l.restore != nil {
+ err = l.restore(ctx, bucket, object, opts)
+ } else {
+ err = l.ObjectLayer.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ l.done <- err
+ return err
+}
+
+// restoreTestWait bounds how long a restore test waits for a step that does
+// real drive work: metadata writes with synchronous persistence and renames
+// on every drive. It is a liveness bound, not a latency contract; a loaded
+// machine has stalled such a step past ten seconds.
+const restoreTestWait = time.Minute
+
+// wait returns the result of the next background restore.
+func (l *restoreIsolationLayer) wait(t *testing.T) error {
+ t.Helper()
+ select {
+ case err := <-l.done:
+ return err
+ case <-time.After(restoreTestWait):
+ buf := make([]byte, 1<<20)
+ t.Fatalf("background restore did not finish in %v; goroutines:\n%s", restoreTestWait, buf[:runtime.Stack(buf, true)])
+ return nil
+ }
+}
+
+func restoreIsolationPOST(t *testing.T, r *restoreTest, layer ObjectLayer, versionID string) *httptest.ResponseRecorder {
+ t.Helper()
+ body := []byte(`1`)
+ path := "/" + r.bucket + "/" + r.object + "?restore"
+ if versionID != "" {
+ path += "&versionId=" + url.QueryEscape(versionID)
+ }
+ req, err := newTestSignedRequestV4(http.MethodPost, path, int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := req.ParseForm(); err != nil {
+ t.Fatal(err)
+ }
+ req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object})
+ rec := httptest.NewRecorder()
+ api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return layer }}
+ api.PostRestoreObjectHandler(rec, req)
+ return rec
+}
+
+// restoreIsolationSeed is a transitioned version without the ongoing marker
+// that nativeRestoreSeed leaves for direct restores: POST sets its own.
+func restoreIsolationSeed(t *testing.T, r *restoreTest, pool int, null, multipart bool) (ObjectInfo, []byte) {
+ t.Helper()
+ oi, body := nativeRestoreSeed(t, r, pool, null, multipart)
+ opts := r.opts(oi.VersionID)
+ opts.MTime = oi.ModTime
+ opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+ o.UserDefined[xhttp.AmzRestore] = ""
+ return ReplicateDecision{}, nil
+ }
+ if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil {
+ t.Fatal(err)
+ }
+ return oi, body
+}
+
+// restoreIsolationSettled waits until the version is no longer ongoing.
+func restoreIsolationSettled(t *testing.T, r *restoreTest, versionID string) ObjectInfo {
+ t.Helper()
+ deadline := time.Now().Add(10 * time.Second)
+ for {
+ oi, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(versionID))
+ if err == nil && !oi.RestoreOngoing {
+ return oi
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("version stays ongoing, refusing every later POST: %q, %v", oi.UserDefined[xhttp.AmzRestore], err)
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+}
+
+// restoreIsolationUploads returns the upload directories of the object on
+// each drive of a pool, as stored, whatever listings show.
+func restoreIsolationUploads(t *testing.T, r *restoreTest, pool int) map[string]int {
+ t.Helper()
+ er := r.z.serverPools[pool].getHashedSet(r.object)
+ found := make(map[string]int)
+ for _, disk := range er.getDisks() {
+ entries, err := disk.ListDir(t.Context(), r.bucket, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), -1)
+ if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
+ continue
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, entry := range entries {
+ found[strings.TrimSuffix(entry, SlashSeparator)]++
+ }
+ }
+ return found
+}
+
+// restoreIsolationGateTier holds a download after its first bytes and can
+// fail it once released.
+type restoreIsolationGateTier struct {
+ WarmBackend
+ first int
+ entered chan struct{}
+ release chan struct{}
+ failure error
+ enter sync.Once
+ open sync.Once
+}
+
+func newRestoreIsolationGateTier(first int, failure error) *restoreIsolationGateTier {
+ return &restoreIsolationGateTier{
+ WarmBackend: globalTierConfigMgr.drivercache["WARM"],
+ first: first, failure: failure,
+ entered: make(chan struct{}), release: make(chan struct{}),
+ }
+}
+
+func (b *restoreIsolationGateTier) unblock() { b.open.Do(func() { close(b.release) }) }
+
+func (b *restoreIsolationGateTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+ rd, err := b.WarmBackend.Get(ctx, object, version, opts)
+ if err != nil {
+ return nil, err
+ }
+ return &restoreIsolationGateReader{ReadCloser: rd, tier: b, ctx: ctx, remaining: b.first}, nil
+}
+
+type restoreIsolationGateReader struct {
+ io.ReadCloser
+ tier *restoreIsolationGateTier
+ ctx context.Context
+ remaining int
+}
+
+func (r *restoreIsolationGateReader) Read(p []byte) (int, error) {
+ if r.remaining > 0 {
+ if len(p) > r.remaining {
+ p = p[:r.remaining]
+ }
+ n, err := r.ReadCloser.Read(p)
+ r.remaining -= n
+ return n, err
+ }
+ r.tier.enter.Do(func() { close(r.tier.entered) })
+ select {
+ case <-r.tier.release:
+ case <-r.ctx.Done():
+ return 0, r.ctx.Err()
+ }
+ if r.tier.failure != nil {
+ return 0, r.tier.failure
+ }
+ return r.ReadCloser.Read(p)
+}
+
+func restoreIsolationEntered(t *testing.T, tier *restoreIsolationGateTier) {
+ t.Helper()
+ select {
+ case <-tier.entered:
+ case <-time.After(restoreTestWait):
+ t.Fatal("restore never reached its download gate")
+ }
+}
+
+// restoreIsolationFaultDisk fails metadata reads of the object while
+// failReads is set, and reports writes of its metadata as failed, after
+// they were made, while failWrites is set. Toggles keep the wrapper in
+// place, so no restore retry races a swap of the drives.
+type restoreIsolationFaultDisk struct {
+ StorageAPI
+ bucket, object string
+ failReads *atomic.Bool
+ failWrites *atomic.Bool
+}
+
+func (d restoreIsolationFaultDisk) ReadVersion(ctx context.Context, origvolume, volume, path, version string, opts ReadOptions) (FileInfo, error) {
+ if d.failReads.Load() && volume == d.bucket && path == d.object {
+ return FileInfo{}, errDiskNotFound
+ }
+ return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, version, opts)
+}
+
+func (d restoreIsolationFaultDisk) ReadXL(ctx context.Context, volume, path string, readData bool) (RawFileInfo, error) {
+ if d.failReads.Load() && volume == d.bucket && path == d.object {
+ return RawFileInfo{}, errDiskNotFound
+ }
+ return d.StorageAPI.ReadXL(ctx, volume, path, readData)
+}
+
+func (d restoreIsolationFaultDisk) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error {
+ err := d.StorageAPI.WriteMetadata(ctx, origvolume, volume, path, fi)
+ if err == nil && d.failWrites != nil && d.failWrites.Load() && volume == d.bucket && path == d.object {
+ return errFaultyDisk
+ }
+ return err
+}
+
+// installRestoreIsolationFaults wraps the drives of the object's set in a
+// pool for the rest of the test; failing marks the drives whose writes fail.
+func installRestoreIsolationFaults(r *restoreTest, pool int, failing int) (reads, writes *atomic.Bool) {
+ reads, writes = new(atomic.Bool), new(atomic.Bool)
+ er := r.z.serverPools[pool].getHashedSet(r.object)
+ disks := er.getDisks()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, disk := range disks {
+ d := restoreIsolationFaultDisk{StorageAPI: disk, bucket: r.bucket, object: r.object, failReads: reads}
+ if i < failing {
+ d.failWrites = writes
+ }
+ wrapped[i] = d
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ return reads, writes
+}
+
+// POST records its status on a fresh read under the object lock, so an
+// Object Lock update it did not see survives.
+func TestRestorePOSTKeepsConcurrentProtection(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, kind := range []string{"legal-hold", "retention"} {
+ t.Run(fmt.Sprintf("pools=%d/%s", pools, kind), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ enableBucketObjectLock(t, r.bucket)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ initial := []byte(`OFF`)
+ updated := []byte(`ON`)
+ key := strings.ToLower(xhttp.AmzObjectLockLegalHold)
+ if kind == "retention" {
+ initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour))
+ updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))
+ key = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
+ }
+ if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK {
+ t.Fatalf("initial protection: %d %s", rec.Code, rec.Body.String())
+ }
+ var accepted ObjectInfo
+ layer := newRestoreIsolationLayer(r)
+ layer.afterRead = func() {
+ if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK {
+ t.Fatalf("update: %d %s", rec.Code, rec.Body.String())
+ }
+ var err error
+ if accepted, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code >= 400 {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if accepted.UserDefined[key] == "" || after.UserDefined[key] != accepted.UserDefined[key] {
+ t.Errorf("POST restore overwrote acknowledged Object Lock metadata: %q -> %q", accepted.UserDefined[key], after.UserDefined[key])
+ }
+ if !after.ModTime.Equal(oi.ModTime) || after.ETag != oi.ETag {
+ t.Error("POST restore changed the version identity")
+ }
+ })
+ }
+ }
+}
+
+// A restored copy removed between POST's read and its status write leads to
+// a new restore, never to a completed header without the restored data.
+func TestRestorePOSTRacesCacheExpiry(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi := r.put("restored", false, true)
+ r.restore(oi)
+ layer := newRestoreIsolationLayer(r)
+ layer.afterRead = func() {
+ if err := expireTransitionedObject(t.Context(), r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code >= 400 {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ select {
+ case err := <-layer.done:
+ if err != nil {
+ t.Fatalf("rehydration: %v", err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Error("POST renewed a removed restored copy instead of restoring it")
+ }
+ _, body, err := r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(body, r.bodies["restored"]) {
+ t.Errorf("version unreadable after POST raced cache expiry: %v", err)
+ }
+ })
+ }
+}
+
+// A restore queued with the versioning POST captured writes back the null
+// version it resolved, even after versioning was enabled in between.
+func TestRestoreKeepsCapturedNullIdentity(t *testing.T) {
+ enable := func(t *testing.T, r *restoreTest) {
+ meta, err := globalBucketMetadataSys.Get(r.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ meta.VersioningConfigXML = []byte(`Enabled`)
+ if err := meta.Save(t.Context(), r.z); err != nil {
+ t.Fatal(err)
+ }
+ globalBucketMetadataSys.Set(r.bucket, meta)
+ }
+ for _, mode := range []string{"direct", "post"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, "")
+ initConfigSubsystem(t.Context(), r.z)
+ var oi ObjectInfo
+ var body []byte
+ if mode == "direct" {
+ oi, body = nativeRestoreSeed(t, r, 0, true, false)
+ // These are the options that POST captured while versioning was off.
+ queued := nativeRestoreOptions(r, "")
+ enable(t, r)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, queued); err != nil {
+ t.Fatal(err)
+ }
+ } else {
+ oi, body = restoreIsolationSeed(t, r, 0, true, false)
+ release := make(chan struct{})
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ <-release
+ return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ enable(t, r)
+ close(release)
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ }
+ versions := nativeRestoreSnapshot(t, r)[0]
+ if len(versions) != 1 || versions[0].VersionID != "" {
+ t.Errorf("restore created a UUID alias of the queued null version: %d versions", len(versions))
+ }
+ _, got, err := r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("null original: %v", err)
+ }
+ })
+ }
+}
+
+// A restore that fails before it reads the version, on its lock wait or on
+// its first read, must not leave POST's ongoing marker for every later POST
+// to refuse: bounded retries clear it once the version is readable again.
+func TestRestoreRecoversStrandedMarker(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, stage := range []string{"lock-wait", "initial-read"} {
+ t.Run(fmt.Sprintf("pools=%d/%s", pools, stage), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+ var err error
+ if stage == "lock-wait" {
+ lk := r.z.NewNSLock(r.bucket, r.object)
+ if pools == 1 {
+ lk = r.z.serverPools[0].getHashedSet(r.object).NewNSLock(r.bucket, r.object)
+ }
+ lkctx, lerr := lk.GetLock(t.Context(), globalOperationTimeout)
+ if lerr != nil {
+ t.Fatal(lerr)
+ }
+ ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond)
+ err = r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ cancel()
+ lk.Unlock(lkctx)
+ } else {
+ reads, _ := installRestoreIsolationFaults(r, 0, 0)
+ reads.Store(true)
+ err = r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ reads.Store(false)
+ }
+ if err == nil {
+ t.Fatal("restore succeeded without its lock or its version")
+ }
+ after := restoreIsolationSettled(t, r, oi.VersionID)
+ if !after.IsRemote() || !after.ModTime.Equal(oi.ModTime) {
+ t.Error("recovery changed more than the restore status")
+ }
+ })
+ }
+ }
+}
+
+// A later POST takes over a marker that no running restore holds, and the
+// superseded attempt then neither downloads nor clears the newer status,
+// also when both POSTs share a second, the precision of the request date.
+func TestRestoreSupersededAttempt(t *testing.T) {
+ for _, mode := range []string{"newer-completes", "newer-fails"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ release := make(chan struct{})
+ var releaseOnce sync.Once
+ unblock := func() { releaseOnce.Do(func() { close(release) }) }
+ t.Cleanup(unblock)
+ older := newRestoreIsolationLayer(r)
+ older.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ // The queued worker loses the restore lock POST handed it,
+ // as when its refresh found it gone: the marker then belongs
+ // to no running restore.
+ opts.Transition.owner.release()
+ <-release
+ return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ newer := newRestoreIsolationLayer(r)
+ time.Sleep(time.Until(time.Now().Truncate(time.Second).Add(time.Second + 10*time.Millisecond)))
+ if rec := restoreIsolationPOST(t, r, older, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("first POST: %d %s", rec.Code, rec.Body.String())
+ }
+ first := r.fileInfo(oi.VersionID).Metadata[xhttp.AmzRestoreRequestDate]
+ if mode == "newer-fails" {
+ globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, errors.New("injected remote failure")}
+ }
+ if rec := restoreIsolationPOST(t, r, newer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST over a marker no restore holds: %d %s", rec.Code, rec.Body.String())
+ }
+ second := r.fileInfo(oi.VersionID).Metadata[xhttp.AmzRestoreRequestDate]
+ t.Logf("request dates %q %q, same second: %v", first, second, first == second)
+ newerErr := newer.wait(t)
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ before := r.fileInfo(oi.VersionID)
+ gets := tier.gets.Load()
+ unblock()
+ if err := older.wait(t); err == nil {
+ t.Error("superseded restore reported success")
+ }
+ if tier.gets.Load() != gets {
+ t.Error("superseded restore downloaded the version again")
+ }
+ after := r.fileInfo(oi.VersionID)
+ if !reflect.DeepEqual(before.Metadata, after.Metadata) || before.DataDir != after.DataDir {
+ t.Errorf("superseded restore changed the version: %q -> %q", before.Metadata[xhttp.AmzRestore], after.Metadata[xhttp.AmzRestore])
+ }
+ info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil || info.RestoreOngoing {
+ t.Fatalf("version left ongoing: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+ }
+ if mode == "newer-completes" {
+ if newerErr != nil || info.IsRemote() {
+ t.Fatalf("newer restore: %v remote=%v", newerErr, info.IsRemote())
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) || tier.gets.Load() != gets {
+ t.Errorf("restored contents: %v", err)
+ }
+ return
+ }
+ if newerErr == nil || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" {
+ t.Fatalf("failed newer restore is not retryable: %v %q", newerErr, info.UserDefined[xhttp.AmzRestore])
+ }
+ if rec := restoreIsolationPOST(t, r, newer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("retry POST: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := newer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Errorf("retried restore contents: %v", err)
+ }
+ })
+ }
+}
+
+// A restart loses the restore that POST started; a POST to the restarted
+// server takes the stranded marker over and restores the version.
+func TestRestoreRestartTakeover(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ lost := newRestoreIsolationLayer(r)
+ lost.restore = func(context.Context, string, string, ObjectOptions) error { return nil }
+ if rec := restoreIsolationPOST(t, r, lost, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST before restart: %d %s", rec.Code, rec.Body.String())
+ }
+ lost.wait(t)
+ if info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err != nil || !info.RestoreOngoing {
+ t.Fatalf("fixture: %q %v", info.UserDefined[xhttp.AmzRestore], err)
+ }
+
+ endpoints := make(EndpointServerPools, len(r.z.serverPools))
+ for i, pool := range r.z.serverPools {
+ endpoints[i] = pool.endpoints
+ }
+ obj, err := newErasureServerPools(t.Context(), endpoints)
+ if err != nil {
+ t.Fatal(err)
+ }
+ restarted := obj.(*erasureServerPools)
+ t.Cleanup(func() { restarted.Shutdown(context.Background()) })
+ r.z = restarted
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST after restart: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ info := restoreIsolationSettled(t, r, oi.VersionID)
+ if info.IsRemote() || info.UserDefined[restoreIsolationAttemptKey] != "" {
+ t.Errorf("restore after restart incomplete: remote=%v attempt=%q", info.IsRemote(), info.UserDefined[restoreIsolationAttemptKey])
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Errorf("contents after restart: %v", err)
+ }
+}
+
+// A null version replaced while its restore was queued is another version:
+// the restore must not rehydrate it, even if it is transitioned as well.
+func TestRestoreReplacementNullObject(t *testing.T) {
+ for _, state := range []versioning.State{"", versioning.Suspended} {
+ t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, state)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := restoreIsolationSeed(t, r, 0, true, false)
+ release := make(chan struct{})
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ <-release
+ return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code != http.StatusOK {
+ close(release)
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ replacement := []byte("replacement null object")
+ repl, err := r.z.serverPools[0].PutObject(t.Context(), r.bucket, r.object,
+ mustGetPutObjReader(t, bytes.NewReader(replacement), int64(len(replacement)), "", ""),
+ ObjectOptions{VersionSuspended: state == versioning.Suspended})
+ if err != nil {
+ close(release)
+ t.Fatal(err)
+ }
+ topts := r.opts(nullVersionID)
+ topts.MTime = repl.ModTime
+ topts.Transition = TransitionOptions{Tier: "WARM", ETag: repl.ETag}
+ if err := r.z.serverPools[0].TransitionObject(t.Context(), r.bucket, r.object, topts); err != nil {
+ close(release)
+ t.Fatal(err)
+ }
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ before := r.fileInfo(nullVersionID)
+ gets := tier.gets.Load()
+ close(release)
+ layer.wait(t)
+ after := r.fileInfo(nullVersionID)
+ if tier.gets.Load() != gets || after.Metadata[xhttp.AmzRestore] != "" || !reflect.DeepEqual(before.Metadata, after.Metadata) {
+ t.Errorf("queued restore rehydrated the replacement null object: %q", after.Metadata[xhttp.AmzRestore])
+ }
+ if after.ModTime.Equal(oi.ModTime) || !after.ModTime.Equal(repl.ModTime) {
+ t.Fatal("fixture did not replace the null version")
+ }
+ if _, got, err := r.read(nullVersionID); err != nil || !bytes.Equal(got, replacement) {
+ t.Errorf("replacement contents: %v", err)
+ }
+ })
+ }
+}
+
+// POST's own status write can fail after its ongoing marker reached the
+// drives; the marker must not stay for every later POST to refuse.
+func TestRestorePOSTStatusWritePartialFailure(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ _, writes := installRestoreIsolationFaults(r, 0, 2)
+ layer := newRestoreIsolationLayer(r)
+ writes.Store(true)
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ writes.Store(false)
+ if rec.Code < 400 {
+ t.Fatalf("POST acknowledged a failed status write: %d %s", rec.Code, rec.Body.String())
+ }
+ if info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err == nil && !info.RestoreOngoing {
+ t.Log("the failed status write left no marker in quorum")
+ }
+ restoreIsolationSettled(t, r, oi.VersionID)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST after a failed status write: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Errorf("restore after a failed status write: %v", err)
+ }
+}
+
+// Duplicate copies of a version in two pools can differ in whether they
+// hold a restored copy. A completed restore header must never reach a copy
+// without one; POST renews or restores only the copy that restores serve.
+func TestRestoreDuplicatePoolCacheStates(t *testing.T) {
+ for _, mode := range []string{"suspended-owner", "unrestored-first", "restored-first"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ // The duplicate is the same stored version, metadata only.
+ src := r.z.serverPools[0].getHashedSet(r.object).getDisks()
+ dst := r.z.serverPools[1].getHashedSet(r.object).getDisks()
+ for i := range src {
+ raw, err := src[i].ReadAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := dst[i].WriteAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile), raw); err != nil {
+ t.Fatal(err)
+ }
+ }
+ owner := 0
+ if mode == "unrestored-first" {
+ owner = 1
+ }
+ if err := r.z.serverPools[owner].RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ if mode == "suspended-owner" {
+ r.z.poolMetaMutex.Lock()
+ r.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
+ r.z.poolMetaMutex.Unlock()
+ t.Cleanup(func() {
+ r.z.poolMetaMutex.Lock()
+ r.z.poolMeta.Pools[0].Decommission = nil
+ r.z.poolMetaMutex.Unlock()
+ })
+ }
+ before := nativeRestoreSnapshot(t, r)
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ if mode != "restored-first" {
+ select {
+ case err := <-layer.done:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Error("POST did not restore the copy without restored data")
+ }
+ }
+ after := nativeRestoreSnapshot(t, r)
+ for pool := range r.z.serverPools {
+ gr, err := r.z.serverPools[pool].GetObjectNInfo(t.Context(), r.bucket, r.object, nil, nil, ObjectOptions{VersionID: oi.VersionID})
+ if err != nil {
+ t.Errorf("pool %d copy unreadable, header %q: %v", pool, after[pool][0].Metadata[xhttp.AmzRestore], err)
+ continue
+ }
+ got, err := io.ReadAll(gr)
+ gr.Close()
+ if err != nil || !bytes.Equal(got, body) {
+ t.Errorf("pool %d copy contents: %v", pool, err)
+ }
+ }
+ // The copy that is neither renewed nor restored stays as it was.
+ untouched := 1
+ if mode == "suspended-owner" {
+ untouched = 0
+ }
+ if !reflect.DeepEqual(before[untouched], after[untouched]) {
+ t.Errorf("POST changed the pool %d copy: %q -> %q", untouched, before[untouched][0].Metadata[xhttp.AmzRestore], after[untouched][0].Metadata[xhttp.AmzRestore])
+ }
+ })
+ }
+}
+
+// A restored copy expires only by its own reason, on a fresh read: its
+// restore deadline passed and nothing else. A renewed deadline survives
+// another rule falling due and an expiry checked across all pools.
+func TestExpireRestoredNeedsOwnReason(t *testing.T) {
+ for _, mode := range []string{"other-expiry-due", "checked-all-pools", "ongoing", "renewed", "expired"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi := r.put("restored", false, true)
+ r.restore(oi)
+ before := r.fileInfo(oi.VersionID)
+ if mode == "other-expiry-due" {
+ meta, err := globalBucketMetadataSys.Get(r.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ meta.LifecycleConfigXML = []byte(`deleteEnabled2020-01-01T00:00:00Z`)
+ if err := meta.Save(t.Context(), r.z); err != nil {
+ t.Fatal(err)
+ }
+ globalBucketMetadataSys.Set(r.bucket, meta)
+ }
+ header := completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()
+ switch mode {
+ case "ongoing":
+ header = ongoingRestoreObj().String()
+ case "expired":
+ header = ""
+ }
+ if header != "" {
+ opts := r.opts(oi.VersionID)
+ opts.MTime = oi.ModTime
+ opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+ o.UserDefined[xhttp.AmzRestore] = header
+ return ReplicateDecision{}, nil
+ }
+ if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil {
+ t.Fatal(err)
+ }
+ }
+ var err error
+ if mode == "checked-all-pools" {
+ opts := r.opts(oi.VersionID)
+ opts.Expiration = ExpirationOptions{Expire: true, CheckedAllPools: true}
+ opts.Transition.ExpireRestored = true
+ _, err = r.z.serverPools[0].getHashedSet(r.object).DeleteObject(t.Context(), r.bucket, r.object, opts)
+ } else {
+ err = expireTransitionedObject(t.Context(), r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner)
+ }
+ after := r.fileInfo(oi.VersionID)
+ if mode == "expired" {
+ // Control: an elapsed deadline is the reason itself.
+ if err != nil || after.Metadata[xhttp.AmzRestore] != "" || r.dataDirs(before.DataDir) != 0 {
+ t.Errorf("expired restored copy not removed: %v", err)
+ }
+ return
+ }
+ t.Logf("cleanup=%v header=%q", err, after.Metadata[xhttp.AmzRestore])
+ if err == nil || after.Metadata[xhttp.AmzRestore] != header || r.dataDirs(before.DataDir) != len(r.z.serverPools[0].getHashedSet(r.object).getDisks()) {
+ t.Error("cache cleanup removed a restored copy whose own deadline has not passed")
+ }
+ })
+ }
+}
+
+// Controls: an ordinary POST restore, a renewal, and a client's multipart
+// upload of the same key while a restore runs.
+func TestRestorePOSTControls(t *testing.T) {
+ t.Run("restore", func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil || info.IsRemote() || info.RestoreOngoing || info.UserDefined[restoreIsolationAttemptKey] != "" {
+ t.Fatalf("restored status: %q %q, %v", info.UserDefined[xhttp.AmzRestore], info.UserDefined[restoreIsolationAttemptKey], err)
+ }
+ if !info.ModTime.Equal(oi.ModTime) || info.ETag != oi.ETag || info.UserDefined["x-amz-meta-preserve"] != "keep-me" {
+ t.Error("restore changed the version identity or metadata")
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Errorf("restored contents: %v", err)
+ }
+ if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+ t.Errorf("completed restore left its upload: %v", uploads)
+ }
+ })
+ t.Run("renewal", func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := nativeRestoreSeed(t, r, 0, false, false)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ before := r.fileInfo(oi.VersionID)
+ gets := tier.gets.Load()
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ select {
+ case err := <-layer.done:
+ t.Fatalf("renewal restored again: %v", err)
+ case <-time.After(200 * time.Millisecond):
+ }
+ after := r.fileInfo(oi.VersionID)
+ if after.DataDir != before.DataDir || !after.ModTime.Equal(before.ModTime) || after.Metadata[xhttp.AmzRestore] == "" {
+ t.Error("renewal changed the restored copy")
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) || tier.gets.Load() != gets {
+ t.Errorf("renewed copy not served locally: %v", err)
+ }
+ })
+ t.Run("public-multipart-same-key", func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ tier := newRestoreIsolationGateTier(5<<20, nil)
+ globalTierConfigMgr.drivercache["WARM"] = tier
+ defer tier.unblock()
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ restoreIsolationEntered(t, tier)
+ upload, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 100)
+ listed := false
+ for _, u := range uploads.Uploads {
+ listed = listed || u.UploadID == upload.UploadID
+ }
+ if err != nil || !listed {
+ t.Fatalf("client upload not listed: %+v, %v", uploads.Uploads, err)
+ }
+ tier.unblock()
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ // Single-pool part writes wait for the object lock of a running restore.
+ data := []byte("client data")
+ part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ parts, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, upload.UploadID, 0, 10, ObjectOptions{})
+ if err != nil || len(parts.Parts) != 1 {
+ t.Fatalf("client parts: %+v, %v", parts, err)
+ }
+ newer, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, got, err := r.read(newer.VersionID); err != nil || !bytes.Equal(got, data) {
+ t.Errorf("client version: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Errorf("restored version: %v", err)
+ }
+ })
+}
diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go
new file mode 100644
index 000000000..61733e094
--- /dev/null
+++ b/cmd/restore-native_test.go
@@ -0,0 +1,935 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "crypto/md5"
+ "encoding/base64"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ "github.com/minio/minio/internal/bucket/versioning"
+ "github.com/minio/minio/internal/event"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/minio/minio/internal/pubsub"
+ "github.com/minio/mux"
+)
+
+func nativeRestoreSeed(t *testing.T, r *restoreTest, pool int, null, multipart bool) (ObjectInfo, []byte) {
+ t.Helper()
+ ctx := t.Context()
+ opts := ObjectOptions{Versioned: !null, MTime: UTCNow().Add(-2 * time.Hour), UserDefined: map[string]string{
+ "content-type": "application/x-restore-test", "x-amz-meta-preserve": "keep-me",
+ }}
+ body := bytes.Repeat([]byte("restore-payload"), 100000)
+ var oi ObjectInfo
+ var err error
+ if multipart {
+ parts := [][]byte{bytes.Repeat([]byte("a"), 5<<20), []byte("last-part")}
+ body = bytes.Join(parts, nil)
+ upload, err := r.z.serverPools[pool].NewMultipartUpload(ctx, r.bucket, r.object, opts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var complete []CompletePart
+ for i, data := range parts {
+ p, err := r.z.serverPools[pool].PutObjectPart(ctx, r.bucket, r.object, upload.UploadID, i+1,
+ mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ complete = append(complete, CompletePart{PartNumber: i + 1, ETag: p.ETag})
+ }
+ oi, err = r.z.serverPools[pool].CompleteMultipartUpload(ctx, r.bucket, r.object, upload.UploadID, complete, opts)
+ if err == nil && len(oi.Parts) != 2 {
+ t.Fatalf("fixture has %d parts, want 2", len(oi.Parts))
+ }
+ } else {
+ oi, err = r.z.serverPools[pool].PutObject(ctx, r.bucket, r.object,
+ mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), opts)
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ oi.VersionID = restoreTestVersionKey(oi.VersionID)
+ topts := r.opts(oi.VersionID)
+ topts.MTime = oi.ModTime
+ topts.Transition = TransitionOptions{Tier: "WARM", ETag: oi.ETag}
+ if err := r.z.serverPools[pool].TransitionObject(ctx, r.bucket, r.object, topts); err != nil {
+ t.Fatal(err)
+ }
+ // Use the production metadata writer to mark the request ongoing.
+ topts.Transition = TransitionOptions{}
+ topts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+ o.UserDefined[xhttp.AmzRestore] = ongoingRestoreObj().String()
+ return ReplicateDecision{}, nil
+ }
+ if _, err := r.z.PutObjectMetadata(ctx, r.bucket, r.object, topts); err != nil {
+ t.Fatal(err)
+ }
+ return oi, body
+}
+
+func nativeRestoreSnapshot(t *testing.T, r *restoreTest) [][]FileInfo {
+ t.Helper()
+ result := make([][]FileInfo, len(r.z.serverPools))
+ for p, pool := range r.z.serverPools {
+ raw, err := pool.getHashedSet(r.object).getDisks()[0].ReadXL(t.Context(), r.bucket, r.object, false)
+ if errors.Is(err, errFileNotFound) {
+ continue
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ infos, err := getFileInfoVersions(raw.Buf, r.bucket, r.object, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ result[p] = append(infos.Versions, infos.FreeVersions...)
+ }
+ return result
+}
+
+func nativeRestoreOptions(r *restoreTest, id string) ObjectOptions {
+ opts := r.opts(id)
+ opts.Transition = TransitionOptions{RestoreRequest: &RestoreObjectRequest{Days: 1}}
+ return opts
+}
+
+type nativeRestoreFailTier struct {
+ WarmBackend
+ failure error
+}
+
+func (b nativeRestoreFailTier) Get(context.Context, string, remoteVersionID, WarmBackendGetOpts) (io.ReadCloser, error) {
+ return nil, b.failure
+}
+
+func TestNativeRestorePools(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, multipart := range []bool{false, true} {
+ for _, fail := range []bool{false, true} {
+ t.Run(fmt.Sprintf("pools=%d/multipart=%v/failure=%v", pools, multipart, fail), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ oi, body := nativeRestoreSeed(t, r, 0, false, multipart)
+ before := nativeRestoreSnapshot(t, r)
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ failure := errors.New("injected remote download failure")
+ if fail {
+ globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, failure}
+ }
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+ err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ if fail {
+ if !errors.Is(err, failure) {
+ t.Fatalf("restore error=%v, want remote failure", err)
+ }
+ got, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil || got.UserDefined[xhttp.AmzRestore] != "" {
+ t.Fatalf("failed restore did not clear ongoing status: %v, %q", err, got.UserDefined[xhttp.AmzRestore])
+ }
+ } else {
+ if err != nil {
+ t.Fatal(err)
+ }
+ reads := tier.gets.Load()
+ _, got, err := r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents differ: %v", err)
+ }
+ if tier.gets.Load() != reads {
+ t.Error("restored read still fetched the remote tier")
+ }
+ }
+ after := nativeRestoreSnapshot(t, r)
+ if len(after[0]) != len(before[0]) || after[0][0].VersionID != before[0][0].VersionID || after[0][0].TransitionedObjName != before[0][0].TransitionedObjName {
+ t.Fatal("restore changed the version stack or remote object identity")
+ }
+ if after[0][0].Metadata["content-type"] != "application/x-restore-test" || after[0][0].Metadata["x-amz-meta-preserve"] != "keep-me" {
+ t.Error("restore lost original metadata")
+ }
+ })
+ }
+ }
+ }
+}
+
+func TestNativeRestoreVersionRouting(t *testing.T) {
+ for _, pool := range []int{0, 1} {
+ for _, null := range []bool{false, true} {
+ t.Run(fmt.Sprintf("target-pool=%d/null=%v", pool, null), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+ oi, body := nativeRestoreSeed(t, r, pool, null, false)
+ latest := putConsistencyObject(t, r.z, r.bucket, r.object, 1-pool, "latest-stays", ObjectOptions{Versioned: true, MTime: oi.ModTime.Add(time.Hour)})
+ before := nativeRestoreSnapshot(t, r)
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+ if err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ _, got, err := r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("requested version not restored: %v", err)
+ }
+ after := nativeRestoreSnapshot(t, r)
+ if !reflect.DeepEqual(before[1-pool], after[1-pool]) {
+ t.Error("restore changed the latest version in another pool")
+ }
+ if len(after[pool]) != 1 || after[pool][0].VersionID != before[pool][0].VersionID {
+ t.Error("restore created a version alias")
+ }
+ gotInfo, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{})
+ if err != nil || gotInfo.VersionID != latest.VersionID {
+ t.Errorf("latest changed: %v", err)
+ }
+ before = nativeRestoreSnapshot(t, r)
+ if err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, mustGetUUID())); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+ t.Errorf("missing version: %v", err)
+ }
+ if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
+ t.Error("missing version restore mutated metadata")
+ }
+ })
+ }
+ }
+}
+
+func TestNativeRestoreInitialFailurePreservesMetadata(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi := putConsistencyObject(t, r.z, r.bucket, r.object, 0, "original-latest", ObjectOptions{Versioned: true, UserDefined: map[string]string{"content-type": "application/x-original", "x-amz-meta-preserve": "keep"}})
+ before := nativeRestoreSnapshot(t, r)
+ for _, missing := range []string{mustGetUUID(), nullVersionID} {
+ err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, missing))
+ if !isErrVersionNotFound(err) {
+ t.Errorf("missing restore %s: %v", missing, err)
+ }
+ if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
+ t.Error("initial restore failure changed stored metadata or version stack")
+ }
+ }
+ _, body, err := r.read(oi.VersionID)
+ if err != nil || string(body) != "original-latest" {
+ t.Errorf("initial restore failure changed content: %v", err)
+ }
+}
+
+func TestNativeRestoreWithoutOuterLock(t *testing.T) {
+ for _, mode := range []string{"single-part", "multipart", "failure-metadata"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, mode == "multipart")
+ tier := globalTierConfigMgr.drivercache["WARM"]
+ failure := errors.New("injected remote failure")
+ if mode == "failure-metadata" {
+ globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, failure}
+ }
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ lock := er.NewNSLock(r.bucket, r.object)
+ lkctx, err := lock.GetLock(t.Context(), globalOperationTimeout)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer lock.Unlock(lkctx)
+ before := nativeRestoreSnapshot(t, r)
+ ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+ defer cancel()
+ err = er.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ if err == nil {
+ t.Error("restore wrote through another caller's lock")
+ }
+ if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
+ t.Error("NoLock=false failed to protect the stored version")
+ }
+ })
+ }
+}
+
+type nativeRestoreHTTPResult struct {
+ opts ObjectOptions
+ err error
+}
+type nativeRestoreHTTPObserver struct {
+ ObjectLayer
+ done chan nativeRestoreHTTPResult
+}
+
+func (o nativeRestoreHTTPObserver) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+ err := o.ObjectLayer.RestoreTransitionedObject(ctx, bucket, object, opts)
+ o.done <- nativeRestoreHTTPResult{opts, err}
+ return err
+}
+
+func TestNativeRestorePOSTVersionIdentity(t *testing.T) {
+ for _, state := range []versioning.State{versioning.Enabled, versioning.Suspended} {
+ for _, target := range []int{0, 1} {
+ for _, split := range []bool{false, true} {
+ for _, multipart := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/target=%d/split=%v/multipart=%v", state, target, split, multipart), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 2, state)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := nativeRestoreSeed(t, r, target, !multipart, multipart)
+ metaOpts := r.opts(oi.VersionID)
+ metaOpts.MTime = oi.ModTime
+ metaOpts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+ o.UserDefined[xhttp.AmzRestore] = ""
+ return ReplicateDecision{}, nil
+ }
+ if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, metaOpts); err != nil {
+ t.Fatal(err)
+ }
+ if split {
+ putConsistencyObject(t, r.z, r.bucket, r.object, 1-target, "newer-version", ObjectOptions{Versioned: true, MTime: oi.ModTime.Add(time.Hour)})
+ }
+ before := nativeRestoreSnapshot(t, r)
+ observed := nativeRestoreHTTPObserver{r.z, make(chan nativeRestoreHTTPResult, 1)}
+ api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return observed }}
+ events := make(chan event.Event, 1)
+ done := make(chan struct{})
+ defer close(done)
+ if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRestoreCompleted), events, done, nil); err != nil {
+ t.Fatal(err)
+ }
+ bodyXML := []byte(`1`)
+ path := "/" + r.bucket + "/" + r.object + "?restore"
+ if split {
+ path += "&versionId=" + url.QueryEscape(oi.VersionID)
+ }
+ req, err := newTestSignedRequestV4(http.MethodPost, path, int64(len(bodyXML)), bytes.NewReader(bodyXML), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err = req.ParseForm(); err != nil {
+ t.Fatal(err)
+ }
+ req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object})
+ rec := httptest.NewRecorder()
+ api.PostRestoreObjectHandler(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("POST restore: %d %s", rec.Code, rec.Body.String())
+ }
+ select {
+ case result := <-observed.done:
+ if result.err != nil {
+ t.Fatal(result.err)
+ }
+ if result.opts.VersionID != oi.VersionID || result.opts.Versioned != (state == versioning.Enabled) || result.opts.VersionSuspended != (state == versioning.Suspended) {
+ t.Errorf("background options lost version context: %+v", result.opts)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("background restore did not return")
+ }
+ select {
+ case <-events:
+ case <-time.After(5 * time.Second):
+ t.Fatal("no restore completion event")
+ }
+ after := nativeRestoreSnapshot(t, r)
+ if len(after[target]) != 1 || after[target][0].VersionID != before[target][0].VersionID || after[target][0].TransitionedObjName != before[target][0].TransitionedObjName {
+ t.Fatal("POST restore added a version alias")
+ }
+ if !reflect.DeepEqual(before[1-target], after[1-target]) {
+ t.Error("POST changed another pool's version")
+ }
+ _, got, err := r.read(oi.VersionID)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("POST-restored contents differ: %v", err)
+ }
+ })
+ }
+ }
+ }
+ }
+}
+
+// A restore request is internal context, not an exception for multipart
+// overwrites: a native restore never completes an upload, so no completion,
+// whatever restore context it carries, writes an existing version back.
+func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) {
+ for _, mode := range []string{"matching-restore", "ordinary-complete", "restore-without-attempt", "other-attempt", "not-tiered", "changed-time", "changed-size", "changed-etag"} {
+ t.Run(mode, func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, body := nativeRestoreSeed(t, r, 0, false, true)
+ pool := r.z.serverPools[0]
+ if mode == "not-tiered" {
+ er := pool.getHashedSet(r.object)
+ fi, _, _, err := er.getObjectFileInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID), false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fi.TransitionStatus = lifecycle.TransitionPending
+ for _, disk := range er.getDisks() {
+ if err := disk.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ current, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ opts := putRestoreOpts(&RestoreObjectRequest{Days: 1}, current, r.opts(oi.VersionID))
+ opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1}
+ opts.PreserveETag = current.ETag
+ if mode == "ordinary-complete" {
+ opts.Transition.RestoreRequest = nil
+ }
+ if mode == "changed-time" {
+ opts.MTime = opts.MTime.Add(time.Second)
+ }
+ // Hold the ETag constant for the size case, so only size can reject it.
+ if mode == "changed-size" {
+ body = append(body, 'x')
+ }
+ if mode == "changed-etag" {
+ opts.PreserveETag = "different-etag"
+ }
+ // The attempt claims what its upload claims; only the matching
+ // mode claims exactly the stored version.
+ if mode != "ordinary-complete" && mode != "restore-without-attempt" {
+ opts.Transition.attempt = &restoreAttempt{ID: mustGetUUID(), VersionID: oi.VersionID, ETag: opts.PreserveETag, Size: current.Size, ModTime: opts.MTime}
+ }
+ uopts := opts
+ uopts.MTime = time.Time{}
+ upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, uopts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var complete []CompletePart
+ for i, data := range [][]byte{body[:5<<20], body[5<<20:]} {
+ part, err := pool.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, i+1,
+ mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Transition: opts.Transition})
+ if err != nil {
+ t.Fatal(err)
+ }
+ complete = append(complete, CompletePart{PartNumber: i + 1, ETag: part.ETag})
+ }
+ if mode != "changed-size" && getCompleteMultipartMD5(complete) != current.ETag {
+ t.Fatal("fixture multipart ETag does not match original")
+ }
+ if mode == "other-attempt" {
+ other := *opts.Transition.attempt
+ other.ID = mustGetUUID()
+ opts.Transition.attempt = &other
+ }
+ before := nativeRestoreSnapshot(t, r)
+ _, err = pool.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, complete, opts)
+ if !errors.Is(err, errAuthentication) {
+ t.Fatalf("existing version overwrite: %v", err)
+ }
+ if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) {
+ t.Error("rejected completion changed the target")
+ }
+ })
+ }
+}
+
+// Hold the download after the restore snapshot has been read. Object Lock
+// updates must serialize with both successful rehydration and failure cleanup.
+type nativeRestoreBlockedTier struct {
+ WarmBackend
+ entered chan struct{}
+ release chan struct{}
+ once sync.Once
+ failure error
+ afterFirstPart bool
+}
+
+func (b *nativeRestoreBlockedTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+ if b.afterFirstPart {
+ r, err := b.WarmBackend.Get(ctx, object, version, opts)
+ if err != nil {
+ return nil, err
+ }
+ return &nativeRestorePartGateReader{ReadCloser: r, gate: b, ctx: ctx, remaining: 5 << 20}, nil
+ }
+ b.once.Do(func() { close(b.entered) })
+ select {
+ case <-b.release:
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+ if b.failure != nil {
+ return nil, b.failure
+ }
+ return b.WarmBackend.Get(ctx, object, version, opts)
+}
+
+func nativeRestoreLockRequest(ctx context.Context, t *testing.T, r *restoreTest, id, kind string, body []byte) *httptest.ResponseRecorder {
+ t.Helper()
+ sum := md5.Sum(body)
+ req, err := newTestSignedRequestV4(http.MethodPut, "/"+r.bucket+"/"+r.object+"?"+kind+"&versionId="+url.QueryEscape(id), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := req.ParseForm(); err != nil {
+ t.Fatal(err)
+ }
+ req = mux.SetURLVars(req.WithContext(ctx), map[string]string{"bucket": r.bucket, "object": r.object})
+ rec := httptest.NewRecorder()
+ api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return r.z }}
+ if kind == "legal-hold" {
+ api.PutObjectLegalHoldHandler(rec, req)
+ } else {
+ api.PutObjectRetentionHandler(rec, req)
+ }
+ return rec
+}
+
+// A native restore downloads without the object lock. An Object Lock
+// update made meanwhile commits at once, and the restore, whether it
+// completes or fails, keeps it.
+func TestNativeRestoreKeepsObjectLockUpdate(t *testing.T) {
+ for _, kind := range []string{"legal-hold", "retention"} {
+ for _, fail := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/failure=%v", kind, fail), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ enableBucketObjectLock(t, r.bucket)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, true)
+ initial := []byte(`OFF`)
+ updated := []byte(`ON`)
+ keys := []string{strings.ToLower(xhttp.AmzObjectLockLegalHold), ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp}
+ if kind == "retention" {
+ initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour))
+ updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))
+ keys = []string{strings.ToLower(xhttp.AmzObjectLockMode), strings.ToLower(xhttp.AmzObjectLockRetainUntilDate), ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp}
+ }
+ if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK {
+ t.Fatalf("initial protection: %d %s", rec.Code, rec.Body.String())
+ }
+ before, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ blocked := &nativeRestoreBlockedTier{WarmBackend: globalTierConfigMgr.drivercache["WARM"], entered: make(chan struct{}), release: make(chan struct{})}
+ if fail {
+ blocked.failure = errors.New("injected download failure after snapshot")
+ }
+ globalTierConfigMgr.drivercache["WARM"] = blocked
+ var release sync.Once
+ unblock := func() { release.Do(func() { close(blocked.release) }) }
+ defer unblock()
+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ defer cancel()
+ result := make(chan error, 1)
+ go func() {
+ result <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ }()
+ select {
+ case <-blocked.entered:
+ case <-ctx.Done():
+ t.Fatal("restore never started download")
+ }
+ // The download holds no object lock: an Object Lock update
+ // commits at once, and the restore's commit keeps it.
+ updateCtx, updateCancel := context.WithTimeout(t.Context(), 5*time.Second)
+ rec := nativeRestoreLockRequest(updateCtx, t, r, oi.VersionID, kind, updated)
+ updateCancel()
+ if rec.Code != http.StatusOK {
+ t.Fatalf("Object Lock update during the restore download: %d %s", rec.Code, rec.Body.String())
+ }
+ readOpts := r.opts(oi.VersionID)
+ readOpts.NoLock = true
+ during, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, readOpts)
+ if err != nil {
+ t.Fatal(err)
+ }
+ unblock()
+ select {
+ case err := <-result:
+ if !errors.Is(err, blocked.failure) {
+ t.Fatalf("restore returned %v, want %v", err, blocked.failure)
+ }
+ case <-ctx.Done():
+ t.Fatal("restore failed to finish")
+ }
+ after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, key := range keys {
+ if during.UserDefined[key] == "" || after.UserDefined[key] != during.UserDefined[key] {
+ t.Errorf("restore changed protection %s: %q -> %q", key, during.UserDefined[key], after.UserDefined[key])
+ }
+ }
+ changed := keys[0]
+ if kind == "retention" {
+ changed = keys[1]
+ }
+ if after.UserDefined[changed] == before.UserDefined[changed] {
+ t.Error("the update made during the download was not kept")
+ }
+ if !after.ModTime.Equal(before.ModTime) || after.ETag != before.ETag || after.Size != before.Size {
+ t.Error("Object Lock update changed content identity")
+ }
+ })
+ }
+ }
+}
+
+// The POST handler marks a version ongoing before its background restore
+// takes the object lock, and only that restore clears the marker. A restore
+// queued behind another version's download must keep waiting: one that gave up
+// after a lock timeout would leave the marker for every later POST to refuse.
+// The wait still ends with its context.
+func TestNativeRestoreQueuedBehindDownload(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ first, _ := nativeRestoreSeed(t, r, 0, false, false)
+ queued, body := nativeRestoreSeed(t, r, pools-1, false, false)
+ prev := globalRestoreLockTimeout
+ globalRestoreLockTimeout = newDynamicTimeout(50*time.Millisecond, 50*time.Millisecond)
+ t.Cleanup(func() { globalRestoreLockTimeout = prev })
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ blocked := &nativeRestoreBlockedTier{WarmBackend: tier, entered: make(chan struct{}), release: make(chan struct{})}
+ globalTierConfigMgr.drivercache["WARM"] = blocked
+ var once sync.Once
+ unblock := func() { once.Do(func() { close(blocked.release) }) }
+ defer unblock()
+ ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
+ defer cancel()
+ firstDone := make(chan error, 1)
+ go func() {
+ firstDone <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, first.VersionID))
+ }()
+ select {
+ case <-blocked.entered:
+ case <-ctx.Done():
+ t.Fatal("first restore never reached the tier")
+ }
+ short, shortCancel := context.WithTimeout(ctx, 200*time.Millisecond)
+ defer shortCancel()
+ ended := make(chan error, 1)
+ go func() {
+ ended <- r.z.RestoreTransitionedObject(short, r.bucket, r.object, nativeRestoreOptions(r, queued.VersionID))
+ }()
+ select {
+ case err := <-ended:
+ if err == nil {
+ t.Fatal("restore finished without the object lock")
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("restore kept waiting for the lock after its context ended")
+ }
+ queuedDone := make(chan error, 1)
+ go func() {
+ queuedDone <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, queued.VersionID))
+ }()
+ select {
+ case err := <-queuedDone:
+ t.Fatalf("queued restore gave up behind the download: %v", err)
+ case <-time.After(10 * 50 * time.Millisecond):
+ }
+ unblock()
+ for _, done := range []chan error{firstDone, queuedDone} {
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatalf("restore: %v", err)
+ }
+ case <-ctx.Done():
+ t.Fatal("restore did not finish")
+ }
+ }
+ info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(queued.VersionID))
+ if err != nil || info.RestoreOngoing {
+ t.Fatalf("queued version still ongoing after restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+ }
+ reads := tier.gets.Load()
+ if _, got, err := r.read(queued.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("queued restore contents: %v", err)
+ }
+ if tier.gets.Load() != reads {
+ t.Error("queued version was not restored locally")
+ }
+ })
+ }
+}
+
+type nativeRestorePartGateReader struct {
+ io.ReadCloser
+ gate *nativeRestoreBlockedTier
+ ctx context.Context
+ remaining int
+}
+
+func (r *nativeRestorePartGateReader) Read(p []byte) (int, error) {
+ if r.remaining > 0 {
+ if len(p) > r.remaining {
+ p = p[:r.remaining]
+ }
+ n, err := r.ReadCloser.Read(p)
+ r.remaining -= n
+ return n, err
+ }
+ r.gate.once.Do(func() { close(r.gate.entered) })
+ select {
+ case <-r.gate.release:
+ case <-r.ctx.Done():
+ return 0, r.ctx.Err()
+ }
+ return r.ReadCloser.Read(p)
+}
+
+// Invalid upload paths must retain the backend's normal error precedence,
+// rather than reacquiring the object lock as the upload lock.
+func TestMultipartCompletionLockAlias(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, id := range []string{"", ".", "./"} {
+ t.Run(fmt.Sprintf("pools=%d/id=%q", pools, id), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ for _, missingBucket := range []bool{false, true} {
+ bucket := r.bucket
+ if missingBucket {
+ bucket = "missing-bucket"
+ }
+ ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond)
+ _, err := r.z.CompleteMultipartUpload(ctx, bucket, r.object, id, nil, ObjectOptions{})
+ cancel()
+ switch {
+ case id != "":
+ if !errors.As(err, &MalformedUploadID{}) {
+ t.Errorf("malformed upload: %v", err)
+ }
+ case missingBucket:
+ if !isErrBucketNotFound(err) {
+ t.Errorf("missing bucket: %v", err)
+ }
+ default:
+ if !errors.As(err, &InvalidUploadID{}) {
+ t.Errorf("invalid upload: %v", err)
+ }
+ }
+ }
+ })
+ }
+ }
+}
+
+type nativeRestoreReadHook struct {
+ io.ReadCloser
+ remaining int
+ mutate func()
+}
+
+func (r *nativeRestoreReadHook) Read(p []byte) (int, error) {
+ n, err := r.ReadCloser.Read(p)
+ r.remaining -= n
+ if r.remaining == 0 && r.mutate != nil {
+ f := r.mutate
+ r.mutate = nil
+ f()
+ }
+ return n, err
+}
+
+type nativeRestoreReadHookTier struct {
+ WarmBackend
+ size int
+ mutate func()
+}
+
+func (b nativeRestoreReadHookTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
+ rd, err := b.WarmBackend.Get(ctx, object, version, opts)
+ if err != nil {
+ return nil, err
+ }
+ return &nativeRestoreReadHook{rd, b.size, b.mutate}, nil
+}
+
+// Remove only the addressed entry from a majority of disks, keeping shards
+// and the healthy newer version. Return errors to the synchronous caller.
+func nativeRestoreLoseMetadata(ctx context.Context, disks []StorageAPI, bucket, object, id string) error {
+ for _, disk := range disks[1:] {
+ raw, err := disk.ReadXL(ctx, bucket, object, true)
+ if err != nil {
+ return err
+ }
+ var meta xlMetaV2
+ if err := meta.Load(raw.Buf); err != nil {
+ return err
+ }
+ del := FileInfo{VersionID: id}
+ del.SetSkipTierFreeVersion()
+ if _, err := meta.DeleteVersion(del); err != nil {
+ return err
+ }
+ data, err := meta.AppendTo(nil)
+ if err != nil {
+ return err
+ }
+ if err := disk.WriteAll(ctx, bucket, filepath.Join(object, xlStorageFormatFile), data); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// A target can lose metadata quorum after the restore snapshot. Nested
+// completion/retention reads must not turn rehydration into remote GC.
+func TestNativeRestoreCompletionReadSafety(t *testing.T) {
+ for _, null := range []bool{false, true} {
+ for _, multipart := range []bool{false, true} {
+ t.Run(fmt.Sprintf("null=%v/multipart=%v", null, multipart), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, body := nativeRestoreSeed(t, r, 0, null, multipart)
+ latest := r.put("newer", false, false)
+ enableBucketObjectLock(t, r.bucket)
+ disks := r.z.serverPools[0].getHashedSet(r.object).getDisks()
+ var mutationErr error
+ mutated := false
+ mutate := func() {
+ mutated = true
+ mutationErr = nativeRestoreLoseMetadata(t.Context(), disks, r.bucket, r.object, oi.VersionID)
+ }
+ tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier)
+ globalTierConfigMgr.drivercache["WARM"] = nativeRestoreReadHookTier{tier, len(body), mutate}
+ ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second)
+ defer cancel()
+ err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ if ctx.Err() != nil {
+ t.Fatalf("restore timed out: %v", err)
+ }
+ if !mutated || mutationErr != nil {
+ t.Fatalf("invalid injection: mutated=%v err=%v", mutated, mutationErr)
+ }
+ t.Logf("restore after addressed metadata lost: %v", err)
+ for i, disk := range disks {
+ raw, e := disk.ReadXL(t.Context(), r.bucket, r.object, true)
+ if e != nil {
+ t.Fatal(e)
+ }
+ versions, e := getFileInfoVersions(raw.Buf, r.bucket, r.object, false)
+ if e != nil {
+ t.Fatal(e)
+ }
+ if len(versions.FreeVersions) > 0 {
+ t.Errorf("disk %d restore manufactured %d remote GC entries", i, len(versions.FreeVersions))
+ }
+ }
+ if err == nil {
+ reads := tier.gets.Load()
+ _, got, readErr := r.read(oi.VersionID)
+ if readErr != nil || !bytes.Equal(got, body) || tier.gets.Load() != reads {
+ t.Errorf("successful restore did not rehydrate its own content: %v", readErr)
+ }
+ }
+ _, got, readErr := r.read(latest.VersionID)
+ if readErr != nil || !bytes.Equal(got, r.bodies["newer"]) {
+ t.Errorf("restore changed the other version: %v", readErr)
+ }
+ })
+ }
+ }
+}
+
+// nativeRestoreStatusHook runs afterRead once POST has read the version and
+// before POST records its status on it.
+type nativeRestoreStatusHook struct {
+ ObjectLayer
+ afterRead func()
+ readCalled bool
+ backgroundCalled atomic.Bool
+}
+
+func (h *nativeRestoreStatusHook) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+ oi, err := h.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts)
+ if err == nil && !h.readCalled {
+ h.readCalled = true
+ h.afterRead()
+ }
+ return oi, err
+}
+
+func (h *nativeRestoreStatusHook) RestoreTransitionedObject(context.Context, string, string, ObjectOptions) error {
+ h.backgroundCalled.Store(true)
+ return errors.New("unexpected background restore after failed status update")
+}
+
+func TestNativeRestorePOSTReadSafety(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, stage := range []string{"initial", "status-copy"} {
+ t.Run(fmt.Sprintf("pools=%d/%s", pools, stage), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+ metaOpts := r.opts(oi.VersionID)
+ metaOpts.MTime = oi.ModTime
+ metaOpts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) {
+ o.UserDefined[xhttp.AmzRestore] = ""
+ return ReplicateDecision{}, nil
+ }
+ if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, metaOpts); err != nil {
+ t.Fatal(err)
+ }
+ r.put("newer", false, false)
+ disks := r.z.serverPools[0].getHashedSet(r.object).getDisks()
+ var before [][]byte
+ mutate := func() {
+ if err := nativeRestoreLoseMetadata(t.Context(), disks, r.bucket, r.object, oi.VersionID); err != nil {
+ t.Fatal(err)
+ }
+ before = restoreBoundarySnapshot(t, r)
+ }
+ hook := &nativeRestoreStatusHook{ObjectLayer: r.z, afterRead: mutate}
+ if stage == "initial" {
+ mutate()
+ }
+ api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return hook }}
+ body := []byte(`1`)
+ req, err := newTestSignedRequestV4(http.MethodPost, "/"+r.bucket+"/"+r.object+"?restore&versionId="+url.QueryEscape(oi.VersionID), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := req.ParseForm(); err != nil {
+ t.Fatal(err)
+ }
+ req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object})
+ rec := httptest.NewRecorder()
+ api.PostRestoreObjectHandler(rec, req)
+ if rec.Code < 400 {
+ t.Fatalf("POST acknowledged failed read: %d %s", rec.Code, rec.Body.String())
+ }
+ if hook.readCalled != (stage == "status-copy") || hook.backgroundCalled.Load() {
+ t.Fatal("request followed the wrong failure path")
+ }
+ if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) {
+ t.Error("failed POST read changed stored metadata or free versions")
+ }
+ })
+ }
+ }
+}
diff --git a/cmd/restore-stage.go b/cmd/restore-stage.go
new file mode 100644
index 000000000..1bc2a443e
--- /dev/null
+++ b/cmd/restore-stage.go
@@ -0,0 +1,900 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "maps"
+ "net/http"
+ "slices"
+ "strconv"
+ "sync"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/minio/minio/internal/bucket/lifecycle"
+ xhttp "github.com/minio/minio/internal/http"
+ "github.com/pgsty/silo-pkg/v3/sync/errgroup"
+)
+
+// A native restore runs in two phases. It downloads the tier copy of a
+// version and erasure-codes it into the tmp volume of the version's set
+// without the object lock O, then takes O only to check the version again
+// and publish the staged copy drive by drive, as heal does. Hot versions of
+// the key stay readable and writable during the download.
+//
+// The restore lock R of the version tells whether a restore of it runs: a
+// POST takes R before O, and the worker that rehydrates the version holds R
+// for its whole run. Nothing else takes R, so nothing holds O while it waits
+// for R. R is a liveness hint, not an exclusion: a lost refresh can let two
+// workers overlap. Only the checks under O decide what is written.
+
+// restoreOwnerPrefix names the restore locks under minioMetaBucket.
+const restoreOwnerPrefix = "restore"
+
+// restorePublicationDelays are the in-memory retries of a convergence after
+// the one under the commit's own lock hold, see recoverRestorePublication.
+var restorePublicationDelays = []time.Duration{100 * time.Millisecond, time.Second, 5 * time.Second, 30 * time.Second, 2 * time.Minute}
+
+var (
+ // errRestoreMoved: the version moved to another pool after the restore
+ // staged its copy in this one.
+ errRestoreMoved = errors.New("restored version moved to another pool")
+ // errRestoreNotConverged: a publication that reached some drives but not
+ // a write quorum could not be brought back to the unrestored version yet.
+ errRestoreNotConverged = errors.New("restore publication did not converge")
+)
+
+// restoreOwner is the restore lock R of one version, held by a POST that
+// registers an attempt and then by the worker that runs it. ctx ends when the
+// lock is released or when its holder learns that it lost it.
+type restoreOwner struct {
+ lk RWLocker
+ lkctx LockContext
+ ctx context.Context
+ cancel context.CancelFunc
+ once sync.Once
+}
+
+// release unlocks R once. It is a no-op on a nil owner.
+func (o *restoreOwner) release() {
+ if o == nil {
+ return
+ }
+ o.once.Do(func() {
+ o.cancel()
+ o.lk.Unlock(o.lkctx)
+ })
+}
+
+// restoreOwnerResource is the name of the restore lock of a version. Bucket
+// names hold no slash, object names no ".." element, and the version ID has
+// a fixed form, so the name is unambiguous.
+func restoreOwnerResource(bucket, object, versionID string) string {
+ return pathJoin(restoreOwnerPrefix, bucket, object, restoreVersionID(versionID))
+}
+
+// takeRestoreOwner takes the restore lock of a version. With a nil timeout it
+// waits for as long as ctx lives, as a direct caller does; otherwise it gives
+// up after timeout. ctx must outlive the restore that the lock is held for.
+func (z *erasureServerPools) takeRestoreOwner(ctx context.Context, bucket, object, versionID string, timeout *dynamicTimeout) (*restoreOwner, error) {
+ for {
+ // A new instance for each try: a distributed unlock retries in the
+ // background and could release a later try of the same instance.
+ lk := z.NewNSLock(minioMetaBucket, restoreOwnerResource(bucket, object, versionID))
+ wait := timeout
+ if wait == nil {
+ wait = globalRestoreLockTimeout
+ }
+ lkctx, err := lk.GetLock(ctx, wait)
+ if err != nil {
+ if timeout == nil && errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil {
+ continue
+ }
+ return nil, err
+ }
+ octx, cancel := context.WithCancel(lkctx.Context())
+ return &restoreOwner{lk: lk, lkctx: lkctx, ctx: octx, cancel: cancel}, nil
+ }
+}
+
+// restoreStage is the copy of one version that a native restore encoded into
+// the tmp volume of the version's set, under tmpID/dataDir, before it takes
+// the object lock.
+type restoreStage struct {
+ versionID string
+ // fi is the version as the download read it: the commit only publishes
+ // the copy for a version that still has this content, see
+ // restoreSameContent.
+ fi FileInfo
+ tmpID string
+ dataDir string
+ // staged tells, in the order of the version's erasure distribution,
+ // which drives hold every part of the copy.
+ staged []bool
+ expiry time.Time
+ // request is set for an attempt that POST registered: days and date
+ // are what it recorded. A direct caller keeps the version's own.
+ request bool
+ days int
+ date string
+}
+
+// restoreSameContent reports whether two reads of a version name the same
+// encoded bytes, laid out the same way: the version, its tier copy, its parts
+// and its erasure layout. The tier copy is never rewritten under the same
+// name and remote version, so these decide the bytes. Every other field, such
+// as tags, Object Lock, replication state or a rewrapped object key, is taken
+// from the version as the commit reads it.
+func restoreSameContent(a, b FileInfo) bool {
+ if a.VersionID != b.VersionID || a.Deleted || b.Deleted || !a.ModTime.Equal(b.ModTime) ||
+ a.Metadata["etag"] != b.Metadata["etag"] || a.Size != b.Size ||
+ a.TransitionStatus != lifecycle.TransitionComplete || b.TransitionStatus != lifecycle.TransitionComplete ||
+ a.TransitionTier != b.TransitionTier || a.TransitionedObjName != b.TransitionedObjName ||
+ a.TransitionVersionID != b.TransitionVersionID {
+ return false
+ }
+ if len(a.Parts) != len(b.Parts) {
+ return false
+ }
+ for i := range a.Parts {
+ if a.Parts[i].Number != b.Parts[i].Number || a.Parts[i].Size != b.Parts[i].Size {
+ return false
+ }
+ }
+ ea, eb := a.Erasure, b.Erasure
+ return ea.Algorithm == eb.Algorithm && ea.DataBlocks == eb.DataBlocks && ea.ParityBlocks == eb.ParityBlocks &&
+ ea.BlockSize == eb.BlockSize && slices.Equal(ea.Distribution, eb.Distribution)
+}
+
+// restoreInlineKey marks a version whose data lives in its xl.meta.
+const restoreInlineKey = ReservedMetadataPrefixLower + "inline-data"
+
+// stageRestore downloads the tier copy of the version opts address and
+// erasure-codes it, part by part, into the tmp volume of the drives of this
+// set, as heal writes the parts it rebuilds. It holds no object lock. A drive
+// that fails to write any part, or that misses a part afterwards, for
+// example because its tmp volume was moved away when its server restarted,
+// takes no further part and no part in the commit.
+func (er erasureObjects) stageRestore(ctx context.Context, bucket, object string, opts ObjectOptions) (*restoreStage, error) {
+ rreq := opts.Transition.RestoreRequest
+ if rreq == nil {
+ return nil, errInvalidArgument
+ }
+ gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
+ if err != nil {
+ return nil, toObjectErr(err, bucket, object, opts.VersionID)
+ }
+ fi := gen.fi
+ if fi.Deleted {
+ return nil, toObjectErr(errMethodNotAllowed, bucket, object)
+ }
+ if a := opts.Transition.attempt; a != nil && !a.ownsGeneration(gen, bucket, object) {
+ // A later POST took the version over, or it was replaced.
+ return nil, errRestoreSuperseded
+ }
+ disks := er.getDisks()
+ var total int64
+ for _, p := range fi.Parts {
+ if p.Size < 0 {
+ return nil, toObjectErr(errFileCorrupt, bucket, object)
+ }
+ total += p.Size
+ }
+ if fi.TransitionStatus != lifecycle.TransitionComplete || len(fi.Parts) == 0 || total != fi.Size ||
+ len(fi.Erasure.Distribution) != len(disks) {
+ return nil, InvalidObjectState{Bucket: bucket, Object: object}
+ }
+ erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
+ if err != nil {
+ return nil, toObjectErr(err, bucket, object)
+ }
+ writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
+ st := &restoreStage{
+ versionID: restoreVersionID(fi.VersionID),
+ fi: fi,
+ tmpID: mustGetUUID(),
+ dataDir: mustGetUUID(),
+ expiry: lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days),
+ }
+ if opts.Transition.restoreDate != "" {
+ st.request, st.days, st.date = true, rreq.Days, opts.Transition.restoreDate
+ }
+ staged := false
+ defer func() {
+ if !staged {
+ er.deleteAll(context.Background(), minioMetaTmpBucket, st.tmpID)
+ }
+ }()
+
+ // The tier copy is the encoded stream of the parts, encrypted or
+ // compressed as stored: the parts get exactly these bytes back.
+ oi := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
+ gr, err := getTransitionedObjectReader(ctx, bucket, object, nil, http.Header{}, oi, opts)
+ if err != nil {
+ return nil, toObjectErr(err, bucket, object)
+ }
+ defer gr.Close()
+
+ var buffer []byte
+ pool := globalBytePoolCap.Load()
+ if int64(pool.Width()) < fi.Erasure.BlockSize {
+ buffer = make([]byte, fi.Erasure.BlockSize)
+ } else {
+ buffer = pool.Get()
+ defer pool.Put(buffer)
+ }
+ buffer = buffer[:fi.Erasure.BlockSize]
+
+ // Slots follow the erasure distribution: the drive in slot s gets shard
+ // s+1 of every part, and the commit gives it erasure index s+1.
+ slots := shuffleDisks(disks, fi.Erasure.Distribution)
+ for _, part := range fi.Parts {
+ partPath := pathJoin(st.tmpID, st.dataDir, fmt.Sprintf("part.%d", part.Number))
+ writers := make([]io.Writer, len(slots))
+ for s, disk := range slots {
+ if disk != nil {
+ writers[s] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, partPath,
+ erasure.ShardFileSize(part.Size), DefaultBitrotAlgorithm, erasure.ShardSize())
+ }
+ }
+ n, err := erasure.Encode(ctx, io.LimitReader(gr, part.Size), writers, buffer, writeQuorum)
+ closeErrs := closeBitrotWriters(writers)
+ if err != nil {
+ return nil, toObjectErr(err, bucket, object)
+ }
+ if n != part.Size {
+ return nil, IncompleteBody{Bucket: bucket, Object: object}
+ }
+ for s := range slots {
+ if writers[s] == nil || closeErrs[s] != nil {
+ slots[s] = nil
+ }
+ }
+ if countOnlineDisks(slots) < writeQuorum {
+ return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
+ }
+ }
+ // The tier copy must end with the last part.
+ var extra [1]byte
+ if n, _ := io.ReadFull(gr, extra[:]); n != 0 {
+ return nil, InvalidObjectState{Bucket: bucket, Object: object}
+ }
+
+ // A tmp volume that was moved away and recreated by a later part lacks the
+ // earlier parts: find them before anything is published. CheckParts sees a
+ // part that is present and not shorter than its shards; a staged drive
+ // that wrote every part without an error has exactly that.
+ check := fi
+ check.DataDir = st.dataDir
+ g := errgroup.WithNErrs(len(slots))
+ for s, disk := range slots {
+ if disk == nil {
+ continue
+ }
+ g.Go(func() error {
+ resp, err := disk.CheckParts(ctx, minioMetaTmpBucket, st.tmpID, check)
+ if err != nil {
+ return err
+ }
+ for _, result := range resp.Results {
+ if result != checkPartSuccess {
+ return errFileCorrupt
+ }
+ }
+ return nil
+ }, s)
+ }
+ st.staged = make([]bool, len(slots))
+ count := 0
+ for s, err := range g.Wait() {
+ if slots[s] != nil && err == nil {
+ st.staged[s] = true
+ count++
+ }
+ }
+ if count < writeQuorum {
+ return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
+ }
+ staged = true
+ return st, nil
+}
+
+// fileInfoRevisions are the revisions the entry that writing fi creates can
+// have, as readRestoreGeneration and the convergence identify the entries
+// they read. The header flag that tells whether a transitioned version uses
+// its data directory follows whether its restored copy has expired at the
+// moment a drive writes the entry, so a restore whose copy expires while it
+// publishes can find either value: both are this entry. Every other field of
+// the header and the full metadata digest still have to match.
+func fileInfoRevisions(fi FileInfo) ([]revisionKey, error) {
+ var x xlMetaV2
+ if err := x.AddVersion(fi); err != nil {
+ return nil, err
+ }
+ buf, err := x.AppendTo(nil)
+ if err != nil {
+ return nil, err
+ }
+ var y xlMetaV2
+ if err := y.Load(buf); err != nil {
+ return nil, err
+ }
+ if len(y.versions) != 1 {
+ return nil, errFileCorrupt
+ }
+ ver := y.versions[0]
+ other := ver
+ other.header.Flags ^= xlFlagUsesDataDir
+ return []revisionKey{versionRevision(ver, y.metaV), versionRevision(other, y.metaV)}, nil
+}
+
+// restorePublication remembers, for a commit that published the staged copy
+// on some drives but not on a write quorum, what each drive held before the
+// commit (Sᵢ), what the commit wrote there (Pᵢ) and what a convergence writes
+// there (Vᵢ), in the order of the version's erasure distribution. Only the
+// worker that published knows them; a convergence touches a version only
+// while every drive holds one of them.
+type restorePublication struct {
+ bucket, object string
+ versionID string
+ uv [16]byte
+ dataDir string
+ attempt *restoreAttempt
+ quorum int
+ dist []int
+ before []*revisionKey
+ anyBefore map[revisionKey]bool
+ published map[revisionKey]bool
+ conv []FileInfo
+ convKeys map[revisionKey]bool
+}
+
+// restorePublished is the entry a commit publishes on the drive in slot s:
+// the version as the commit read it on that drive, with the staged data
+// directory and a completed restore. Restore fields are written alike on
+// every drive: the request POST recorded, or for a direct caller the one
+// the authoritative generation auth holds.
+func restorePublished(base, auth FileInfo, st *restoreStage, slot int) FileInfo {
+ fi := base
+ fi.Metadata = maps.Clone(base.Metadata)
+ if fi.Metadata == nil {
+ fi.Metadata = make(map[string]string)
+ }
+ for _, k := range restoreFieldKeys {
+ delete(fi.Metadata, k)
+ }
+ // An earlier restore may have inlined the data, and expiring that copy
+ // leaves the inline marker behind; this copy lives in its data directory.
+ delete(fi.Metadata, restoreInlineKey)
+ fi.Metadata[xhttp.AmzRestore] = completedRestoreObj(st.expiry).String()
+ if st.request {
+ fi.Metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(st.days)
+ fi.Metadata[xhttp.AmzRestoreRequestDate] = st.date
+ } else {
+ for _, k := range []string{xhttp.AmzRestoreExpiryDays, xhttp.AmzRestoreRequestDate} {
+ if v, ok := auth.Metadata[k]; ok {
+ fi.Metadata[k] = v
+ }
+ }
+ }
+ fi.DataDir = st.dataDir
+ fi.Data = nil
+ fi.Parts = slices.Clone(base.Parts)
+ fi.Erasure.Index = slot + 1
+ // The staged parts use the default streaming bitrot algorithm.
+ fi.Erasure.Checksums = nil
+ return fi
+}
+
+// restoreUnrestored is the entry a convergence writes on the drive in slot s:
+// the version as the commit read it, with no restore fields, so that its tier
+// copy serves reads again and a later POST can restore it.
+func restoreUnrestored(base FileInfo, slot int) FileInfo {
+ fi := base
+ fi.Metadata = maps.Clone(base.Metadata)
+ for _, k := range restoreFieldKeys {
+ delete(fi.Metadata, k)
+ }
+ delete(fi.Metadata, restoreInlineKey)
+ fi.Data = nil
+ fi.Erasure.Index = slot + 1
+ return fi
+}
+
+// commitRestore publishes a staged copy while the caller holds the object
+// lock. It reads the version again, checks that the attempt still owns it and
+// that it still has the content the copy was staged for, then publishes the
+// copy on every staged drive that holds the version, with RenameData on each
+// drive, as heal does: never through erasure renameData, whose undo of a
+// replaced version could delete the new data and keep metadata that points
+// at it.
+//
+// It returns nil when the copy reached a write quorum. Otherwise it returns
+// an error, and a publication when some drive may already have the copy; the
+// caller must then converge, see convergeRestore.
+func (er erasureObjects) commitRestore(ctx context.Context, bucket, object string, st *restoreStage, opts ObjectOptions, placed func(context.Context) error) (*restorePublication, error) {
+ if placed != nil {
+ if err := placed(ctx); err != nil {
+ return nil, err
+ }
+ }
+ gen, err := er.readRestoreGeneration(ctx, bucket, object, st.versionID)
+ if err != nil {
+ return nil, toObjectErr(err, bucket, object, st.versionID)
+ }
+ if gen.fi.Deleted {
+ return nil, toObjectErr(errMethodNotAllowed, bucket, object)
+ }
+ a := opts.Transition.attempt
+ if a != nil && !a.ownsGeneration(gen, bucket, object) {
+ return nil, errRestoreSuperseded
+ }
+ if !restoreSameContent(st.fi, gen.fi) {
+ // Replaced, or its tier copy or layout changed since the download.
+ return nil, InvalidObjectState{Bucket: bucket, Object: object}
+ }
+ disks := er.getDisks()
+ dist := gen.fi.Erasure.Distribution
+ if len(dist) != len(disks) || len(gen.revs) != len(disks) {
+ return nil, InvalidObjectState{Bucket: bucket, Object: object}
+ }
+ n := len(disks)
+ writeQuorum := gen.fi.WriteQuorum(er.defaultWQuorum())
+
+ pub := &restorePublication{
+ bucket: bucket, object: object, versionID: st.versionID, dataDir: st.dataDir,
+ attempt: a, quorum: writeQuorum, dist: slices.Clone(dist),
+ before: make([]*revisionKey, n),
+ anyBefore: make(map[revisionKey]bool),
+ published: make(map[revisionKey]bool),
+ conv: make([]FileInfo, n),
+ convKeys: make(map[revisionKey]bool),
+ }
+ if st.versionID != nullVersionID {
+ id, err := uuid.Parse(st.versionID)
+ if err != nil {
+ return nil, toObjectErr(errFileVersionNotFound, bucket, object, st.versionID)
+ }
+ pub.uv = id
+ }
+ targets := make([]StorageAPI, n)
+ files := make([]FileInfo, n)
+ for i := range disks {
+ s := dist[i] - 1
+ if s < 0 || s >= n {
+ return nil, InvalidObjectState{Bucket: bucket, Object: object}
+ }
+ if gen.revs[i] != nil {
+ pub.before[s] = gen.revs[i]
+ pub.anyBefore[*gen.revs[i]] = true
+ }
+ v := restoreUnrestored(gen.fi, s)
+ keys, err := fileInfoRevisions(v)
+ if err != nil {
+ return nil, toObjectErr(err, bucket, object)
+ }
+ pub.conv[s] = v
+ // The unrestored entry has no restore header: its flag never varies.
+ pub.convKeys[keys[0]] = true
+ if gen.disks[i] == nil || !st.staged[s] {
+ continue
+ }
+ fi := restorePublished(gen.drives[i], gen.fi, st, s)
+ if keys, err = fileInfoRevisions(fi); err != nil {
+ return nil, toObjectErr(err, bucket, object)
+ }
+ for _, key := range keys {
+ pub.published[key] = true
+ }
+ targets[s], files[s] = gen.disks[i], fi
+ }
+ if countOnlineDisks(targets) < writeQuorum {
+ // Nothing is published.
+ return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
+ }
+
+ errs := make([]error, n)
+ olds := make([]string, n)
+ var wg sync.WaitGroup
+ for s, disk := range targets {
+ if disk == nil {
+ errs[s] = errDiskNotFound
+ continue
+ }
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ resp, err := disk.RenameData(ctx, minioMetaTmpBucket, st.tmpID, files[s], bucket, object, RenameOptions{})
+ olds[s], errs[s] = resp.OldDataDir, err
+ }()
+ }
+ wg.Wait()
+
+ if countErrs(errs, nil) < writeQuorum {
+ return pub, toObjectErr(errErasureWriteQuorum, bucket, object)
+ }
+ // The data directory of the replaced entry, which held a former restored
+ // copy, goes only when every drive that published reports the same one:
+ // RenameData reports none where another version shares it, and then the
+ // directory stays everywhere. It goes only from the drives that reported
+ // it.
+ old := ""
+ cleanup := make([]StorageAPI, n)
+ for s := range targets {
+ if errs[s] != nil {
+ continue
+ }
+ if olds[s] == "" || (old != "" && olds[s] != old) {
+ old = ""
+ break
+ }
+ old = olds[s]
+ cleanup[s] = targets[s]
+ }
+ if old != "" && old != st.dataDir {
+ er.commitRenameDataDir(ctx, bucket, object, old, cleanup, 1)
+ }
+ if countErrs(errs, nil) < n {
+ er.addPartial(bucket, object, gen.fi.VersionID)
+ }
+ return nil, nil
+}
+
+// restoreConvergence is the outcome of one convergence pass.
+type restoreConvergence int
+
+const (
+ // restoreIncomplete: the drives are not settled yet; try again.
+ restoreIncomplete restoreConvergence = iota
+ // restoreConverged: every drive that holds the version holds it
+ // unrestored again.
+ restoreConverged
+ // restoreCommitted: the copy reached a write quorum after all, such as
+ // when a drive applied a publication whose answer was lost.
+ restoreCommitted
+ // restoreStopped: some drive holds an entry that neither this attempt
+ // nor the version before it wrote: someone else changed the version.
+ restoreStopped
+)
+
+// classify reads the entry of the version on every drive and sorts it into
+// the slot of its drive: held tells which slots hold an entry this
+// publication knows, published and converged count the drives that hold
+// what the commit and the convergence write. A drive that cannot be read is
+// left out. An entry the publication does not know stops it.
+func (pub *restorePublication) classify(ctx context.Context, disks []StorageAPI) (held []bool, published, converged int, err error) {
+ raws, errs := readAllRawFileInfo(ctx, disks, pub.bucket, pub.object, false)
+ held = make([]bool, len(disks))
+ for i := range disks {
+ if errs[i] != nil || i >= len(pub.dist) {
+ continue
+ }
+ var xl xlMetaV2
+ if xl.LoadOrConvert(raws[i].Buf) != nil {
+ continue
+ }
+ s := pub.dist[i] - 1
+ for _, ver := range xl.versions {
+ if ver.header.VersionID != pub.uv || ver.header.FreeVersion() {
+ continue
+ }
+ key := versionRevision(ver, xl.metaV)
+ switch {
+ case pub.published[key]:
+ published++
+ case pub.convKeys[key]:
+ converged++
+ case pub.before[s] != nil && *pub.before[s] == key:
+ case pub.before[s] == nil && pub.anyBefore[key]:
+ default:
+ return nil, 0, 0, errRestoreSuperseded
+ }
+ held[s] = true
+ break
+ }
+ }
+ return held, published, converged, nil
+}
+
+// convergeRestore settles a publication while the caller holds the object
+// lock. When every drive still holds what the version held before the commit,
+// what the commit wrote, or what an earlier pass wrote, it either finds the
+// copy on a write quorum, or writes the unrestored version to every drive
+// that holds the version. It deletes no directory: an unreferenced staged
+// directory is left to abandoned-data cleanup. Any other entry stops it, so
+// that it never overwrites what someone else wrote between two passes.
+func (er erasureObjects) convergeRestore(ctx context.Context, pub *restorePublication) (restoreConvergence, error) {
+ disks := er.getDisks()
+ held, published, _, err := pub.classify(ctx, disks)
+ if err != nil {
+ return restoreStopped, err
+ }
+ if published >= pub.quorum {
+ er.addPartial(pub.bucket, pub.object, pub.versionID)
+ return restoreCommitted, nil
+ }
+ slots := shuffleDisks(disks, pub.dist)
+ files := make([]FileInfo, len(slots))
+ for s := range slots {
+ if !held[s] {
+ slots[s] = nil
+ continue
+ }
+ files[s] = pub.conv[s]
+ }
+ _, werr := writeUniqueFileInfo(ctx, slots, "", pub.bucket, pub.object, files, pub.quorum)
+
+ held, published, converged, err := pub.classify(ctx, disks)
+ if err != nil {
+ return restoreStopped, err
+ }
+ holding := 0
+ for _, h := range held {
+ if h {
+ holding++
+ }
+ }
+ if published == 0 && converged == holding && converged >= pub.quorum {
+ return restoreConverged, nil
+ }
+ if werr == nil {
+ werr = errErasureWriteQuorum
+ }
+ return restoreIncomplete, werr
+}
+
+// recoverRestorePublication retries a convergence in memory, for a bounded
+// time, each pass under the object lock taken afresh. A restart loses the
+// retries; the version then stays as the drives hold it, see the design's
+// excluded residuals.
+func (er erasureObjects) recoverRestorePublication(pub *restorePublication, lock func(context.Context) (LockContext, func(), error)) {
+ go func() {
+ err := errRestoreNotConverged
+ for _, delay := range restorePublicationDelays {
+ select {
+ case <-GlobalContext.Done():
+ return
+ case <-time.After(delay):
+ }
+ ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
+ lkctx, unlock, lerr := lock(ctx)
+ if lerr != nil {
+ cancel()
+ err = lerr
+ continue
+ }
+ var res restoreConvergence
+ res, err = er.convergeRestore(lkctx.Context(), pub)
+ unlock()
+ cancel()
+ if res == restoreConverged || res == restoreCommitted {
+ return
+ }
+ if res == restoreStopped {
+ break
+ }
+ }
+ er.logRestoreNotConverged(pub, err)
+ }()
+}
+
+func (er erasureObjects) logRestoreNotConverged(pub *restorePublication, err error) {
+ attempt := ""
+ if pub.attempt != nil {
+ attempt = pub.attempt.ID
+ }
+ storageLogIf(GlobalContext, fmt.Errorf("%w: %s/%s(%s), attempt %q, data dir %s: %v",
+ errRestoreNotConverged, pub.bucket, pub.object, pub.versionID, attempt, pub.dataDir, err))
+}
+
+// restoreHooks tie a restore of one set to its caller.
+type restoreHooks struct {
+ // lock takes the object lock O of the version, in the lock domain of the
+ // object's writes, and returns its release.
+ lock func(ctx context.Context) (LockContext, func(), error)
+ // placed tells, under O, whether the version still lives in this set.
+ placed func(ctx context.Context) error
+ // clear clears the attempt's ongoing marker after a failure that
+ // published nothing, taking O itself, and retries in the background.
+ clear func()
+}
+
+// restoreVersion rehydrates the transitioned version opts address in this
+// set: it stages the copy without the object lock, then commits it under the
+// lock. A failure that published nothing clears the attempt's marker. A
+// publication that reached some drives but not a write quorum converges
+// under the same lock hold, and in bounded background retries after it.
+func (er erasureObjects) restoreVersion(ctx context.Context, bucket, object string, opts ObjectOptions, h restoreHooks) error {
+ st, err := er.stageRestore(ctx, bucket, object, opts)
+ if err != nil {
+ return restoreAbandon(err, h)
+ }
+ defer er.deleteAll(context.Background(), minioMetaTmpBucket, st.tmpID)
+
+ lkctx, unlock, err := h.lock(ctx)
+ if err != nil {
+ return restoreAbandon(err, h)
+ }
+ pub, err := er.commitRestore(lkctx.Context(), bucket, object, st, opts, h.placed)
+ if err == nil {
+ unlock()
+ return nil
+ }
+ if pub == nil {
+ unlock()
+ return restoreAbandon(err, h)
+ }
+ res, cerr := er.convergeRestore(lkctx.Context(), pub)
+ unlock()
+ switch res {
+ case restoreCommitted:
+ return nil
+ case restoreConverged:
+ return err
+ case restoreStopped:
+ er.logRestoreNotConverged(pub, cerr)
+ return err
+ default:
+ er.recoverRestorePublication(pub, h.lock)
+ return err
+ }
+}
+
+// restoreAbandon clears the marker of an attempt that published nothing,
+// unless another attempt owns the version. A not-found error may come from the
+// tier rather than from the version, and the clear checks the marker's owner
+// and finds nothing to do for a version that is gone, so it runs then too.
+func restoreAbandon(err error, h restoreHooks) error {
+ if errors.Is(err, errRestoreSuperseded) {
+ return err
+ }
+ if h.clear != nil {
+ h.clear()
+ }
+ return err
+}
+
+// RestoreTransitionedObject - restore transitioned object content locally on
+// this cluster. This is similar to PostObjectRestore from AWS GLACIER storage
+// class. When PostObjectRestore API is called, a temporary copy of the object
+// is restored locally to the bucket on source cluster until the restore
+// expiry date. The copy that was transitioned continues to reside in the
+// transitioned tier.
+//
+// The set entry serves direct callers of one set. It takes the object lock in
+// the set's own domain and no restore lock; native restores of a cluster go
+// through erasureServerPools.
+func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ lock := func(ctx context.Context) (LockContext, func(), error) {
+ lk := er.NewNSLock(bucket, object)
+ lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+ if err != nil {
+ return LockContext{}, nil, err
+ }
+ return lkctx, func() { lk.Unlock(lkctx) }, nil
+ }
+ clearMarker := func() {
+ ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
+ defer cancel()
+ lkctx, unlock, err := lock(ctx)
+ if err != nil {
+ storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, err))
+ return
+ }
+ defer unlock()
+ versionID := opts.VersionID
+ if a := opts.Transition.attempt; a != nil {
+ versionID = a.VersionID
+ }
+ if err := er.clearRestoreStatus(lkctx.Context(), bucket, object, versionID, opts.Transition.attempt); err != nil &&
+ !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !isErrMethodNotAllowed(err) {
+ storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, versionID, err))
+ }
+ }
+ return er.restoreVersion(ctx, bucket, object, opts, restoreHooks{lock: lock, clear: clearMarker})
+}
+
+// RestoreTransitionedObject - restore transitioned object content locally on
+// this cluster, see erasureObjects.RestoreTransitionedObject.
+//
+// A POST hands over the restore lock R of the version it registered in
+// opts.Transition.owner; a direct caller takes R here, waiting while ctx
+// lives. The restore runs while R is held, and releases it when it returns.
+func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ object = encodeDirObject(object)
+ owner := opts.Transition.owner
+ opts.Transition.owner = nil
+ if owner != nil && owner.ctx.Err() != nil {
+ // The lock POST handed over ended before the restore ran, such as
+ // when its holder lost it while the restore was queued: take it again
+ // as a direct caller does. The attempt still has to own the version.
+ owner = nil
+ }
+ defer func() { owner.release() }()
+
+ // Avoid restoring object from a pool being decommissioned.
+ opts.SkipDecommissioned = true
+ opts.NoLock = true
+ clearMarker := func() {
+ if err := z.resetRestoreStatus(bucket, object, opts); err != nil {
+ z.recoverRestoreStatus(bucket, object, opts)
+ }
+ }
+ idx, err := z.restorePoolIdx(ctx, bucket, object, opts)
+ if err != nil {
+ var reset restoreResetError
+ if errors.As(err, &reset) {
+ clearMarker()
+ return reset.error
+ }
+ if isErrObjectNotFound(err) && opts.VersionID != "" {
+ err = VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID}
+ }
+ return err
+ }
+ // A server may have been replaced by an older binary while the restore
+ // was queued: such a server knows neither the restore lock nor the
+ // staged commit.
+ if err := nativeRestoreAvailable(ctx); err != nil {
+ clearMarker()
+ return err
+ }
+ er := z.serverPools[idx].getHashedSet(object)
+ if owner == nil {
+ if opts.VersionID == "" {
+ // Restore the version that is latest now, by its own ID.
+ gen, err := er.readRestoreGeneration(ctx, bucket, object, "")
+ if err != nil {
+ return toObjectErr(err, bucket, object)
+ }
+ opts.VersionID = restoreVersionID(gen.fi.VersionID)
+ }
+ if owner, err = z.takeRestoreOwner(ctx, bucket, object, opts.VersionID, nil); err != nil {
+ return err
+ }
+ }
+ return er.restoreVersion(owner.ctx, bucket, object, opts, restoreHooks{
+ lock: func(ctx context.Context) (LockContext, func(), error) {
+ lk := z.restoreLock(bucket, object)
+ lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+ if err != nil {
+ return LockContext{}, nil, err
+ }
+ return lkctx, func() { lk.Unlock(lkctx) }, nil
+ },
+ placed: func(ctx context.Context) error {
+ got, err := z.restorePoolIdx(ctx, bucket, object, opts)
+ var reset restoreResetError
+ if errors.As(err, &reset) {
+ return reset.error
+ }
+ if err != nil {
+ return err
+ }
+ if got != idx {
+ return errRestoreMoved
+ }
+ return nil
+ },
+ clear: clearMarker,
+ })
+}
diff --git a/cmd/restore-two-phase_test.go b/cmd/restore-two-phase_test.go
new file mode 100644
index 000000000..ba3e981f1
--- /dev/null
+++ b/cmd/restore-two-phase_test.go
@@ -0,0 +1,858 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio/internal/bucket/versioning"
+ "github.com/minio/minio/internal/config/storageclass"
+ xhttp "github.com/minio/minio/internal/http"
+)
+
+// A native restore downloads its tier copy without the object lock: while
+// the download of an old version is paused, the hot versions of the same key
+// stay readable and writable. Only the short commit takes the object lock.
+func TestRestoreDownloadKeepsKeyAvailable(t *testing.T) {
+ for _, pools := range []int{1, 2} {
+ for _, multipart := range []bool{false, true} {
+ t.Run(fmt.Sprintf("pools=%d/multipart=%v", pools, multipart), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, pools, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := nativeRestoreSeed(t, r, 0, false, multipart)
+ hot := putConsistencyObject(t, r.z, r.bucket, r.object, 0, "hot latest", ObjectOptions{Versioned: true, MTime: oi.ModTime.Add(time.Hour)})
+ putConsistencyObject(t, r.z, r.bucket, "unrelated", 0, "unrelated hot object", ObjectOptions{Versioned: true})
+ gate := newRestoreIsolationGateTier(1, nil)
+ globalTierConfigMgr.drivercache["WARM"] = gate
+ defer gate.unblock()
+ done := make(chan error, 1)
+ go func() {
+ done <- r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ }()
+ restoreIsolationEntered(t, gate)
+
+ within := func(name string, fn func(ctx context.Context) error) {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ defer cancel()
+ start := time.Now()
+ if err := fn(ctx); err != nil {
+ t.Fatalf("%s during a paused restore download: %v after %s", name, err, time.Since(start))
+ }
+ }
+ for _, key := range []string{"unrelated", r.object} {
+ within("GET "+key, func(ctx context.Context) error {
+ gr, err := r.z.GetObjectNInfo(ctx, r.bucket, key, nil, http.Header{}, ObjectOptions{})
+ if err != nil {
+ return err
+ }
+ defer gr.Close()
+ _, err = io.ReadAll(gr)
+ return err
+ })
+ }
+ within("HEAD hot latest", func(ctx context.Context) error {
+ got, err := r.z.GetObjectInfo(ctx, r.bucket, r.object, ObjectOptions{})
+ if err == nil && got.VersionID != hot.VersionID {
+ err = fmt.Errorf("latest is %s, want %s", got.VersionID, hot.VersionID)
+ }
+ return err
+ })
+ within("PutObjectTags hot latest", func(ctx context.Context) error {
+ _, err := r.z.PutObjectTags(ctx, r.bucket, r.object, "during=restore", ObjectOptions{VersionID: hot.VersionID, Versioned: true})
+ return err
+ })
+ // The download publishes nothing: no upload, public or not.
+ if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 {
+ t.Errorf("restore created uploads: %v", uploads)
+ }
+ within("PUT new version", func(ctx context.Context) error {
+ data := []byte("written during restore")
+ _, err := r.z.PutObject(ctx, r.bucket, r.object,
+ mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true})
+ return err
+ })
+
+ gate.unblock()
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(time.Minute):
+ t.Fatal("restore did not complete after releasing the tier")
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored version mismatch: %v", err)
+ }
+ })
+ }
+ }
+}
+
+// restoreTwoPhaseStagedCommitFailure makes a multipart completion stage its
+// object metadata in the upload and then fail before moving any data.
+type restoreTwoPhaseStagedCommitFailure struct {
+ StorageAPI
+ bucket, object string
+}
+
+func (d restoreTwoPhaseStagedCommitFailure) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
+ if sv == minioMetaMultipartBucket && dv == d.bucket && dp == d.object {
+ if err := d.WriteMetadata(ctx, "", sv, sp, fi); err != nil {
+ return RenameDataResp{}, err
+ }
+ return RenameDataResp{}, errFaultyDisk
+ }
+ return d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
+}
+
+// An ordinary upload whose completion failed after staging stays usable as
+// on main: the client re-uploads its parts and completes, or aborts.
+func TestMultipartFailedCommitRetriesLikeMain(t *testing.T) {
+ for _, abort := range []bool{false, true} {
+ t.Run(fmt.Sprintf("abort=%v", abort), func(t *testing.T) {
+ z, bucket := consistencyPools(t)
+ initConfigSubsystem(t.Context(), z)
+ const object = "public-upload"
+ er := z.serverPools[0].getHashedSet(object)
+ ctx := t.Context()
+ mp, err := er.NewMultipartUpload(ctx, bucket, object, ObjectOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ body := []byte("ordinary multipart data")
+ putPart := func() (PartInfo, error) {
+ return er.PutObjectPart(ctx, bucket, object, mp.UploadID, 1,
+ mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
+ }
+ p, err := putPart()
+ if err != nil {
+ t.Fatal(err)
+ }
+ disks := er.getDisks()
+ wrapped := make([]StorageAPI, len(disks))
+ for i, d := range disks {
+ wrapped[i] = restoreTwoPhaseStagedCommitFailure{d, bucket, object}
+ }
+ previous := er.getDisks
+ er.getDisks = func() []StorageAPI { return wrapped }
+ _, err = er.CompleteMultipartUpload(ctx, bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: p.ETag}}, ObjectOptions{})
+ er.getDisks = previous
+ if err == nil {
+ t.Fatal("fault injection failed")
+ }
+ if _, err := er.GetMultipartInfo(ctx, bucket, object, mp.UploadID, ObjectOptions{}); err != nil {
+ t.Fatalf("upload after a failed commit: %v", err)
+ }
+ if abort {
+ if err := er.AbortMultipartUpload(ctx, bucket, object, mp.UploadID, ObjectOptions{}); err != nil {
+ t.Fatalf("abort after a failed commit: %v", err)
+ }
+ if _, err := er.GetMultipartInfo(ctx, bucket, object, mp.UploadID, ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) {
+ t.Fatalf("aborted upload still answers: %v", err)
+ }
+ return
+ }
+ if p, err = putPart(); err != nil {
+ t.Fatalf("re-upload a part after a failed commit: %v", err)
+ }
+ if _, err := er.CompleteMultipartUpload(ctx, bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: p.ETag}}, ObjectOptions{}); err != nil {
+ t.Fatalf("complete after re-uploading: %v", err)
+ }
+ gr, err := er.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer gr.Close()
+ if got, err := io.ReadAll(gr); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("completed object = %q, %v", got, err)
+ }
+ })
+ }
+}
+
+// restoreTwoPhaseFaults are the faults restoreTwoPhaseDisk injects, shared by
+// a test and its drives so that the test can turn them off.
+type restoreTwoPhaseFaults struct {
+ // rename: "" passes the publication through, "fail" fails it without
+ // applying it, "apply-fail" applies it and then reports a failure.
+ rename atomic.Value
+ // oldDataDir, when set, replaces the old data directory a publication
+ // reports.
+ oldDataDir atomic.Pointer[string]
+ // writeFails fails this many metadata writes of the object.
+ writeFails atomic.Int32
+ // writes counts the metadata writes of the object.
+ writes atomic.Int32
+ // recreateTmp drops the staging directory before the second part is
+ // written, as a server restart moves the tmp volume away.
+ recreateTmp atomic.Bool
+}
+
+func (f *restoreTwoPhaseFaults) renameMode() string {
+ m, _ := f.rename.Load().(string)
+ return m
+}
+
+type restoreTwoPhaseDisk struct {
+ StorageAPI
+ bucket, object string
+ f *restoreTwoPhaseFaults
+}
+
+func (d restoreTwoPhaseDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
+ if sv != minioMetaTmpBucket || dv != d.bucket || dp != d.object {
+ return d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
+ }
+ switch d.f.renameMode() {
+ case "fail":
+ return RenameDataResp{}, errFaultyDisk
+ case "apply-fail":
+ if _, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts); err != nil {
+ return RenameDataResp{}, err
+ }
+ return RenameDataResp{}, errFaultyDisk
+ }
+ resp, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
+ if old := d.f.oldDataDir.Load(); old != nil {
+ resp.OldDataDir = *old
+ }
+ return resp, err
+}
+
+func (d restoreTwoPhaseDisk) WriteMetadata(ctx context.Context, orig, vol, path string, fi FileInfo) error {
+ if vol == d.bucket && path == d.object {
+ d.f.writes.Add(1)
+ if d.f.writeFails.Load() > 0 {
+ d.f.writeFails.Add(-1)
+ return errFaultyDisk
+ }
+ }
+ return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi)
+}
+
+func (d restoreTwoPhaseDisk) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) error {
+ if volume == minioMetaTmpBucket && strings.HasSuffix(path, "/part.2") && d.f.recreateTmp.Load() {
+ tmpID, _, _ := strings.Cut(path, "/")
+ if err := os.RemoveAll(filepath.Join(d.Endpoint().Path, minioMetaTmpBucket, tmpID)); err != nil {
+ return err
+ }
+ }
+ return d.StorageAPI.CreateFile(ctx, origvolume, volume, path, size, reader)
+}
+
+// restoreTwoPhaseFaulty wraps the drives of the object's set in the first
+// pool; faults[i] is nil for a drive left as it is. It returns the plain
+// drives, for reads that see what is stored.
+func restoreTwoPhaseFaulty(t *testing.T, r *restoreTest, faults ...*restoreTwoPhaseFaults) []StorageAPI {
+ t.Helper()
+ er := r.z.serverPools[0].getHashedSet(r.object)
+ disks := er.getDisks()
+ previous := er.getDisks
+ t.Cleanup(func() { er.getDisks = previous })
+ wrapped := append([]StorageAPI(nil), disks...)
+ for i, f := range faults {
+ if f != nil {
+ wrapped[i] = restoreTwoPhaseDisk{StorageAPI: disks[i], bucket: r.bucket, object: r.object, f: f}
+ }
+ }
+ er.getDisks = func() []StorageAPI { return wrapped }
+ return disks
+}
+
+// restoreTwoPhaseEntries reads the version from each drive.
+func restoreTwoPhaseEntries(t *testing.T, r *restoreTest, disks []StorageAPI, versionID string) []FileInfo {
+ t.Helper()
+ infos, errs := readAllFileInfo(t.Context(), disks, "", r.bucket, r.object, restoreTestVersionKey(versionID), false, false)
+ for i, err := range errs {
+ if err != nil {
+ t.Fatalf("drive %d: %v", i, err)
+ }
+ }
+ return infos
+}
+
+func restoreTwoPhaseUnrestored(fi FileInfo) bool {
+ return fi.Metadata[xhttp.AmzRestore] == "" && fi.Metadata[restoreAttemptKey] == "" && fi.IsRemote()
+}
+
+func restoreTwoPhaseRestored(fi FileInfo) bool {
+ st, err := parseRestoreObjStatus(fi.Metadata[xhttp.AmzRestore])
+ return err == nil && !st.Ongoing() && !fi.IsRemote() && fi.Metadata[restoreAttemptKey] == ""
+}
+
+// restoreTwoPhaseDelays shortens the convergence retries for a test.
+func restoreTwoPhaseDelays(t *testing.T, delays ...time.Duration) {
+ previous := restorePublicationDelays
+ restorePublicationDelays = delays
+ t.Cleanup(func() { restorePublicationDelays = previous })
+}
+
+// A publication that reaches fewer drives than the write quorum is brought
+// back to the unrestored version on every drive, under the same lock hold.
+// No directory is deleted: the staged one is left to abandoned-data cleanup.
+// The version restores again once the drives work.
+func TestRestorePublicationBelowQuorumReverts(t *testing.T) {
+ for _, multipart := range []bool{false, true} {
+ t.Run(fmt.Sprintf("multipart=%v", multipart), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, body := nativeRestoreSeed(t, r, 0, false, multipart)
+ faults := &restoreTwoPhaseFaults{}
+ faults.rename.Store("fail")
+ disks := restoreTwoPhaseFaulty(t, r, faults, faults)
+ before := restoreTwoPhaseEntries(t, r, disks, oi.VersionID)
+ err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ if err == nil {
+ t.Fatal("a publication on two of four drives succeeded")
+ }
+ for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+ if !restoreTwoPhaseUnrestored(fi) || fi.DataDir != before[i].DataDir {
+ t.Errorf("drive %d after a failed publication: restore %q, remote %v, data dir %s (was %s)",
+ i, fi.Metadata[xhttp.AmzRestore], fi.IsRemote(), fi.DataDir, before[i].DataDir)
+ }
+ }
+ // The drives that published keep the staged directory.
+ for i := 2; i < len(disks); i++ {
+ entries, err := os.ReadDir(filepath.Join(disks[i].Endpoint().Path, r.bucket, r.object))
+ if err != nil {
+ t.Fatal(err)
+ }
+ dirs := 0
+ for _, e := range entries {
+ if e.IsDir() && e.Name() != before[i].DataDir {
+ dirs++
+ }
+ }
+ if dirs == 0 {
+ t.Errorf("drive %d: the published data directory was deleted", i)
+ }
+ }
+ faults.rename.Store("")
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatalf("restore after the drives recovered: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ })
+ }
+}
+
+// A publication whose drives applied it but did not answer is found on a
+// write quorum when the worker reads the drives again: the restore stands.
+func TestRestorePublicationUncertainCommits(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, body := nativeRestoreSeed(t, r, 0, false, true)
+ faults := &restoreTwoPhaseFaults{}
+ faults.rename.Store("apply-fail")
+ disks := restoreTwoPhaseFaulty(t, r, faults, faults)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatalf("restore whose answers were lost: %v", err)
+ }
+ for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+ if !restoreTwoPhaseRestored(fi) {
+ t.Errorf("drive %d: not restored: %q", i, fi.Metadata[xhttp.AmzRestore])
+ }
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+}
+
+// A convergence that cannot write every drive under the commit's lock hold
+// finishes in a background retry.
+func TestRestoreConvergenceRetries(t *testing.T) {
+ restoreTwoPhaseDelays(t, 50*time.Millisecond, 50*time.Millisecond, 50*time.Millisecond)
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+ failing, flaky := &restoreTwoPhaseFaults{}, &restoreTwoPhaseFaults{}
+ failing.rename.Store("fail")
+ flaky.writeFails.Store(1)
+ disks := restoreTwoPhaseFaulty(t, r, failing, failing, flaky)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err == nil {
+ t.Fatal("a publication on two of four drives succeeded")
+ }
+ deadline := time.Now().Add(restoreTestWait)
+ for {
+ settled := true
+ for _, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+ settled = settled && restoreTwoPhaseUnrestored(fi)
+ }
+ if settled {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("background convergence did not finish")
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+}
+
+// A convergence retry never overwrites what someone else wrote between two
+// lock holds: a metadata update or a newer attempt's marker stops it.
+func TestRestoreConvergenceStopsOnOtherWrites(t *testing.T) {
+ for _, other := range []string{"tags", "successor"} {
+ t.Run(other, func(t *testing.T) {
+ restoreTwoPhaseDelays(t, 300*time.Millisecond, 300*time.Millisecond, 300*time.Millisecond)
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, false)
+ failing, stuck := &restoreTwoPhaseFaults{}, &restoreTwoPhaseFaults{}
+ failing.rename.Store("fail")
+ stuck.writeFails.Store(1 << 20)
+ disks := restoreTwoPhaseFaulty(t, r, failing, failing, stuck)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err == nil {
+ t.Fatal("a publication on two of four drives succeeded")
+ }
+ var successor string
+ release := make(chan struct{})
+ layer := newRestoreIsolationLayer(r)
+ switch other {
+ case "tags":
+ if _, err := r.z.PutObjectTags(t.Context(), r.bucket, r.object, "kept=yes", r.opts(oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ case "successor":
+ layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ <-release
+ return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ close(release)
+ t.Fatalf("successor POST: %d %s", rec.Code, rec.Body.String())
+ }
+ successor = restoreTwoPhaseEntries(t, r, disks, oi.VersionID)[0].Metadata[restoreAttemptKey]
+ if successor == "" {
+ close(release)
+ t.Fatal("successor POST recorded no attempt")
+ }
+ }
+ writes := stuck.writes.Load()
+ time.Sleep(time.Second)
+ if got := stuck.writes.Load(); got != writes {
+ t.Errorf("a stopped convergence kept writing: %d -> %d writes", writes, got)
+ }
+ for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+ if i == 2 {
+ continue // the drive the convergence could not write
+ }
+ switch other {
+ case "tags":
+ if fi.Metadata[xhttp.AmzObjectTagging] != "kept=yes" {
+ t.Errorf("drive %d: the tags written between retries were overwritten: %q", i, fi.Metadata[xhttp.AmzObjectTagging])
+ }
+ case "successor":
+ if fi.Metadata[restoreAttemptKey] != successor {
+ t.Errorf("drive %d: the successor's marker was overwritten: %q", i, fi.Metadata[restoreAttemptKey])
+ }
+ }
+ }
+ if other == "successor" {
+ failing.rename.Store("")
+ stuck.writeFails.Store(0)
+ close(release)
+ if err := layer.wait(t); err != nil {
+ t.Fatalf("successor restore: %v", err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ }
+ })
+ }
+}
+
+// The data directory of the replaced entry goes only where every drive that
+// reports it agrees on it; a drive that does not report it, as when another
+// version shares the directory, keeps the directory everywhere.
+func TestRestoreRemovesOldDataDirOnlyWhenUnanimous(t *testing.T) {
+ for _, mixed := range []bool{false, true} {
+ t.Run(fmt.Sprintf("mixed=%v", mixed), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, body := nativeRestoreSeed(t, r, 0, false, true)
+ old := r.fileInfo(oi.VersionID).DataDir
+ if old == "" {
+ t.Fatal("fixture: the transitioned version has no data directory")
+ }
+ faults := &restoreTwoPhaseFaults{}
+ if mixed {
+ none := ""
+ faults.oldDataDir.Store(&none)
+ }
+ restoreTwoPhaseFaulty(t, r, faults)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ n := r.dataDirs(old)
+ if mixed && n != 4 {
+ t.Errorf("drives disagreed on the old data directory, but %d of 4 kept it", n)
+ }
+ if !mixed && n != 0 {
+ t.Errorf("the unanimous old data directory remained on %d drives", n)
+ }
+ })
+ }
+}
+
+// An earlier restore of a small object may have inlined its data, and the
+// expiry of that copy leaves the inline marker behind. A new restore writes
+// a data directory and must not publish the marker with it.
+func TestRestoreDropsStaleInlineMarker(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, body := nativeRestoreSeed(t, r, 0, false, false)
+ disks := r.z.serverPools[0].getHashedSet(r.object).getDisks()
+ for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+ fi.SetInlineData()
+ if err := disks[i].WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatal(err)
+ }
+ for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+ if fi.InlineData() {
+ t.Errorf("drive %d: the restored copy is marked inline", i)
+ }
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+}
+
+// A drive whose staging directory was moved away and recreated by a later
+// part misses the earlier parts: it is left out of the commit, and when too
+// few drives are left nothing is published at all.
+func TestRestoreStagingDropsRecreatedTmp(t *testing.T) {
+ for _, recreated := range []int{1, 2} {
+ t.Run(fmt.Sprintf("drives=%d", recreated), func(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, body := nativeRestoreSeed(t, r, 0, false, true)
+ faults := &restoreTwoPhaseFaults{}
+ faults.recreateTmp.Store(true)
+ wrap := make([]*restoreTwoPhaseFaults, recreated)
+ for i := range wrap {
+ wrap[i] = faults
+ }
+ disks := restoreTwoPhaseFaulty(t, r, wrap...)
+ err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ entries := restoreTwoPhaseEntries(t, r, disks, oi.VersionID)
+ if recreated == 2 {
+ if err == nil {
+ t.Fatal("a copy staged completely on two of four drives was published")
+ }
+ for i, fi := range entries {
+ if !restoreTwoPhaseUnrestored(fi) {
+ t.Errorf("drive %d after a failed staging: %q", i, fi.Metadata[xhttp.AmzRestore])
+ }
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if restoreTwoPhaseRestored(entries[0]) {
+ t.Error("the drive with missing parts got the restored copy")
+ }
+ for i := 1; i < len(entries); i++ {
+ if !restoreTwoPhaseRestored(entries[i]) {
+ t.Errorf("drive %d: not restored", i)
+ }
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+ })
+ }
+}
+
+// A POST while the worker of an earlier POST holds the restore lock answers
+// that a restore is in progress; it starts no second worker.
+func TestRestoreSecondPOSTWhileRunning(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, body := restoreIsolationSeed(t, r, 0, false, true)
+ release := make(chan struct{})
+ layer := newRestoreIsolationLayer(r)
+ layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+ <-release
+ return r.z.RestoreTransitionedObject(ctx, bucket, object, opts)
+ }
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("first POST: %d %s", rec.Code, rec.Body.String())
+ }
+ rec := restoreIsolationPOST(t, r, layer, oi.VersionID)
+ close(release)
+ if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "RestoreAlreadyInProgress") {
+ t.Errorf("second POST: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := layer.wait(t); err != nil {
+ t.Fatal(err)
+ }
+ select {
+ case err := <-layer.done:
+ t.Fatalf("a second restore ran: %v", err)
+ default:
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("restored contents: %v", err)
+ }
+}
+
+// A version deleted while its tier copy downloads is not brought back.
+func TestRestoreTargetDeletedDuringDownload(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+ gate := newRestoreIsolationGateTier(1, nil)
+ globalTierConfigMgr.drivercache["WARM"] = gate
+ defer gate.unblock()
+ done := make(chan error, 1)
+ go func() {
+ done <- r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
+ }()
+ restoreIsolationEntered(t, gate)
+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ _, err := r.z.DeleteObject(ctx, r.bucket, r.object, r.opts(oi.VersionID))
+ cancel()
+ if err != nil {
+ t.Fatalf("delete during the download: %v", err)
+ }
+ gate.unblock()
+ select {
+ case err := <-done:
+ if err == nil {
+ t.Fatal("the restore of a deleted version succeeded")
+ }
+ case <-time.After(restoreTestWait):
+ t.Fatal("restore did not finish")
+ }
+ if _, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+ t.Fatalf("deleted version after the restore: %v", err)
+ }
+}
+
+// With several pools, abandoned-data cleanup takes the pools lock that every
+// write of the object takes, also where a pool's sets have lockers of their
+// own: while a writer holds it, cleanup removes nothing, such as a data
+// directory that a commit moved in place and has not yet referenced.
+func TestCheckAbandonedPartsTakesPoolsLock(t *testing.T) {
+ r := newRestoreSafetyTest(t, 2, versioning.Enabled)
+ nativeRestoreSeed(t, r, 1, false, false)
+ for _, set := range r.z.serverPools[1].sets {
+ previous := set.nsMutex
+ set.nsMutex = newNSLock(false)
+ t.Cleanup(func() { set.nsMutex = previous })
+ }
+ disks := r.z.serverPools[1].getHashedSet(r.object).getDisks()
+ moved := mustGetUUID()
+ for _, disk := range disks {
+ dir := filepath.Join(disk.Endpoint().Path, r.bucket, r.object, moved)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, "part.1"), []byte("staged"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ present := func() int {
+ n := 0
+ for _, disk := range disks {
+ if _, err := os.Stat(filepath.Join(disk.Endpoint().Path, r.bucket, r.object, moved)); err == nil {
+ n++
+ }
+ }
+ return n
+ }
+ lk := r.z.NewObjectNSLock(r.bucket, r.object)
+ lkctx, err := lk.GetLock(t.Context(), globalOperationTimeout)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx, cancel := context.WithTimeout(t.Context(), time.Second)
+ r.z.CheckAbandonedParts(ctx, r.bucket, r.object, madmin.HealOpts{Remove: true})
+ cancel()
+ held := present()
+ lk.Unlock(lkctx)
+ if held != len(disks) {
+ t.Fatalf("abandoned-data cleanup removed a moved directory from %d drives while a writer held the object lock", len(disks)-held)
+ }
+ if err := r.z.CheckAbandonedParts(t.Context(), r.bucket, r.object, madmin.HealOpts{Remove: true}); err != nil {
+ t.Fatalf("abandoned-data cleanup without a writer: %v", err)
+ }
+ if n := present(); n != 0 {
+ t.Fatalf("abandoned-data cleanup left the unreferenced directory on %d drives", n)
+ }
+}
+
+// The header flag that tells whether a transitioned version uses its data
+// directory follows whether its restored copy has expired when a drive writes
+// the entry. A publication's own entry is recognized with either value.
+func TestRestorePublishedRevisionAcrossExpiry(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ oi, _ := nativeRestoreSeed(t, r, 0, false, false)
+ fi := r.fileInfo(oi.VersionID)
+ expiry := time.Now().Add(2 * time.Second).Truncate(time.Second).Add(time.Second)
+ st := &restoreStage{dataDir: mustGetUUID(), expiry: expiry}
+ published := restorePublished(fi, fi, st, 0)
+ keys, err := fileInfoRevisions(published)
+ if err != nil {
+ t.Fatal(err)
+ }
+ stored := func() revisionKey {
+ var x xlMetaV2
+ if err := x.AddVersion(published); err != nil {
+ t.Fatal(err)
+ }
+ buf, err := x.AppendTo(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var y xlMetaV2
+ if err := y.Load(buf); err != nil {
+ t.Fatal(err)
+ }
+ return versionRevision(y.versions[0], y.metaV)
+ }
+ before := stored()
+ time.Sleep(time.Until(expiry) + 100*time.Millisecond)
+ after := stored()
+ if before == after {
+ t.Fatal("fixture: the stored flag did not change at the expiry")
+ }
+ for _, got := range []revisionKey{before, after} {
+ if got != keys[0] && got != keys[1] {
+ t.Errorf("a publication's own entry is not recognized: %v not in %v", got, keys)
+ }
+ }
+}
+
+// A tier that lost the copy answers not found. That is no reason to keep the
+// version's ongoing marker: the attempt clears it, and a later POST can
+// restore the version again.
+func TestRestoreTierNotFoundClearsMarker(t *testing.T) {
+ r := newRestoreSafetyTest(t, 1, versioning.Enabled)
+ initConfigSubsystem(t.Context(), r.z)
+ oi, _ := restoreIsolationSeed(t, r, 0, false, false)
+ tier := globalTierConfigMgr.drivercache["WARM"]
+ globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, ObjectNotFound{Bucket: r.bucket, Object: r.object}}
+ layer := newRestoreIsolationLayer(r)
+ if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ if err := layer.wait(t); !isErrObjectNotFound(err) {
+ t.Fatalf("restore from a tier without the copy: %v", err)
+ }
+ info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID))
+ if err != nil || info.RestoreOngoing || info.UserDefined[xhttp.AmzRestore] != "" {
+ t.Fatalf("version after the failed restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err)
+ }
+}
+
+// newRestoreWideTest is newRestoreSafetyTest with one pool of twelve drives,
+// whose default layout is eight data and four parity shards: a version's
+// metadata reaches its read quorum on seven drives, one short of the write
+// quorum of eight that its shards need.
+func newRestoreWideTest(t *testing.T) *restoreTest {
+ t.Helper()
+ prevTier := globalTierConfigMgr
+ t.Cleanup(func() { globalTierConfigMgr = prevTier })
+ dirs, err := getRandomDisks(12)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { removeRoots(dirs) })
+ obj, _, err := initObjectLayer(t.Context(), mustGetPoolEndpoints(0, dirs...))
+ if err != nil {
+ t.Fatal(err)
+ }
+ z := obj.(*erasureServerPools)
+ t.Cleanup(func() { z.Shutdown(context.Background()) })
+ // Other tests leave a storage class behind; fix the layout to 8+4.
+ saved := globalStorageClass
+ globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
+ t.Cleanup(func() { globalStorageClass.Update(saved) })
+ globalTierConfigMgr = NewTierConfigMgr()
+ globalTierConfigMgr.drivercache["WARM"] = &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
+ r := &restoreTest{t: t, z: z, bucket: getRandomBucketName(), object: "obj", labels: map[string]string{}, bodies: map[string][]byte{}}
+ if err := z.MakeBucket(t.Context(), r.bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
+ t.Fatal(err)
+ }
+ meta, err := globalBucketMetadataSys.Get(r.bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ meta.LifecycleConfigXML = []byte(restoreSafetyRules)
+ if err := meta.Save(t.Context(), z); err != nil {
+ t.Fatal(err)
+ }
+ globalBucketMetadataSys.Set(r.bucket, meta)
+ return r
+}
+
+// On twelve drives with eight data shards, a publication that reached seven
+// drives has the metadata read quorum but not the eight shards a read needs.
+// It is not a restore: the worker brings every drive back to the unrestored
+// version, and the version reads from its tier copy.
+func TestRestorePublicationBelowWriteQuorumOnWideSet(t *testing.T) {
+ r := newRestoreWideTest(t)
+ oi, body := nativeRestoreSeed(t, r, 0, false, true)
+ fi := r.fileInfo(oi.VersionID)
+ if fi.Erasure.DataBlocks != 8 || fi.Erasure.ParityBlocks != 4 {
+ t.Fatalf("fixture layout %d+%d, want 8+4", fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks)
+ }
+ faults := &restoreTwoPhaseFaults{}
+ faults.rename.Store("fail")
+ disks := restoreTwoPhaseFaulty(t, r, faults, faults, faults, faults, faults)
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err == nil {
+ t.Fatal("a publication on seven of twelve drives succeeded")
+ }
+ for i, e := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+ if !restoreTwoPhaseUnrestored(e) || e.DataDir != fi.DataDir {
+ t.Errorf("drive %d after a publication on seven drives: restore %q, remote %v", i, e.Metadata[xhttp.AmzRestore], e.IsRemote())
+ }
+ }
+ if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("the version no longer reads from its tier copy: %v", err)
+ }
+ faults.rename.Store("")
+ if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil {
+ t.Fatalf("restore after the drives recovered: %v", err)
+ }
+ for i, e := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) {
+ if !restoreTwoPhaseRestored(e) {
+ t.Errorf("drive %d: not restored", i)
+ }
+ }
+}
diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go
index 5a054f86e..5e4f56d65 100644
--- a/cmd/xl-storage-format-v2.go
+++ b/cmd/xl-storage-format-v2.go
@@ -1372,6 +1372,26 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) {
}
}
+ if fi.ExpireRestored {
+ i, ver, err := x.findVersion(uv)
+ if err != nil {
+ return "", err
+ }
+ // Check the stored type and transition before any marker, legacy or
+ // replication mutation. Only completed tiered objects own a cache.
+ if ver.Type != ObjectType || string(ver.ObjectV2.MetaSys[metaTierStatus]) != lifecycle.TransitionComplete {
+ return "", nil
+ }
+ ver.ObjectV2.RemoveRestoreHdrs()
+ if err := x.setIdx(i, *ver); err != nil {
+ return "", err
+ }
+ if x.SharedDataDirCount(ver.ObjectV2.VersionID, ver.ObjectV2.DataDir) > 0 {
+ return "", nil
+ }
+ return uuid.UUID(ver.ObjectV2.DataDir).String(), nil
+ }
+
var ventry xlMetaV2Version
if fi.Deleted {
ventry = xlMetaV2Version{
@@ -1496,11 +1516,8 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) {
if err != nil {
return "", err
}
- switch {
- case fi.ExpireRestored:
- ver.ObjectV2.RemoveRestoreHdrs()
- err = x.setIdx(i, *ver)
- case fi.TransitionStatus == lifecycle.TransitionComplete:
+ switch fi.TransitionStatus {
+ case lifecycle.TransitionComplete:
ver.ObjectV2.SetTransition(fi)
ver.ObjectV2.ResetInlineData()
err = x.setIdx(i, *ver)
diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go
index 81e415553..0758396b6 100644
--- a/cmd/xl-storage.go
+++ b/cmd/xl-storage.go
@@ -1371,6 +1371,9 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
}
if legacyJSON {
+ if fi.ExpireRestored {
+ return nil // Legacy objects cannot contain a completed transition.
+ }
// Delete the meta file, if there are no more versions the
// top level parent is automatically removed.
return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true, false)
diff --git a/internal/grid/handlers.go b/internal/grid/handlers.go
index 1d20fa9b9..0f2c39d7e 100644
--- a/internal/grid/handlers.go
+++ b/internal/grid/handlers.go
@@ -116,6 +116,7 @@ const (
HandlerRenamePart
HandlerClearUploadID
HandlerCheckParts3
+ HandlerRestoreCapability
// Add more above here ^^^
// If all handlers are used, the type of Handler can be changed.
@@ -200,6 +201,7 @@ var handlerPrefixes = [handlerLast]string{
HandlerCheckParts3: storagePrefix,
HandlerRenamePart: storagePrefix,
HandlerClearUploadID: peerPrefix,
+ HandlerRestoreCapability: peerPrefix,
}
const (
diff --git a/internal/grid/handlers_string.go b/internal/grid/handlers_string.go
index d6c6b4c38..aea3f45fb 100644
--- a/internal/grid/handlers_string.go
+++ b/internal/grid/handlers_string.go
@@ -86,14 +86,15 @@ func _() {
_ = x[HandlerRenamePart-75]
_ = x[HandlerClearUploadID-76]
_ = x[HandlerCheckParts3-77]
- _ = x[handlerTest-78]
- _ = x[handlerTest2-79]
- _ = x[handlerLast-80]
+ _ = x[HandlerRestoreCapability-78]
+ _ = x[handlerTest-79]
+ _ = x[handlerTest2-80]
+ _ = x[handlerLast-81]
}
-const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3handlerTesthandlerTest2handlerLast"
+const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3RestoreCapabilityhandlerTesthandlerTest2handlerLast"
-var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 971, 983, 994}
+var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 977, 988, 1000, 1011}
func (i HandlerID) String() string {
idx := int(i) - 0