From 347d9763fd08ce42f4f8c382147eae22b991e050 Mon Sep 17 00:00:00 2001 From: Denis Babchuk Date: Mon, 2 Nov 2026 19:58:46 +0300 Subject: [PATCH 01/26] fix(ilm): recheck expiries decided in one pool against the versions of the object in all pools Signed-off-by: Denis Babchuk --- cmd/erasure-object.go | 2 +- cmd/erasure-server-pool-lifecycle.go | 252 +++++++++++++ cmd/erasure-server-pool-lifecycle_test.go | 416 ++++++++++++++++++++++ cmd/erasure-server-pool.go | 28 ++ cmd/object-api-interface.go | 4 + cmd/object-api-interface_gen.go | 15 +- cmd/object-handlers-common.go | 1 + 7 files changed, 714 insertions(+), 4 deletions(-) create mode 100644 cmd/erasure-server-pool-lifecycle.go create mode 100644 cmd/erasure-server-pool-lifecycle_test.go diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 5c9ee5d80..8ceae79d4 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2151,7 +2151,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string } } - if opts.Expiration.Expire { + if opts.Expiration.Expire && !opts.Expiration.CheckedAllPools { if gerr == nil { evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi) var isErr bool diff --git a/cmd/erasure-server-pool-lifecycle.go b/cmd/erasure-server-pool-lifecycle.go new file mode 100644 index 000000000..800b39d92 --- /dev/null +++ b/cmd/erasure-server-pool-lifecycle.go @@ -0,0 +1,252 @@ +// Copyright (c) 2026 Denis Babchuk +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "sort" + "sync" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" +) + +// ILM decides on the versions of an object that one pool holds: the scanner +// and the drive heal read the drives of one set, decommission and rebalance +// walk one pool, and a GET or HEAD of a version reads the pool that holds it. +// IsLatest, NumVersions and SuccessorModTime count only these. An object can +// have versions in several pools: from two concurrent first writes, from +// overwrites that went to another pool, from data movement. The expiries +// these decisions run delete in every pool, so a version that is the latest +// in its own pool but not of the object had a newer version hidden under a +// delete marker or deleted with all versions, and a delete marker without +// noncurrent versions in its pool was removed over the versions of another +// pool. Deciding on fewer versions only delays the expiry of a noncurrent +// version: its successor in the pool is not older than the one of the object, +// and fewer newer versions count against NewerNoncurrentVersions. +// +// Before an expiry deletes in any pool, it is evaluated again with the +// versions of the object in all pools, as the scanner would evaluate it if +// one pool held them all. An expiry that no longer applies is not run, as the +// recheck of a set does it: the scanner queues it again once the rule +// applies to the object. + +// objectVersions returns the versions of object that a read quorum of the +// drives of the set agree on, newest first, without free versions. It returns +// no versions when the set does not hold the object, and an error when too +// few drives answer to tell. +func (er erasureObjects) objectVersions(ctx context.Context, bucket, object string) ([]FileInfo, error) { + disks := er.getDisks() + rawFileInfos, errs := readAllRawFileInfo(ctx, disks, bucket, object, false) + readQuorum := (len(disks) + 1) / 2 + + var ( + metas [][]xlMetaV2ShallowVersion + metaV uint8 + unknown int + ) + for i, rf := range rawFileInfos { + if errs[i] != nil { + if !errors.Is(errs[i], errFileNotFound) && !errors.Is(errs[i], errVolumeNotFound) { + unknown++ + } + continue + } + var xl xlMetaV2 + if err := xl.LoadOrConvert(rf.Buf); err != nil { + unknown++ + continue + } + metas = append(metas, xl.versions) + metaV = xl.metaV + } + if len(metas) < readQuorum { + if len(metas)+unknown < readQuorum { + return nil, nil // not in this set + } + return nil, toObjectErr(errErasureReadQuorum, bucket, object) + } + + merged := mergeXLV2Versions(readQuorum, false, 0, metas...) + if len(merged) == 0 { + for _, versions := range metas { + if len(versions) > 0 { + return nil, toObjectErr(errErasureReadQuorum, bucket, object) + } + } + return nil, nil + } + all, err := xlMetaV2{versions: merged, metaV: metaV}.ListVersions(bucket, object, false) + if err != nil { + return nil, toObjectErr(err, bucket, object) + } + versions := all[:0] + for _, fi := range all { + if !fi.TierFreeVersion() { + versions = append(versions, fi) + } + } + return versions, nil +} + +// lifecycleEventsAllPools returns the versions of object in all pools, +// newest first, and the lifecycle event of each as the scanner evaluates it +// on them. A version with copies in several pools, as an interrupted move +// leaves it, counts once. +func (z *erasureServerPools) lifecycleEventsAllPools(ctx context.Context, bucket, object string) ([]ObjectInfo, []lifecycle.Event, error) { + lc, err := globalLifecycleSys.Get(bucket) + if err != nil { + if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) { + return nil, nil, nil + } + return nil, nil, err + } + lr, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + return nil, nil, err + } + rcfg, err := getReplicationConfig(ctx, bucket) + if err != nil { + return nil, nil, err + } + + poolVersions := make([][]FileInfo, len(z.serverPools)) + errs := make([]error, len(z.serverPools)) + var wg sync.WaitGroup + for i, pool := range z.serverPools { + wg.Add(1) + go func(i int, set *erasureObjects) { + defer wg.Done() + poolVersions[i], errs[i] = set.objectVersions(ctx, bucket, object) + }(i, pool.getHashedSet(object)) + } + wg.Wait() + for _, err := range errs { + if err != nil { + return nil, nil, err + } + } + + var all []FileInfo + for _, versions := range poolVersions { + all = append(all, versions...) + } + // Each pool lists its versions newest first. On equal times the lower + // pool comes first, as reads choose it. + sort.SliceStable(all, func(i, j int) bool { return all[i].ModTime.After(all[j].ModTime) }) + seen := make(map[string]struct{}, len(all)) + versions := all[:0] + for _, fi := range all { + if _, ok := seen[fi.VersionID]; ok { + continue + } + seen[fi.VersionID] = struct{}{} + versions = append(versions, fi) + } + + vc, _ := globalBucketVersioningSys.Get(bucket) + versioned := vc != nil && vc.Versioned(decodeDirObject(object)) + objInfos := make([]ObjectInfo, len(versions)) + objOpts := make([]lifecycle.ObjectOpts, len(versions)) + for i := range versions { + fi := versions[i] + fi.IsLatest = i == 0 + fi.NumVersions = len(versions) + fi.SuccessorModTime = time.Time{} + if i > 0 { + fi.SuccessorModTime = versions[i-1].ModTime + } + objInfos[i] = fi.ToObjectInfo(bucket, object, versioned) + objOpts[i] = objInfos[i].ToLifecycleOpts() + } + events, err := lifecycle.NewEvaluator(*lc).WithLockRetention(&lr).WithReplicationConfig(rcfg).Eval(objOpts) + if err != nil { + return nil, nil, err + } + return objInfos, events, nil +} + +// lifecycleAllPools holds the versions of an object in all pools and their +// lifecycle events, see lifecycleEventsAllPools. +type lifecycleAllPools struct { + versions []ObjectInfo + events []lifecycle.Event + err error +} + +// checkExpiryAllPools returns nil when the ILM expiry that opts describe +// still applies to object with the versions it has in all pools, see +// lifecycleAllPools.expiryApplies. +func (z *erasureServerPools) checkExpiryAllPools(ctx context.Context, bucket, object string, opts ObjectOptions) error { + var all lifecycleAllPools + all.versions, all.events, all.err = z.lifecycleEventsAllPools(ctx, bucket, object) + return all.expiryApplies(bucket, object, opts) +} + +// expiryApplies returns nil when the ILM expiry that opts describe applies +// to the versions, and otherwise the error a set returns for an expiry that +// no longer applies: +// - an expiry of all versions (DeletePrefix) needs the latest version to +// expire with all versions; +// - a delete marker (no VersionID) hides the latest version, which has to +// expire; +// - a version (VersionID) has to expire by itself: a noncurrent version, +// or a delete marker without noncurrent versions. +func (all lifecycleAllPools) expiryApplies(bucket, object string, opts ObjectOptions) error { + if all.err != nil { + if _, ok := all.err.(InsufficientReadQuorum); ok { + return InsufficientWriteQuorum{} + } + return all.err + } + name := decodeDirObject(object) + switch { + case opts.DeletePrefix: + if len(all.events) > 0 && all.events[0].Action.DeleteAll() { + return nil + } + case opts.VersionID == "": + if len(all.events) > 0 && (all.events[0].Action == lifecycle.DeleteAction || all.events[0].Action == lifecycle.DeleteAllVersionsAction) { + return nil + } + default: + for i, oi := range all.versions { + if oi.VersionID == opts.VersionID || oi.VersionID == "" && opts.VersionID == nullVersionID { + if all.events[i].Action == lifecycle.DeleteVersionAction { + return nil + } + break + } + } + return VersionNotFound{Bucket: bucket, Object: name, VersionID: opts.VersionID} + } + return ObjectNotFound{Bucket: bucket, Object: name} +} + +// expiryNeedsAllPools reports whether an ILM expiry in opts deletes or hides +// versions by a rule that depends on the other versions of the object. A +// restored copy expires by its own restore date. Without versions each copy +// of an object is a write of its own, and a set rechecks the rule on its +// copy (deleteObjectFromAllPools). +func (z *erasureServerPools) expiryNeedsAllPools(opts ObjectOptions) bool { + if !opts.Expiration.Expire || z.SinglePool() || opts.Transition.ExpireRestored { + return false + } + return opts.DeletePrefix || opts.VersionID != "" || opts.Versioned || opts.VersionSuspended +} diff --git a/cmd/erasure-server-pool-lifecycle_test.go b/cmd/erasure-server-pool-lifecycle_test.go new file mode 100644 index 000000000..0e346e33f --- /dev/null +++ b/cmd/erasure-server-pool-lifecycle_test.go @@ -0,0 +1,416 @@ +// Copyright (c) 2026 Denis Babchuk +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "io" + "slices" + "strings" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/versioning" +) + +// The scanner, the drive heal and a GET or HEAD of a version evaluate ILM on +// the versions of a key in one pool: IsLatest, NumVersions and SuccessorModTime +// count only these. A key can have versions in several pools: two concurrent +// first writes, overwrites that went to another pool. The expiry they decide +// runs in every pool. + +// splitILM lays out the versions of keys in the two pools of migrationILM. +type splitILM struct { + *migrationILM + ids map[string]string // object and label -> version ID + bodies map[string]string // body -> label +} + +func newSplitILM(t *testing.T, status versioning.State, rules string) *splitILM { + t.Helper() + return &splitILM{migrationILM: newMigrationILM(t, status, false, rules), ids: map[string]string{}, bodies: map[string]string{}} +} + +// splitVersion is a version of a key in a test layout. +type splitVersion struct { + pools []int // the pools that hold a copy of it + label string + days float64 // its age + marker bool + tiered bool // its data is in a remote tier + null bool // a null version (unversioned or suspended bucket) + size int // its size, at least the size of its label +} + +func (s *splitILM) write(object string, v splitVersion) { + s.t.Helper() + ctx := s.t.Context() + mtime := UTCNow().Add(-time.Duration(v.days * float64(24*time.Hour))) + versionID := mustGetUUID() + if v.null { + versionID = "" + } + for _, pool := range v.pools { + if v.marker { + _, err := s.z.serverPools[pool].DeleteObject(ctx, s.bucket, object, + ObjectOptions{Versioned: true, VersionID: versionID, DeleteMarker: true, MTime: mtime}) + if err != nil { + s.t.Fatal(err) + } + continue + } + opts := ObjectOptions{Versioned: !v.null, VersionID: versionID, MTime: mtime} + if v.null && s.status() == versioning.Suspended { + opts = ObjectOptions{VersionSuspended: true, MTime: mtime} + } + body := object + " " + v.label + if len(body) < v.size { + body += strings.Repeat(".", v.size-len(body)) + } + putConsistencyObject(s.t, s.z, s.bucket, object, pool, body, opts) + s.bodies[body] = v.label + if v.tiered { + vid := versionID + if vid == "" { + vid = nullVersionID + } + infos, errs := readAllFileInfo(ctx, s.z.serverPools[pool].getHashedSet(object).getDisks(), "", s.bucket, object, vid, false, false) + if errs[0] != nil { + s.t.Fatal(errs[0]) + } + fi := infos[0] + fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", "remote-"+v.label + if err := s.z.serverPools[pool].DecomTieredObject(ctx, s.bucket, object, fi, ObjectOptions{}); err != nil { + s.t.Fatal(err) + } + } + } + if v.marker { + s.markers[s.key(object, versionID)] = v.label + } else { + s.labels[s.key(object, versionID)] = v.label + } + s.ids[object+" "+v.label] = versionID + s.track(object) +} + +func (s *splitILM) status() versioning.State { + vc, _ := globalBucketVersioningSys.Get(s.bucket) + if vc == nil { + return "" + } + return vc.Status +} + +// scan applies ILM to every object in a pool as the scanner of a drive of +// that pool does, from the xl.meta of that drive, and runs the queued +// expiries as an expiry worker does. +func (s *splitILM) scan(pool int) { + s.t.Helper() + ctx := s.t.Context() + lc, err := globalLifecycleSys.Get(s.bucket) + if err != nil { + s.t.Fatal(err) + } + lr, err := globalBucketObjectLockSys.Get(s.bucket) + if err != nil { + s.t.Fatal(err) + } + vc, _ := globalBucketVersioningSys.Get(s.bucket) + for _, object := range s.objects { + disk := s.z.serverPools[pool].getHashedSet(object).getDisks()[0] + rf, err := disk.ReadXL(ctx, s.bucket, object, false) + if err != nil { + continue // not in this pool + } + fivs, err := getFileInfoVersions(rf.Buf, s.bucket, object, false) + if err != nil { + s.t.Fatal(err) + } + objInfos := make([]ObjectInfo, len(fivs.Versions)) + for i, fi := range fivs.Versions { + objInfos[i] = fi.ToObjectInfo(s.bucket, object, vc != nil && vc.Versioned(object)) + } + item := scannerItem{bucket: s.bucket, objectName: object, lifeCycle: lc} + item.applyActions(ctx, s.z, objInfos, lr, &sizeSummary{}, nil) + } + s.runQueue() +} + +// head applies ILM to a version as a HEAD or GET with its versionId does. +func (s *splitILM) head(object, label string) lifecycle.Action { + s.t.Helper() + ctx := s.t.Context() + oi, err := s.z.GetObjectInfo(ctx, s.bucket, object, ObjectOptions{VersionID: s.ids[object+" "+label]}) + if err != nil { + s.t.Fatal(err) + } + lc, err := globalLifecycleSys.Get(s.bucket) + if err != nil { + s.t.Fatal(err) + } + lr, err := globalBucketObjectLockSys.Get(s.bucket) + if err != nil { + s.t.Fatal(err) + } + evt := evalActionFromLifecycle(ctx, *lc, lr, nil, oi) + if evt.Action.Delete() { + applyExpiryRule(evt, lcEventSrc_s3HeadObject, oi) + } + s.runQueue() + return evt.Action +} + +// runQueue runs the queued expiries with an expiry worker. +func (s *splitILM) runQueue() { + done := make(chan struct{}) + go func() { + defer close(done) + globalExpiryState.Worker(s.queue) + }() + s.queue <- nil // the worker returns once it ran every expiry queued before + <-done +} + +// get returns the label of the version a GET of object reads, or "-" when it +// reads as deleted. +func (s *splitILM) get(object string) string { + s.t.Helper() + ctx := s.t.Context() + oi, err := s.z.GetObjectInfo(ctx, s.bucket, object, ObjectOptions{}) + if err != nil { + if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrMethodNotAllowed(err) { + return "-" + } + return fmt.Sprintf("<%v>", err) + } + if oi.VersionID != "" && oi.VersionID != nullVersionID { + return s.label(object, oi.VersionID, false) + } + // Copies of a null version in two pools are told apart by their data. + gr, err := s.z.GetObjectNInfo(ctx, s.bucket, object, nil, nil, ObjectOptions{}) + if err != nil { + return fmt.Sprintf("<%v>", err) + } + defer gr.Close() + data, err := io.ReadAll(gr) + if err != nil { + s.t.Fatal(err) + } + if l, ok := s.bodies[string(data)]; ok { + return l + } + return "?" + string(data) +} + +// views returns the versions of each object in each pool, newest first. +func (s *splitILM) views() []map[string][]string { + views := make([]map[string][]string, len(s.z.serverPools)) + for pool := range s.z.serverPools { + views[pool] = s.view(pool) + } + return views +} + +type splitKey struct { + name string + versions []splitVersion + before string // what a GET reads before ILM ran + after string // what a GET reads once ILM ran + want [][]string // versions in each pool once ILM ran, nil to skip +} + +// run lays out keys, runs two scanner cycles over all pools and checks that +// a GET of each key never reads anything but before or after, and the +// versions each pool holds at the end. The second cycle must not change +// anything. +func (s *splitILM) run(keys []splitKey) { + s.t.Helper() + for _, k := range keys { + for _, v := range k.versions { + s.write(k.name, v) + } + if got := s.get(k.name); got != k.before { + s.t.Fatalf("%s: fixture reads %s, want %s", k.name, got, k.before) + } + } + var first []map[string][]string + for cycle := 1; cycle <= 2; cycle++ { + for pool := range s.z.serverPools { + s.scan(pool) + for _, k := range keys { + if got := s.get(k.name); got != k.before && got != k.after { + s.t.Errorf("%s: reads %s after the scanner of pool %d (cycle %d), want %s", k.name, got, pool, cycle, k.after) + } + } + } + if cycle == 1 { + first = s.views() + } + } + last := s.views() + for _, k := range keys { + if got := s.get(k.name); got != k.after { + s.t.Errorf("%s: reads %s once ILM ran, want %s", k.name, got, k.after) + } + for pool := range last { + if !slices.Equal(first[pool][k.name], last[pool][k.name]) { + s.t.Errorf("%s: the second scanner cycle changed pool %d from %v to %v", k.name, pool, first[pool][k.name], last[pool][k.name]) + } + if k.want != nil && !slices.Equal(last[pool][k.name], k.want[pool]) { + s.t.Errorf("%s: pool %d holds %v, want %v", k.name, pool, last[pool][k.name], k.want[pool]) + } + } + } +} + +const splitILMRules = "expEnabledexp-" + + "1" + + "eavEnabledeav-" + + "1true" + + "edmEnablededm-" + + "true" + + "dmeEnableddme-" + + "1" + + "ncvEnabledncv-" + + "1" + + "exsEnabledexs-100" + + "1" + + "nnvEnablednnv-" + + "11" + +func splitAt(label string, days float64, pools ...int) splitVersion { + return splitVersion{pools: pools, label: label, days: days} +} + +func splitMarker(label string, days float64, pools ...int) splitVersion { + return splitVersion{pools: pools, label: label, days: days, marker: true} +} + +func splitTiered(label string, days float64, pools ...int) splitVersion { + return splitVersion{pools: pools, label: label, days: days, tiered: true} +} + +func splitNull(label string, days float64, pools ...int) splitVersion { + return splitVersion{pools: pools, label: label, days: days, null: true} +} + +// splitILMOnePool are keys in one pool. ILM applies to them as usual. +func splitILMOnePool(pool int) []splitKey { + in := func(vs ...string) [][]string { + want := [][]string{nil, nil} + want[pool] = vs + return want + } + return []splitKey{ + {"exp-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool)}, "v2", "-", in("DM", "v2", "v1")}, + {"exp-one-tiered", []splitVersion{splitAt("v1", 5, pool), splitTiered("v2", 4, pool)}, "v2", "-", in("DM", "v2", "v1")}, + {"eav-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool)}, "v2", "-", in()}, + {"edm-one", []splitVersion{splitMarker("dm", 3, pool)}, "-", "-", in()}, + {"dme-one", []splitVersion{splitAt("v1", 5, pool), splitMarker("dm", 3, pool)}, "-", "-", in()}, + {"ncv-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool), splitAt("v3", 0.01, pool)}, "v3", "v3", in("v3", "v2")}, + {"nnv-one", []splitVersion{splitAt("v1", 5, pool), splitAt("v2", 4, pool), splitAt("v3", 3, pool), splitAt("v4", 2, pool)}, "v4", "v4", in("v4", "v3")}, + } +} + +func TestPoolsLifecycleSplitObject(t *testing.T) { + t.Run("versioned", func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + s.run([]splitKey{ + // Expiration with Days applies to the current version v2 only. + {"exp-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, + {"exp-old-in-1", []splitVersion{splitAt("v2", 0.01, 0), splitAt("v1", 5, 1)}, "v2", "v2", [][]string{{"v2"}, {"v1"}}}, + {"exp-tiered-old-in-0", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, + // v2 is old enough: one delete marker hides the key. + {"exp-both-old", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1)}, "v2", "-", [][]string{{"DM", "v1"}, {"v2"}}}, + // The rule applies to v2 only by its size. + {"exs-big-in-1", []splitVersion{splitAt("v1", 5, 0), {pools: []int{1}, label: "v2", days: 4, size: 200}}, "v2", "-", [][]string{{"DM", "v1"}, {"v2"}}}, + // The marker hides v1 and is not a marker without noncurrent + // versions; v1 is not the current version. + {"exp-marker-in-0", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", [][]string{{"dm"}, {"v1"}}}, + // ExpiredObjectAllVersions applies to the current version v2 only. + {"eav-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, + {"eav-old-in-1", []splitVersion{splitAt("v2", 0.01, 0), splitAt("v1", 5, 1)}, "v2", "v2", [][]string{{"v2"}, {"v1"}}}, + {"eav-tiered-old-in-0", []splitVersion{splitTiered("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, + {"eav-both-old", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1)}, "v2", "-", [][]string{nil, nil}}, + // A copy of v1 in both pools, as an interrupted move leaves it. + {"eav-copies", []splitVersion{splitAt("v1", 5, 0, 1)}, "v1", "-", [][]string{nil, nil}}, + // The marker hides v1 in the other pool. + {"edm-marker-in-0", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", [][]string{{"dm"}, {"v1"}}}, + {"edm-marker-in-1", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 1)}, "-", "-", [][]string{{"v1"}, {"dm"}}}, + {"edm-copies", []splitVersion{splitMarker("dm", 3, 0, 1)}, "-", "-", [][]string{nil, nil}}, + // v2 in the other pool is the current version, the marker does + // not expire. + {"dme-new-in-1", []splitVersion{splitMarker("dm", 3, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"dm"}, {"v2"}}}, + {"dme-new-in-0", []splitVersion{splitAt("v2", 0.01, 0), splitMarker("dm", 3, 1)}, "v2", "v2", [][]string{{"v2"}, {"dm"}}}, + // The marker is the current version and expires with v1. + {"dme-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 1)}, "-", "-", [][]string{nil, nil}}, + // v1 is noncurrent since v2 was written, v2 since v3 was. + {"ncv-split", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 1), splitAt("v3", 2, 0)}, "v3", "v3", [][]string{{"v3"}, {"v2"}}}, + }) + }) + t.Run("versioned/one-pool", func(t *testing.T) { + for pool := range 2 { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + s.run(splitILMOnePool(pool)) + } + }) + t.Run("versioned/head-version", func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + keys := []splitKey{ + {"exp-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, + {"eav-old-in-0", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, + } + for _, k := range keys { + for _, v := range k.versions { + s.write(k.name, v) + } + act := s.head(k.name, "v1") + t.Logf("%s: HEAD of v1 evaluates %s", k.name, act) + if got := s.get(k.name); got != k.after { + t.Errorf("%s: reads %s after a HEAD of v1, want %s", k.name, got, k.after) + } + views := s.views() + for pool := range views { + if !slices.Equal(views[pool][k.name], k.want[pool]) { + t.Errorf("%s: pool %d holds %v, want %v", k.name, pool, views[pool][k.name], k.want[pool]) + } + } + } + }) + t.Run("suspended", func(t *testing.T) { + s := newSplitILM(t, versioning.Suspended, splitILMRules) + s.run([]splitKey{ + {"exp-null-old-in-0", []splitVersion{splitNull("v1", 5, 0), splitAt("v2", 0.01, 1)}, "v2", "v2", [][]string{{"v1"}, {"v2"}}}, + }) + }) + t.Run("unversioned", func(t *testing.T) { + s := newSplitILM(t, "", splitILMRules) + s.run([]splitKey{ + {"eav-old-in-0", []splitVersion{splitNull("old", 5, 0), splitNull("new", 0.01, 1)}, "new", "new", nil}, + {"exp-old-in-0", []splitVersion{splitNull("old", 5, 0), splitNull("new", 0.01, 1)}, "new", "new", nil}, + }) + views := s.views() + for object, want := range map[string]int{"eav-old-in-0": 1, "exp-old-in-0": 0} { + if len(views[0][object]) != want || len(views[1][object]) != 1 { + t.Errorf("%s: pools hold %v and %v, want %d and 1 copies", object, views[0][object], views[1][object], want) + } + } + }) +} diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index d1e83bb71..c79a0f5b5 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1248,6 +1248,16 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob ctx = lkctx.Context() defer lk.Unlock(lkctx) + // An ILM expiry is decided on the versions in one pool. Under the lock, + // check it with the versions in all pools; the sets then take it as + // decided, without a recheck by the versions each one holds. + if z.expiryNeedsAllPools(opts) { + if err = z.checkExpiryAllPools(ctx, bucket, object, opts); err != nil { + return ObjectInfo{Name: decodeDirObject(object), VersionID: opts.VersionID}, err + } + opts.Expiration.CheckedAllPools = true + } + if opts.DeletePrefix { return ObjectInfo{}, z.deletePrefix(ctx, bucket, object) } @@ -1517,10 +1527,28 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o batch = make([]ObjectToDelete, 0, len(objects)) batchIdx = make([]int, 0, len(objects)) data, dataIdx := make([]ObjectToDelete, 0, len(objects)), make([]int, 0, len(objects)) + ilm := make(map[string]lifecycleAllPools) for i, obj := range objects { if derrs[i] != nil { continue } + // A version that ILM expires is checked with the versions of + // its object in all pools, see checkExpiryAllPools. When the + // expiry no longer applies, nothing is deleted and nothing is + // left to replicate. + vopts := ObjectOptions{Expiration: opts.Expiration, VersionID: obj.VersionID} + if obj.VersionID != "" && z.expiryNeedsAllPools(vopts) { + all, ok := ilm[obj.ObjectName] + if !ok { + all.versions, all.events, all.err = z.lifecycleEventsAllPools(ctx, bucket, obj.ObjectName) + ilm[obj.ObjectName] = all + } + if err := all.expiryApplies(bucket, obj.ObjectName, vopts); err != nil { + dobjects[i] = DeletedObject{ObjectName: obj.ObjectName, VersionID: obj.VersionID} + derrs[i] = err + continue + } + } if deletesData(obj, opts) { if z.copyWithoutDeleteQuorum(ctx, bucket, obj.ObjectName, obj.VersionID) { derrs[i] = InsufficientWriteQuorum{} diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 964a01149..658e30f9b 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -159,6 +159,10 @@ type WalkOptions struct { // ExpirationOptions represents object options for object expiration at objectLayer. type ExpirationOptions struct { Expire bool + // CheckedAllPools: the expiry was checked with the versions of the + // object in all pools (checkExpiryAllPools). A set does not check it + // again with the versions it holds alone. + CheckedAllPools bool } // TransitionOptions represents object options for transition ObjectLayer operation diff --git a/cmd/object-api-interface_gen.go b/cmd/object-api-interface_gen.go index 919cdb174..8637f5498 100644 --- a/cmd/object-api-interface_gen.go +++ b/cmd/object-api-interface_gen.go @@ -79,10 +79,13 @@ func (z BucketOptions) Msgsize() (s int) { // MarshalMsg implements msgp.Marshaler func (z ExpirationOptions) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 1 + // map header, size 2 // string "Expire" - o = append(o, 0x81, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65) + o = append(o, 0x82, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65) o = msgp.AppendBool(o, z.Expire) + // string "CheckedAllPools" + o = append(o, 0xaf, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x65, 0x64, 0x41, 0x6c, 0x6c, 0x50, 0x6f, 0x6f, 0x6c, 0x73) + o = msgp.AppendBool(o, z.CheckedAllPools) return } @@ -110,6 +113,12 @@ func (z *ExpirationOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Expire") return } + case "CheckedAllPools": + z.CheckedAllPools, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CheckedAllPools") + return + } default: bts, err = msgp.Skip(bts) if err != nil { @@ -124,7 +133,7 @@ func (z *ExpirationOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z ExpirationOptions) Msgsize() (s int) { - s = 1 + 7 + msgp.BoolSize + s = 1 + 7 + msgp.BoolSize + 16 + msgp.BoolSize return } diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go index ddfc8dde3..acd7ebcb2 100644 --- a/cmd/object-handlers-common.go +++ b/cmd/object-handlers-common.go @@ -467,6 +467,7 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD deletedObjs, errs := o.DeleteObjects(ctx, bucket, toDel, ObjectOptions{ PrefixEnabledFn: vc.PrefixEnabled, VersionSuspended: vc.Suspended(), + Expiration: ExpirationOptions{Expire: true}, }) for i, dobj := range deletedObjs { From 8c1473a618f89fda3b89462c69c2a3c1a320d4c6 Mon Sep 17 00:00:00 2001 From: Denis Babchuk Date: Mon, 2 Nov 2026 19:59:13 +0300 Subject: [PATCH 02/26] fix(pools): move versions whose ILM expiry does not hold with the versions in other pools Signed-off-by: Denis Babchuk --- cmd/erasure-server-pool-decom.go | 4 + cmd/erasure-server-pool-lifecycle.go | 29 ++++++++ cmd/erasure-server-pool-lifecycle_test.go | 89 ++++++++++++++++++++++- cmd/erasure-server-pool-rebalance.go | 4 + 4 files changed, 124 insertions(+), 2 deletions(-) diff --git a/cmd/erasure-server-pool-decom.go b/cmd/erasure-server-pool-decom.go index bc5e1c514..19e6a01e8 100644 --- a/cmd/erasure-server-pool-decom.go +++ b/cmd/erasure-server-pool-decom.go @@ -819,6 +819,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool // a noncurrent version and has to be moved like any other. The // scanner applies the expiry once it lives in another pool. return lifecycle.NoneAction + case evt.Action.Delete() && !z.migrationExpiryApplies(ctx, objInfo, evt): + // Another pool holds versions with which the rule does not + // apply. The version moves like any other. + return lifecycle.NoneAction case evt.Action.Delete(): return evt.Action default: diff --git a/cmd/erasure-server-pool-lifecycle.go b/cmd/erasure-server-pool-lifecycle.go index 800b39d92..16079e534 100644 --- a/cmd/erasure-server-pool-lifecycle.go +++ b/cmd/erasure-server-pool-lifecycle.go @@ -250,3 +250,32 @@ func (z *erasureServerPools) expiryNeedsAllPools(opts ObjectOptions) bool { } return opts.DeletePrefix || opts.VersionID != "" || opts.Versioned || opts.VersionSuspended } + +// migrationExpiryApplies reports whether the ILM expiry evt, which a +// decommission or rebalance decided for oi on the versions of the pool it +// walks, applies with the versions of the object in all pools too. Only then +// may the walk leave the version to the expiry instead of moving it. A +// version that is the latest in the walked pool can be a noncurrent version +// of the object, and a delete marker alone in that pool can hide versions in +// another one. Such a version is moved like any other, and the scanner +// applies the rule in the pool that holds the object then. +func (z *erasureServerPools) migrationExpiryApplies(ctx context.Context, oi ObjectInfo, evt lifecycle.Event) bool { + // The options the expiry deletes with, see + // applyExpiryOnNonTransitionedObjects and expireTransitionedObject. + opts := ObjectOptions{ + Expiration: ExpirationOptions{Expire: true}, + Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name), + } + if evt.Action.DeleteVersioned() { + opts.VersionID = oi.VersionID + } + if evt.Action.DeleteAll() { + opts.DeletePrefix = true + } + opts.Transition.ExpireRestored = evt.Action.DeleteRestored() + if !z.expiryNeedsAllPools(opts) { + return true + } + return z.checkExpiryAllPools(ctx, oi.Bucket, encodeDirObject(oi.Name), opts) == nil +} diff --git a/cmd/erasure-server-pool-lifecycle_test.go b/cmd/erasure-server-pool-lifecycle_test.go index 0e346e33f..ea39d2819 100644 --- a/cmd/erasure-server-pool-lifecycle_test.go +++ b/cmd/erasure-server-pool-lifecycle_test.go @@ -29,8 +29,8 @@ import ( "github.com/minio/minio/internal/bucket/versioning" ) -// The scanner, the drive heal and a GET or HEAD of a version evaluate ILM on -// the versions of a key in one pool: IsLatest, NumVersions and SuccessorModTime +// The scanner and the walks of decommission and rebalance evaluate ILM on the +// versions of a key in one pool: IsLatest, NumVersions and SuccessorModTime // count only these. A key can have versions in several pools: two concurrent // first writes, overwrites that went to another pool. The expiry they decide // runs in every pool. @@ -414,3 +414,88 @@ func TestPoolsLifecycleSplitObject(t *testing.T) { } }) } + +// walk moves pool 0 out with a decommission or a rebalance pass, as move +// does, but reports a decommission that did not complete instead of stopping +// the test, so the reads are still checked. +func (s *splitILM) walk(kind string) { + s.t.Helper() + ctx := s.t.Context() + if kind == "rebalance" { + migrationTestWalk(s.t, s.z, s.bucket, 0, kind) + s.z.rebalMu.Lock() + s.z.rebalMeta.PoolStats[0].Info.Status = rebalCompleted + s.z.rebalMu.Unlock() + return + } + if err := s.z.StartDecommission(ctx, 0); err != nil { + s.t.Fatal(err) + } + s.z.doDecommissionInRoutine(ctx, 0) + s.z.poolMetaMutex.RLock() + info := *s.z.poolMeta.Pools[0].Decommission + s.z.poolMetaMutex.RUnlock() + if !info.Complete || info.Failed || info.ItemsDecommissionFailed > 0 { + s.t.Errorf("decommission did not complete: complete=%t failed=%t itemsFailed=%d; pool 0 holds %v", + info.Complete, info.Failed, info.ItemsDecommissionFailed, s.view(0)) + } +} + +// Rules that expire a whole object (DelMarkerExpiration, +// ExpiredObjectAllVersions) or a delete marker without noncurrent versions +// (ExpiredObjectDeleteMarker) depend on every version of the object. A walk +// sees the versions in its own pool only. +func TestPoolsMigrationLifecycleWholeObject(t *testing.T) { + recent := 1.0 / 24 + objects := []struct { + name string + versions []splitVersion // oldest first + before string // what a GET reads before the walk + after string // what a GET reads once ILM caught up + want []string // versions in pool 1 once ILM caught up, newest first + }{ + // The marker expires and takes v1 with it. + {"dme-one-pool", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 0)}, "-", "-", nil}, + {"dme-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", nil}, + // v2 is the current version, the marker does not expire. + {"dme-newer-in-pool-1", []splitVersion{splitMarker("dm", 3, 0), splitAt("v2", recent, 1)}, "v2", "v2", []string{"v2", "dm"}}, + // v2 expires and takes v1 with it. + {"eav-one-pool", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 0)}, "v2", "-", nil}, + {"eav-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitAt("v2", 4, 0)}, "v2", "-", nil}, + // v2 is the current version and does not expire. + {"eav-newer-in-pool-1", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", recent, 1)}, "v2", "v2", []string{"v2", "v1"}}, + // The marker hides v1, it is not a marker without noncurrent versions. + {"edm-older-in-pool-1", []splitVersion{splitAt("v1", 5, 1), splitMarker("dm", 3, 0)}, "-", "-", []string{"dm", "v1"}}, + {"keep-marker", []splitVersion{splitAt("v1", 5, 0), splitMarker("dm", 3, 0)}, "-", "-", []string{"dm", "v1"}}, + {"keep-versions", []splitVersion{splitAt("v1", 5, 0), splitAt("v2", 4, 0)}, "v2", "v2", []string{"v2", "v1"}}, + } + for _, kind := range []string{"decommission", "rebalance"} { + t.Run(kind, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + for _, o := range objects { + for _, v := range o.versions { + s.write(o.name, v) + } + if got := s.get(o.name); got != o.before { + t.Fatalf("%s: fixture reads %s, want %s", o.name, got, o.before) + } + } + s.walk(kind) + for _, o := range objects { + if got := s.get(o.name); got != o.before && got != o.after { + t.Errorf("%s: reads %s after the %s, before ILM caught up; want %s or %s", o.name, got, kind, o.before, o.after) + } + } + s.runQueue() + s.scan(1) + want := map[string][]string{} + for _, o := range objects { + want[o.name] = o.want + if got := s.get(o.name); got != o.after { + t.Errorf("%s: reads %s once ILM caught up, want %s", o.name, got, o.after) + } + } + s.check(want) + }) + } +} diff --git a/cmd/erasure-server-pool-rebalance.go b/cmd/erasure-server-pool-rebalance.go index a23a9eb3d..10a40bd86 100644 --- a/cmd/erasure-server-pool-rebalance.go +++ b/cmd/erasure-server-pool-rebalance.go @@ -605,6 +605,10 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, // a noncurrent version and has to be moved like any other. The // scanner applies the expiry once it lives in another pool. return lifecycle.NoneAction + case evt.Action.Delete() && !z.migrationExpiryApplies(ctx, objInfo, evt): + // Another pool holds versions with which the rule does not + // apply. The version moves like any other. + return lifecycle.NoneAction case evt.Action.Delete(): return evt.Action } From 001078e9cae9c1e036941f7421cf52474f1c7b9e Mon Sep 17 00:00:00 2001 From: Denis Babchuk Date: Mon, 2 Nov 2026 21:41:37 +0300 Subject: [PATCH 03/26] fix(ilm): expire a restored copy by the version it belongs to Signed-off-by: Denis Babchuk --- cmd/bucket-lifecycle-restore_test.go | 451 +++++++++++++++++++++++++++ cmd/bucket-lifecycle.go | 11 + 2 files changed, 462 insertions(+) create mode 100644 cmd/bucket-lifecycle-restore_test.go diff --git a/cmd/bucket-lifecycle-restore_test.go b/cmd/bucket-lifecycle-restore_test.go new file mode 100644 index 000000000..e435a1f9e --- /dev/null +++ b/cmd/bucket-lifecycle-restore_test.go @@ -0,0 +1,451 @@ +// Copyright (c) 2026 Denis Babchuk +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/versioning" + xhttp "github.com/minio/minio/internal/http" +) + +// restoreTestTier is a remote tier held in memory that counts its reads. +type restoreTestTier struct { + *tierExpiryTestBackend + gets atomic.Int64 +} + +func (b *restoreTestTier) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) { + b.gets.Add(1) + return b.tierExpiryTestBackend.Get(ctx, object, rv, opts) +} + +// restoreTest is a cluster with a remote tier WARM and a bucket whose +// versions are transitioned to it and restored. +type restoreTest struct { + t *testing.T + z *erasureServerPools + bucket string + object string + queue chan expiryOp + labels map[string]string // version ID ("null" for the null version) -> label + bodies map[string][]byte // label -> data +} + +func restoreTestVersionKey(versionID string) string { + if versionID == "" { + return nullVersionID + } + return versionID +} + +// put writes a version of the object to the first pool. A restored version +// is 1 MiB, so that its restored copy has a data directory of its own. +func (r *restoreTest) put(label string, null, restored bool) ObjectInfo { + r.t.Helper() + body := []byte(r.object + " " + label) + if restored { + body = bytes.Repeat(body, (1<<20)/len(body)+1) + } + opts := ObjectOptions{Versioned: !null} + if null { + opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object) + } + oi, err := r.z.serverPools[0].PutObject(r.t.Context(), r.bucket, r.object, + mustGetPutObjReader(r.t, bytes.NewReader(body), int64(len(body)), "", ""), opts) + if err != nil { + r.t.Fatal(err) + } + r.labels[restoreTestVersionKey(oi.VersionID)] = label + r.bodies[label] = body + return oi +} + +// opts returns the options of a request on a version of the object, as the +// S3 handlers set them. +func (r *restoreTest) opts(versionID string) ObjectOptions { + return ObjectOptions{ + VersionID: versionID, + Versioned: globalBucketVersioningSys.PrefixEnabled(r.bucket, r.object), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object), + } +} + +// restore transitions a version to the tier, restores it, and moves the +// expiry date of the restore into the past. +func (r *restoreTest) restore(oi ObjectInfo) { + r.t.Helper() + ctx := r.t.Context() + vopts := r.opts(oi.VersionID) + topts := vopts + topts.MTime = oi.ModTime + topts.Transition = TransitionOptions{Tier: "WARM", ETag: oi.ETag} + if err := r.z.TransitionObject(ctx, r.bucket, r.object, topts); err != nil { + r.t.Fatal(err) + } + ropts := vopts + ropts.Transition = TransitionOptions{RestoreRequest: &RestoreObjectRequest{Days: 1}} + // Through the pool, as with a single pool: with more than one pool the + // pools layer holds the object lock that the write of the restored copy + // waits for. + if err := r.z.serverPools[0].RestoreTransitionedObject(ctx, r.bucket, r.object, ropts); err != nil { + r.t.Fatal(err) + } + expired := completedRestoreObj(time.Now().Add(-24 * time.Hour).UTC()).String() + mopts := vopts + mopts.MTime = oi.ModTime + mopts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[xhttp.AmzRestore] = expired + return ReplicateDecision{}, nil + } + if _, err := r.z.PutObjectMetadata(ctx, r.bucket, r.object, mopts); err != nil { + r.t.Fatal(err) + } +} + +// fileInfo returns the version of the object from a drive of the first pool. +func (r *restoreTest) fileInfo(versionID string) FileInfo { + r.t.Helper() + infos, errs := readAllFileInfo(r.t.Context(), r.z.serverPools[0].getHashedSet(r.object).getDisks(), "", r.bucket, r.object, restoreTestVersionKey(versionID), false, false) + if errs[0] != nil { + r.t.Fatal(errs[0]) + } + return infos[0] +} + +// dataDirs returns how many drives of the first pool hold the data +// directory of a version. +func (r *restoreTest) dataDirs(dataDir string) int { + r.t.Helper() + n := 0 + for _, disk := range r.z.serverPools[0].getHashedSet(r.object).getDisks() { + _, err := os.Stat(filepath.Join(disk.Endpoint().Path, r.bucket, r.object, dataDir)) + if err == nil { + n++ + } else if !os.IsNotExist(err) { + r.t.Fatal(err) + } + } + return n +} + +// versions returns the versions of the object in each pool, newest first, +// and its free versions in all pools. +func (r *restoreTest) versions() (views []string, free int) { + r.t.Helper() + for _, pool := range r.z.serverPools { + rf, err := pool.getHashedSet(r.object).getDisks()[0].ReadXL(r.t.Context(), r.bucket, r.object, false) + if errors.Is(err, errFileNotFound) { + views = append(views, "") + continue + } + if err != nil { + r.t.Fatal(err) + } + fivs, err := getFileInfoVersions(rf.Buf, r.bucket, r.object, false) + if err != nil { + r.t.Fatal(err) + } + var labels []string + for _, fi := range fivs.Versions { + if fi.Deleted { + labels = append(labels, "DM") + } else { + labels = append(labels, r.labels[restoreTestVersionKey(fi.VersionID)]) + } + } + views = append(views, strings.Join(labels, " ")) + free += len(fivs.FreeVersions) + } + return views, free +} + +// read returns the label and the data of a version, or of the latest version +// when versionID is empty, or the error of the read. +func (r *restoreTest) read(versionID string) (string, []byte, error) { + r.t.Helper() + gr, err := r.z.GetObjectNInfo(r.t.Context(), r.bucket, r.object, nil, nil, ObjectOptions{VersionID: versionID}) + if err != nil { + return "", nil, err + } + defer gr.Close() + data, err := io.ReadAll(gr) + if err != nil { + return "", nil, err + } + return r.labels[restoreTestVersionKey(gr.ObjInfo.VersionID)], data, nil +} + +// runQueue runs the queued expiries with an expiry worker. +func (r *restoreTest) runQueue() { + done := make(chan struct{}) + go func() { + defer close(done) + globalExpiryState.Worker(r.queue) + }() + r.queue <- nil // the worker returns once it ran every expiry queued before + <-done +} + +// objInfos returns the versions of the object as a drive of the +// first pool holds them, as the scanner and the walks of decommission and +// the drive heal see them. +func (r *restoreTest) objInfos() []ObjectInfo { + r.t.Helper() + rf, err := r.z.serverPools[0].getHashedSet(r.object).getDisks()[0].ReadXL(r.t.Context(), r.bucket, r.object, false) + if err != nil { + r.t.Fatal(err) + } + fivs, err := getFileInfoVersions(rf.Buf, r.bucket, r.object, false) + if err != nil { + r.t.Fatal(err) + } + versioned := globalBucketVersioningSys.PrefixEnabled(r.bucket, r.object) || globalBucketVersioningSys.PrefixSuspended(r.bucket, r.object) + objInfos := make([]ObjectInfo, len(fivs.Versions)) + for i, fi := range fivs.Versions { + objInfos[i] = fi.ToObjectInfo(r.bucket, r.object, versioned) + } + return objInfos +} + +// The restored copy of a transitioned version expires on the expiry date of +// its restore. The expiry removes the copy and nothing else: the version +// stays where it is in the version stack, transitioned, and is read from the +// tier again. Every path that applies ILM runs the expiry: the scanner and a +// GET or HEAD queue it for an expiry worker, decommission and the drive heal +// run it at once. The expiry of the restored copy of the latest version used +// to delete the object instead: a versioned bucket got a delete marker that +// hid it. With versioning suspended it went to the null version, removed its +// data if there was one and failed if not, and the restored copy stayed. +func TestExpireRestoredCopy(t *testing.T) { + prevTier := globalTierConfigMgr + t.Cleanup(func() { globalTierConfigMgr = prevTier }) + prevExpiry := globalExpiryState + t.Cleanup(func() { globalExpiryState = prevExpiry }) + + // Nothing here is old enough to transition; the rule makes the bucket + // one with a tier, as restored versions are in. + const rules = `tierEnabled` + + `1000WARM` + + type version struct { + label string + null bool // the null version + restored bool // transitioned and restored, the restore has expired + } + testCases := []struct { + name string + versioning versioning.State // "", Enabled or Suspended + versions []version // oldest first + }{ + {name: "versioned", versioning: versioning.Enabled, versions: []version{{label: "v1"}, {label: "v2", restored: true}}}, + {name: "versioned-noncurrent", versioning: versioning.Enabled, versions: []version{{label: "v1", restored: true}, {label: "v2"}}}, + {name: "versioned-null", versioning: versioning.Enabled, versions: []version{{label: "v1", null: true, restored: true}}}, + {name: "suspended", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true}, {label: "v2", restored: true}}}, + {name: "suspended-no-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", restored: true}}}, + {name: "suspended-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", null: true, restored: true}}}, + {name: "unversioned", versions: []version{{label: "v1", null: true, restored: true}}}, + } + + sources := []struct { + name string + expire func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) + }{ + {"scanner", func(r *restoreTest, _ ObjectInfo, _ lifecycle.Event) { + lc, err := globalLifecycleSys.Get(r.bucket) + if err != nil { + r.t.Fatal(err) + } + lr, err := globalBucketObjectLockSys.Get(r.bucket) + if err != nil { + r.t.Fatal(err) + } + item := scannerItem{bucket: r.bucket, objectName: r.object, lifeCycle: lc} + item.applyActions(r.t.Context(), r.z, r.objInfos(), lr, &sizeSummary{}, nil) + r.runQueue() + }}, + {"head", func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) { + applyExpiryRule(evt, lcEventSrc_s3HeadObject, oi) + r.runQueue() + }}, + {"list", func(r *restoreTest, _ ObjectInfo, _ lifecycle.Event) { + // ListObjects of the key with max-keys=1 reads its latest + // version without the versioning of the bucket, a listing of + // versions reads them all. + if _, err := r.z.ListObjects(r.t.Context(), r.bucket, r.object, "", "", 1); err != nil { + r.t.Fatal(err) + } + if _, err := r.z.ListObjectVersions(r.t.Context(), r.bucket, r.object, "", "", "", 100); err != nil { + r.t.Fatal(err) + } + r.runQueue() + }}, + {"decom", func(r *restoreTest, oi ObjectInfo, evt lifecycle.Event) { + for _, o := range r.objInfos() { + if o.VersionID == oi.VersionID { + applyExpiryOnTransitionedObject(r.t.Context(), r.z, o, evt, lcEventSrc_Decom) + return + } + } + r.t.Fatal("version not found on the drive") + }}, + } + + for _, pools := range []int{1, 2} { + t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) { + ctx := t.Context() + dirs, err := getRandomDisks(4 * pools) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { removeRoots(dirs) }) + var endpoints EndpointServerPools + for i := range pools { + endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...) + } + obj, _, err := initObjectLayer(ctx, endpoints) + if err != nil { + t.Fatal(err) + } + z := obj.(*erasureServerPools) + // After initObjectLayer, which sets up a new tier config. + tier := &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}} + globalTierConfigMgr = NewTierConfigMgr() + globalTierConfigMgr.drivercache["WARM"] = tier + + for _, tc := range testCases { + for _, src := range sources { + t.Run(tc.name+"/"+src.name, func(t *testing.T) { + r := &restoreTest{ + t: t, z: z, + bucket: "restore-" + tc.name + "-" + src.name, object: "obj", + queue: make(chan expiryOp, 100), + labels: map[string]string{}, bodies: map[string][]byte{}, + } + if err := z.MakeBucket(ctx, r.bucket, MakeBucketOptions{VersioningEnabled: tc.versioning != ""}); err != nil { + t.Fatal(err) + } + meta, err := globalBucketMetadataSys.Get(r.bucket) + if err != nil { + t.Fatal(err) + } + if tc.versioning == versioning.Suspended { + meta.VersioningConfigXML = []byte("Suspended") + } + meta.LifecycleConfigXML = []byte(rules) + if err := meta.Save(ctx, z); err != nil { + t.Fatal(err) + } + globalBucketMetadataSys.Set(r.bucket, meta) + es := &expiryState{ctx: ctx, objAPI: z} + workers := []chan expiryOp{r.queue} + es.workers.Store(&workers) + globalExpiryState = es + + var restored ObjectInfo + for _, v := range tc.versions { + oi := r.put(v.label, v.null, v.restored) + if v.restored { + r.restore(oi) + restored = oi + } + } + rlabel := r.labels[restoreTestVersionKey(restored.VersionID)] + + // The fixture: the restored copy is on the drives, + // its restore has expired, ILM expires it. + fi := r.fileInfo(restored.VersionID) + if fi.TransitionStatus != lifecycle.TransitionComplete || fi.DataDir == "" || r.dataDirs(fi.DataDir) == 0 { + t.Fatalf("fixture: %s transition status %q, restored copy on %d drives", rlabel, fi.TransitionStatus, r.dataDirs(fi.DataDir)) + } + oi, err := z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(restored.VersionID)) + if err != nil { + t.Fatal(err) + } + lc, err := globalLifecycleSys.Get(r.bucket) + if err != nil { + t.Fatal(err) + } + lr, err := globalBucketObjectLockSys.Get(r.bucket) + if err != nil { + t.Fatal(err) + } + evt := evalActionFromLifecycle(ctx, *lc, lr, nil, oi) + if !evt.Action.DeleteRestored() || oi.RestoreExpires.IsZero() { + t.Fatalf("fixture: %s restore expires %v, action %v", rlabel, oi.RestoreExpires, evt.Action) + } + before, _ := r.versions() + for key, label := range r.labels { + if _, data, err := r.read(key); err != nil || !bytes.Equal(data, r.bodies[label]) { + t.Fatalf("fixture: GET %s: %v", label, err) + } + } + + src.expire(r, oi, evt) + + after, free := r.versions() + if fmt.Sprint(after) != fmt.Sprint(before) || free != 0 { + t.Errorf("versions in each pool: before %q, after %q with %d free versions", before, after, free) + } + if label, _, err := r.read(""); err != nil || label != tc.versions[len(tc.versions)-1].label { + t.Errorf("GET reads %q, %v; want %q", label, err, tc.versions[len(tc.versions)-1].label) + } + if n := r.dataDirs(fi.DataDir); n != 0 { + t.Errorf("restored copy of %s still on %d drives", rlabel, n) + } + oi, err = z.GetObjectInfo(ctx, r.bucket, r.object, r.opts(restored.VersionID)) + if err != nil { + t.Fatalf("HEAD %s: %v", rlabel, err) + } + if oi.TransitionedObject.Status != lifecycle.TransitionComplete || oi.UserDefined[xhttp.AmzRestore] != "" || !oi.RestoreExpires.IsZero() { + t.Errorf("HEAD %s: transition status %q, x-amz-restore %q", rlabel, oi.TransitionedObject.Status, oi.UserDefined[xhttp.AmzRestore]) + } + for _, v := range tc.versions { + for key, label := range r.labels { + if label != v.label { + continue + } + gets := tier.gets.Load() + _, data, err := r.read(key) + switch { + case err != nil: + t.Errorf("GET %s: %v", v.label, err) + case !bytes.Equal(data, r.bodies[v.label]): + t.Errorf("GET %s reads %d bytes that differ", v.label, len(data)) + case v.restored && tier.gets.Load() == gets: + t.Errorf("GET %s does not read the tier", v.label) + } + } + } + }) + } + } + }) + } +} diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go index e7a6d99e5..015c9c410 100644 --- a/cmd/bucket-lifecycle.go +++ b/cmd/bucket-lifecycle.go @@ -622,6 +622,17 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob // delete locally restored copy of object or object version // from the source, while leaving metadata behind. The data on // transitioned tier lies untouched and still accessible + // + // The restored copy belongs to this version, also when it is the + // latest one (DeleteRestoredAction). Without its version ID the + // delete applies to the object: a versioned bucket gets a delete + // marker that hides it, a suspended one strips the null version. + // An empty version ID is the null version, as callers that read + // it without the versioning of the bucket leave it. + opts.VersionID = oi.VersionID + if opts.VersionID == "" { + opts.VersionID = nullVersionID + } opts.Transition.ExpireRestored = true _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts) if err == nil { From 3a1e132f16e432cc06754e74991d9d5fec7d4633 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 20:40:04 +0800 Subject: [PATCH 04/26] fix(ilm): preserve all-pool approval for tiered batch expiry Signed-off-by: Feng Ruohang --- cmd/erasure-server-pool-consistency.go | 4 + ...rasure-server-pool-lifecycle-batch_test.go | 209 ++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 cmd/erasure-server-pool-lifecycle-batch_test.go diff --git a/cmd/erasure-server-pool-consistency.go b/cmd/erasure-server-pool-consistency.go index ad3e24d38..44f3dfed6 100644 --- a/cmd/erasure-server-pool-consistency.go +++ b/cmd/erasure-server-pool-consistency.go @@ -760,6 +760,10 @@ func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket } deleteOpts := opts deleteOpts.VersionID, deleteOpts.NoLock = obj.VersionID, true + // DeleteObjects already checked ordinary ILM UUID/explicit null + // versions against all pools. Empty IDs and user deletes have no such + // authorization; restored-copy expiry uses the single-object path. + deleteOpts.Expiration.CheckedAllPools = opts.Expiration.Expire && obj.VersionID != "" deleteOpts.DeleteReplication = obj.ReplicationState() if opts.PrefixEnabledFn != nil { deleteOpts.Versioned = opts.PrefixEnabledFn(obj.ObjectName) diff --git a/cmd/erasure-server-pool-lifecycle-batch_test.go b/cmd/erasure-server-pool-lifecycle-batch_test.go new file mode 100644 index 000000000..bfb5c63cc --- /dev/null +++ b/cmd/erasure-server-pool-lifecycle-batch_test.go @@ -0,0 +1,209 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "net/http" + "testing" + + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" +) + +// Reuse the author's scanner/worker fixture. A second configured pool is +// sufficient to trigger the tiered NNV regression; no versions need move there. +func TestPoolsLifecycleTieredNoncurrentBatch(t *testing.T) { + for _, tiered := range []bool{false, true} { + name := "plain" + if tiered { + name = "tiered" + } + t.Run(name, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + old := func(label string, days float64) splitVersion { + v := splitAt(label, days, 0) + v.tiered = tiered + return v + } + s.run([]splitKey{ + {"nnv-all-in-0", []splitVersion{old("v1", 5), old("v2", 4), splitAt("v3", 3, 0), splitAt("v4", 2, 0)}, "v4", "v4", [][]string{{"v4", "v3"}, nil}}, + {"ncv-all-in-0", []splitVersion{old("v1", 5), old("v2", 4), splitAt("v3", 0.01, 0)}, "v3", "v3", [][]string{{"v3", "v2"}, nil}}, + }) + }) + } +} + +// Observe the actual pool results consumed by the expiry worker, without +// replacing the storage deletion or the replication enqueue path. +type lifecycleBatchResult struct { + ObjectLayer + deleted []DeletedObject + errs []error +} + +func (o *lifecycleBatchResult) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) { + o.deleted, o.errs = o.ObjectLayer.DeleteObjects(ctx, bucket, objects, opts) + return o.deleted, o.errs +} + +func TestPoolsLifecycleTieredBatchReplication(t *testing.T) { + for _, identity := range []string{"uuid", "null"} { + t.Run(identity, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + const key = "nnv-replicated" + oldest := splitTiered("v1", 5, 0) + oldest.null = identity == "null" + for _, v := range []splitVersion{oldest, splitTiered("v2", 4, 0), splitTiered("v3", 3, 0), splitAt("v4", 2, 0)} { + s.write(key, v) + } + target := newILMTestTarget(t) + registerILMTestTarget(t, s.z, s.bucket, target, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Errorf("unexpected target method: %s", r.Method) + w.WriteHeader(500) + return + } + target.deletes.Add(1) + opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true, ReplicationRequest: true} + opts.SetReplicaStatus(replication.Replica) + if _, err := s.z.DeleteObject(r.Context(), target.bucket, key, opts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + t.Errorf("target purge: %v", err) + w.WriteHeader(500) + return + } + w.WriteHeader(http.StatusNoContent) + }) + setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket) + stats := NewReplicationStats(t.Context(), nil) + previous := globalReplicationStats.Swap(stats) + t.Cleanup(func() { globalReplicationStats.Store(previous) }) + restore, _ := newILMTestReplicationPool(t.Context(), s.z, stats) + t.Cleanup(restore) + var entries []ObjectToDelete + for _, label := range []string{"v1", "v3"} { + vid := s.ids[key+" "+label] + if vid == "" { + vid = nullVersionID + } + oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: true}) + if err != nil { + t.Fatal(err) + } + entry := ilmPurgeToDelete(t.Context(), s.bucket, oi, true) + if entry.VersionPurgeStatus != replication.VersionPurgePending { + t.Fatalf("fixture lacks purge decision: %+v", entry) + } + entries = append(entries, entry) + putConsistencyObject(t, s.z, target.bucket, key, 0, label, ObjectOptions{Versioned: identity != "null" || label != "v1", VersionID: vid}) + } + result := &lifecycleBatchResult{ObjectLayer: s.z} + deleteObjectVersions(t.Context(), result, s.bucket, entries, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}}) + if result.errs[0] != nil { + t.Errorf("eligible delete: %v", result.errs[0]) + } + if !isErrVersionNotFound(result.errs[1]) { + t.Errorf("retained version result: %v", result.errs[1]) + } + if result.deleted[1].VersionPurgeStatus() != "" || result.deleted[1].ReplicationState.ReplicateDecisionStr != "" { + t.Errorf("rejected entry retained replication state: %+v", result.deleted[1]) + } + source, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: entries[0].VersionID, Versioned: true}) + if !isErrMethodNotAllowed(err) || source.VersionPurgeStatus != replication.VersionPurgePending { + t.Errorf("source must be a pending purge tombstone before enqueue: status=%q err=%v", source.VersionPurgeStatus, err) + } + retained, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: entries[1].VersionID, Versioned: true}) + if err != nil || retained.VersionPurgeStatus != "" { + t.Errorf("rejected source changed: %+v, %v", retained, err) + } + queue := globalReplicationPool.Get().workers[0] + if len(queue) != 1 { + t.Fatalf("queued %d purges, want only eligible v1", len(queue)) + } + deletion, ok := (<-queue).(DeletedObjectReplicationInfo) + if !ok || deletion.VersionID != entries[0].VersionID || deletion.EventType != ReplicateILMDelete || deletion.VersionPurgeStatus() != replication.VersionPurgePending { + t.Fatalf("wrong purge queued: %+v", deletion) + } + if result := replicateDelete(t.Context(), deletion, s.z); result.VersionPurgeStatus() != replication.VersionPurgeComplete { + t.Fatalf("replicate purge: %+v", result) + } + if target.deletes.Load() != 1 { + t.Errorf("target delete count=%d", target.deletes.Load()) + } + for _, bucket := range []string{s.bucket, target.bucket} { + if _, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: entries[0].VersionID}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + t.Errorf("purged v1 remains in %s: %v", bucket, err) + } + if _, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: entries[1].VersionID}); err != nil { + t.Errorf("retained v3 missing from %s: %v", bucket, err) + } + } + }) + } +} + +func TestPoolsLifecycleTieredBatchIdentity(t *testing.T) { + for _, tc := range []struct { + name string + versioned bool + id string + expire bool + wantGone bool + }{ + {"uuid-ilm", true, "uuid", true, true}, + {"null-ilm", true, "null", true, true}, + {"empty-unversioned-ilm", false, "", true, false}, + {"uuid-user-delete", true, "uuid", false, true}, + {"empty-user-delete", false, "", false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + state := versioning.State("") + if tc.versioned { + state = versioning.Enabled + } + s := newSplitILM(t, state, splitILMRules) + const key = "nnv-identity" + v := splitTiered("v1", 5, 0) + v.null = tc.id != "uuid" + s.write(key, v) + if tc.versioned { + s.write(key, splitAt("v2", 4, 0)) + s.write(key, splitAt("v3", 3, 0)) + } + vid := tc.id + if vid == "uuid" { + vid = s.ids[key+" v1"] + } + vc, _ := globalBucketVersioningSys.Get(s.bucket) + _, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: vid}}}, ObjectOptions{PrefixEnabledFn: vc.PrefixEnabled, Expiration: ExpirationOptions{Expire: tc.expire}}) + if errs[0] != nil { + t.Fatalf("delete: %v", errs[0]) + } + query := vid + if query == "" { + query = nullVersionID + } + _, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: query}) + gone := isErrVersionNotFound(err) || isErrObjectNotFound(err) + if gone != tc.wantGone || (!gone && err != nil) { + t.Fatalf("gone=%t err=%v, want gone=%t", gone, err, tc.wantGone) + } + }) + } +} From e7103d77cb1d520005a592aec99014d792269d16 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 20:55:22 +0800 Subject: [PATCH 05/26] fix(ilm): constrain restored-copy cleanup to verified tiered versions Signed-off-by: Feng Ruohang --- cmd/bucket-lifecycle-restore-safety_test.go | 284 ++++++++++++++++++++ cmd/bucket-lifecycle-restore_test.go | 2 + cmd/bucket-lifecycle.go | 6 +- cmd/erasure-object.go | 4 +- cmd/xl-storage-format-v2.go | 5 + 5 files changed, 297 insertions(+), 4 deletions(-) create mode 100644 cmd/bucket-lifecycle-restore-safety_test.go diff --git a/cmd/bucket-lifecycle-restore-safety_test.go b/cmd/bucket-lifecycle-restore-safety_test.go new file mode 100644 index 000000000..9facbfd0d --- /dev/null +++ b/cmd/bucket-lifecycle-restore-safety_test.go @@ -0,0 +1,284 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/versioning" + xhttp "github.com/minio/minio/internal/http" +) + +const restoreSafetyRules = `tierEnabled1000WARM` + +// Reuse the original restore-copy fixture with a small, isolated real cluster. +func newRestoreSafetyTest(t *testing.T, pools int, state versioning.State) *restoreTest { + t.Helper() + prevTier := globalTierConfigMgr + t.Cleanup(func() { globalTierConfigMgr = prevTier }) + dirs, err := getRandomDisks(4 * pools) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { removeRoots(dirs) }) + var endpoints EndpointServerPools + for i := range pools { + endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...) + } + obj, _, err := initObjectLayer(t.Context(), endpoints) + if err != nil { + t.Fatal(err) + } + z := obj.(*erasureServerPools) + t.Cleanup(func() { z.Shutdown(context.Background()) }) + globalTierConfigMgr = NewTierConfigMgr() + globalTierConfigMgr.drivercache["WARM"] = &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}} + r := &restoreTest{t: t, z: z, bucket: getRandomBucketName(), object: "obj", labels: map[string]string{}, bodies: map[string][]byte{}} + if err := z.MakeBucket(t.Context(), r.bucket, MakeBucketOptions{VersioningEnabled: state != ""}); err != nil { + t.Fatal(err) + } + meta, err := globalBucketMetadataSys.Get(r.bucket) + if err != nil { + t.Fatal(err) + } + if state == versioning.Suspended { + meta.VersioningConfigXML = []byte(`Suspended`) + } + meta.LifecycleConfigXML = []byte(restoreSafetyRules) + if err := meta.Save(t.Context(), z); err != nil { + t.Fatal(err) + } + globalBucketMetadataSys.Set(r.bucket, meta) + return r +} + +type restoreCleanupFaultDisk struct { + consistencyReadFaultDisk + deletes *atomic.Int64 +} + +func (d restoreCleanupFaultDisk) DeleteVersion(ctx context.Context, bucket, object string, fi FileInfo, force bool, opts DeleteOptions) error { + if bucket == d.bucket && object == d.object { + d.deletes.Add(1) + } + return d.StorageAPI.DeleteVersion(ctx, bucket, object, fi, force, opts) +} + +func TestExpireRestoredRequiresFreshMetadata(t *testing.T) { + for _, restored := range []bool{true, false} { + t.Run(fmt.Sprintf("expire-restored=%v", restored), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, "") + oi := r.put("v1", true, true) + r.restore(oi) + before := r.fileInfo(oi.VersionID) + er := r.z.serverPools[0].getHashedSet(r.object) + original := er.getDisks + disks := original() + var deletes atomic.Int64 + faulty := make([]StorageAPI, len(disks)) + for i, d := range disks { + faulty[i] = restoreCleanupFaultDisk{consistencyReadFaultDisk{d, r.bucket, r.object}, &deletes} + } + er.getDisks = func() []StorageAPI { return faulty } + // Restore the callback before any fixture reads and on a fatal assertion. + defer func() { er.getDisks = original }() + opts := ObjectOptions{VersionID: nullVersionID, Transition: TransitionOptions{ExpireRestored: restored}, Expiration: ExpirationOptions{Expire: restored}} + _, err := er.DeleteObject(t.Context(), r.bucket, r.object, opts) + er.getDisks = original + if restored { + var quorum InsufficientWriteQuorum + if !errors.As(err, &quorum) { + t.Errorf("cleanup without fresh metadata: got %v, want write-quorum error", err) + } + if n := deletes.Load(); n != 0 { + t.Errorf("cleanup attempted %d blind disk deletes", n) + } + if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before, after) { + t.Error("failed metadata read changed the stored version") + } + if n := r.dataDirs(before.DataDir); n != len(disks) { + t.Errorf("cleanup removed data: %d/%d directories remain", n, len(disks)) + } + } else { + // Ordinary unversioned DELETE retains the existing tryDel behavior. + if err != nil || deletes.Load() != int64(len(disks)) { + t.Fatalf("ordinary DELETE: err=%v, deletes=%d", err, deletes.Load()) + } + if _, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{}); !isErrObjectNotFound(err) { + t.Fatalf("ordinary DELETE left the object: %v", err) + } + } + }) + } +} + +func TestExpireRestoredRenewedOrMissing(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi := r.put("v1", false, true) + r.restore(oi) + stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil { + t.Fatal(err) + } + before := r.fileInfo(oi.VersionID) + renewed := completedRestoreObj(time.Now().Add(48 * time.Hour)).String() + opts := r.opts(oi.VersionID) + opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[xhttp.AmzRestore] = renewed + return ReplicateDecision{}, nil + } + if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil { + t.Fatal(err) + } + err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner) + if !isErrVersionNotFound(err) { + t.Errorf("stale task should be rejected, got %v", err) + } + if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed || got.DataDir != before.DataDir { + t.Error("stale task changed renewed restore") + } + if n := r.dataDirs(before.DataDir); n != 4 { + t.Errorf("renewed copy has %d directories, want 4", n) + } + if _, data, err := r.read(oi.VersionID); err != nil || !bytes.Equal(data, r.bodies["v1"]) { + t.Fatalf("renewed copy not readable: %v", err) + } + + for _, id := range []string{mustGetUUID(), nullVersionID} { + opts := r.opts(id) + opts.Expiration.Expire = true + opts.Transition.ExpireRestored = true + if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, opts); !isErrVersionNotFound(err) { + t.Errorf("missing version %s: %v", id, err) + } + } + missing := r.opts(nullVersionID) + missing.Expiration.Expire = true + missing.Transition.ExpireRestored = true + if _, err := r.z.DeleteObject(t.Context(), r.bucket, "missing-object", missing); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + t.Errorf("missing object: %v", err) + } + if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed { + t.Error("missing-target cleanup changed existing restore") + } +} + +func TestExpireRestoredStorageBoundary(t *testing.T) { + for _, status := range []string{"", lifecycle.TransitionPending, lifecycle.TransitionComplete} { + for _, inline := range []bool{false, true} { + for _, requestStatus := range []string{"", lifecycle.TransitionComplete} { + t.Run(fmt.Sprintf("stored=%s/inline=%v/request=%s", status, inline, requestStatus), func(t *testing.T) { + disk, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatal(err) + } + const bucket, object = "restore-storage", "obj" + ctx := t.Context() + if err := disk.MakeVol(ctx, bucket); err != nil { + t.Fatal(err) + } + fi := newFileInfo(object, 2, 2) + fi.VersionID, fi.DataDir = mustGetUUID(), mustGetUUID() + fi.ModTime, fi.Size = time.Now().UTC(), 1024 + fi.TransitionStatus = status + fi.AddObjectPart(1, "", fi.Size, fi.Size, fi.ModTime, nil, nil) + fi.Metadata = map[string]string{xhttp.AmzRestore: completedRestoreObj(time.Now().Add(-time.Hour)).String(), "x-amz-meta-preserve": "keep"} + payload := []byte("sole local contents") + if inline { + fi.Data = payload + fi.SetInlineData() + } + if err := disk.WriteMetadata(ctx, "", bucket, object, fi); err != nil { + t.Fatal(err) + } + dir := filepath.Join(disk.Endpoint().Path, bucket, object, fi.DataDir) + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "part.1"), payload, 0o600); err != nil { + t.Fatal(err) + } + raw, err := disk.ReadXL(ctx, bucket, object, true) + if err != nil { + t.Fatal(err) + } + var xl xlMetaV2 + if err := xl.Load(raw.Buf); err != nil { + t.Fatal(err) + } + del := FileInfo{VersionID: fi.VersionID, ExpireRestored: true, TransitionStatus: requestStatus} + dataDir, err := xl.DeleteVersion(del) + if err != nil { + t.Fatal(err) + } + wantDir := "" + if status == lifecycle.TransitionComplete { + wantDir = fi.DataDir + } + if dataDir != wantDir { + t.Errorf("format authorized directory %q, want %q", dataDir, wantDir) + } + before, err := getFileInfo(raw.Buf, bucket, object, fi.VersionID, fileInfoOpts{Data: true}) + if err != nil { + t.Fatal(err) + } + if err := disk.DeleteVersion(ctx, bucket, object, del, false, DeleteOptions{}); err != nil { + t.Fatal(err) + } + afterRaw, err := disk.ReadXL(ctx, bucket, object, true) + if err != nil { + t.Fatal(err) + } + after, err := getFileInfo(afterRaw.Buf, bucket, object, fi.VersionID, fileInfoOpts{Data: true}) + if err != nil { + t.Fatal(err) + } + data, dirErr := os.ReadFile(filepath.Join(dir, "part.1")) + if status != lifecycle.TransitionComplete { + if !reflect.DeepEqual(before, after) { + t.Error("non-complete cleanup changed metadata or inline data") + } + if dirErr != nil || !bytes.Equal(data, payload) { + t.Errorf("sole local contents removed: %v", dirErr) + } + } else { + if !os.IsNotExist(dirErr) { + t.Errorf("completed restore directory still present: %v", dirErr) + } + if len(after.Data) != 0 || after.Metadata[xhttp.AmzRestore] != "" { + t.Error("completed restore kept cached data or restore headers") + } + if after.TransitionStatus != status || after.VersionID != before.VersionID || after.Metadata["x-amz-meta-preserve"] != "keep" { + t.Error("cleanup changed the transitioned version identity or user metadata") + } + } + }) + } + } + } +} diff --git a/cmd/bucket-lifecycle-restore_test.go b/cmd/bucket-lifecycle-restore_test.go index e435a1f9e..6917f539d 100644 --- a/cmd/bucket-lifecycle-restore_test.go +++ b/cmd/bucket-lifecycle-restore_test.go @@ -270,6 +270,7 @@ func TestExpireRestoredCopy(t *testing.T) { {name: "suspended", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true}, {label: "v2", restored: true}}}, {name: "suspended-no-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", restored: true}}}, {name: "suspended-null", versioning: versioning.Suspended, versions: []version{{label: "v1"}, {label: "v2", null: true, restored: true}}}, + {name: "suspended-null-noncurrent", versioning: versioning.Suspended, versions: []version{{label: "v1", null: true, restored: true}, {label: "v2"}}}, {name: "unversioned", versions: []version{{label: "v1", null: true, restored: true}}}, } @@ -334,6 +335,7 @@ func TestExpireRestoredCopy(t *testing.T) { t.Fatal(err) } z := obj.(*erasureServerPools) + t.Cleanup(func() { z.Shutdown(context.Background()) }) // After initObjectLayer, which sets up a new tier config. tier := &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}} globalTierConfigMgr = NewTierConfigMgr() diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go index 015c9c410..5c5597587 100644 --- a/cmd/bucket-lifecycle.go +++ b/cmd/bucket-lifecycle.go @@ -611,8 +611,9 @@ func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) { func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) error { traceFn := globalLifecycleSys.trace(*oi) opts := ObjectOptions{ - Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name), - Expiration: ExpirationOptions{Expire: true}, + Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name), + Expiration: ExpirationOptions{Expire: true}, } if lcEvent.Action.DeleteVersioned() { opts.VersionID = oi.VersionID @@ -647,7 +648,6 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob // touching the remote tier. If the lifecycle configuration changed after // the scanner queued this task, DeleteObject rejects the stale action while // the remote object is still protected. - opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name) keepRemote := opts.VersionID == "" && (opts.Versioned || (opts.VersionSuspended && oi.VersionID != "" && oi.VersionID != nullVersionID)) if lcEvent.Action == lifecycle.DeleteAction || lcEvent.Action == lifecycle.DeleteVersionAction { diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 5c9ee5d80..56737e962 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2114,7 +2114,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string tryDel := false if gerr != nil && goi.Name == "" { if _, ok := gerr.(InsufficientReadQuorum); ok { - if opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 { + // A restored-copy expiry needs fresh metadata to confirm its target + // and deadline; ordinary unversioned DELETE may still try a blind delete. + if opts.Transition.ExpireRestored || opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 { // Add an MRF heal for next time. er.addPartial(bucket, object, opts.VersionID) return objInfo, InsufficientWriteQuorum{} diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 5a054f86e..69465c9ab 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -1498,6 +1498,11 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) { } switch { case fi.ExpireRestored: + // Only a completed transition has a remote copy. Trust the selected + // stored version, not the transition status supplied by the caller. + if string(ver.ObjectV2.MetaSys[metaTierStatus]) != lifecycle.TransitionComplete { + return "", nil + } ver.ObjectV2.RemoveRestoreHdrs() err = x.setIdx(i, *ver) case fi.TransitionStatus == lifecycle.TransitionComplete: From 65c59dd860e746513d73e155ee327feb472e2477 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 21:07:55 +0800 Subject: [PATCH 06/26] fix(ilm): recheck read expiry for the resolved version Recheck only local ordinary-delete candidates that need an all-pool lifecycle decision. Match UUID/null identity to the version actually read and queue its global metadata and event; retain a successful read on recheck failure, absence, or refusal. Keep GET reader cleanup, authorization, proxy reads, single-pool reads, restored-copy cleanup, and expiry response headers on their existing paths. Cover signed GET/HEAD, queue contents, metadata faults, missing matches, no-scan boundaries, and reader-lock release. Signed-off-by: Feng Ruohang --- cmd/erasure-server-pool-lifecycle.go | 37 +++ cmd/object-handlers.go | 6 +- cmd/object-lifecycle-read_test.go | 380 +++++++++++++++++++++++++++ 3 files changed, 421 insertions(+), 2 deletions(-) create mode 100644 cmd/object-lifecycle-read_test.go diff --git a/cmd/erasure-server-pool-lifecycle.go b/cmd/erasure-server-pool-lifecycle.go index 16079e534..1f75bacd8 100644 --- a/cmd/erasure-server-pool-lifecycle.go +++ b/cmd/erasure-server-pool-lifecycle.go @@ -251,6 +251,43 @@ func (z *erasureServerPools) expiryNeedsAllPools(opts ObjectOptions) bool { return opts.DeletePrefix || opts.VersionID != "" || opts.Versioned || opts.VersionSuspended } +// recheckReadExpiry only rechecks a local ordinary-delete candidate. The +// global latest expiring does not authorize hiding a different version that +// GET/HEAD has already read. On an incomplete or rejected recheck, keep that +// successful read and do not queue an expiry. Leave response metadata intact. +func recheckReadExpiry(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, evt lifecycle.Event) (ObjectInfo, lifecycle.Event) { + z, ok := objectAPI.(*erasureServerPools) + if !ok || !evt.Action.Delete() || evt.Action.DeleteRestored() || !z.expiryNeedsAllPools(ObjectOptions{ + Expiration: ExpirationOptions{Expire: true}, + VersionID: oi.VersionID, + Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name), + DeletePrefix: evt.Action.DeleteAll(), + }) { + return oi, evt + } + versions, events, err := z.lifecycleEventsAllPools(ctx, oi.Bucket, encodeDirObject(oi.Name)) + if err == nil { + versionID := oi.VersionID + if versionID == "" { + versionID = nullVersionID + } + for i, version := range versions { + id := version.VersionID + if id == "" { + id = nullVersionID + } + if id == versionID { + if events[i].Action.Delete() && !events[i].Action.DeleteRestored() { + return version, events[i] + } + break + } + } + } + return oi, lifecycle.Event{Action: lifecycle.NoneAction} +} + // migrationExpiryApplies reports whether the ILM expiry evt, which a // decommission or rebalance decided for oi on the versions of the pool it // walks, applies with the versions of the object in all pools too. Only then diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 26784ac16..91a21d621 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -492,9 +492,10 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj return } event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo) + expiryInfo, event := recheckReadExpiry(ctx, objectAPI, objInfo, event) if event.Action.Delete() { // apply whatever the expiry rule is. - applyExpiryRule(event, lcEventSrc_s3GetObject, objInfo) + applyExpiryRule(event, lcEventSrc_s3GetObject, expiryInfo) if !event.Action.DeleteRestored() { // If the ILM action is not on restored object return error. writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchKey), r.URL) @@ -972,9 +973,10 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob return } event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo) + expiryInfo, event := recheckReadExpiry(ctx, objectAPI, objInfo, event) if event.Action.Delete() { // apply whatever the expiry rule is. - applyExpiryRule(event, lcEventSrc_s3HeadObject, objInfo) + applyExpiryRule(event, lcEventSrc_s3HeadObject, expiryInfo) if !event.Action.DeleteRestored() { // If the ILM action is not on restored object return error. writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey)) diff --git a/cmd/object-lifecycle-read_test.go b/cmd/object-lifecycle-read_test.go new file mode 100644 index 000000000..449e3d47a --- /dev/null +++ b/cmd/object-lifecycle-read_test.go @@ -0,0 +1,380 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/versioning" + xhttp "github.com/minio/minio/internal/http" +) + +// An explicit-version read uses ReadVersion. Only the added lifecycle scan +// uses ReadXL, so its errors/absence can be injected without failing the read. +type lifecycleReadDisk struct { + StorageAPI + bucket, object string + scans *atomic.Int64 + err error +} + +func (d lifecycleReadDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) { + if bucket == d.bucket && object == d.object { + d.scans.Add(1) + if d.err != nil { + return RawFileInfo{}, d.err + } + } + return d.StorageAPI.ReadXL(ctx, bucket, object, data) +} + +func lifecycleReadDisks(t *testing.T, s *splitILM, object string, failedPool int, err error) *atomic.Int64 { + t.Helper() + var scans atomic.Int64 + for p, pool := range s.z.serverPools { + pool.erasureDisksMu.Lock() + previous := pool.erasureDisks + wrapped := make([][]StorageAPI, len(previous)) + for i, disks := range previous { + wrapped[i] = append([]StorageAPI(nil), disks...) + for j, disk := range disks { + var injected error + if p == failedPool { + injected = err + } + wrapped[i][j] = lifecycleReadDisk{StorageAPI: disk, bucket: s.bucket, object: object, scans: &scans, err: injected} + } + } + pool.erasureDisks = wrapped + pool.erasureDisksMu.Unlock() + t.Cleanup(func() { + pool.erasureDisksMu.Lock() + pool.erasureDisks = previous + pool.erasureDisksMu.Unlock() + }) + } + return &scans +} + +// Real signed GET and HEAD requests use the author's two-pool storage fixture. +// Checking the queue as well as the response catches both false hiding and +// accidentally authorizing the latest version instead of the version read. +func TestPoolsLifecycleReadHTTP(t *testing.T) { + const bothRules = `bothEnabledboth-` + + `11` + for _, state := range []versioning.State{versioning.Enabled, versioning.Suspended} { + t.Run(string(state), func(t *testing.T) { + s := newSplitILM(t, state, splitILMRules+bothRules) + router := initTestAPIEndPoints(s.z, nil) + for _, tc := range []struct { + name, prefix string + latestAge float64 + readLatest bool + implicit bool + action lifecycle.Action + local lifecycle.Action + fault error + wantScan bool + }{ + {name: "old-version-new-latest", prefix: "exp-", latestAge: 0.01, local: lifecycle.DeleteAction, wantScan: true}, + {name: "old-version-expired-latest", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, wantScan: true}, + {name: "old-version-delete-all-latest", prefix: "eav-", latestAge: 3, local: lifecycle.DeleteAllVersionsAction, wantScan: true}, + {name: "expired-current", prefix: "exp-", latestAge: 3, readLatest: true, local: lifecycle.DeleteAction, action: lifecycle.DeleteAction, wantScan: true}, + {name: "expired-current-implicit", prefix: "exp-", latestAge: 3, readLatest: true, implicit: true, local: lifecycle.DeleteAction, action: lifecycle.DeleteAction, wantScan: true}, + {name: "expired-noncurrent", prefix: "both-", latestAge: 3, local: lifecycle.DeleteAction, action: lifecycle.DeleteVersionAction, wantScan: true}, + {name: "extra-read-quorum-error", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, fault: errDiskNotFound, wantScan: true}, + {name: "read-version-missing-from-recheck", prefix: "exp-", latestAge: 3, local: lifecycle.DeleteAction, fault: errFileNotFound, wantScan: true}, + {name: "local-none-global-expired", prefix: "ncv-", latestAge: 3, local: lifecycle.NoneAction}, + } { + for _, null := range []bool{false, true} { + for oldPool := range 2 { + t.Run(fmt.Sprintf("%s/null=%t/old-pool=%d", tc.name, null, oldPool), func(t *testing.T) { + key := fmt.Sprintf("%s%s-%t-%d", tc.prefix, tc.name, null, oldPool) + old := splitAt("v1", 5, oldPool) + old.null = null + s.write(key, old) + s.write(key, splitAt("v2", tc.latestAge, 1-oldPool)) + label := "v1" + if tc.readLatest { + label = "v2" + } + vid := s.ids[key+" "+label] + if vid == "" { + vid = nullVersionID + } + oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: state == versioning.Enabled, VersionSuspended: state == versioning.Suspended}) + if err != nil { + t.Fatal(err) + } + lc, err := globalLifecycleSys.Get(s.bucket) + if err != nil { + t.Fatal(err) + } + if local := lc.Eval(oi.ToLifecycleOpts()); local.Action != tc.local { + t.Fatalf("fixture local event=%v, want %v", local.Action, tc.local) + } + versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key) + if err != nil { + t.Fatal(err) + } + index := 1 + if tc.readLatest { + index = 0 + } + if tc.local == lifecycle.NoneAction && events[index].Action != lifecycle.DeleteVersionAction { + t.Fatal("fixture must be globally expired but locally NoneAction") + } + scans := lifecycleReadDisks(t, s, key, oldPool, tc.fault) + for _, method := range []string{http.MethodGet, http.MethodHead} { + t.Run(method, func(t *testing.T) { + scans.Store(0) + requestID := vid + if tc.implicit { + requestID = "" + } + rec := consistencyRequest(t, router, method, s.bucket, key, requestID) + want := http.StatusOK + if tc.action.Delete() { + want = http.StatusNotFound + } + if rec.Code != want { + t.Errorf("status=%d want=%d body=%s", rec.Code, want, rec.Body.String()) + } + if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != key+" "+label { + t.Errorf("body=%q, want requested version %s", rec.Body.String(), label) + } + if !tc.implicit && (scans.Load() > 0) != tc.wantScan { + t.Errorf("ReadXL calls=%d, want scan=%t", scans.Load(), tc.wantScan) + } + if tc.action.Delete() { + select { + case op := <-s.queue: + task, ok := op.(expiryTask) + src := lcEventSrc_s3GetObject + if method == http.MethodHead { + src = lcEventSrc_s3HeadObject + } + global := versions[index] + if !ok || task.src != src || task.event != events[index] || task.event.Action != tc.action || + task.objInfo.VersionID != global.VersionID || task.objInfo.IsLatest != global.IsLatest || + task.objInfo.NumVersions != global.NumVersions || !task.objInfo.SuccessorModTime.Equal(global.SuccessorModTime) { + t.Errorf("wrong queued expiry: %+v; want %s with global event %+v", task, vid, events[index]) + } + default: + t.Error("missing expiry task") + } + } + if n := len(s.queue); n != 0 { + t.Errorf("unexpected expiry tasks: %d", n) + for len(s.queue) > 0 { + <-s.queue + } + } + }) + } + }) + } + } + } + }) + } +} + +func TestPoolsLifecycleReadAuthorizationAndClose(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + initConfigSubsystem(t.Context(), s.z) + globalIAMSys.Init(t.Context(), s.z, globalEtcdClient, 2*time.Second) + denied := newObjectAttributesAuthzUser(t, "lifecycle-read", s.bucket, `"s3:PutObject"`) + router := initTestAPIEndPoints(s.z, nil) + for _, expire := range []bool{false, true} { + key := fmt.Sprintf("exp-reader-%t", expire) + v := splitAt("v1", 5, 0) + v.size = 1 << 20 // a non-inline reader retains the object read lock + s.write(key, v) + if !expire { + s.write(key, splitAt("v2", 0.01, 1)) + } + vid := s.ids[key+" v1"] + oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: vid, Versioned: true}) + if err != nil || oi.Inlined { + t.Fatalf("fixture must retain a reader lock: inlined=%t err=%v", oi.Inlined, err) + } + scans := lifecycleReadDisks(t, s, key, -1, nil) + for _, method := range []string{http.MethodGet, http.MethodHead} { + req, err := newTestSignedRequestV4(method, getGetObjectURL("", s.bucket, key)+"?versionId="+vid, 0, nil, denied.AccessKey, denied.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden || scans.Load() != 0 || len(s.queue) != 0 { + t.Fatalf("denied %s: status=%d scans=%d queued=%d", method, rec.Code, scans.Load(), len(s.queue)) + } + } + rec := consistencyRequest(t, router, http.MethodGet, s.bucket, key, vid) + want := http.StatusOK + if expire { + want = http.StatusNotFound + } + if rec.Code != want { + t.Errorf("authorized GET status=%d want=%d", rec.Code, want) + } + // The reader owns the multi-pool read lock until Close, including the + // early 404 path. A writer must acquire it immediately after the handler. + lk := s.z.NewNSLock(s.bucket, key) + lkctx, err := lk.GetLock(t.Context(), newDynamicTimeout(time.Second, time.Second)) + if err != nil { + t.Fatalf("GET leaked its reader/lock: %v", err) + } + lk.Unlock(lkctx) + for len(s.queue) > 0 { + <-s.queue + } + } +} + +func TestPoolsLifecycleReadBoundaries(t *testing.T) { + const transitionRules = `transitionEnabledtransition-` + + `1WARM` + for _, tc := range []struct { + name string + state versioning.State + null bool + singlePool bool + restored bool + transition bool + action lifecycle.Action + wantScan bool + }{ + {name: "single-pool", state: versioning.Enabled, singlePool: true, action: lifecycle.DeleteAction}, + {name: "single-pool-null", state: versioning.Enabled, singlePool: true, null: true, action: lifecycle.DeleteAction}, + {name: "unversioned", null: true, action: lifecycle.DeleteAction}, + {name: "current-null-enabled", state: versioning.Enabled, null: true, action: lifecycle.DeleteAction, wantScan: true}, + {name: "current-null-suspended", state: versioning.Suspended, null: true, action: lifecycle.DeleteAction, wantScan: true}, + {name: "transition", state: versioning.Enabled, transition: true, action: lifecycle.TransitionAction}, + {name: "restored", state: versioning.Enabled, restored: true, action: lifecycle.DeleteRestoredAction}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newSplitILM(t, tc.state, splitILMRules+transitionRules) + key := "exp-" + tc.name + if tc.restored || tc.transition { + key = "transition-" + tc.name + } + v := splitAt("v1", 5, 0) + v.null, v.tiered = tc.null, tc.restored + s.write(key, v) + vid := s.ids[key+" v1"] + if vid == "" { + vid = nullVersionID + } + if tc.restored { + globalTierConfigMgr.drivercache["WARM"] = &tierExpiryTestBackend{objects: map[string][]byte{"remote-v1": []byte(key + " v1")}} + opts := ObjectOptions{VersionID: vid, Versioned: true} + opts.EvalMetadataFn = func(oi *ObjectInfo, _ error) (ReplicateDecision, error) { + oi.UserDefined[xhttp.AmzRestore] = completedRestoreObj(UTCNow().Add(-time.Hour)).String() + return ReplicateDecision{}, nil + } + if _, err := s.z.PutObjectMetadata(t.Context(), s.bucket, key, opts); err != nil { + t.Fatal(err) + } + } + z := s.z + if tc.singlePool { + // Exercise the single-pool dispatch with the same storage fixture. + z = &erasureServerPools{serverPools: s.z.serverPools[:1]} + } + router := initTestAPIEndPoints(z, []string{"GetObject", "HeadObject"}) + scans := lifecycleReadDisks(t, s, key, -1, nil) + for _, method := range []string{http.MethodGet, http.MethodHead} { + scans.Store(0) + rec := consistencyRequest(t, router, method, s.bucket, key, vid) + want := http.StatusOK + if tc.action.Delete() && !tc.action.DeleteRestored() { + want = http.StatusNotFound + } + if rec.Code != want || (scans.Load() > 0) != tc.wantScan { + t.Errorf("%s status=%d want=%d scans=%d wantScan=%t body=%s", method, rec.Code, want, scans.Load(), tc.wantScan, rec.Body.String()) + } + if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != key+" v1" { + t.Errorf("%s lost object contents: %q", method, rec.Body.String()) + } + if tc.action.Delete() { + select { + case op := <-s.queue: + task, ok := op.(expiryTask) + if !ok || task.event.Action != tc.action || s.key(key, task.objInfo.VersionID) != s.key(key, vid) { + t.Errorf("%s wrong expiry: %+v", method, task) + } + default: + t.Errorf("%s missing expiry", method) + } + } + if n := len(s.queue); n != 0 { + t.Fatalf("%s queued %d unexpected expiries", method, n) + } + } + }) + } +} + +func TestPoolsLifecycleReadProxy(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + const key, body = "exp-proxied", "remote object remains readable" + vid := mustGetUUID() + target := newILMTestTarget(t) + var remoteRequests atomic.Int64 + registerILMTestTarget(t, s.z, s.bucket, target, func(w http.ResponseWriter, r *http.Request) { + remoteRequests.Add(1) + w.Header().Set(xhttp.ContentLength, fmt.Sprint(len(body))) + w.Header().Set(xhttp.LastModified, UTCNow().Add(-5*24*time.Hour).Format(http.TimeFormat)) + w.Header().Set(xhttp.ETag, `"da39a3ee5e6b4b0d3255bfef95601890"`) + w.Header().Set(xhttp.AmzVersionID, vid) + if r.Method == http.MethodGet { + _, _ = w.Write([]byte(body)) + } + }) + setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket) + // The delete-only target fixture omits credentials; proxy discovery clones + // the target descriptor and requires its credentials field to be present. + globalBucketTargetSys.Lock() + globalBucketTargetSys.targetsMap[s.bucket][0].Credentials = &madmin.Credentials{AccessKey: "proxy-test"} + globalBucketTargetSys.Unlock() + stats := NewReplicationStats(t.Context(), nil) + previous := globalReplicationStats.Swap(stats) + t.Cleanup(func() { globalReplicationStats.Store(previous) }) + scans := lifecycleReadDisks(t, s, key, -1, nil) + router := initTestAPIEndPoints(s.z, nil) + for _, method := range []string{http.MethodGet, http.MethodHead} { + before := remoteRequests.Load() + rec := consistencyRequest(t, router, method, s.bucket, key, vid) + if rec.Code != http.StatusOK || remoteRequests.Load() == before || scans.Load() != 0 || len(s.queue) != 0 { + t.Errorf("proxy %s: status=%d remote=%d scans=%d queued=%d body=%s", method, rec.Code, remoteRequests.Load()-before, scans.Load(), len(s.queue), rec.Body.String()) + } + if method == http.MethodGet && rec.Body.String() != body { + t.Errorf("proxy body=%q", rec.Body.String()) + } + } +} From fa42a098262ed9684d5d8f84bdaf54f423def7d6 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 21:20:24 +0800 Subject: [PATCH 07/26] fix(restore): rehydrate the selected version under its object lock Preserve handler version context, resolve the requested version's pool, and return on the initial read failure without writing metadata. Propagate owned NoLock to single-part writes, multipart completion and failure cleanup in the same patch. Preserve the original modification time on cleanup. Route the restore status copy by version and allow internal multipart rehydration only for a matching completed transition. Single-pool restores own the metadata snapshot through download and cleanup. Keep public completion in object-before-upload order, using the target set's object lock for single-pool calls, so concurrent completion cannot invert locks. Cover real signed POSTs, null/UUID and cross-pool identity, download failures, multipart overwrite guards, Object Lock updates and concurrent completion. This package depends on the restored-copy cleanup safety delivered in R2. Signed-off-by: Feng Ruohang --- cmd/erasure-multipart.go | 8 +- cmd/erasure-object.go | 25 +- cmd/erasure-server-pool.go | 43 ++- cmd/object-handlers.go | 13 +- cmd/restore-native_test.go | 709 +++++++++++++++++++++++++++++++++++++ 5 files changed, 778 insertions(+), 20 deletions(-) create mode 100644 cmd/restore-native_test.go diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 6d717df26..4b2893c08 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -34,6 +34,7 @@ import ( "github.com/google/uuid" "github.com/klauspost/readahead" "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/config/storageclass" "github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/hash" @@ -1697,13 +1698,18 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str return oi, toObjectErr(err, bucket, object) } } else { - _, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{ + current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{ VersionID: fi.VersionID, Versioned: opts.Versioned, VersionSuspended: opts.VersionSuspended, NoLock: true, }) switch { + case gerr == nil && opts.Transition.RestoreRequest != nil && + current.TransitionedObject.Status == lifecycle.TransitionComplete && + current.ETag == fi.Metadata["etag"] && current.Size == fi.Size && current.ModTime.Equal(fi.ModTime): + // The internal restore path rehydrates this same tiered version. + // Ordinary multipart completion must still reject a version overwrite. case gerr == nil, errors.As(gerr, &MethodNotAllowed{}): return oi, toObjectErr(errAuthentication, bucket, object) case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr): diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 56737e962..0451d1c08 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2644,6 +2644,8 @@ func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, obje VersionID: oi.VersionID, }, ObjectOptions{ VersionID: oi.VersionID, + MTime: oi.ModTime, + NoLock: opts.NoLock, }); err != nil { storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err)) return err @@ -2654,6 +2656,19 @@ func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, obje // restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts // as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error { + if !opts.NoLock { + // Single-pool and direct callers must protect the metadata snapshot + // through download, rehydration and failure cleanup, just like pools. + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) + if err != nil { + return err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + opts.NoLock = true + } + setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error { if rerr == nil { return nil // nothing to do; restore object was successful @@ -2665,11 +2680,13 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s // get the file info on disk for transitioned object actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false) if err != nil { - return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + // No version was read, so there is no restore metadata to update. + return toObjectErr(err, bucket, object) } oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi) + ropts.NoLock = opts.NoLock if len(oi.Parts) == 1 { var rs *HTTPRangeSpec gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts) @@ -2719,7 +2736,11 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s }) } _, err = er.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{ - MTime: oi.ModTime, + MTime: oi.ModTime, + NoLock: opts.NoLock, + Versioned: opts.Versioned, + VersionSuspended: opts.VersionSuspended, + Transition: opts.Transition, }) return setRestoreHeaderFn(oi, err) } diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index d1e83bb71..23661c413 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1625,7 +1625,17 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec return oi, err } - poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size) + var poolIdx int + if dstOpts.MetadataChg && cpSrcDstSame && srcInfo.metadataOnly && srcOpts.VersionID == dstOpts.VersionID { + // Restore status updates address an existing version. Keep ordinary + // copy/write allocation unchanged, including its latest-version lookup. + poolIdx, err = z.getPoolIdxExistingWithOpts(ctx, dstBucket, dstObject, ObjectOptions{ + VersionID: dstOpts.VersionID, MetadataChg: true, NoLock: true, + SkipDecommissioned: true, SkipRebalancing: true, + }) + } else { + poolIdx, err = z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size) + } if err != nil { return objInfo, err } @@ -2312,18 +2322,24 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket } }() - if !z.SinglePool() { - if !opts.NoLock { - lk := z.NewNSLock(bucket, encodeDirObject(object)) - lkctx, err := lk.GetLock(ctx, globalOperationTimeout) - if err != nil { - return objInfo, err - } - ctx = lkctx.Context() - defer lk.Unlock(lkctx) + // Keep object -> upload lock ordering for single-pool completion too: + // native restore holds the object lock while publishing upload parts. + if !opts.NoLock { + lk := z.NewNSLock(bucket, encodeDirObject(object)) + if z.SinglePool() { + // Preserve the set-local lock domain used by single-pool writes. + lk = z.serverPools[0].getHashedSet(object).NewNSLock(bucket, encodeDirObject(object)) } - opts.NoLock = true + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return objInfo, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + opts.NoLock = true + if !z.SinglePool() { // A conditional completion must be evaluated against the logical // latest object across pools, under the object write lock held for // this operation. The pool hosting the upload may only hold a stale @@ -3330,7 +3346,10 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck // Avoid restoring object from a pool being decommissioned. opts.SkipDecommissioned = true opts.NoLock = true - idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts) + // Restore an existing version, not the latest version's write location. + lookupOpts := opts + lookupOpts.MetadataChg = true + idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts) if err != nil { return err } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 26784ac16..2843f42d1 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -4006,8 +4006,9 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * if _, err := objectAPI.CopyObject(GlobalContext, bucket, object, bucket, object, objInfo, ObjectOptions{ VersionID: objInfo.VersionID, }, ObjectOptions{ - VersionID: objInfo.VersionID, - MTime: objInfo.ModTime, + VersionID: objInfo.VersionID, + MTime: objInfo.ModTime, + MetadataChg: true, }); err != nil { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL) return @@ -4080,14 +4081,16 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * rreq.SelectParameters.Close() return } - opts := ObjectOptions{ + restoreOpts := ObjectOptions{ Transition: TransitionOptions{ RestoreRequest: rreq, RestoreExpiry: restoreExpiry, }, - VersionID: objInfo.VersionID, + VersionID: objInfo.VersionID, + Versioned: opts.Versioned, + VersionSuspended: opts.VersionSuspended, } - if err := objectAPI.RestoreTransitionedObject(rctx, bucket, object, opts); err != nil { + if err := objectAPI.RestoreTransitionedObject(rctx, bucket, object, restoreOpts); err != nil { s3LogIf(ctx, fmt.Errorf("Unable to restore transitioned bucket/object %s/%s: %w", bucket, object, err)) return } diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go new file mode 100644 index 000000000..d9e6ad8f1 --- /dev/null +++ b/cmd/restore-native_test.go @@ -0,0 +1,709 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/event" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/pubsub" + "github.com/minio/mux" +) + +func nativeRestoreSeed(t *testing.T, r *restoreTest, pool int, null, multipart bool) (ObjectInfo, []byte) { + t.Helper() + ctx := t.Context() + opts := ObjectOptions{Versioned: !null, MTime: UTCNow().Add(-2 * time.Hour), UserDefined: map[string]string{ + "content-type": "application/x-restore-test", "x-amz-meta-preserve": "keep-me", + }} + body := bytes.Repeat([]byte("restore-payload"), 100000) + var oi ObjectInfo + var err error + if multipart { + parts := [][]byte{bytes.Repeat([]byte("a"), 5<<20), []byte("last-part")} + body = bytes.Join(parts, nil) + upload, err := r.z.serverPools[pool].NewMultipartUpload(ctx, r.bucket, r.object, opts) + if err != nil { + t.Fatal(err) + } + var complete []CompletePart + for i, data := range parts { + p, err := r.z.serverPools[pool].PutObjectPart(ctx, r.bucket, r.object, upload.UploadID, i+1, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + complete = append(complete, CompletePart{PartNumber: i + 1, ETag: p.ETag}) + } + oi, err = r.z.serverPools[pool].CompleteMultipartUpload(ctx, r.bucket, r.object, upload.UploadID, complete, opts) + if err == nil && len(oi.Parts) != 2 { + t.Fatalf("fixture has %d parts, want 2", len(oi.Parts)) + } + } else { + oi, err = r.z.serverPools[pool].PutObject(ctx, r.bucket, r.object, + mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), opts) + } + if err != nil { + t.Fatal(err) + } + oi.VersionID = restoreTestVersionKey(oi.VersionID) + topts := r.opts(oi.VersionID) + topts.MTime = oi.ModTime + topts.Transition = TransitionOptions{Tier: "WARM", ETag: oi.ETag} + if err := r.z.serverPools[pool].TransitionObject(ctx, r.bucket, r.object, topts); err != nil { + t.Fatal(err) + } + // Use the production metadata writer to mark the request ongoing. + topts.Transition = TransitionOptions{} + topts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[xhttp.AmzRestore] = ongoingRestoreObj().String() + return ReplicateDecision{}, nil + } + if _, err := r.z.PutObjectMetadata(ctx, r.bucket, r.object, topts); err != nil { + t.Fatal(err) + } + return oi, body +} + +func nativeRestoreSnapshot(t *testing.T, r *restoreTest) [][]FileInfo { + t.Helper() + result := make([][]FileInfo, len(r.z.serverPools)) + for p, pool := range r.z.serverPools { + raw, err := pool.getHashedSet(r.object).getDisks()[0].ReadXL(t.Context(), r.bucket, r.object, false) + if errors.Is(err, errFileNotFound) { + continue + } + if err != nil { + t.Fatal(err) + } + infos, err := getFileInfoVersions(raw.Buf, r.bucket, r.object, false) + if err != nil { + t.Fatal(err) + } + result[p] = append(infos.Versions, infos.FreeVersions...) + } + return result +} + +func nativeRestoreOptions(r *restoreTest, id string) ObjectOptions { + opts := r.opts(id) + opts.Transition = TransitionOptions{RestoreRequest: &RestoreObjectRequest{Days: 1}} + return opts +} + +type nativeRestoreFailTier struct { + WarmBackend + failure error +} + +func (b nativeRestoreFailTier) Get(context.Context, string, remoteVersionID, WarmBackendGetOpts) (io.ReadCloser, error) { + return nil, b.failure +} + +func TestNativeRestorePools(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, multipart := range []bool{false, true} { + for _, fail := range []bool{false, true} { + t.Run(fmt.Sprintf("pools=%d/multipart=%v/failure=%v", pools, multipart, fail), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, 0, false, multipart) + before := nativeRestoreSnapshot(t, r) + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + failure := errors.New("injected remote download failure") + if fail { + globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, failure} + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + if fail { + if !errors.Is(err, failure) { + t.Fatalf("restore error=%v, want remote failure", err) + } + got, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil || got.UserDefined[xhttp.AmzRestore] != "" { + t.Fatalf("failed restore did not clear ongoing status: %v, %q", err, got.UserDefined[xhttp.AmzRestore]) + } + } else { + if err != nil { + t.Fatal(err) + } + reads := tier.gets.Load() + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents differ: %v", err) + } + if tier.gets.Load() != reads { + t.Error("restored read still fetched the remote tier") + } + } + after := nativeRestoreSnapshot(t, r) + if len(after[0]) != len(before[0]) || after[0][0].VersionID != before[0][0].VersionID || after[0][0].TransitionedObjName != before[0][0].TransitionedObjName { + t.Fatal("restore changed the version stack or remote object identity") + } + if after[0][0].Metadata["content-type"] != "application/x-restore-test" || after[0][0].Metadata["x-amz-meta-preserve"] != "keep-me" { + t.Error("restore lost original metadata") + } + }) + } + } + } +} + +func TestNativeRestoreVersionRouting(t *testing.T) { + for _, pool := range []int{0, 1} { + for _, null := range []bool{false, true} { + t.Run(fmt.Sprintf("target-pool=%d/null=%v", pool, null), func(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, pool, null, false) + latest := putConsistencyObject(t, r.z, r.bucket, r.object, 1-pool, "latest-stays", ObjectOptions{Versioned: true, MTime: oi.ModTime.Add(time.Hour)}) + before := nativeRestoreSnapshot(t, r) + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + if err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("requested version not restored: %v", err) + } + after := nativeRestoreSnapshot(t, r) + if !reflect.DeepEqual(before[1-pool], after[1-pool]) { + t.Error("restore changed the latest version in another pool") + } + if len(after[pool]) != 1 || after[pool][0].VersionID != before[pool][0].VersionID { + t.Error("restore created a version alias") + } + gotInfo, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{}) + if err != nil || gotInfo.VersionID != latest.VersionID { + t.Errorf("latest changed: %v", err) + } + before = nativeRestoreSnapshot(t, r) + if err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, mustGetUUID())); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + t.Errorf("missing version: %v", err) + } + if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) { + t.Error("missing version restore mutated metadata") + } + }) + } + } +} + +func TestNativeRestoreInitialFailurePreservesMetadata(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi := putConsistencyObject(t, r.z, r.bucket, r.object, 0, "original-latest", ObjectOptions{Versioned: true, UserDefined: map[string]string{"content-type": "application/x-original", "x-amz-meta-preserve": "keep"}}) + before := nativeRestoreSnapshot(t, r) + for _, missing := range []string{mustGetUUID(), nullVersionID} { + err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, missing)) + if !isErrVersionNotFound(err) { + t.Errorf("missing restore %s: %v", missing, err) + } + if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) { + t.Error("initial restore failure changed stored metadata or version stack") + } + } + _, body, err := r.read(oi.VersionID) + if err != nil || string(body) != "original-latest" { + t.Errorf("initial restore failure changed content: %v", err) + } +} + +func TestNativeRestoreWithoutOuterLock(t *testing.T) { + for _, mode := range []string{"single-part", "multipart", "failure-metadata"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, _ := nativeRestoreSeed(t, r, 0, false, mode == "multipart") + tier := globalTierConfigMgr.drivercache["WARM"] + failure := errors.New("injected remote failure") + if mode == "failure-metadata" { + globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, failure} + } + er := r.z.serverPools[0].getHashedSet(r.object) + lock := er.NewNSLock(r.bucket, r.object) + lkctx, err := lock.GetLock(t.Context(), globalOperationTimeout) + if err != nil { + t.Fatal(err) + } + defer lock.Unlock(lkctx) + before := nativeRestoreSnapshot(t, r) + ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + defer cancel() + err = er.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + if err == nil { + t.Error("restore wrote through another caller's lock") + } + if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) { + t.Error("NoLock=false failed to protect the stored version") + } + }) + } +} + +type nativeRestoreHTTPResult struct { + opts ObjectOptions + err error +} +type nativeRestoreHTTPObserver struct { + ObjectLayer + done chan nativeRestoreHTTPResult +} + +func (o nativeRestoreHTTPObserver) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + ctx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + err := o.ObjectLayer.RestoreTransitionedObject(ctx, bucket, object, opts) + o.done <- nativeRestoreHTTPResult{opts, err} + return err +} + +func TestNativeRestorePOSTVersionIdentity(t *testing.T) { + for _, state := range []versioning.State{versioning.Enabled, versioning.Suspended} { + for _, target := range []int{0, 1} { + for _, split := range []bool{false, true} { + for _, multipart := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/target=%d/split=%v/multipart=%v", state, target, split, multipart), func(t *testing.T) { + r := newRestoreSafetyTest(t, 2, state) + initConfigSubsystem(t.Context(), r.z) + oi, body := nativeRestoreSeed(t, r, target, !multipart, multipart) + metaOpts := r.opts(oi.VersionID) + metaOpts.MTime = oi.ModTime + metaOpts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[xhttp.AmzRestore] = "" + return ReplicateDecision{}, nil + } + if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, metaOpts); err != nil { + t.Fatal(err) + } + if split { + putConsistencyObject(t, r.z, r.bucket, r.object, 1-target, "newer-version", ObjectOptions{Versioned: true, MTime: oi.ModTime.Add(time.Hour)}) + } + before := nativeRestoreSnapshot(t, r) + observed := nativeRestoreHTTPObserver{r.z, make(chan nativeRestoreHTTPResult, 1)} + api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return observed }} + events := make(chan event.Event, 1) + done := make(chan struct{}) + defer close(done) + if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRestoreCompleted), events, done, nil); err != nil { + t.Fatal(err) + } + bodyXML := []byte(`1`) + path := "/" + r.bucket + "/" + r.object + "?restore" + if split { + path += "&versionId=" + url.QueryEscape(oi.VersionID) + } + req, err := newTestSignedRequestV4(http.MethodPost, path, int64(len(bodyXML)), bytes.NewReader(bodyXML), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + if err = req.ParseForm(); err != nil { + t.Fatal(err) + } + req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object}) + rec := httptest.NewRecorder() + api.PostRestoreObjectHandler(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("POST restore: %d %s", rec.Code, rec.Body.String()) + } + select { + case result := <-observed.done: + if result.err != nil { + t.Fatal(result.err) + } + if result.opts.VersionID != oi.VersionID || result.opts.Versioned != (state == versioning.Enabled) || result.opts.VersionSuspended != (state == versioning.Suspended) { + t.Errorf("background options lost version context: %+v", result.opts) + } + case <-time.After(5 * time.Second): + t.Fatal("background restore did not return") + } + select { + case <-events: + case <-time.After(5 * time.Second): + t.Fatal("no restore completion event") + } + after := nativeRestoreSnapshot(t, r) + if len(after[target]) != 1 || after[target][0].VersionID != before[target][0].VersionID || after[target][0].TransitionedObjName != before[target][0].TransitionedObjName { + t.Fatal("POST restore added a version alias") + } + if !reflect.DeepEqual(before[1-target], after[1-target]) { + t.Error("POST changed another pool's version") + } + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("POST-restored contents differ: %v", err) + } + }) + } + } + } + } +} + +// A restore request is internal context, not an exception for normal S3 +// multipart overwrites or for a version that has never completed transition. +func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { + for _, mode := range []string{"matching-restore", "ordinary-complete", "not-tiered", "changed-time", "changed-size", "changed-etag"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, 0, false, true) + pool := r.z.serverPools[0] + if mode == "not-tiered" { + er := pool.getHashedSet(r.object) + fi, _, _, err := er.getObjectFileInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID), false) + if err != nil { + t.Fatal(err) + } + fi.TransitionStatus = lifecycle.TransitionPending + for _, disk := range er.getDisks() { + if err := disk.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + } + current, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil { + t.Fatal(err) + } + opts := putRestoreOpts(r.bucket, r.object, &RestoreObjectRequest{Days: 1}, current) + opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1} + if mode == "ordinary-complete" { + opts.Transition.RestoreRequest = nil + } + if mode == "changed-time" { + opts.MTime = opts.MTime.Add(time.Second) + } + // Hold the ETag constant for the size case, so only size can reject it. + if mode == "changed-size" { + opts.UserDefined["etag"] = current.ETag + body = append(body, 'x') + } + if mode == "changed-etag" { + opts.UserDefined["etag"] = "different-etag" + } + upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, opts) + if err != nil { + t.Fatal(err) + } + var complete []CompletePart + for i, data := range [][]byte{body[:5<<20], body[5<<20:]} { + part, err := pool.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, i+1, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + complete = append(complete, CompletePart{PartNumber: i + 1, ETag: part.ETag}) + } + if mode != "changed-size" && getCompleteMultipartMD5(complete) != current.ETag { + t.Fatal("fixture multipart ETag does not match original") + } + before := nativeRestoreSnapshot(t, r) + _, err = pool.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, complete, opts) + if mode == "matching-restore" { + if err != nil { + t.Fatalf("matching restore rejected: %v", err) + } + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("matching restore contents: %v", err) + } + return + } + if !errors.Is(err, errAuthentication) { + t.Fatalf("existing version overwrite: %v", err) + } + if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) { + t.Error("rejected completion changed the target") + } + }) + } +} + +// Hold the download after the restore snapshot has been read. Object Lock +// updates must serialize with both successful rehydration and failure cleanup. +type nativeRestoreBlockedTier struct { + WarmBackend + entered chan struct{} + release chan struct{} + once sync.Once + failure error + afterFirstPart bool +} + +func (b *nativeRestoreBlockedTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) { + if b.afterFirstPart { + r, err := b.WarmBackend.Get(ctx, object, version, opts) + if err != nil { + return nil, err + } + return &nativeRestorePartGateReader{ReadCloser: r, gate: b, ctx: ctx, remaining: 5 << 20}, nil + } + b.once.Do(func() { close(b.entered) }) + select { + case <-b.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + if b.failure != nil { + return nil, b.failure + } + return b.WarmBackend.Get(ctx, object, version, opts) +} + +func nativeRestoreLockRequest(t *testing.T, ctx context.Context, r *restoreTest, id, kind string, body []byte) *httptest.ResponseRecorder { + t.Helper() + sum := md5.Sum(body) + req, err := newTestSignedRequestV4(http.MethodPut, "/"+r.bucket+"/"+r.object+"?"+kind+"&versionId="+url.QueryEscape(id), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])}) + if err != nil { + t.Fatal(err) + } + if err := req.ParseForm(); err != nil { + t.Fatal(err) + } + req = mux.SetURLVars(req.WithContext(ctx), map[string]string{"bucket": r.bucket, "object": r.object}) + rec := httptest.NewRecorder() + api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return r.z }} + if kind == "legal-hold" { + api.PutObjectLegalHoldHandler(rec, req) + } else { + api.PutObjectRetentionHandler(rec, req) + } + return rec +} + +func TestNativeRestoreSerializesObjectLock(t *testing.T) { + for _, kind := range []string{"legal-hold", "retention"} { + for _, fail := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/failure=%v", kind, fail), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + enableBucketObjectLock(t, r.bucket) + oi, _ := nativeRestoreSeed(t, r, 0, false, true) + initial := []byte(`OFF`) + updated := []byte(`ON`) + keys := []string{strings.ToLower(xhttp.AmzObjectLockLegalHold), ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp} + if kind == "retention" { + initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour)) + updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour)) + keys = []string{strings.ToLower(xhttp.AmzObjectLockMode), strings.ToLower(xhttp.AmzObjectLockRetainUntilDate), ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp} + } + if rec := nativeRestoreLockRequest(t, t.Context(), r, oi.VersionID, kind, initial); rec.Code != http.StatusOK { + t.Fatalf("initial protection: %d %s", rec.Code, rec.Body.String()) + } + before, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil { + t.Fatal(err) + } + blocked := &nativeRestoreBlockedTier{WarmBackend: globalTierConfigMgr.drivercache["WARM"], entered: make(chan struct{}), release: make(chan struct{})} + if fail { + blocked.failure = errors.New("injected download failure after snapshot") + } + globalTierConfigMgr.drivercache["WARM"] = blocked + var release sync.Once + unblock := func() { release.Do(func() { close(blocked.release) }) } + defer unblock() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { + result <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + select { + case <-blocked.entered: + case <-ctx.Done(): + t.Fatal("restore never started download") + } + updateCtx, updateCancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + rec := nativeRestoreLockRequest(t, updateCtx, r, oi.VersionID, kind, updated) + updateCancel() + if rec.Code == http.StatusOK { + t.Error("Object Lock update committed during restore's metadata snapshot") + // On the unfixed path this update commits, and the restore + // must not subsequently erase the acknowledged protection. + readOpts := r.opts(oi.VersionID) + readOpts.NoLock = true + before, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, readOpts) + if err != nil { + t.Fatal(err) + } + } + if rec.Code != http.StatusOK && (rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "RequestTimeout")) { + t.Errorf("update failed for an unrelated reason: %d %s", rec.Code, rec.Body.String()) + } + unblock() + select { + case err := <-result: + if !errors.Is(err, blocked.failure) { + t.Fatalf("restore returned %v, want %v", err, blocked.failure) + } + case <-ctx.Done(): + t.Fatal("restore failed to finish") + } + after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil { + t.Fatal(err) + } + for _, key := range keys { + if before.UserDefined[key] == "" || after.UserDefined[key] != before.UserDefined[key] { + t.Errorf("restore changed protection %s: %q -> %q", key, before.UserDefined[key], after.UserDefined[key]) + } + } + if rec := nativeRestoreLockRequest(t, t.Context(), r, oi.VersionID, kind, updated); rec.Code != http.StatusOK { + t.Fatalf("protection update after restore: %d %s", rec.Code, rec.Body.String()) + } + final, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil { + t.Fatal(err) + } + if !final.ModTime.Equal(before.ModTime) || final.ETag != before.ETag || final.Size != before.Size { + t.Error("Object Lock update changed content identity") + } + changed := keys[0] + if kind == "retention" { + changed = keys[1] + } + if final.UserDefined[changed] == before.UserDefined[changed] || final.UserDefined[keys[len(keys)-1]] == before.UserDefined[keys[len(keys)-1]] { + t.Error("new protection or its timestamp was not stored") + } + }) + } + } +} + +func TestNativeRestoreConcurrentCompletion(t *testing.T) { + for _, separateSets := range []bool{false, true} { + t.Run(fmt.Sprintf("separate-set-locks=%v", separateSets), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + if separateSets { + // Local production sets share one map. Independent maps model the + // distinct locker groups of distributed sets while reusing test disks. + pool := r.z.serverPools[0] + sets, count := pool.sets, pool.setCount + t.Cleanup(func() { pool.sets, pool.setCount = sets, count }) + first, second := *sets[0], *sets[0] + first.nsMutex, second.nsMutex = newNSLock(false), newNSLock(false) + second.setIndex = 1 + pool.sets, pool.setCount = []*erasureObjects{&first, &second}, 2 + for i := 0; ; i++ { + r.object = fmt.Sprintf("second-set-%d", i) + if pool.getHashedSetIndex(r.object) == 1 { + break + } + } + } + oi, body := nativeRestoreSeed(t, r, 0, false, true) + blocked := &nativeRestoreBlockedTier{WarmBackend: globalTierConfigMgr.drivercache["WARM"], entered: make(chan struct{}), release: make(chan struct{}), afterFirstPart: true} + globalTierConfigMgr.drivercache["WARM"] = blocked + var once sync.Once + unblock := func() { once.Do(func() { close(blocked.release) }) } + defer unblock() + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + restored := make(chan error, 1) + go func() { + restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + select { + case <-blocked.entered: + case <-ctx.Done(): + t.Fatal("restore never started") + } + // A client's exact-prefix listing can discover the internal upload. It can + // try to complete the first part while native restore owns the object. + uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10) + if err != nil || len(uploads.Uploads) != 1 { + t.Fatalf("restore upload discovery: %+v, %v", uploads, err) + } + id := uploads.Uploads[0].UploadID + parts, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{}) + if err != nil || len(parts.Parts) != 1 { + t.Fatalf("restore must have published its first part: %+v, %v", parts, err) + } + part := parts.Parts[0] + completed := make(chan struct{}) + var completeErr error + go func() { + _, completeErr = r.z.CompleteMultipartUpload(ctx, r.bucket, r.object, id, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, r.opts(oi.VersionID)) + close(completed) + }() + waitForListingReadLock(t, r.z.serverPools[0].getHashedSet(r.object).nsMutex, r.bucket, r.object, completed) + unblock() + select { + case err := <-restored: + if err != nil { + t.Errorf("restore blocked by concurrent completion: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("restore did not finish") + } + select { + case <-completed: + case <-time.After(5 * time.Second): + t.Fatal("completion did not finish") + } + if !errors.As(completeErr, &InvalidUploadID{}) { + t.Errorf("completion after native restore: %v, want InvalidUploadID", completeErr) + } + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents after concurrent completion: %v", err) + } + }) + } +} + +type nativeRestorePartGateReader struct { + io.ReadCloser + gate *nativeRestoreBlockedTier + ctx context.Context + remaining int +} + +func (r *nativeRestorePartGateReader) Read(p []byte) (int, error) { + if r.remaining > 0 { + if len(p) > r.remaining { + p = p[:r.remaining] + } + n, err := r.ReadCloser.Read(p) + r.remaining -= n + return n, err + } + r.gate.once.Do(func() { close(r.gate.entered) }) + select { + case <-r.gate.release: + case <-r.ctx.Done(): + return 0, r.ctx.Err() + } + return r.ReadCloser.Read(p) +} From 3c8699e2167d1a2bffdc95d132e594e9cc4aefa2 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 21:20:24 +0800 Subject: [PATCH 08/26] ci: race-check lifecycle expiry and native restore together Signed-off-by: Feng Ruohang --- .github/workflows/go.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2bf27c930..7e26840f0 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -96,6 +96,11 @@ jobs: - name: Run multi-pool delete consistency tests under race detector run: go test -race ./cmd -run '^Test(Pools(Delete|BulkDelete|RetiringCopyPreservesSharedTierObject)|BulkDeleteQuorumEvidence)' -count=1 -timeout=5m + - name: Run lifecycle expiry and native restore tests under race detector + env: + MINIO_API_REQUESTS_MAX: "10000" + run: go test -race -tags kqueue,dev ./cmd -run '^Test(PoolsLifecycle|PoolsMigrationLifecycle|ExpireRestored|NativeRestore)' -count=1 -timeout=5m + - name: Run multipart listing, cancellation and cleanup tests under race detector env: MINIO_API_REQUESTS_MAX: "10000" From 5ac27f75336ad7ff8db8a4266daad279d3081bf9 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 21:21:46 +0800 Subject: [PATCH 09/26] test(restore): follow context-first helper convention Signed-off-by: Feng Ruohang --- cmd/restore-native_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go index d9e6ad8f1..577607ced 100644 --- a/cmd/restore-native_test.go +++ b/cmd/restore-native_test.go @@ -481,7 +481,7 @@ func (b *nativeRestoreBlockedTier) Get(ctx context.Context, object string, versi return b.WarmBackend.Get(ctx, object, version, opts) } -func nativeRestoreLockRequest(t *testing.T, ctx context.Context, r *restoreTest, id, kind string, body []byte) *httptest.ResponseRecorder { +func nativeRestoreLockRequest(ctx context.Context, t *testing.T, r *restoreTest, id, kind string, body []byte) *httptest.ResponseRecorder { t.Helper() sum := md5.Sum(body) req, err := newTestSignedRequestV4(http.MethodPut, "/"+r.bucket+"/"+r.object+"?"+kind+"&versionId="+url.QueryEscape(id), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])}) @@ -518,7 +518,7 @@ func TestNativeRestoreSerializesObjectLock(t *testing.T) { updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour)) keys = []string{strings.ToLower(xhttp.AmzObjectLockMode), strings.ToLower(xhttp.AmzObjectLockRetainUntilDate), ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp} } - if rec := nativeRestoreLockRequest(t, t.Context(), r, oi.VersionID, kind, initial); rec.Code != http.StatusOK { + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK { t.Fatalf("initial protection: %d %s", rec.Code, rec.Body.String()) } before, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) @@ -545,7 +545,7 @@ func TestNativeRestoreSerializesObjectLock(t *testing.T) { t.Fatal("restore never started download") } updateCtx, updateCancel := context.WithTimeout(t.Context(), 200*time.Millisecond) - rec := nativeRestoreLockRequest(t, updateCtx, r, oi.VersionID, kind, updated) + rec := nativeRestoreLockRequest(updateCtx, t, r, oi.VersionID, kind, updated) updateCancel() if rec.Code == http.StatusOK { t.Error("Object Lock update committed during restore's metadata snapshot") @@ -579,7 +579,7 @@ func TestNativeRestoreSerializesObjectLock(t *testing.T) { t.Errorf("restore changed protection %s: %q -> %q", key, before.UserDefined[key], after.UserDefined[key]) } } - if rec := nativeRestoreLockRequest(t, t.Context(), r, oi.VersionID, kind, updated); rec.Code != http.StatusOK { + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK { t.Fatalf("protection update after restore: %d %s", rec.Code, rec.Body.String()) } final, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) From 8ae19de7a85126c150d5cceb83cbee5823d8240c Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 21:39:36 +0800 Subject: [PATCH 10/26] fix(restore): avoid duplicate completion locks for empty upload IDs Keep the storage layer error precedence for missing buckets and uploads while retaining every real upload lock. Cover the alias with short deadlines and race-check the original complete-multipart suite in CI. Signed-off-by: Feng Ruohang --- .github/workflows/go.yml | 2 +- cmd/erasure-server-pool.go | 16 ++++++++++------ cmd/restore-native_test.go | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2bf27c930..44afacf74 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -99,7 +99,7 @@ jobs: - name: Run multipart listing, cancellation and cleanup tests under race detector env: MINIO_API_REQUESTS_MAX: "10000" - run: go test -race ./cmd -run '^Test(Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m + run: go test -race ./cmd -run '^Test(ObjectCompleteMultipartUpload|Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m - name: Run listing metadata recovery, pagination and cancellation tests under race detector run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity|ListObjectVersionsCrossParityNull|MigrationCrossParity)' -count=1 -timeout=5m diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 23661c413..db2973717 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -2391,13 +2391,17 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket // Hold write locks to verify uploaded parts, also disallows any // parallel PutObjectPart() requests. - uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID)) - wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout) - if err != nil { - return objInfo, err + // An empty upload ID aliases the object lock already held above. + // Let the storage layer return its normal missing bucket/upload error. + if uploadID != "" { + uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID)) + wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + return objInfo, err + } + ctx = wlkctx.Context() + defer uploadIDLock.Unlock(wlkctx) } - ctx = wlkctx.Context() - defer uploadIDLock.Unlock(wlkctx) if z.SinglePool() { return z.serverPools[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts) diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go index 577607ced..706dbc8a4 100644 --- a/cmd/restore-native_test.go +++ b/cmd/restore-native_test.go @@ -707,3 +707,35 @@ func (r *nativeRestorePartGateReader) Read(p []byte) (int, error) { } return r.ReadCloser.Read(p) } + +// Invalid upload paths must retain the backend's normal error precedence, +// rather than reacquiring the object lock as the upload lock. +func TestMultipartCompletionLockAlias(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, id := range []string{"", ".", "./"} { + t.Run(fmt.Sprintf("pools=%d/id=%q", pools, id), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + for _, missingBucket := range []bool{false, true} { + bucket := r.bucket + if missingBucket { + bucket = "missing-bucket" + } + ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + _, err := r.z.CompleteMultipartUpload(ctx, bucket, r.object, id, nil, ObjectOptions{}) + cancel() + if id != "" { + if !errors.As(err, &MalformedUploadID{}) { + t.Errorf("malformed upload: %v", err) + } + } else if missingBucket { + if !isErrBucketNotFound(err) { + t.Errorf("missing bucket: %v", err) + } + } else if !errors.As(err, &InvalidUploadID{}) { + t.Errorf("invalid upload: %v", err) + } + } + }) + } + } +} From 50a234d6c339f495907ef471c16cab59e62d514d Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 21:40:59 +0800 Subject: [PATCH 11/26] test(restore): clarify invalid-upload error cases Signed-off-by: Feng Ruohang --- cmd/restore-native_test.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go index 706dbc8a4..dfe7fe6f4 100644 --- a/cmd/restore-native_test.go +++ b/cmd/restore-native_test.go @@ -723,16 +723,19 @@ func TestMultipartCompletionLockAlias(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) _, err := r.z.CompleteMultipartUpload(ctx, bucket, r.object, id, nil, ObjectOptions{}) cancel() - if id != "" { + switch { + case id != "": if !errors.As(err, &MalformedUploadID{}) { t.Errorf("malformed upload: %v", err) } - } else if missingBucket { + case missingBucket: if !isErrBucketNotFound(err) { t.Errorf("missing bucket: %v", err) } - } else if !errors.As(err, &InvalidUploadID{}) { - t.Errorf("invalid upload: %v", err) + default: + if !errors.As(err, &InvalidUploadID{}) { + t.Errorf("invalid upload: %v", err) + } } } }) From 4f72d0ea9493d91b140d7683c6e11989b7e23d7c Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 22:16:47 +0800 Subject: [PATCH 12/26] fix(restore): preserve encoded object identity and part layout Signed-off-by: Feng Ruohang --- cmd/erasure-object.go | 20 ++- cmd/restore-native-encoding_test.go | 195 ++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 4 deletions(-) create mode 100644 cmd/restore-native-encoding_test.go diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 0451d1c08..b8a610b7b 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2687,14 +2687,23 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi) ropts.NoLock = opts.NoLock - if len(oi.Parts) == 1 { + // Rehydration copies encoded bytes without changing their content identity. + ropts.PreserveETag = oi.ETag + // Even a one-part multipart upload must retain its original part number: + // encrypted multipart data uses that number to derive its decryption key. + if len(oi.Parts) == 1 && !oi.isMultipart() { + actualSize, err := oi.GetActualSize() + if err != nil { + return setRestoreHeaderFn(oi, err) + } + ropts.IndexCB = func() []byte { return oi.Parts[0].Index } var rs *HTTPRangeSpec gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts) if err != nil { return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) } defer gr.Close() - hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size) + hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", actualSize) if err != nil { return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) } @@ -2719,11 +2728,14 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s // rehydrate the parts back on disk as per the original xl.meta prior to transition for _, partInfo := range oi.Parts { - hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size) + // Stored and logical lengths differ for compressed/encrypted data. + hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.ActualSize) if err != nil { return setRestoreHeaderFn(oi, err) } - pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{}) + pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{ + IndexCB: func() []byte { return partInfo.Index }, + }) if err != nil { return setRestoreHeaderFn(oi, err) } diff --git a/cmd/restore-native-encoding_test.go b/cmd/restore-native-encoding_test.go new file mode 100644 index 000000000..1ee83a927 --- /dev/null +++ b/cmd/restore-native-encoding_test.go @@ -0,0 +1,195 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/pubsub" + "github.com/minio/mux" +) + +// Exercise the stored representation produced by real S3 uploads. Native +// restore copies encoded bytes, but must retain logical sizes and identity. +func TestNativeRestoreEncodedObjects(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, encoding := range []string{"plain", "compressed", "ssec", "s3-compressed"} { + for _, layout := range []string{"single", "multipart", "sparse-one-part"} { + t.Run(fmt.Sprintf("pools=%d/%s/%s", pools, encoding, layout), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + r.object = "restore-encoded.txt" + previousKMS, previousTLS := GlobalKMS, globalIsTLS + t.Cleanup(func() { GlobalKMS, globalIsTLS = previousKMS, previousTLS }) + globalIsTLS = true + headers := make(map[string]string) + readHeaders := make(map[string]string) + if encoding == "compressed" || encoding == "s3-compressed" { + enableAttributesPartsCompression(t) + } + if encoding == "ssec" { + headers = ssecTestHeaders(0x53) + readHeaders = ssecTestHeaders(0x53) + } + if encoding == "s3-compressed" { + var err error + GlobalKMS, err = kms.NewBuiltin("restore-test-key", bytes.Repeat([]byte{0x59}, 32)) + if err != nil { + t.Fatal(err) + } + headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES + } + previousObjectAPI := newObjectLayerFn() + t.Cleanup(func() { setObjectLayer(previousObjectAPI) }) + router := initTestAPIEndPoints(r.z, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "PutObject", "GetObject"}) + bodies := [][]byte{bytes.Repeat([]byte("encoded-restore-"), compMinIndexSize/16+1)} + partNumbers := []int{1} + switch layout { + case "multipart": + bodies = append(bodies, []byte("last encoded part")) + partNumbers = []int{1, 2} + case "sparse-one-part": + partNumbers = []int{5} + } + want := bytes.Join(bodies, nil) + if layout == "single" { + headers[xhttp.AmzChecksumCRC32] = mustChecksum(t, hash.ChecksumCRC32, want) + putCopyChecksumSource(t, router, globalActiveCred, r.bucket, r.object, want, headers) + } else { + attributesPartsUpload(t, router, globalActiveCred, r.bucket, r.object, headers, bodies, partNumbers) + } + before, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if before.IsCompressed() != (encoding == "compressed" || encoding == "s3-compressed") { + t.Fatalf("unexpected compression state: %v", before.IsCompressed()) + } + if before.IsCompressed() && len(before.Parts[0].Index) == 0 { + t.Fatal("fixture must have a compression index") + } + if layout == "single" && len(before.Checksum) == 0 { + t.Fatal("fixture must have a stored checksum") + } + objectURL := getGetObjectURL("", r.bucket, r.object) + "?versionId=" + url.QueryEscape(before.VersionID) + beforeGet := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, objectURL, nil, readHeaders) + if beforeGet.Code != http.StatusOK || !bytes.Equal(beforeGet.Body.Bytes(), want) { + t.Fatalf("fixture GET: %d bytes=%d want=%d", beforeGet.Code, beforeGet.Body.Len(), len(want)) + } + beforeRaw := nativeRestoreSnapshot(t, r) + topts := r.opts(before.VersionID) + topts.MTime = before.ModTime + topts.Transition = TransitionOptions{Tier: "WARM", ETag: before.ETag} + if err := r.z.TransitionObject(t.Context(), r.bucket, r.object, topts); err != nil { + t.Fatal(err) + } + observed := nativeRestoreHTTPObserver{r.z, make(chan nativeRestoreHTTPResult, 1)} + api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return observed }} + events, done := make(chan event.Event, 1), make(chan struct{}) + defer close(done) + if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRestoreCompleted), events, done, nil); err != nil { + t.Fatal(err) + } + restoreXML := []byte(`1`) + req, err := newTestSignedRequestV4(http.MethodPost, objectURL+"&restore", int64(len(restoreXML)), bytes.NewReader(restoreXML), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + if err := req.ParseForm(); err != nil { + t.Fatal(err) + } + req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object}) + rec := httptest.NewRecorder() + api.PostRestoreObjectHandler(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("restore POST: %d %s", rec.Code, rec.Body.String()) + } + select { + case result := <-observed.done: + if result.err != nil { + t.Fatalf("background restore: %v", result.err) + } + case <-time.After(5 * time.Second): + t.Fatal("restore did not finish") + } + select { + case <-events: + case <-time.After(5 * time.Second): + t.Fatal("restore completion event missing") + } + after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(before.VersionID)) + if err != nil { + t.Fatal(err) + } + if after.ETag != before.ETag || after.Size != before.Size || !after.ModTime.Equal(before.ModTime) { + t.Error("restore changed content identity") + } + afterRaw := nativeRestoreSnapshot(t, r) + for pool := range beforeRaw { + if len(afterRaw[pool]) != len(beforeRaw[pool]) { + t.Fatalf("pool %d version count changed", pool) + } + for i, fi := range beforeRaw[pool] { + if !bytes.Equal(fi.Checksum, afterRaw[pool][i].Checksum) { + t.Error("restore lost stored checksum") + } + } + } + if len(after.Parts) != len(before.Parts) { + t.Fatal("restore changed part count") + } + for i, part := range after.Parts { + old := before.Parts[i] + if part.Number != old.Number || part.Size != old.Size || part.ActualSize != old.ActualSize || !bytes.Equal(part.Index, old.Index) { + t.Errorf("part %d layout or compression index changed", i) + } + } + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + reads := tier.gets.Load() + got := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, objectURL, nil, readHeaders) + if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), want) { + t.Errorf("restored GET: %d bytes=%d want=%d", got.Code, got.Body.Len(), len(want)) + } + if got.Header().Get(xhttp.ETag) != beforeGet.Header().Get(xhttp.ETag) { + t.Error("restore changed client ETag") + } + for i, number := range partNumbers { + part := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, objectURL+fmt.Sprintf("&partNumber=%d", number), nil, readHeaders) + if part.Code != http.StatusPartialContent || !bytes.Equal(part.Body.Bytes(), bodies[i]) { + t.Errorf("restored part %d: %d bytes=%d want=%d", number, part.Code, part.Body.Len(), len(bodies[i])) + } + } + if tier.gets.Load() != reads { + t.Error("restored GET still used remote tier") + } + }) + } + } + } +} From 85794665578d91ad12c5660b439d86837873df50 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 22:32:33 +0800 Subject: [PATCH 13/26] fix(restore): preserve versions across cleanup and failed reads Reject replacement markers and legacy objects before restored-copy cleanup can mutate their type or data. Prevent restore lookups and failure cleanup from purging dangling versions, preserve Transition on metadata rereads, and return without writing when the initial restore read fails. Add real-disk regressions for stale null tasks, minority metadata, initial and cleanup read failures, and stored marker/legacy types. Keep ordinary dangling cleanup and marker purge as positive controls. Signed-off-by: Feng Ruohang --- cmd/bucket-lifecycle-restore-boundary_test.go | 278 ++++++++++++++++++ cmd/erasure-object.go | 17 +- cmd/xl-storage-format-v2.go | 32 +- cmd/xl-storage.go | 3 + 4 files changed, 318 insertions(+), 12 deletions(-) create mode 100644 cmd/bucket-lifecycle-restore-boundary_test.go diff --git a/cmd/bucket-lifecycle-restore-boundary_test.go b/cmd/bucket-lifecycle-restore-boundary_test.go new file mode 100644 index 000000000..14967cf6a --- /dev/null +++ b/cmd/bucket-lifecycle-restore-boundary_test.go @@ -0,0 +1,278 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/versioning" +) + +// Read the actual metadata from every disk without object-layer read repair. +func restoreBoundarySnapshot(t *testing.T, r *restoreTest) [][]byte { + t.Helper() + var result [][]byte + for _, pool := range r.z.serverPools { + for _, disk := range pool.getHashedSet(r.object).getDisks() { + raw, err := disk.ReadXL(t.Context(), r.bucket, r.object, true) + if err != nil && err != errFileNotFound { + t.Fatal(err) + } + result = append(result, bytes.Clone(raw.Buf)) + } + } + return result +} + +func TestExpireRestoredKeepsReplacementDeleteMarker(t *testing.T) { + for _, pools := range []int{1, 2} { + t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Suspended) + r.put("old", false, false) + oi := r.put("null", true, true) + r.restore(oi) + stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(nullVersionID)) + if err != nil { + t.Fatal(err) + } + if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, r.opts("")); err != nil { + t.Fatal(err) + } + if !r.fileInfo(nullVersionID).Deleted { + t.Fatal("ordinary DELETE did not replace null with a delete marker") + } + before := restoreBoundarySnapshot(t, r) + err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner) + if !isErrVersionNotFound(err) { + t.Errorf("stale cleanup should reject the replacement marker: %v", err) + } + if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) { + t.Error("stale cleanup changed stored versions or their data") + } + if _, _, err := r.read(""); !isErrObjectNotFound(err) { + t.Errorf("stale cleanup exposed the old version: %v", err) + } + // An explicit user purge must still remove that same marker. + if _, err := r.z.DeleteObject(t.Context(), r.bucket, r.object, r.opts(nullVersionID)); err != nil { + t.Fatal(err) + } + if label, data, err := r.read(""); err != nil || label != "old" || !bytes.Equal(data, r.bodies["old"]) { + t.Fatalf("ordinary marker purge: label=%s err=%v", label, err) + } + }) + } +} + +func TestExpireRestoredMetadataReadKeepsMinorityVersion(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, operation := range []string{"cleanup", "restore", "restore-cleanup", "lookup-cleanup", "lookup-restore", "get", "delete"} { + t.Run(fmt.Sprintf("pools=%d/%s", pools, operation), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + oi := r.put("target", false, true) + r.restore(oi) + fi := r.fileInfo(oi.VersionID) + // A readable latest version lets pools lookup finish before the + // set reads the specifically addressed, incomplete old version. + r.put("newer", false, false) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + for _, disk := range disks[1:] { + raw, err := disk.ReadXL(t.Context(), r.bucket, r.object, true) + if err != nil { + t.Fatal(err) + } + var meta xlMetaV2 + if err := meta.Load(raw.Buf); err != nil { + t.Fatal(err) + } + del := FileInfo{VersionID: oi.VersionID} + del.SetSkipTierFreeVersion() + if _, err := meta.DeleteVersion(del); err != nil { + t.Fatal(err) + } + buf, err := meta.AppendTo(nil) + if err != nil { + t.Fatal(err) + } + // Remove only this metadata entry; retain every real shard. + if err := disk.WriteAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile), buf); err != nil { + t.Fatal(err) + } + } + before := restoreBoundarySnapshot(t, r) + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + opts := r.opts(oi.VersionID) + var err error + switch operation { + case "cleanup": + err = expireTransitionedObject(ctx, r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredVersionAction}, lcEventSrc_Scanner) + case "restore": + opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1} + err = r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, opts) + case "restore-cleanup": + // The initial read succeeded before remote download failed; + // cleanup must not purge a now-incomplete target on reread. + opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1} + err = er.updateRestoreMetadata(ctx, r.bucket, r.object, fi.ToObjectInfo(r.bucket, r.object, true), opts) + case "lookup-cleanup", "lookup-restore": + // Version-sensitive lookup also covers native restore routing. + opts.MetadataChg = true + if operation == "lookup-cleanup" { + opts.Transition.ExpireRestored = true + } else { + opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1} + } + _, _, err = r.z.getPoolInfoExistingWithOpts(ctx, r.bucket, r.object, opts) + case "get": + _, err = r.z.GetObjectInfo(ctx, r.bucket, r.object, opts) + case "delete": + _, err = r.z.DeleteObject(ctx, r.bucket, r.object, opts) + } + if ctx.Err() != nil { + t.Fatalf("%s timed out instead of returning its read error: %v", operation, err) + } + if err == nil { + t.Fatal("incomplete target unexpectedly readable") + } + if operation != "get" && operation != "delete" { + if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) { + t.Error("failed restore read changed metadata, inline data or free versions") + } + if n := r.dataDirs(fi.DataDir); n != len(disks) { + t.Errorf("failed restore read removed shards: %d/%d directories remain", n, len(disks)) + } + return + } + // Ordinary GET/DELETE retain their existing dangling cleanup. + raw, err := disks[0].ReadXL(t.Context(), r.bucket, r.object, false) + if err != nil { + t.Fatal(err) + } + versions, err := getFileInfoVersions(raw.Buf, r.bucket, r.object, false) + if err != nil { + t.Fatal(err) + } + if len(versions.Versions) != 1 || len(versions.FreeVersions) != 1 || r.dataDirs(fi.DataDir) != len(disks)-1 { + t.Fatal("ordinary dangling cleanup no longer removed the minority version") + } + }) + } + } +} + +func TestNativeRestoreInitialMissingReadDoesNotWrite(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + r.put("live", false, false) + before := restoreBoundarySnapshot(t, r) + opts := r.opts(mustGetUUID()) + opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1} + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, opts); !isErrVersionNotFound(err) { + t.Fatalf("missing restore target: %v", err) + } + if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) { + t.Fatal("initial restore failure wrote metadata without a resolved version") + } +} + +func TestExpireRestoredStorageRejectsOtherTypes(t *testing.T) { + for _, kind := range []string{"null-marker", "uuid-marker", "legacy", "legacy-json"} { + t.Run(kind, func(t *testing.T) { + disk, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatal(err) + } + ctx := t.Context() + const bucket, object = "restore-types", "obj" + if err := disk.MakeVol(ctx, bucket); err != nil { + t.Fatal(err) + } + var meta xlMetaV2 + versionID, metaName, dataDir := nullVersionID, xlStorageFormatFile, "" + var buf []byte + switch kind { + case "null-marker", "uuid-marker": + if kind == "uuid-marker" { + versionID = mustGetUUID() + } + err = meta.AddVersion(FileInfo{VersionID: versionID, Deleted: true, ModTime: time.Now().UTC()}) + case "legacy", "legacy-json": + legacy := getSampleXLMeta(1) + legacy.DataDir, dataDir = "legacy", "legacy" + if kind == "legacy-json" { + metaName = xlStorageFormatFileV1 + buf, err = json.Marshal(legacy) + disk.storage.Lock() + disk.storage.formatLegacy = true + disk.storage.Unlock() + } else { + err = meta.AddLegacy(&legacy) + } + } + if err != nil { + t.Fatal(err) + } + if buf == nil { + buf, err = meta.AppendTo(nil) + if err != nil { + t.Fatal(err) + } + } + if err := disk.WriteAll(ctx, bucket, pathJoin(object, metaName), buf); err != nil { + t.Fatal(err) + } + payload := []byte("sole legacy contents") + dataPath := filepath.Join(disk.Endpoint().Path, bucket, object, dataDir, "part.1") + if dataDir != "" { + if err := os.MkdirAll(filepath.Dir(dataPath), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dataPath, payload, 0o600); err != nil { + t.Fatal(err) + } + } + if err := disk.DeleteVersion(ctx, bucket, object, FileInfo{VersionID: versionID, ExpireRestored: true, TransitionStatus: lifecycle.TransitionComplete}, false, DeleteOptions{}); err != nil { + t.Fatal(err) + } + after, err := disk.ReadAll(ctx, bucket, pathJoin(object, metaName)) + if err != nil || !bytes.Equal(buf, after) { + t.Errorf("restore cleanup changed %s metadata: %v", kind, err) + } + if dataDir != "" { + if data, err := os.ReadFile(dataPath); err != nil || !bytes.Equal(data, payload) { + t.Errorf("restore cleanup removed legacy contents: %v", err) + } + } + if err := disk.DeleteVersion(ctx, bucket, object, FileInfo{VersionID: versionID}, false, DeleteOptions{}); err != nil { + t.Fatal(err) + } + if _, err := disk.ReadAll(ctx, bucket, pathJoin(object, metaName)); err != errFileNotFound { + t.Errorf("ordinary purge left %s: %v", kind, err) + } + }) + } +} diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 56737e962..76c6c176b 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -509,6 +509,11 @@ func joinErrs(errs []error) string { } func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int, opts ObjectOptions) (FileInfo, error) { + // Restore and restored-copy cleanup must preserve the version on read + // failure, including lookups and failure cleanup that reread metadata. + if opts.Transition.ExpireRestored || opts.Transition.RestoreRequest != nil { + return FileInfo{}, errErasureReadQuorum + } m, ok := isObjectDangling(metaArr, errs, dataErrsByPart) if !ok { // We only come here if we cannot figure out if the object @@ -2136,6 +2141,12 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string } } + // A queued null restore may now address a replacement delete marker. + // Such errors carry ObjectInfo, but never authorize cache cleanup. + if opts.Transition.ExpireRestored && (gerr != nil || goi.DeleteMarker) { + return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID} + } + if opts.EvalMetadataFn != nil { dsc, err := opts.EvalMetadataFn(&goi, gerr) if err != nil { @@ -2641,7 +2652,8 @@ func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, obje delete(oi.UserDefined, xhttp.AmzRestore) if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{ - VersionID: oi.VersionID, + VersionID: oi.VersionID, + Transition: opts.Transition, }, ObjectOptions{ VersionID: oi.VersionID, }); err != nil { @@ -2665,7 +2677,8 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s // get the file info on disk for transitioned object actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false) if err != nil { - return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + // No version was read, so there is no restore metadata to update. + return toObjectErr(err, bucket, object) } oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 69465c9ab..5e4f56d65 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -1372,6 +1372,26 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) { } } + if fi.ExpireRestored { + i, ver, err := x.findVersion(uv) + if err != nil { + return "", err + } + // Check the stored type and transition before any marker, legacy or + // replication mutation. Only completed tiered objects own a cache. + if ver.Type != ObjectType || string(ver.ObjectV2.MetaSys[metaTierStatus]) != lifecycle.TransitionComplete { + return "", nil + } + ver.ObjectV2.RemoveRestoreHdrs() + if err := x.setIdx(i, *ver); err != nil { + return "", err + } + if x.SharedDataDirCount(ver.ObjectV2.VersionID, ver.ObjectV2.DataDir) > 0 { + return "", nil + } + return uuid.UUID(ver.ObjectV2.DataDir).String(), nil + } + var ventry xlMetaV2Version if fi.Deleted { ventry = xlMetaV2Version{ @@ -1496,16 +1516,8 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) { if err != nil { return "", err } - switch { - case fi.ExpireRestored: - // Only a completed transition has a remote copy. Trust the selected - // stored version, not the transition status supplied by the caller. - if string(ver.ObjectV2.MetaSys[metaTierStatus]) != lifecycle.TransitionComplete { - return "", nil - } - ver.ObjectV2.RemoveRestoreHdrs() - err = x.setIdx(i, *ver) - case fi.TransitionStatus == lifecycle.TransitionComplete: + switch fi.TransitionStatus { + case lifecycle.TransitionComplete: ver.ObjectV2.SetTransition(fi) ver.ObjectV2.ResetInlineData() err = x.setIdx(i, *ver) diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 81e415553..0758396b6 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -1371,6 +1371,9 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F } if legacyJSON { + if fi.ExpireRestored { + return nil // Legacy objects cannot contain a completed transition. + } // Delete the meta file, if there are no more versions the // top level parent is automatically removed. return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true, false) From d9837fb319868690d20fca18e114ab25b76c0a5b Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 22:32:55 +0800 Subject: [PATCH 14/26] fix(restore): retain legacy part-size compatibility Signed-off-by: Feng Ruohang --- cmd/erasure-object.go | 10 ++++-- cmd/restore-native-encoding_test.go | 55 ++++++++++++++++++++++++++++- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index b8a610b7b..f3cd7e562 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2728,8 +2728,14 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s // rehydrate the parts back on disk as per the original xl.meta prior to transition for _, partInfo := range oi.Parts { - // Stored and logical lengths differ for compressed/encrypted data. - hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.ActualSize) + // Old uncompressed parts may not record ActualSize. Encrypted parts + // derive it from the written ciphertext in PutObjectPart; compressed + // parts need their recorded logical length, including valid zero sizes. + actualSize := partInfo.Size + if oi.IsCompressed() { + actualSize = partInfo.ActualSize + } + hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", actualSize) if err != nil { return setRestoreHeaderFn(oi, err) } diff --git a/cmd/restore-native-encoding_test.go b/cmd/restore-native-encoding_test.go index 1ee83a927..6a4fa3ba9 100644 --- a/cmd/restore-native-encoding_test.go +++ b/cmd/restore-native-encoding_test.go @@ -19,6 +19,7 @@ package cmd import ( "bytes" + "context" "fmt" "net/http" "net/http/httptest" @@ -102,6 +103,10 @@ func TestNativeRestoreEncodedObjects(t *testing.T) { if beforeGet.Code != http.StatusOK || !bytes.Equal(beforeGet.Body.Bytes(), want) { t.Fatalf("fixture GET: %d bytes=%d want=%d", beforeGet.Code, beforeGet.Body.Len(), len(want)) } + beforeETags := beforeGet.Header()[xhttp.ETag] + if len(beforeETags) != 1 || beforeETags[0] == "" { + t.Fatal("fixture GET must expose a nonempty ETag") + } beforeRaw := nativeRestoreSnapshot(t, r) topts := r.opts(before.VersionID) topts.MTime = before.ModTime @@ -176,7 +181,8 @@ func TestNativeRestoreEncodedObjects(t *testing.T) { if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), want) { t.Errorf("restored GET: %d bytes=%d want=%d", got.Code, got.Body.Len(), len(want)) } - if got.Header().Get(xhttp.ETag) != beforeGet.Header().Get(xhttp.ETag) { + afterETags := got.Header()[xhttp.ETag] + if len(afterETags) != 1 || afterETags[0] == "" || afterETags[0] != beforeETags[0] { t.Error("restore changed client ETag") } for i, number := range partNumbers { @@ -193,3 +199,50 @@ func TestNativeRestoreEncodedObjects(t *testing.T) { } } } + +// XL metadata predating compression has no ActualSize. Conversion and healing +// preserve those zero values; a plain multipart restore must derive the size. +func TestNativeRestoreLegacyPartSizes(t *testing.T) { + r := newRestoreSafetyTest(t, 1, "") + oi, body := nativeRestoreSeed(t, r, 0, true, true) + for _, disk := range r.z.serverPools[0].getHashedSet(r.object).getDisks() { + fi, err := disk.ReadVersion(t.Context(), r.bucket, r.bucket, r.object, oi.VersionID, ReadOptions{}) + if err != nil { + t.Fatal(err) + } + for i := range fi.Parts { + fi.Parts[i].ActualSize = 0 + } + delete(fi.Metadata, ReservedMetadataPrefix+"actual-size") + if err := disk.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + before, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil { + t.Fatal(err) + } + for _, part := range before.Parts { + if part.ActualSize != 0 { + t.Fatal("legacy fixture was normalized") + } + } + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("legacy fixture read: %v", err) + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + if err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + reads := tier.gets.Load() + _, got, err = r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("legacy restored read: %v", err) + } + if tier.gets.Load() != reads { + t.Error("legacy restored read used the remote tier") + } +} From ba0b59d0b29e4b4dbae308e84f3527969ed9f3db Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 22:38:21 +0800 Subject: [PATCH 15/26] fix(ilm): preserve duplicate metadata and reject unapplied purges Merge lock and tag revisions for duplicate versions before all-pool lifecycle evaluation using the existing metadata reconciliation rules. Distinguish ordinary lifecycle refusal from object absence and clear rejected batch replication state so a late policy withdrawal cannot schedule a purge. Preserve restored-copy cleanup and real absence. Cover single and batch protections, legitimate metadata removals, and actual replication queue and HTTP target behavior. Signed-off-by: Feng Ruohang --- cmd/data-scanner.go | 4 +- cmd/erasure-object.go | 9 + cmd/erasure-server-pool-consistency.go | 6 + ...rasure-server-pool-lifecycle-batch_test.go | 9 +- ...asure-server-pool-lifecycle-safety_test.go | 353 ++++++++++++++++++ cmd/erasure-server-pool-lifecycle.go | 64 ++-- 6 files changed, 416 insertions(+), 29 deletions(-) create mode 100644 cmd/erasure-server-pool-lifecycle-safety_test.go diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 329772df9..77d265367 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -1204,7 +1204,7 @@ func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) (ok bool) { timeILM := globalScannerMetrics.timeILM(lcEvent.Action) if err := expireTransitionedObject(ctx, objLayer, &obj, lcEvent, src); err != nil { - if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + if errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) { return false } ilmLogIf(ctx, fmt.Errorf("expireTransitionedObject(%s, %s): %w", obj.Bucket, obj.Name, err)) @@ -1271,7 +1271,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts) if err != nil { - if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + if errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) { traceFn(ILMExpiry, nil, nil) return false } diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 8ceae79d4..dee11cc8c 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2045,6 +2045,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string // expiration attempted on a bucket with no lifecycle // rules shall be rejected. if lc == nil && opts.Expiration.Expire { + if !opts.Transition.ExpireRestored { + return objInfo, errLifecycleNotApplicable + } if opts.VersionID != "" { return objInfo, VersionNotFound{ Bucket: bucket, @@ -2089,6 +2092,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string isErr = true } if isErr { + if !opts.Transition.ExpireRestored { + return goi, errLifecycleNotApplicable + } if goi.VersionID != "" { return goi, VersionNotFound{ Bucket: bucket, @@ -2162,6 +2168,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string isErr = true } if isErr { + if !opts.Transition.ExpireRestored { + return goi, errLifecycleNotApplicable + } if goi.VersionID != "" { return goi, VersionNotFound{ Bucket: bucket, diff --git a/cmd/erasure-server-pool-consistency.go b/cmd/erasure-server-pool-consistency.go index 44f3dfed6..320fb57af 100644 --- a/cmd/erasure-server-pool-consistency.go +++ b/cmd/erasure-server-pool-consistency.go @@ -19,6 +19,7 @@ package cmd import ( "context" + "errors" "maps" "sort" "strings" @@ -769,6 +770,11 @@ func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket deleteOpts.Versioned = opts.PrefixEnabledFn(obj.ObjectName) } _, err := z.deleteObjectReconciled(ctx, bucket, obj.ObjectName, deleteOpts) + if errors.Is(err, errLifecycleNotApplicable) { + // A later lifecycle rejection did not commit a local purge. Do + // not let the worker replicate the request's pending decision. + deleted[i].ReplicationState = ReplicationState{} + } if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { errs[i] = err } else { diff --git a/cmd/erasure-server-pool-lifecycle-batch_test.go b/cmd/erasure-server-pool-lifecycle-batch_test.go index bfb5c63cc..b2e838078 100644 --- a/cmd/erasure-server-pool-lifecycle-batch_test.go +++ b/cmd/erasure-server-pool-lifecycle-batch_test.go @@ -19,6 +19,7 @@ package cmd import ( "context" + "errors" "net/http" "testing" @@ -118,7 +119,7 @@ func TestPoolsLifecycleTieredBatchReplication(t *testing.T) { if result.errs[0] != nil { t.Errorf("eligible delete: %v", result.errs[0]) } - if !isErrVersionNotFound(result.errs[1]) { + if !errors.Is(result.errs[1], errLifecycleNotApplicable) { t.Errorf("retained version result: %v", result.errs[1]) } if result.deleted[1].VersionPurgeStatus() != "" || result.deleted[1].ReplicationState.ReplicateDecisionStr != "" { @@ -192,7 +193,11 @@ func TestPoolsLifecycleTieredBatchIdentity(t *testing.T) { } vc, _ := globalBucketVersioningSys.Get(s.bucket) _, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: vid}}}, ObjectOptions{PrefixEnabledFn: vc.PrefixEnabled, Expiration: ExpirationOptions{Expire: tc.expire}}) - if errs[0] != nil { + if tc.expire && !tc.wantGone { + if !errors.Is(errs[0], errLifecycleNotApplicable) { + t.Fatalf("ineligible expiry: %v", errs[0]) + } + } else if errs[0] != nil { t.Fatalf("delete: %v", errs[0]) } query := vid diff --git a/cmd/erasure-server-pool-lifecycle-safety_test.go b/cmd/erasure-server-pool-lifecycle-safety_test.go new file mode 100644 index 000000000..27791813f --- /dev/null +++ b/cmd/erasure-server-pool-lifecycle-safety_test.go @@ -0,0 +1,353 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "maps" + "net/http" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" + xhttp "github.com/minio/minio/internal/http" +) + +// A partial metadata update can leave the newest lock/tag revision on a copy +// other than the one selected by content ModTime. Keep the data identity intact. +func setLifecycleCopyMetadata(t *testing.T, s *splitILM, key, id string, pool int, metadata map[string]string, advance bool) { + t.Helper() + oi, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}) + if err != nil { + t.Fatal(err) + } + opts := ObjectOptions{VersionID: id, Versioned: true, MTime: oi.ModTime} + if advance { + opts.MTime = opts.MTime.Add(time.Hour) + } + opts.EvalMetadataFn = func(oi *ObjectInfo, _ error) (ReplicateDecision, error) { + maps.Copy(oi.UserDefined, metadata) + return ReplicateDecision{}, nil + } + if _, err := s.z.serverPools[pool].PutObjectMetadata(t.Context(), s.bucket, key, opts); err != nil { + t.Fatal(err) + } +} + +func TestPoolsLifecycleDuplicateLockMetadata(t *testing.T) { + old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano) + recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano) + for _, tc := range []struct { + name string + copies [2]map[string]string + mode, hold string + delete bool + newerPool1 bool + }{ + {name: "hold-on", copies: [2]map[string]string{poolLockMetadata("", "OFF", old, old), poolLockMetadata("", "ON", old, recent)}, hold: "ON"}, + {name: "held-earlier-pool", copies: [2]map[string]string{poolLockMetadata("", "ON", old, recent), poolLockMetadata("", "OFF", old, old)}, hold: "ON", newerPool1: true}, + {name: "retention-on", copies: [2]map[string]string{poolLockMetadata("", "OFF", old, old), poolLockMetadata("GOVERNANCE", "OFF", recent, old)}, mode: "GOVERNANCE", hold: "OFF"}, + {name: "independent-fields", copies: [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", recent, old), poolLockMetadata("", "ON", old, recent)}, mode: "GOVERNANCE", hold: "ON"}, + {name: "hold-released", copies: [2]map[string]string{poolLockMetadata("", "ON", old, old), poolLockMetadata("", "OFF", old, recent)}, hold: "OFF", delete: true}, + {name: "retention-released", copies: [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", old, old), poolLockMetadata("", "OFF", recent, old)}, hold: "OFF", delete: true}, + } { + for _, batch := range []bool{false, true} { + path := "single" + if batch { + path = "batch" + } + t.Run(tc.name+"/"+path, func(t *testing.T) { + s := &splitILM{migrationILM: newMigrationILM(t, versioning.Enabled, true, splitILMRules), ids: map[string]string{}, bodies: map[string]string{}} + const key = "ncv-copy-lock" + v := splitAt("old", 5, 0, 1) + v.tiered = batch + s.write(key, v) + s.write(key, splitAt("new", 3, 0, 1)) + id := s.ids[key+" old"] + for pool, metadata := range tc.copies { + metadata = maps.Clone(metadata) + if metadata["x-amz-object-lock-mode"] != "" { + metadata["x-amz-object-lock-retain-until-date"] = UTCNow().AddDate(1, 0, 0).Format(time.RFC3339) + } else { + metadata["x-amz-object-lock-retain-until-date"] = "" + } + setLifecycleCopyMetadata(t, s, key, id, pool, metadata, tc.newerPool1 && pool == 1) + } + versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key) + if err != nil { + t.Fatal(err) + } + if len(versions) != 2 || versions[1].VersionID != id || versions[1].NumVersions != 2 { + t.Fatalf("wrong version history: %+v", versions) + } + state := storedObjectLockState(versions[1].UserDefined) + if state.mode != tc.mode || state.legalHold != tc.hold { + t.Errorf("merged state=%+v, want mode=%s hold=%s", state, tc.mode, tc.hold) + } + want := lifecycle.NoneAction + if tc.delete { + want = lifecycle.DeleteVersionAction + } + // Read-side expiry rechecks consume the same event producer. + if events[1].Action != want { + t.Errorf("expiry event=%s want=%s", events[1].Action, want) + } + opts := ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}} + if batch { + _, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: id}}}, opts) + err = errs[0] + } else { + opts.VersionID = id + _, err = s.z.DeleteObject(t.Context(), s.bucket, key, opts) + } + if tc.delete && err != nil || !tc.delete && !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("delete error=%v, wantDelete=%t", err, tc.delete) + } + for pool := range s.z.serverPools { + _, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}) + // Single DeleteObject retains its existing first-copy dispatch. + gone := tc.delete && (batch || pool == 0) + if gone && !isErrVersionNotFound(err) || !gone && err != nil { + t.Errorf("pool=%d err=%v wantGone=%t", pool, err, gone) + } + } + }) + } + } +} + +func TestPoolsLifecycleDuplicateTagMetadata(t *testing.T) { + const rule = `taggedEnabledexpireyes1` + for _, tc := range []struct { + name, stale, fresh string + expire bool + }{ + {"removed", "expire=yes", "", false}, + {"changed", "expire=yes", "expire=no", false}, + {"enabled", "expire=no", "expire=yes", true}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, rule) + const key = "tagged-copy" + s.write(key, splitTiered("old", 5, 0, 1)) + s.write(key, splitAt("new", 3, 0, 1)) + id := s.ids[key+" old"] + old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano) + recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano) + for pool, tags := range []string{tc.stale, tc.fresh} { + stamp := old + if pool == 1 { + stamp = recent + } + setLifecycleCopyMetadata(t, s, key, id, pool, map[string]string{xhttp.AmzObjectTagging: tags, ReservedMetadataPrefixLower + TaggingTimestamp: stamp}, false) + } + versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key) + if err != nil { + t.Fatal(err) + } + if len(versions) != 2 || versions[1].UserTags != tc.fresh || versions[1].UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] != recent { + t.Fatalf("lost newest tag revision: %+v", versions) + } + want := lifecycle.NoneAction + if tc.expire { + want = lifecycle.DeleteVersionAction + } + if events[1].Action != want { + t.Errorf("shared read/delete expiry event=%s want=%s", events[1].Action, want) + } + _, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: id}}}, ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}}) + if tc.expire && errs[0] != nil || !tc.expire && !errors.Is(errs[0], errLifecycleNotApplicable) { + t.Errorf("delete error=%v", errs[0]) + } + for pool := range s.z.serverPools { + _, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}) + if tc.expire && !isErrVersionNotFound(err) || !tc.expire && err != nil { + t.Errorf("pool=%d err=%v wantGone=%t", pool, err, tc.expire) + } + } + }) + } +} + +// All-pool validation uses ReadXL. Run once at the first addressed read in +// batch dispatch, after validation and before the set's lifecycle check. +type lifecycleDispatchDisk struct { + StorageAPI + bucket, key string + once *sync.Once + scans *atomic.Int64 + beforeRead func() +} + +func (d lifecycleDispatchDisk) ReadXL(ctx context.Context, bucket, key string, data bool) (RawFileInfo, error) { + if bucket == d.bucket && key == d.key { + d.scans.Add(1) + } + return d.StorageAPI.ReadXL(ctx, bucket, key, data) +} + +func (d lifecycleDispatchDisk) ReadVersion(ctx context.Context, orig, bucket, key, id string, opts ReadOptions) (FileInfo, error) { + if bucket == d.bucket && key == d.key { + d.once.Do(d.beforeRead) + } + return d.StorageAPI.ReadVersion(ctx, orig, bucket, key, id, opts) +} + +func TestPoolsLifecycleBatchRejectionReplication(t *testing.T) { + for _, mode := range []string{"withdraw-before-validation", "withdraw-after-validation", "already-absent", "normal"} { + t.Run(mode, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + const key = "ncv-dispatch-recheck" + s.write(key, splitTiered("old", 5, 0)) + s.write(key, splitAt("new", 3, 0)) + id := s.ids[key+" old"] + target := newILMTestTarget(t) + registerILMTestTarget(t, s.z, s.bucket, target, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete || r.URL.Query().Get("versionId") != id { + t.Errorf("unexpected purge: %s %s", r.Method, r.URL) + w.WriteHeader(500) + return + } + target.deletes.Add(1) + opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true, ReplicationRequest: true} + opts.SetReplicaStatus(replication.Replica) + if _, err := s.z.DeleteObject(r.Context(), target.bucket, key, opts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + t.Error(err) + w.WriteHeader(500) + return + } + w.WriteHeader(http.StatusNoContent) + }) + setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket) + putConsistencyObject(t, s.z, target.bucket, key, 0, "replica", ObjectOptions{Versioned: true, VersionID: id}) + stats := NewReplicationStats(t.Context(), nil) + previous := globalReplicationStats.Swap(stats) + t.Cleanup(func() { globalReplicationStats.Store(previous) }) + restore, _ := newILMTestReplicationPool(t.Context(), s.z, stats) + t.Cleanup(restore) + oi, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}) + if err != nil { + t.Fatal(err) + } + entry := ilmPurgeToDelete(t.Context(), s.bucket, oi, true) + if entry.VersionPurgeStatus != replication.VersionPurgePending { + t.Fatal("fixture lacks pending purge") + } + withdraw := func() { + meta, err := globalBucketMetadataSys.Get(s.bucket) + if err != nil { + t.Error(err) + return + } + meta.lifecycleConfig = nil + meta.LifecycleConfigXML = nil + globalBucketMetadataSys.Set(s.bucket, meta) + } + if mode == "withdraw-before-validation" { + withdraw() + } + originals := s.z.serverPools[0].getHashedSet(key).getDisks() + var once sync.Once + var scans atomic.Int64 + var dispatched atomic.Bool + for _, pool := range s.z.serverPools { + pool.erasureDisksMu.Lock() + prior := pool.erasureDisks + wrapped := make([][]StorageAPI, len(prior)) + for i, disks := range prior { + wrapped[i] = append([]StorageAPI(nil), disks...) + for j, disk := range disks { + wrapped[i][j] = lifecycleDispatchDisk{StorageAPI: disk, bucket: s.bucket, key: key, once: &once, scans: &scans, beforeRead: func() { + dispatched.Store(true) + if mode != "withdraw-before-validation" && scans.Load() == 0 { + t.Error("dispatch injection ran before all-pool validation") + } + switch mode { + case "withdraw-after-validation": + withdraw() + case "already-absent": + fi := FileInfo{VersionID: id} + fi.SetTierFreeVersionID(mustGetUUID()) + for _, disk := range originals { + if err := disk.DeleteVersion(t.Context(), s.bucket, key, fi, false, DeleteOptions{}); err != nil { + t.Error(err) + } + } + } + }} + } + } + pool.erasureDisks = wrapped + pool.erasureDisksMu.Unlock() + t.Cleanup(func() { pool.erasureDisksMu.Lock(); pool.erasureDisks = prior; pool.erasureDisksMu.Unlock() }) + } + result := &lifecycleBatchResult{ObjectLayer: s.z} + deleteObjectVersions(t.Context(), result, s.bucket, []ObjectToDelete{entry}, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}}) + if mode != "withdraw-before-validation" && !dispatched.Load() { + t.Fatal("dispatch injection did not run") + } + queue := globalReplicationPool.Get().workers[0] + reject := mode == "withdraw-before-validation" || mode == "withdraw-after-validation" + if reject { + if !errors.Is(result.errs[0], errLifecycleNotApplicable) || result.deleted[0].found || result.deleted[0].VersionPurgeStatus() != "" || result.deleted[0].ReplicationState.ReplicateDecisionStr != "" { + t.Errorf("rejection acknowledged a purge: %+v, %v", result.deleted[0], result.errs[0]) + } + if len(queue) != 0 { + t.Fatalf("rejected delete queued %d remote purges", len(queue)) + } + for _, bucket := range []string{s.bucket, target.bucket} { + oi, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: id}) + if err != nil || oi.VersionPurgeStatus != "" { + t.Errorf("rejection changed %s: purge=%s err=%v", bucket, oi.VersionPurgeStatus, err) + } + } + if target.deletes.Load() != 0 { + t.Fatal("rejected delete reached remote target") + } + return + } + if result.errs[0] != nil || len(queue) != 1 { + t.Fatalf("valid/absent purge: error=%v queued=%d", result.errs[0], len(queue)) + } + source, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}) + if mode == "normal" && (!isErrMethodNotAllowed(err) || source.VersionPurgeStatus != replication.VersionPurgePending) { + t.Fatalf("purge not committed locally: %+v %v", source, err) + } + if mode == "already-absent" && !isErrVersionNotFound(err) { + t.Fatalf("source must actually be absent: %v", err) + } + deletion := (<-queue).(DeletedObjectReplicationInfo) + if result := replicateDelete(t.Context(), deletion, s.z); result.VersionPurgeStatus() != replication.VersionPurgeComplete { + t.Fatalf("purge failed: %+v", result) + } + if target.deletes.Load() != 1 { + t.Fatalf("remote deletes=%d", target.deletes.Load()) + } + for _, bucket := range []string{s.bucket, target.bucket} { + if _, err := s.z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + t.Errorf("purged version remains in %s: %v", bucket, err) + } + } + }) + } +} diff --git a/cmd/erasure-server-pool-lifecycle.go b/cmd/erasure-server-pool-lifecycle.go index 16079e534..024c48c8a 100644 --- a/cmd/erasure-server-pool-lifecycle.go +++ b/cmd/erasure-server-pool-lifecycle.go @@ -27,6 +27,10 @@ import ( "github.com/minio/minio/internal/bucket/lifecycle" ) +// An ordinary ILM rejection is not evidence that the version is absent. In +// particular it must not acknowledge a pending replication purge. +var errLifecycleNotApplicable = errors.New("lifecycle expiry no longer applies") + // ILM decides on the versions of an object that one pool holds: the scanner // and the drive heal read the drives of one set, decommission and rebalance // walk one pool, and a GET or HEAD of a version reads the pool that holds it. @@ -113,7 +117,7 @@ func (z *erasureServerPools) lifecycleEventsAllPools(ctx context.Context, bucket lc, err := globalLifecycleSys.Get(bucket) if err != nil { if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) { - return nil, nil, nil + return nil, nil, errLifecycleNotApplicable } return nil, nil, err } @@ -143,37 +147,45 @@ func (z *erasureServerPools) lifecycleEventsAllPools(ctx context.Context, bucket } } - var all []FileInfo - for _, versions := range poolVersions { - all = append(all, versions...) + vc, _ := globalBucketVersioningSys.Get(bucket) + versioned := vc != nil && vc.Versioned(decodeDirObject(object)) + var all []PoolObjInfo + for pool, versions := range poolVersions { + for _, fi := range versions { + all = append(all, PoolObjInfo{Index: pool, ObjInfo: fi.ToObjectInfo(bucket, object, versioned)}) + } } // Each pool lists its versions newest first. On equal times the lower // pool comes first, as reads choose it. - sort.SliceStable(all, func(i, j int) bool { return all[i].ModTime.After(all[j].ModTime) }) - seen := make(map[string]struct{}, len(all)) - versions := all[:0] - for _, fi := range all { - if _, ok := seen[fi.VersionID]; ok { + sort.SliceStable(all, func(i, j int) bool { return all[i].ObjInfo.ModTime.After(all[j].ObjInfo.ModTime) }) + seen := make(map[string]int, len(all)) + var versions [][]PoolObjInfo + for _, copy := range all { + if i, ok := seen[copy.ObjInfo.VersionID]; ok { + versions[i] = append(versions[i], copy) continue } - seen[fi.VersionID] = struct{}{} - versions = append(versions, fi) + seen[copy.ObjInfo.VersionID] = len(versions) + versions = append(versions, []PoolObjInfo{copy}) } - vc, _ := globalBucketVersioningSys.Get(bucket) - versioned := vc != nil && vc.Versioned(decodeDirObject(object)) objInfos := make([]ObjectInfo, len(versions)) objOpts := make([]lifecycle.ObjectOpts, len(versions)) - for i := range versions { - fi := versions[i] - fi.IsLatest = i == 0 - fi.NumVersions = len(versions) - fi.SuccessorModTime = time.Time{} - if i > 0 { - fi.SuccessorModTime = versions[i-1].ModTime + for i, copies := range versions { + oi := copies[0].ObjInfo + if len(copies) > 1 { + // Count the version once, but retain independently ordered lock + // and tag revisions, including explicit removals, from every copy. + oi = mergedPoolObjectInfo(copies) } - objInfos[i] = fi.ToObjectInfo(bucket, object, versioned) - objOpts[i] = objInfos[i].ToLifecycleOpts() + oi.IsLatest = i == 0 + oi.NumVersions = len(versions) + oi.SuccessorModTime = time.Time{} + if i > 0 { + oi.SuccessorModTime = objInfos[i-1].ModTime + } + objInfos[i] = oi + objOpts[i] = oi.ToLifecycleOpts() } events, err := lifecycle.NewEvaluator(*lc).WithLockRetention(&lr).WithReplicationConfig(rcfg).Eval(objOpts) if err != nil { @@ -200,8 +212,7 @@ func (z *erasureServerPools) checkExpiryAllPools(ctx context.Context, bucket, ob } // expiryApplies returns nil when the ILM expiry that opts describe applies -// to the versions, and otherwise the error a set returns for an expiry that -// no longer applies: +// to the versions. A missing version and a rejected expiry remain distinct: // - an expiry of all versions (DeletePrefix) needs the latest version to // expire with all versions; // - a delete marker (no VersionID) hides the latest version, which has to @@ -231,11 +242,14 @@ func (all lifecycleAllPools) expiryApplies(bucket, object string, opts ObjectOpt if all.events[i].Action == lifecycle.DeleteVersionAction { return nil } - break + return errLifecycleNotApplicable } } return VersionNotFound{Bucket: bucket, Object: name, VersionID: opts.VersionID} } + if len(all.versions) > 0 { + return errLifecycleNotApplicable + } return ObjectNotFound{Bucket: bucket, Object: name} } From 496eab4b7637d8c8a7cac9a95511d96c080b4bf1 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 22:46:03 +0800 Subject: [PATCH 16/26] fix(restore): preserve safe read context through nested lookups Carry restore context through POST status reads, metadata copy routing, null-version Object Lock checks and multipart completion. A failed read must preserve the target and must not enqueue its remote object for garbage collection. Cover initial/status POST failures and metadata loss during single and multipart rehydration on real disks. Signed-off-by: Feng Ruohang --- cmd/bucket-object-lock.go | 6 +- cmd/erasure-multipart.go | 3 +- cmd/erasure-object.go | 12 +- cmd/erasure-server-pool.go | 8 +- cmd/object-handlers.go | 7 +- cmd/object-retention-overwrite_test.go | 2 +- cmd/restore-native_test.go | 192 +++++++++++++++++++++++++ 7 files changed, 218 insertions(+), 12 deletions(-) diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go index 397b039ef..293042fc3 100644 --- a/cmd/bucket-object-lock.go +++ b/cmd/bucket-object-lock.go @@ -251,8 +251,9 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec // checkObjectLockedForOverwrite re-checks under the object write lock whether an // in-place write to the null version (such as a directory object) would -// overwrite a locked version in an Object Lock bucket. -func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, getObjectInfoFn GetObjectInfoFn) error { +// overwrite a locked version in an Object Lock bucket. Restore context keeps +// failed reads from deleting the version being rehydrated. +func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, getObjectInfoFn GetObjectInfoFn, transition TransitionOptions) error { if isMinioMetaBucketName(bucket) { return nil } @@ -265,6 +266,7 @@ func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, g } objInfo, err := getObjectInfoFn(ctx, bucket, object, ObjectOptions{ VersionID: nullVersionID, + Transition: transition, NoLock: true, NoAuditLog: true, }) diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 4b2893c08..c68fbc446 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -1694,7 +1694,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str getObjectInfo = opts.replicaObjectInfo } if fi.VersionID == "" || fi.VersionID == nullVersionID { - if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo); err != nil { + if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo, opts.Transition); err != nil { return oi, toObjectErr(err, bucket, object) } } else { @@ -1702,6 +1702,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str VersionID: fi.VersionID, Versioned: opts.Versioned, VersionSuspended: opts.VersionSuspended, + Transition: opts.Transition, NoLock: true, }) switch { diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index abeeee82d..a8b9ba1f2 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -156,7 +156,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d if dstOpts.replicaObjectInfo != nil { getObjectInfo = dstOpts.replicaObjectInfo } - if err := checkObjectLockedForOverwrite(ctx, dstBucket, dstObject, getObjectInfo); err != nil { + if err := checkObjectLockedForOverwrite(ctx, dstBucket, dstObject, getObjectInfo, dstOpts.Transition); err != nil { return oi, toObjectErr(err, dstBucket, dstObject) } } @@ -1673,7 +1673,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st if opts.replicaObjectInfo != nil { getObjectInfo = opts.replicaObjectInfo } - if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo); err != nil { + if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo, opts.Transition); err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } } @@ -2655,9 +2655,10 @@ func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, obje VersionID: oi.VersionID, Transition: opts.Transition, }, ObjectOptions{ - VersionID: oi.VersionID, - MTime: oi.ModTime, - NoLock: opts.NoLock, + VersionID: oi.VersionID, + MTime: oi.ModTime, + NoLock: opts.NoLock, + Transition: opts.Transition, }); err != nil { storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err)) return err @@ -2699,6 +2700,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi) ropts.NoLock = opts.NoLock + ropts.Transition = opts.Transition // Rehydration copies encoded bytes without changing their content identity. ropts.PreserveETag = oi.ETag // Even a one-part multipart upload must retain its original part number: diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index db2973717..52949f849 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1630,8 +1630,12 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec // Restore status updates address an existing version. Keep ordinary // copy/write allocation unchanged, including its latest-version lookup. poolIdx, err = z.getPoolIdxExistingWithOpts(ctx, dstBucket, dstObject, ObjectOptions{ - VersionID: dstOpts.VersionID, MetadataChg: true, NoLock: true, - SkipDecommissioned: true, SkipRebalancing: true, + VersionID: dstOpts.VersionID, + MetadataChg: true, + NoLock: true, + Transition: dstOpts.Transition, + SkipDecommissioned: true, + SkipRebalancing: true, }) } else { poolIdx, err = z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size) diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 2843f42d1..42b4cd0e0 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -3951,6 +3951,8 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * return } + // Preserve a failed restore read even before the request body is parsed. + opts.Transition.RestoreRequest = &RestoreObjectRequest{} objInfo, err := getObjectInfo(ctx, bucket, object, opts) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) @@ -3976,6 +3978,7 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * writeErrorResponse(ctx, w, apiErr, r.URL) return } + opts.Transition.RestoreRequest = rreq statusCode := http.StatusOK alreadyRestored := false if err == nil { @@ -4004,9 +4007,11 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * } objInfo.UserDefined = metadata if _, err := objectAPI.CopyObject(GlobalContext, bucket, object, bucket, object, objInfo, ObjectOptions{ - VersionID: objInfo.VersionID, + VersionID: objInfo.VersionID, + Transition: opts.Transition, }, ObjectOptions{ VersionID: objInfo.VersionID, + Transition: opts.Transition, MTime: objInfo.ModTime, MetadataChg: true, }); err != nil { diff --git a/cmd/object-retention-overwrite_test.go b/cmd/object-retention-overwrite_test.go index 83ce43ebe..7c864e733 100644 --- a/cmd/object-retention-overwrite_test.go +++ b/cmd/object-retention-overwrite_test.go @@ -57,7 +57,7 @@ func TestObjectLockOverwriteConfigUnavailable(t *testing.T) { err := checkObjectLockedForOverwrite(t.Context(), "lock-config-unavailable", "object", func(context.Context, string, string, ObjectOptions) (ObjectInfo, error) { t.Fatal("object must not be read without a bucket configuration") return ObjectInfo{}, nil - }) + }, TransitionOptions{}) if !errors.Is(err, errServerNotInitialized) { t.Fatalf("unavailable lock configuration must fail closed, got %v", err) } diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go index dfe7fe6f4..b01656867 100644 --- a/cmd/restore-native_test.go +++ b/cmd/restore-native_test.go @@ -28,9 +28,11 @@ import ( "net/http" "net/http/httptest" "net/url" + "path/filepath" "reflect" "strings" "sync" + "sync/atomic" "testing" "time" @@ -742,3 +744,193 @@ func TestMultipartCompletionLockAlias(t *testing.T) { } } } + +type nativeRestoreReadHook struct { + io.ReadCloser + remaining int + mutate func() +} + +func (r *nativeRestoreReadHook) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + r.remaining -= n + if r.remaining == 0 && r.mutate != nil { + f := r.mutate + r.mutate = nil + f() + } + return n, err +} + +type nativeRestoreReadHookTier struct { + WarmBackend + size int + mutate func() +} + +func (b nativeRestoreReadHookTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) { + rd, err := b.WarmBackend.Get(ctx, object, version, opts) + if err != nil { + return nil, err + } + return &nativeRestoreReadHook{rd, b.size, b.mutate}, nil +} + +// Remove only the addressed entry from a majority of disks, keeping shards +// and the healthy newer version. Return errors to the synchronous caller. +func nativeRestoreLoseMetadata(ctx context.Context, disks []StorageAPI, bucket, object, id string) error { + for _, disk := range disks[1:] { + raw, err := disk.ReadXL(ctx, bucket, object, true) + if err != nil { + return err + } + var meta xlMetaV2 + if err := meta.Load(raw.Buf); err != nil { + return err + } + del := FileInfo{VersionID: id} + del.SetSkipTierFreeVersion() + if _, err := meta.DeleteVersion(del); err != nil { + return err + } + data, err := meta.AppendTo(nil) + if err != nil { + return err + } + if err := disk.WriteAll(ctx, bucket, filepath.Join(object, xlStorageFormatFile), data); err != nil { + return err + } + } + return nil +} + +// A target can lose metadata quorum after the restore snapshot. Nested +// completion/retention reads must not turn rehydration into remote GC. +func TestNativeRestoreCompletionReadSafety(t *testing.T) { + for _, null := range []bool{false, true} { + for _, multipart := range []bool{false, true} { + t.Run(fmt.Sprintf("null=%v/multipart=%v", null, multipart), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, 0, null, multipart) + latest := r.put("newer", false, false) + enableBucketObjectLock(t, r.bucket) + disks := r.z.serverPools[0].getHashedSet(r.object).getDisks() + var mutationErr error + mutated := false + mutate := func() { + mutated = true + mutationErr = nativeRestoreLoseMetadata(t.Context(), disks, r.bucket, r.object, oi.VersionID) + } + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + globalTierConfigMgr.drivercache["WARM"] = nativeRestoreReadHookTier{tier, len(body), mutate} + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + err := r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + if ctx.Err() != nil { + t.Fatalf("restore timed out: %v", err) + } + if !mutated || mutationErr != nil { + t.Fatalf("invalid injection: mutated=%v err=%v", mutated, mutationErr) + } + t.Logf("restore after addressed metadata lost: %v", err) + for i, disk := range disks { + raw, e := disk.ReadXL(t.Context(), r.bucket, r.object, true) + if e != nil { + t.Fatal(e) + } + versions, e := getFileInfoVersions(raw.Buf, r.bucket, r.object, false) + if e != nil { + t.Fatal(e) + } + if len(versions.FreeVersions) > 0 { + t.Errorf("disk %d restore manufactured %d remote GC entries", i, len(versions.FreeVersions)) + } + } + if err == nil { + reads := tier.gets.Load() + _, got, readErr := r.read(oi.VersionID) + if readErr != nil || !bytes.Equal(got, body) || tier.gets.Load() != reads { + t.Errorf("successful restore did not rehydrate its own content: %v", readErr) + } + } + _, got, readErr := r.read(latest.VersionID) + if readErr != nil || !bytes.Equal(got, r.bodies["newer"]) { + t.Errorf("restore changed the other version: %v", readErr) + } + }) + } + } +} + +type nativeRestoreStatusHook struct { + ObjectLayer + beforeCopy func() + copyCalled bool + backgroundCalled atomic.Bool +} + +func (h *nativeRestoreStatusHook) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, info ObjectInfo, srcOpts, dstOpts ObjectOptions) (ObjectInfo, error) { + h.copyCalled = true + h.beforeCopy() + return h.ObjectLayer.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, info, srcOpts, dstOpts) +} + +func (h *nativeRestoreStatusHook) RestoreTransitionedObject(context.Context, string, string, ObjectOptions) error { + h.backgroundCalled.Store(true) + return errors.New("unexpected background restore after failed status update") +} + +func TestNativeRestorePOSTReadSafety(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, stage := range []string{"initial", "status-copy"} { + t.Run(fmt.Sprintf("pools=%d/%s", pools, stage), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + metaOpts := r.opts(oi.VersionID) + metaOpts.MTime = oi.ModTime + metaOpts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[xhttp.AmzRestore] = "" + return ReplicateDecision{}, nil + } + if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, metaOpts); err != nil { + t.Fatal(err) + } + r.put("newer", false, false) + disks := r.z.serverPools[0].getHashedSet(r.object).getDisks() + var before [][]byte + mutate := func() { + if err := nativeRestoreLoseMetadata(t.Context(), disks, r.bucket, r.object, oi.VersionID); err != nil { + t.Fatal(err) + } + before = restoreBoundarySnapshot(t, r) + } + hook := &nativeRestoreStatusHook{ObjectLayer: r.z, beforeCopy: mutate} + if stage == "initial" { + mutate() + } + api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return hook }} + body := []byte(`1`) + req, err := newTestSignedRequestV4(http.MethodPost, "/"+r.bucket+"/"+r.object+"?restore&versionId="+url.QueryEscape(oi.VersionID), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + if err := req.ParseForm(); err != nil { + t.Fatal(err) + } + req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object}) + rec := httptest.NewRecorder() + api.PostRestoreObjectHandler(rec, req) + if rec.Code < 400 { + t.Fatalf("POST acknowledged failed read: %d %s", rec.Code, rec.Body.String()) + } + if hook.copyCalled != (stage == "status-copy") || hook.backgroundCalled.Load() { + t.Fatal("request followed the wrong failure path") + } + if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) { + t.Error("failed POST read changed stored metadata or free versions") + } + }) + } + } +} From 0c5a3095072cca2599a6a97b9ef1c6fce89ade4b Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Tue, 3 Nov 2026 22:56:05 +0800 Subject: [PATCH 17/26] test(ilm): assert scanner refusal without claiming absence Keep the specialized scanner routing test aligned with the explicit lifecycle rejection result, and verify the addressed version and payload remain readable. This distinguishes refused expiry from an idempotent missing version. Signed-off-by: Feng Ruohang --- cmd/erasure-server-pool-consistency_test.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/erasure-server-pool-consistency_test.go b/cmd/erasure-server-pool-consistency_test.go index 0dec71fd5..8c94fb49b 100644 --- a/cmd/erasure-server-pool-consistency_test.go +++ b/cmd/erasure-server-pool-consistency_test.go @@ -522,10 +522,20 @@ func TestPoolsDeleteVersionSpecialCalls(t *testing.T) { opts := ObjectOptions{Versioned: true, VersionID: oi.VersionID, InclFreeVersions: !expiration, Expiration: ExpirationOptions{Expire: expiration}} _, err := z.DeleteObject(t.Context(), bucket, object, opts) if expiration { - // No lifecycle rule authorizes expiration. Preserve its existing - // version-not-found result, even with an unrelated unreadable pool. - if !isErrVersionNotFound(err) { - t.Errorf("expiration changed its scanner contract: %v", err) + // No rule authorizes expiration. Distinguish refusal from real + // absence, even with an unrelated unreadable pool, so a worker + // cannot acknowledge or replicate a purge that never happened. + if !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("expiration did not preserve its rejection: %v", err) + } + gr, err := z.serverPools[0].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: oi.VersionID}) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(gr) + gr.Close() + if err != nil || string(data) != "payload" || gr.ObjInfo.VersionID != oi.VersionID { + t.Errorf("rejected expiration changed the stored version: %v", err) } } else if err != nil { t.Errorf("free-version cleanup was forced through all-pool resolution: %v", err) From 82dd12faf8f2439a2a84ed6a985e55d97cff7276 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 4 Nov 2026 16:37:42 +0800 Subject: [PATCH 18/26] fix(restore): keep queued restores waiting for the object lock The POST handler marks a version ongoing before its background restore takes the object lock, and only the restore clears that marker. Native restore holds the lock through the tier download, so a restore of another version of the same key waits behind it. Both restore entry points gave up after one delete-operation timeout and returned before any cleanup, leaving ongoing-request="true" with no owner. Every later POST for that version was refused with 409 and nothing expired the marker. Retry the lock while the restore's context lives, with a dedicated dynamic timeout so repeated waits do not lengthen delete timeouts. Shutdown or a caller deadline still ends the wait. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/erasure-object.go | 15 ++++++- cmd/erasure-server-pool.go | 2 +- cmd/globals.go | 2 + cmd/restore-native_test.go | 80 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 2 deletions(-) diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index e3171aaca..64658aa50 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2675,6 +2675,19 @@ func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, obje return nil } +// getRestoreLock waits for a restore's object lock for as long as ctx lives. +// The POST handler has already marked the version ongoing, and only the +// restore clears that marker. A restore queued behind another version's +// download must therefore keep waiting rather than give up after one timeout. +func getRestoreLock(ctx context.Context, lk RWLocker) (LockContext, error) { + for { + lkctx, err := lk.GetLock(ctx, globalRestoreLockTimeout) + if !errors.As(err, &OperationTimedOut{}) || ctx.Err() != nil { + return lkctx, err + } + } +} + // restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts // as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error { @@ -2682,7 +2695,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s // Single-pool and direct callers must protect the metadata snapshot // through download, rehydration and failure cleanup, just like pools. lk := er.NewNSLock(bucket, object) - lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) + lkctx, err := getRestoreLock(ctx, lk) if err != nil { return err } diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index e8678ad92..2f9f0d7b4 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -3371,7 +3371,7 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck if !opts.NoLock { // Acquire write lock before restoring transitioned object lk := z.NewNSLock(bucket, object) - lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) + lkctx, err := getRestoreLock(ctx, lk) if err != nil { return err } diff --git a/cmd/globals.go b/cmd/globals.go index e71294508..607a2ecf5 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -319,6 +319,8 @@ var ( globalOperationTimeout = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops globalDeleteOperationTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute) // default time for delete ops + // Queued restores retry this wait; keep their failures out of delete timeouts. + globalRestoreLockTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute) // Listing recovery has its own failure history and a five-second caller budget. globalListingOperationTimeout = newDynamicTimeout(5*time.Second, 5*time.Second) diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go index b01656867..884b88d24 100644 --- a/cmd/restore-native_test.go +++ b/cmd/restore-native_test.go @@ -603,6 +603,86 @@ func TestNativeRestoreSerializesObjectLock(t *testing.T) { } } +// The POST handler marks a version ongoing before its background restore +// takes the object lock, and only that restore clears the marker. A restore +// queued behind another version's download must keep waiting: one that gave up +// after a lock timeout would leave the marker for every later POST to refuse. +// The wait still ends with its context. +func TestNativeRestoreQueuedBehindDownload(t *testing.T) { + for _, pools := range []int{1, 2} { + t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + first, _ := nativeRestoreSeed(t, r, 0, false, false) + queued, body := nativeRestoreSeed(t, r, pools-1, false, false) + prev := globalRestoreLockTimeout + globalRestoreLockTimeout = newDynamicTimeout(50*time.Millisecond, 50*time.Millisecond) + t.Cleanup(func() { globalRestoreLockTimeout = prev }) + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + blocked := &nativeRestoreBlockedTier{WarmBackend: tier, entered: make(chan struct{}), release: make(chan struct{})} + globalTierConfigMgr.drivercache["WARM"] = blocked + var once sync.Once + unblock := func() { once.Do(func() { close(blocked.release) }) } + defer unblock() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + firstDone := make(chan error, 1) + go func() { + firstDone <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, first.VersionID)) + }() + select { + case <-blocked.entered: + case <-ctx.Done(): + t.Fatal("first restore never reached the tier") + } + short, shortCancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer shortCancel() + ended := make(chan error, 1) + go func() { + ended <- r.z.RestoreTransitionedObject(short, r.bucket, r.object, nativeRestoreOptions(r, queued.VersionID)) + }() + select { + case err := <-ended: + if err == nil { + t.Fatal("restore finished without the object lock") + } + case <-time.After(5 * time.Second): + t.Fatal("restore kept waiting for the lock after its context ended") + } + queuedDone := make(chan error, 1) + go func() { + queuedDone <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, queued.VersionID)) + }() + select { + case err := <-queuedDone: + t.Fatalf("queued restore gave up behind the download: %v", err) + case <-time.After(10 * 50 * time.Millisecond): + } + unblock() + for _, done := range []chan error{firstDone, queuedDone} { + select { + case err := <-done: + if err != nil { + t.Fatalf("restore: %v", err) + } + case <-ctx.Done(): + t.Fatal("restore did not finish") + } + } + info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(queued.VersionID)) + if err != nil || info.RestoreOngoing { + t.Fatalf("queued version still ongoing after restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err) + } + reads := tier.gets.Load() + if _, got, err := r.read(queued.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("queued restore contents: %v", err) + } + if tier.gets.Load() != reads { + t.Error("queued version was not restored locally") + } + }) + } +} + func TestNativeRestoreConcurrentCompletion(t *testing.T) { for _, separateSets := range []bool{false, true} { t.Run(fmt.Sprintf("separate-set-locks=%v", separateSets), func(t *testing.T) { From f321f7f1c7196b60709fe293ea2be81b6392ab33 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 4 Nov 2026 16:37:42 +0800 Subject: [PATCH 19/26] fix(ilm): do not announce batch expiries the recheck rejected The expiry worker now asks DeleteObjects to recheck ILM against all pools. A rejected entry keeps its version and returns errLifecycleNotApplicable, but the worker still wrote an ILMExpiry audit record, an ILM trace and an s3:ObjectRemoved:Delete event for it. Event delivery drops the error element, so consumers saw an ordinary removal, repeated on every scan while the versions stay split across pools. Skip rejected entries; eligible entries of the same batch are still reported. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- ...rasure-server-pool-lifecycle-batch_test.go | 56 +++++++++++++++++++ cmd/object-handlers-common.go | 6 ++ 2 files changed, 62 insertions(+) diff --git a/cmd/erasure-server-pool-lifecycle-batch_test.go b/cmd/erasure-server-pool-lifecycle-batch_test.go index b2e838078..363a8768a 100644 --- a/cmd/erasure-server-pool-lifecycle-batch_test.go +++ b/cmd/erasure-server-pool-lifecycle-batch_test.go @@ -23,9 +23,12 @@ import ( "net/http" "testing" + "github.com/minio/madmin-go/v3" "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/replication" "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/pubsub" ) // Reuse the author's scanner/worker fixture. A second configured pool is @@ -212,3 +215,56 @@ func TestPoolsLifecycleTieredBatchIdentity(t *testing.T) { }) } } + +// A batch entry whose expiry the all-pools recheck rejects is kept. The worker +// must not report it as removed, by event, ILM trace or audit, while eligible +// entries of the same batch are still reported. +func TestPoolsLifecycleBatchRejectionNotAnnounced(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + const key = "nnv-announce" + for _, v := range []splitVersion{splitTiered("v1", 5, 0), splitTiered("v2", 4, 0), splitTiered("v3", 3, 0), splitAt("v4", 2, 0)} { + s.write(key, v) + } + eligible, retained := s.ids[key+" v1"], s.ids[key+" v3"] + done := make(chan struct{}) + defer close(done) + events := make(chan event.Event, 8) + if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedDelete), events, done, nil); err != nil { + t.Fatal(err) + } + traces := make(chan madmin.TraceInfo, 8) + if err := globalTrace.Subscribe(madmin.TraceILM, traces, done, nil); err != nil { + t.Fatal(err) + } + result := &lifecycleBatchResult{ObjectLayer: s.z} + deleteObjectVersions(t.Context(), result, s.bucket, []ObjectToDelete{ + {ObjectV: ObjectV{ObjectName: key, VersionID: eligible}}, + {ObjectV: ObjectV{ObjectName: key, VersionID: retained}}, + }, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}, {Action: lifecycle.DeleteVersionAction}}) + if result.errs[0] != nil || !errors.Is(result.errs[1], errLifecycleNotApplicable) { + t.Fatalf("fixture results: eligible=%v retained=%v", result.errs[0], result.errs[1]) + } + if _, err := s.z.GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: retained, Versioned: true}); err != nil { + t.Fatalf("rejected version must remain: %v", err) + } + // Both publishers deliver synchronously, so everything sent is buffered. + var notified, traced []string + for drained := false; !drained; { + select { + case ev := <-events: + notified = append(notified, ev.S3.Object.VersionID) + case tr := <-traces: + if tr.FuncName == ILMExpiry { + traced = append(traced, tr.Custom["version-id"]) + } + default: + drained = true + } + } + if len(notified) != 1 || notified[0] != eligible { + t.Errorf("removal events for %v, want only eligible %s (retained %s)", notified, eligible, retained) + } + if len(traced) != 1 || traced[0] != eligible { + t.Errorf("ILM expiry traces for %v, want only eligible %s (retained %s)", traced, eligible, retained) + } +} diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go index acd7ebcb2..903e82731 100644 --- a/cmd/object-handlers-common.go +++ b/cmd/object-handlers-common.go @@ -19,6 +19,7 @@ package cmd import ( "context" + "errors" "fmt" "net/http" "regexp" @@ -471,6 +472,11 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD }) for i, dobj := range deletedObjs { + if errors.Is(errs[i], errLifecycleNotApplicable) { + // The recheck kept this version. Do not audit or announce a + // removal that did not happen; the next scan re-evaluates it. + continue + } oi := ObjectInfo{ Bucket: bucket, Name: dobj.ObjectName, From 9f7b49369d4cbc1a5506af4bff05df05c8c8b6d3 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 00:16:27 +0800 Subject: [PATCH 20/26] fix(ilm): require an authoritative cross-pool history before lifecycle expiry The merged all-pool check from #279 decided on the versions that a read quorum of each set merged non-strictly. A drive that missed one metadata update could therefore supply the revision used for the decision, and a partially unreadable set could drop a committed version from the history. Both let an expiry delete data the rule did not cover, including a version under legal hold. - objectVersions accepts a revision only when max(n/2+1, the version's metadata read quorum) drives hold it, identifies revisions by the header plus a sha256 over a canonical, length-prefixed encoding of the normalized metadata, filters non-accepted revisions before the strict merge, and fails closed on omitted versions, too many unreadable drives, or cancellation. - Lock fields are read with the evaluator's key precedence. Cross-pool copies that disagree on hold or retention must be strictly ordered by valid timestamps; copies with different ModTime are separate writes and an older protected write refuses the object. The set safeguard re-checks protection and pending purges on its fresh read for checked version deletes. - Duplicate copies with semantically different replication or purge state refuse the object. - Batch expiry announces and audits only completed or already-absent removals. Reviewed adversarially over seven rounds; the final round approved with no remaining findings in scope. All-versions expiry through deletePrefix and tiered delete-all dispatch are tracked separately. Signed-off-by: Feng Ruohang Co-Authored-By: Claude Opus 5.5 --- cmd/erasure-object.go | 12 +- cmd/erasure-server-pool-consistency.go | 7 +- ...re-server-pool-lifecycle-authority_test.go | 1399 +++++++++++++++++ ...asure-server-pool-lifecycle-safety_test.go | 40 +- cmd/erasure-server-pool-lifecycle.go | 511 +++++- cmd/erasure-server-pool.go | 13 +- cmd/object-api-interface.go | 4 + cmd/object-handlers-common.go | 9 +- 8 files changed, 1940 insertions(+), 55 deletions(-) create mode 100644 cmd/erasure-server-pool-lifecycle-authority_test.go diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 64658aa50..9307aae81 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2118,6 +2118,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object) } + // An ILM version expiry that the pools checked with the versions in all + // pools; the set still guards the protection it reads, see + // checkedExpiryRefused. + checkedExpiry := opts.Expiration.Expire && opts.Expiration.CheckedAllPools && opts.VersionID != "" && !opts.Transition.ExpireRestored + storageDisks := er.getDisks() versionFound := true objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response. @@ -2126,8 +2131,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string if gerr != nil && goi.Name == "" { if _, ok := gerr.(InsufficientReadQuorum); ok { // A restored-copy expiry needs fresh metadata to confirm its target - // and deadline; ordinary unversioned DELETE may still try a blind delete. - if opts.Transition.ExpireRestored || opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 { + // and deadline, a checked expiry to guard its protection; ordinary + // unversioned DELETE may still try a blind delete. + if opts.Transition.ExpireRestored || checkedExpiry || opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 { // Add an MRF heal for next time. er.addPartial(bucket, object, opts.VersionID) return objInfo, InsufficientWriteQuorum{} @@ -2197,6 +2203,8 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string } } } + } else if checkedExpiry && checkedExpiryRefused(*lc, rcfg, replcfg, goi, opts.Expiration.ApprovedProtection) { + return goi, errLifecycleNotApplicable } // Determine whether to mark object deleted for replication diff --git a/cmd/erasure-server-pool-consistency.go b/cmd/erasure-server-pool-consistency.go index 320fb57af..7d449b60a 100644 --- a/cmd/erasure-server-pool-consistency.go +++ b/cmd/erasure-server-pool-consistency.go @@ -671,7 +671,9 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket, // deleteObjectsReconciled removes data while retaining each item's authoritative // copy until every other pool has completed its cleanup. The caller holds the // pools-layer write locks and separates marker creation from this batch. -func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) { +// approved holds the Object Lock state each checked ILM expiry was approved +// with, see ExpirationOptions.ApprovedProtection. +func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket string, objects []ObjectToDelete, approved []objectLockState, opts ObjectOptions) ([]DeletedObject, []error) { deleted := make([]DeletedObject, len(objects)) errs := make([]error, len(objects)) primary := make([]int, len(objects)) @@ -765,6 +767,9 @@ func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket // versions against all pools. Empty IDs and user deletes have no such // authorization; restored-copy expiry uses the single-object path. deleteOpts.Expiration.CheckedAllPools = opts.Expiration.Expire && obj.VersionID != "" + if deleteOpts.Expiration.CheckedAllPools { + deleteOpts.Expiration.ApprovedProtection = approved[i] + } deleteOpts.DeleteReplication = obj.ReplicationState() if opts.PrefixEnabledFn != nil { deleteOpts.Versioned = opts.PrefixEnabledFn(obj.ObjectName) diff --git a/cmd/erasure-server-pool-lifecycle-authority_test.go b/cmd/erasure-server-pool-lifecycle-authority_test.go new file mode 100644 index 000000000..d75525356 --- /dev/null +++ b/cmd/erasure-server-pool-lifecycle-authority_test.go @@ -0,0 +1,1399 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "maps" + "slices" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/event" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/pubsub" +) + +// An all-pools ILM approval is only as sound as the history each set reports +// and the protection it reads under the lock. These cases leave drives with +// stale, partial or unreadable metadata, as failed or interrupted writes and +// offline drives do, and check that no expiry deletes what they hide. + +const authorityNoncurrentRule = `ncvEnabledncv-1` + +var authorityHoldKey = strings.ToLower(xhttp.AmzObjectLockLegalHold) + +func newLockedSplitILM(t *testing.T, rules string) *splitILM { + t.Helper() + return &splitILM{migrationILM: newMigrationILM(t, versioning.Enabled, true, rules), ids: map[string]string{}, bodies: map[string]string{}} +} + +// authorityParity pins new objects to 12+4 on the 16 drives of a set. Other +// tests can leave another storage class configured. Without one, 4 is the +// default; Update cannot restore an unconfigured class, so leave it then. +func authorityParity(t *testing.T) { + t.Helper() + if globalStorageClass.GetParityForSC("") < 0 { + return + } + previous := globalStorageClass + globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}}) + t.Cleanup(func() { globalStorageClass.Update(previous) }) +} + +// authorityDisk fails the history scan (ReadXL) or the version removal of +// one key on a drive. +type authorityDisk struct { + StorageAPI + bucket, key string + failRead, failWrite bool +} + +func (d authorityDisk) ReadXL(ctx context.Context, bucket, key string, data bool) (RawFileInfo, error) { + if d.failRead && bucket == d.bucket && key == d.key { + return RawFileInfo{}, errDiskNotFound + } + return d.StorageAPI.ReadXL(ctx, bucket, key, data) +} + +func (d authorityDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error { + if d.failWrite && volume == d.bucket && path == d.key { + return errDiskNotFound + } + return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts) +} + +func (d authorityDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error { + errs := make([]error, len(versions)) + var pass []FileInfoVersions + var idx []int + for i, v := range versions { + if d.failWrite && volume == d.bucket && v.Name == d.key { + errs[i] = errDiskNotFound + continue + } + pass = append(pass, v) + idx = append(idx, i) + } + if len(pass) > 0 { + for j, err := range d.StorageAPI.DeleteVersions(ctx, volume, pass, opts) { + errs[idx[j]] = err + } + } + return errs +} + +// wrapAuthorityDisks makes the drives of pool that drive selects fail reads +// or writes of key. +func wrapAuthorityDisks(t *testing.T, s *splitILM, pool int, key string, failRead, failWrite bool, drive func(int) bool) { + t.Helper() + p := s.z.serverPools[pool] + p.erasureDisksMu.Lock() + prior := p.erasureDisks + wrapped := make([][]StorageAPI, len(prior)) + for i, disks := range prior { + wrapped[i] = append([]StorageAPI(nil), disks...) + for j := range disks { + if drive(j) { + wrapped[i][j] = authorityDisk{StorageAPI: disks[j], bucket: s.bucket, key: key, failRead: failRead, failWrite: failWrite} + } + } + } + p.erasureDisks = wrapped + p.erasureDisksMu.Unlock() + t.Cleanup(func() { + p.erasureDisksMu.Lock() + p.erasureDisks = prior + p.erasureDisksMu.Unlock() + }) +} + +func authorityDrives(from, to int) func(int) bool { + return func(i int) bool { return i >= from && i < to } +} + +// removeAuthorityCopies removes version id of key from the drives of pool +// that drive selects, as a write that did not reach them leaves it. +func removeAuthorityCopies(t *testing.T, s *splitILM, pool int, key, id string, drive func(int) bool) { + t.Helper() + for i, disk := range s.z.serverPools[pool].getHashedSet(key).getDisks() { + if !drive(i) { + continue + } + fi := FileInfo{VersionID: id} + fi.SetSkipTierFreeVersion() + if err := disk.DeleteVersion(t.Context(), s.bucket, key, fi, false, DeleteOptions{}); err != nil { + t.Fatal(err) + } + } +} + +func authorityHeader(t *testing.T, disk StorageAPI, bucket, key, id string) xlMetaV2VersionHeader { + t.Helper() + rf, err := disk.ReadXL(t.Context(), bucket, key, false) + if err != nil { + t.Fatal(err) + } + var xl xlMetaV2 + if err := xl.LoadOrConvert(rf.Buf); err != nil { + t.Fatal(err) + } + for _, ver := range xl.versions { + if uuid.UUID(ver.header.VersionID).String() == id { + return ver.header + } + } + t.Fatalf("drive has no version %s", id) + return xlMetaV2VersionHeader{} +} + +func authorityExpireVersion(t *testing.T, s *splitILM, key, id string) error { + t.Helper() + _, err := s.z.DeleteObject(t.Context(), s.bucket, key, ObjectOptions{Versioned: true, VersionID: id, Expiration: ExpirationOptions{Expire: true}}) + return err +} + +func authorityPresent(t *testing.T, s *splitILM, pool int, key, id string) { + t.Helper() + if _, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}); err != nil { + t.Fatalf("pool %d lost version %s: %v", pool, id, err) + } +} + +// minorityRevision writes the versions old and new of key to pool 0, applies +// metadata to old, and leaves its last drive with the revision before, as a +// drive that missed the update does. It reports whether that stale revision +// sorts before the fifteen others. +func minorityRevision(t *testing.T, s *splitILM, key string, metadata map[string]string) (id string, staleFirst bool) { + t.Helper() + ctx := t.Context() + s.write(key, splitAt("old", 5, 0)) + s.write(key, splitAt("new", 3, 0)) + id = s.ids[key+" old"] + disks := s.z.serverPools[0].getHashedSet(key).getDisks() + last := disks[len(disks)-1] + stale, err := last.ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{}) + if err != nil { + t.Fatal(err) + } + setLifecycleCopyMetadata(t, s, key, id, 0, metadata, false) + if err := last.WriteMetadata(ctx, "", s.bucket, key, stale); err != nil { + t.Fatal(err) + } + return id, authorityHeader(t, last, s.bucket, key, id).sortsBefore(authorityHeader(t, disks[0], s.bucket, key, id)) +} + +// One drive missed a legal hold that the other fifteen committed. Its stale +// revision must not be selected, and must not keep the fifteen from +// deciding, whichever signature sorts first. +func TestPoolsLifecycleMinorityLockRevision(t *testing.T) { + authorityParity(t) + s := newLockedSplitILM(t, authorityNoncurrentRule) + ctx := t.Context() + for attempt := range 32 { + key := fmt.Sprintf("ncv-lock-%d", attempt) + stamp := UTCNow().Format(time.RFC3339Nano) + id, staleFirst := minorityRevision(t, s, key, poolLockMetadata("", "ON", stamp, stamp)) + versions, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key) + switch { + case err != nil: + t.Errorf("attempt %d (stale sorts first: %t): history: %v", attempt, staleFirst, err) + case len(versions) != 2: + t.Fatalf("attempt %d: merged %d versions", attempt, len(versions)) + case versions[1].UserDefined[authorityHoldKey] != "ON" || events[1].Action != lifecycle.NoneAction: + t.Errorf("attempt %d (stale sorts first: %t): merged hold=%q event=%s", attempt, staleFirst, versions[1].UserDefined[authorityHoldKey], events[1].Action) + } + if err := authorityExpireVersion(t, s, key, id); !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("attempt %d (stale sorts first: %t): expiry of a held version: %v", attempt, staleFirst, err) + } + oi, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}) + if err != nil || oi.UserDefined[authorityHoldKey] != "ON" { + t.Fatalf("held version changed: hold=%q err=%v", oi.UserDefined[authorityHoldKey], err) + } + if staleFirst { + return + } + } + t.Fatal("no attempt where the stale revision sorts first") +} + +// The same layout without protection: a drive missed an ordinary metadata +// update of a version that expires. The fifteen still decide its expiry. +func TestPoolsLifecycleMinorityStaleRevision(t *testing.T) { + authorityParity(t) + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + for attempt := range 32 { + key := fmt.Sprintf("ncv-stale-%d", attempt) + id, staleFirst := minorityRevision(t, s, key, map[string]string{"x-amz-meta-note": fmt.Sprint(attempt)}) + _, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key) + if err != nil || len(events) != 2 || events[1].Action != lifecycle.DeleteVersionAction { + t.Errorf("attempt %d (stale sorts first: %t): events=%v err=%v", attempt, staleFirst, events, err) + } + if err := authorityExpireVersion(t, s, key, id); err != nil { + t.Fatalf("attempt %d (stale sorts first: %t): expiry: %v", attempt, staleFirst, err) + } + for i, disk := range s.z.serverPools[0].getHashedSet(key).getDisks() { + if _, err := disk.ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{}); !errors.Is(err, errFileVersionNotFound) { + t.Fatalf("attempt %d: drive %d still holds the expired version: %v", attempt, i, err) + } + } + if staleFirst { + return + } + } + t.Fatal("no attempt where the stale revision sorts first") +} + +// authorityRevision encodes a 12+4 object version as drive index holds it, +// with the legal hold status hold. +func authorityRevision(t *testing.T, id [16]byte, modTime int64, index int, hold string) xlMetaV2ShallowVersion { + t.Helper() + v := xlMetaV2Version{Type: ObjectType, ObjectV2: &xlMetaV2Object{ + VersionID: id, DataDir: [16]byte{1}, ErasureAlgorithm: ReedSolomon, ErasureM: 12, ErasureN: 4, + ErasureBlockSize: blockSizeV2, ErasureIndex: index, ErasureDist: make([]uint8, 16), BitrotChecksumAlgo: HighwayHash, + PartNumbers: []int{1}, PartSizes: []int64{5}, Size: 5, ModTime: modTime, + MetaUser: map[string]string{authorityHoldKey: hold, "etag": "e", "content-type": "text/plain"}, + MetaSys: map[string][]byte{"x-minio-internal-a": {1}, "x-minio-internal-b": {2}, "x-minio-internal-c": {3}}, + }} + for i := range v.ObjectV2.ErasureDist { + v.ObjectV2.ErasureDist[i] = uint8(i + 1) + } + meta, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + return xlMetaV2ShallowVersion{header: v.header(), meta: meta} +} + +// The header Signature is a 32-bit fold of weak hashes, so two revisions can +// share a header. Revisions are told apart by their metadata: drives that +// hold byte-for-byte different metadata under one header are not one +// revision, and the merged history only carries the accepted metadata. +func TestPoolsLifecycleRevisionIdentity(t *testing.T) { + newID, oldID := [16]byte{2}, [16]byte{1} + history := func(off func(drive int) bool) [][]xlMetaV2ShallowVersion { + metas := make([][]xlMetaV2ShallowVersion, 16) + for i := range metas { + old := authorityRevision(t, oldID, 1, i+1, "ON") + if off(i) { + // The stale revision under the header of the held one. + header := old.header + old = authorityRevision(t, oldID, 1, i+1, "OFF") + old.header = header + } + metas[i] = []xlMetaV2ShallowVersion{authorityRevision(t, newID, 2, i+1, ""), old} + } + return metas + } + metaVs := slices.Repeat([]uint8{xlMetaVersion}, 16) + readQuorum := func(xlMetaV2ShallowVersion, uint8) int { return 12 } + + t.Run("revision-key", func(t *testing.T) { + on, onOther := authorityRevision(t, oldID, 1, 1, "ON"), authorityRevision(t, oldID, 1, 7, "ON") + off := authorityRevision(t, oldID, 1, 1, "OFF") + off.header = on.header + if versionRevision(on, xlMetaVersion) != versionRevision(onOther, xlMetaVersion) { + t.Error("one revision on two drives has two keys") + } + if versionRevision(on, xlMetaVersion) == versionRevision(off, xlMetaVersion) { + t.Error("different metadata under one header has one key") + } + }) + t.Run("stale-minority-first", func(t *testing.T) { + merged, err := authoritativeHistory(t.Context(), history(func(i int) bool { return i == 0 }), metaVs, xlMetaVersion, 9, readQuorum) + if err != nil || len(merged) != 2 { + t.Fatalf("history: err=%v versions=%d", err, len(merged)) + } + var v xlMetaV2Version + if _, err := v.unmarshalV(xlMetaVersion, merged[1].meta); err != nil { + t.Fatal(err) + } + if hold := v.ObjectV2.MetaUser[authorityHoldKey]; hold != "ON" { + t.Fatalf("merged history carries hold=%q", hold) + } + }) + t.Run("split-8-8", func(t *testing.T) { + if merged, err := authoritativeHistory(t.Context(), history(func(i int) bool { return i%2 == 0 }), metaVs, xlMetaVersion, 9, readQuorum); !errors.Is(err, errErasureReadQuorum) { + t.Fatalf("8/8 metadata split: %d versions, err=%v", len(merged), err) + } + }) + // The user and the system map are separate sections of the digest: an + // entry that moves between them changes what the evaluator reads. + t.Run("map-boundary", func(t *testing.T) { + for _, entry := range [][2]string{ + {authorityHoldKey, "ON"}, + {strings.ToLower(xhttp.AmzObjectLockRetainUntilDate), "2099-01-01T00:00:00Z"}, + {xhttp.AmzObjectTagging, "expire=yes"}, + } { + ver := authorityRevision(t, oldID, 1, 1, "") + var v xlMetaV2Version + if _, err := v.unmarshalV(xlMetaVersion, ver.meta); err != nil { + t.Fatal(err) + } + v.ObjectV2.MetaUser, v.ObjectV2.MetaSys = map[string]string{entry[0]: entry[1]}, nil + user, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + v.ObjectV2.MetaUser, v.ObjectV2.MetaSys = nil, map[string][]byte{entry[0]: []byte(entry[1])} + sys, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + if versionRevision(xlMetaV2ShallowVersion{header: ver.header, meta: user}, xlMetaVersion) == + versionRevision(xlMetaV2ShallowVersion{header: ver.header, meta: sys}, xlMetaVersion) { + t.Errorf("%s moved between the maps keeps the revision", entry[0]) + } + } + }) + t.Run("canceled", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := authoritativeHistory(ctx, history(func(int) bool { return false }), metaVs, xlMetaVersion, 9, readQuorum); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled history: %v", err) + } + }) +} + +// authorityMovedHold serves a drive whose metadata of version id carries the +// legal hold in the system map instead of the user map, under the header it +// had: the evaluator no longer reads the hold there. +type authorityMovedHold struct { + StorageAPI + bucket, key, id string +} + +func (d authorityMovedHold) ReadXL(ctx context.Context, bucket, key string, data bool) (RawFileInfo, error) { + rf, err := d.StorageAPI.ReadXL(ctx, bucket, key, data) + if err != nil || bucket != d.bucket || key != d.key { + return rf, err + } + var xl xlMetaV2 + if err = xl.LoadOrConvert(rf.Buf); err != nil { + return rf, err + } + for i := range xl.versions { + if uuid.UUID(xl.versions[i].header.VersionID).String() != d.id { + continue + } + var v xlMetaV2Version + if _, err = v.unmarshalV(xl.metaV, xl.versions[i].meta); err != nil { + return rf, err + } + v.ObjectV2.MetaSys[authorityHoldKey] = []byte(v.ObjectV2.MetaUser[authorityHoldKey]) + delete(v.ObjectV2.MetaUser, authorityHoldKey) + if xl.versions[i].meta, err = v.MarshalMsg(nil); err != nil { + return rf, err + } + } + rf.Buf, err = xl.AppendTo(nil) + return rf, err +} + +// One drive's metadata moves the hold of a held version to the system map +// under the same header. Its revision is not the held one, and the bulk +// expiry keeps the version. +func TestPoolsLifecycleMovedHoldKeepsVersion(t *testing.T) { + s := newLockedSplitILM(t, authorityNoncurrentRule) + ctx := t.Context() + const key = "ncv-moved-hold" + s.write(key, splitAt("old", 5, 0)) + s.write(key, splitAt("new", 3, 0)) + id := s.ids[key+" old"] + stamp := UTCNow().Format(time.RFC3339Nano) + setLifecycleCopyMetadata(t, s, key, id, 0, map[string]string{authorityHoldKey: "ON", ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: stamp}, false) + p := s.z.serverPools[0] + p.erasureDisksMu.Lock() + prior := p.erasureDisks + wrapped := make([][]StorageAPI, len(prior)) + for i, disks := range prior { + wrapped[i] = append([]StorageAPI(nil), disks...) + wrapped[i][0] = authorityMovedHold{StorageAPI: disks[0], bucket: s.bucket, key: key, id: id} + } + p.erasureDisks = wrapped + p.erasureDisksMu.Unlock() + t.Cleanup(func() { p.erasureDisksMu.Lock(); p.erasureDisks = prior; p.erasureDisksMu.Unlock() }) + + versions, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key) + if err != nil || versions[1].UserDefined[authorityHoldKey] != "ON" || events[1].Action != lifecycle.NoneAction { + t.Errorf("history: events=%v err=%v", events, err) + } + if err := authorityDeletePath(t, s, key, id, "bulk"); !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("bulk expiry of a held version: %v", err) + } + authorityPresent(t, s, 0, key, id) +} + +// A legacy version can carry several content checksums for each part. The +// map is encoded in iteration order, and must not change the revision of +// identical metadata; a different checksum does change it (control). +func TestPoolsLifecycleLegacyPartChecksums(t *testing.T) { + legacy := func(index int, sha256Value string) []byte { + m := getSampleXLMeta(1) + m.Stat.ModTime, m.Stat.Size, m.Parts[0].Size = time.Unix(1700000000, 0).UTC(), 20, 20 + m.Erasure.DataBlocks, m.Erasure.ParityBlocks, m.Erasure.Index = 12, 4, index + m.Erasure.Distribution = []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} + m.Parts[0].Checksums = map[string]string{"CRC32": "AAAAAA==", "CRC32C": "AAAAAQ==", "SHA1": "AAAAAg==", "SHA256": sha256Value} + buf, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return buf + } + revision := func(buf []byte) revisionKey { + var xl xlMetaV2 + if err := xl.LoadOrConvert(buf); err != nil { + t.Fatal(err) + } + return versionRevision(xl.versions[0], xl.metaV) + } + keys := make(map[revisionKey]bool) + for i := range 32 { + keys[revision(legacy(i%16+1, "AAAAAw=="))] = true + } + if len(keys) != 1 { + t.Fatalf("identical legacy metadata has %d revisions", len(keys)) + } + if revision(legacy(1, "AAAABA==")) == revision(legacy(1, "AAAAAw==")) { + t.Fatal("a different part checksum keeps the revision") + } + disks := make([]StorageAPI, 16) + for i := range disks { + disks[i] = authorityRawDisk{buf: legacy(i+1, "AAAAAw==")} + } + er := erasureObjects{getDisks: func() []StorageAPI { return disks }, defaultParityCount: 4} + if versions, err := er.objectVersions(t.Context(), "bucket", "key"); err != nil || len(versions) != 1 { + t.Fatalf("healthy legacy history: %d versions, err=%v", len(versions), err) + } +} + +// authorityRawDisk serves the same xl.meta for every key. +type authorityRawDisk struct { + StorageAPI + buf []byte +} + +func (d authorityRawDisk) ReadXL(context.Context, string, string, bool) (RawFileInfo, error) { + return RawFileInfo{Buf: d.buf}, nil +} + +// A legacy (xl.json) version stores the bitrot checksum of each drive's own +// shard. Healthy drives differ there and still hold one revision. +func TestPoolsLifecycleLegacyHistory(t *testing.T) { + for _, distinct := range []bool{false, true} { + name := "same-checksums" // control + if distinct { + name = "per-shard-checksums" + } + t.Run(name, func(t *testing.T) { + disks := make([]StorageAPI, 16) + for i := range disks { + m := getSampleXLMeta(1) + m.Stat.ModTime, m.Stat.Size, m.Parts[0].Size = time.Unix(1700000000, 0).UTC(), 20, 20 + m.Erasure.DataBlocks, m.Erasure.ParityBlocks, m.Erasure.Index = 12, 4, i+1 + m.Erasure.Distribution = []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} + shard := []byte{1, 1} + if distinct { + shard[0] = byte(i + 1) + } + sum := sha256.Sum256(shard) + m.Erasure.Checksums[0] = ChecksumInfo{PartNumber: 1, Algorithm: SHA256, Hash: sum[:]} + buf, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + disks[i] = authorityRawDisk{buf: buf} + } + er := erasureObjects{getDisks: func() []StorageAPI { return disks }, defaultParityCount: 4} + if versions, err := er.objectVersions(t.Context(), "bucket", "key"); err != nil || len(versions) != 1 { + t.Fatalf("healthy legacy history: %d versions, err=%v", len(versions), err) + } + }) + } +} + +// A fresh version committed to 15 of 16 drives, with eight other drives +// unreadable: only seven answering drives show it. +func TestPoolsLifecyclePartialHistory(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "eav-partial-history" + s.write(key, splitAt("old", 5, 0, 1)) + s.write(key, splitAt("fresh", 0, 1)) + id := s.ids[key+" fresh"] + removeAuthorityCopies(t, s, 1, key, id, authorityDrives(0, 1)) + wrapAuthorityDisks(t, s, 1, key, true, false, authorityDrives(1, 9)) + + if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) { + t.Errorf("partial history: %v", err) + } + _, err := s.z.DeleteObject(ctx, s.bucket, key, ObjectOptions{Versioned: true, DeletePrefix: true, DeletePrefixObject: true, Expiration: ExpirationOptions{Expire: true}}) + if !isErrWriteQuorum(err) { + t.Errorf("DeleteAll on a partial history: %v", err) + } + authorityPresent(t, s, 1, key, id) +} + +// Eight drives hold one revision of a version and eight another. Neither is +// authoritative, so neither may decide the expiry. +func TestPoolsLifecycleConflictingRevisions(t *testing.T) { + s := newLockedSplitILM(t, authorityNoncurrentRule) + ctx := t.Context() + const key = "ncv-split-revision" + s.write(key, splitAt("old", 5, 0)) + s.write(key, splitAt("new", 3, 0)) + id := s.ids[key+" old"] + disks := s.z.serverPools[0].getHashedSet(key).getDisks() + half := len(disks) / 2 + stale := make([]FileInfo, len(disks)) + for i := half; i < len(disks); i++ { + fi, err := disks[i].ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{}) + if err != nil { + t.Fatal(err) + } + stale[i] = fi + } + stamp := UTCNow().Format(time.RFC3339Nano) + setLifecycleCopyMetadata(t, s, key, id, 0, poolLockMetadata("", "ON", stamp, stamp), false) + for i := half; i < len(disks); i++ { + if err := disks[i].WriteMetadata(ctx, "", s.bucket, key, stale[i]); err != nil { + t.Fatal(err) + } + } + + if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) { + t.Errorf("8/8 revisions: %v", err) + } + if err := authorityExpireVersion(t, s, key, id); !isErrWriteQuorum(err) { + t.Errorf("expiry on 8/8 revisions: %v", err) + } + for i, disk := range disks { + if _, err := disk.ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{}); err != nil { + t.Fatalf("drive %d lost the version: %v", i, err) + } + } +} + +// Every drive answers and the old version is complete, but a fresh version +// that some drives show reached too few of them to be authoritative. +func TestPoolsLifecycleMissingVersion(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "eav-missing-version" + s.write(key, splitAt("old", 5, 0, 1)) + s.write(key, splitAt("fresh", 0, 1)) + id := s.ids[key+" fresh"] + removeAuthorityCopies(t, s, 1, key, id, authorityDrives(0, 9)) + + if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) { + t.Errorf("history without an observed version: %v", err) + } + _, err := s.z.DeleteObject(ctx, s.bucket, key, ObjectOptions{Versioned: true, DeletePrefix: true, DeletePrefixObject: true, Expiration: ExpirationOptions{Expire: true}}) + if !isErrWriteQuorum(err) { + t.Errorf("DeleteAll: %v", err) + } + disks := s.z.serverPools[1].getHashedSet(key).getDisks() + for i := 9; i < len(disks); i++ { + if _, err := disks[i].ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{}); err != nil { + t.Fatalf("drive %d lost the fresh version: %v", i, err) + } + } +} + +// The completeness checks also hold where a set has fewer answering drives +// than a read quorum or no version reaches one. +func TestPoolsLifecycleHistoryEarlyReturns(t *testing.T) { + deleteAll := ObjectOptions{Versioned: true, DeletePrefix: true, DeletePrefixObject: true, Expiration: ExpirationOptions{Expire: true}} + + // Seven drives hold the key, the other nine do not have it at all. + t.Run("minority-copy-only", func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "eav-minority-only" + s.write(key, splitAt("old", 5, 0)) + s.write(key, splitAt("fresh", 0, 1)) + id := s.ids[key+" fresh"] + removeAuthorityCopies(t, s, 1, key, id, authorityDrives(0, 9)) + disks := s.z.serverPools[1].getHashedSet(key).getDisks() + if _, err := disks[0].ReadXL(ctx, s.bucket, key, false); !errors.Is(err, errFileNotFound) { + t.Fatalf("fixture: drive 0 still holds the key: %v", err) + } + if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) { + t.Errorf("minority copy: %v", err) + } + if _, err := s.z.DeleteObject(ctx, s.bucket, key, deleteAll); !isErrWriteQuorum(err) { + t.Errorf("DeleteAll: %v", err) + } + for i := 9; i < len(disks); i++ { + if _, err := disks[i].ReadVersion(ctx, "", s.bucket, key, id, ReadOptions{}); err != nil { + t.Fatalf("drive %d lost the fresh version: %v", i, err) + } + } + }) + + // Control: two versions on seven drives each, none reaches a quorum. + t.Run("disjoint-minorities", func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "eav-disjoint" + s.write(key, splitAt("old", 5, 0)) + s.write(key, splitAt("a", 2, 1)) + s.write(key, splitAt("b", 1, 1)) + removeAuthorityCopies(t, s, 1, key, s.ids[key+" a"], authorityDrives(7, 16)) + removeAuthorityCopies(t, s, 1, key, s.ids[key+" b"], func(i int) bool { return i < 7 || i >= 14 }) + if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) { + t.Fatalf("disjoint minorities: %v", err) + } + if _, err := s.z.DeleteObject(ctx, s.bucket, key, deleteAll); !isErrWriteQuorum(err) { + t.Fatalf("DeleteAll: %v", err) + } + }) +} + +// A set without an unknown drive less than half of its drives still tells +// its history; half of them unknown could hide a version. +func TestPoolsLifecycleUnknownDriveThreshold(t *testing.T) { + for _, tc := range []struct { + name string + pool int // the pool with unreadable drives + unknown int + refused bool + }{ + {"history/7", 0, 7, false}, + {"history/8", 0, 8, true}, + {"absent/7", 1, 7, false}, + {"absent/8", 1, 8, true}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "ncv-threshold" + // Empty versions need only a simple majority of the drives, so + // seven unknown drives leave enough copies. + var ids []string + for _, days := range []int{5, 3} { + id := mustGetUUID() + putConsistencyObject(t, s.z, s.bucket, key, 0, "", ObjectOptions{Versioned: true, VersionID: id, MTime: UTCNow().AddDate(0, 0, -days)}) + ids = append(ids, id) + } + wrapAuthorityDisks(t, s, tc.pool, key, true, false, authorityDrives(0, tc.unknown)) + err := authorityExpireVersion(t, s, key, ids[0]) + if tc.refused { + if !isErrWriteQuorum(err) { + t.Errorf("%d unknown drives: %v", tc.unknown, err) + } + authorityPresent(t, s, 0, key, ids[0]) + return + } + if err != nil { + t.Fatalf("%d unknown drives: %v", tc.unknown, err) + } + if _, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: ids[0]}); !isErrVersionNotFound(err) { + t.Fatalf("expired version remains: %v", err) + } + }) + } +} + +// Control: free versions are tier GC records. Residue of one on a few drives +// must not block an expiry. +func TestPoolsLifecycleFreeVersionResidue(t *testing.T) { + for _, tc := range []struct { + name string + residue int // the pool with the residue + }{ + {"beside-history", 0}, + {"alone", 1}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "ncv-free-residue" + s.write(key, splitTiered("gone", 6, tc.residue)) + s.write(key, splitAt("old", 5, 0)) + s.write(key, splitAt("new", 3, 0)) + gone := s.ids[key+" gone"] + disks := s.z.serverPools[tc.residue].getHashedSet(key).getDisks() + fvID := mustGetUUID() + for i, disk := range disks { + fi := FileInfo{VersionID: gone} + if i < 3 { + fi.SetTierFreeVersionID(fvID) + } else { + fi.SetSkipTierFreeVersion() + } + if err := disk.DeleteVersion(ctx, s.bucket, key, fi, false, DeleteOptions{}); err != nil { + t.Fatal(err) + } + } + if h := authorityHeader(t, disks[0], s.bucket, key, fvID); !h.FreeVersion() { + t.Fatalf("fixture lacks a free version: %v", h) + } + versions, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key) + if err != nil || len(versions) != 2 || events[1].Action != lifecycle.DeleteVersionAction { + t.Fatalf("history with free-version residue: %d versions, events=%v, err=%v", len(versions), events, err) + } + if err := authorityExpireVersion(t, s, key, s.ids[key+" old"]); err != nil { + t.Fatalf("expiry beside free-version residue: %v", err) + } + }) + } +} + +// An interrupted 12+4 write left nine matching copies of a newer version: +// a majority of the drives, but less than its read quorum. It must not make +// the last committed version noncurrent. +func TestPoolsLifecycleReducedParityPartialVersion(t *testing.T) { + for _, path := range []string{"single", "bulk"} { + t.Run(path, func(t *testing.T) { + authorityParity(t) + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "ncv-partial-write" + s.write(key, splitAt("committed", 5, 0)) + s.write(key, splitAt("partial", 3, 0)) + committed, partial := s.ids[key+" committed"], s.ids[key+" partial"] + oi, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: partial}) + if err != nil || oi.DataBlocks != 12 || oi.ParityBlocks != 4 { + t.Fatalf("fixture is not 12+4: data=%d parity=%d err=%v", oi.DataBlocks, oi.ParityBlocks, err) + } + removeAuthorityCopies(t, s, 0, key, partial, authorityDrives(9, 16)) + + if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !isErrReadQuorum(err) { + t.Errorf("history with a partial version: %v", err) + } + if err := authorityDeletePath(t, s, key, committed, path); !isErrWriteQuorum(err) { + t.Errorf("expiry of the last committed version: %v", err) + } + authorityPresent(t, s, 0, key, committed) + }) + } +} + +// Control: two drives do not answer, every version still has its quorum. +func TestPoolsLifecycleDegradedHistory(t *testing.T) { + authorityParity(t) + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "ncv-degraded" + s.write(key, splitAt("old", 5, 0)) + s.write(key, splitAt("new", 3, 0)) + id := s.ids[key+" old"] + wrapAuthorityDisks(t, s, 0, key, true, false, authorityDrives(0, 2)) + if err := authorityExpireVersion(t, s, key, id); err != nil { + t.Fatalf("degraded expiry: %v", err) + } + if _, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) { + t.Fatalf("expired version remains: %v", err) + } +} + +// authorityPoolLock sets the Object Lock metadata of the copy of version id +// in pool. +func authorityPoolLock(t *testing.T, s *splitILM, key, id string, pool int, metadata map[string]string) { + t.Helper() + metadata = maps.Clone(metadata) + if metadata[strings.ToLower(xhttp.AmzObjectLockMode)] != "" { + metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = UTCNow().AddDate(1, 0, 0).Format(time.RFC3339) + } else { + metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = "" + } + setLifecycleCopyMetadata(t, s, key, id, pool, metadata, false) +} + +// authorityDeletePath expires version id by the single path or by the bulk +// path, which takes the ordinary or the tiered route as the version is. +func authorityDeletePath(t *testing.T, s *splitILM, key, id, path string) error { + t.Helper() + if path == "single" { + return authorityExpireVersion(t, s, key, id) + } + _, errs := s.z.DeleteObjects(t.Context(), s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: id}}}, ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}}) + return errs[0] +} + +var authorityPaths = []string{"single", "bulk", "tiered-bulk"} + +// The copies of a version in two pools carry protection revisions that do not +// tell which is the newest. Refuse instead of guessing. +func TestPoolsLifecycleAmbiguousProtection(t *testing.T) { + stamp := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano) + old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano) + for _, tc := range []struct { + name string + copies [2]map[string]string + }{ + {"hold-equal-stamps", [2]map[string]string{poolLockMetadata("", "OFF", old, stamp), poolLockMetadata("", "ON", old, stamp)}}, + {"hold-missing-stamp", [2]map[string]string{poolLockMetadata("", "ON", old, ""), poolLockMetadata("", "OFF", old, stamp)}}, + {"hold-malformed-stamp", [2]map[string]string{poolLockMetadata("", "ON", old, "yesterday"), poolLockMetadata("", "OFF", old, stamp)}}, + {"retention-equal-stamps", [2]map[string]string{poolLockMetadata("", "OFF", stamp, old), poolLockMetadata("GOVERNANCE", "OFF", stamp, old)}}, + } { + for _, path := range authorityPaths { + t.Run(tc.name+"/"+path, func(t *testing.T) { + s := newLockedSplitILM(t, authorityNoncurrentRule) + const key = "ncv-ambiguous" + v := splitAt("old", 5, 0, 1) + v.tiered = path == "tiered-bulk" + s.write(key, v) + s.write(key, splitAt("new", 3, 0, 1)) + id := s.ids[key+" old"] + for pool, metadata := range tc.copies { + authorityPoolLock(t, s, key, id, pool, metadata) + } + if _, _, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key); !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("ambiguous protection: %v", err) + } + if err := authorityDeletePath(t, s, key, id, path); !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("expiry with ambiguous protection: %v", err) + } + for pool := range s.z.serverPools { + authorityPresent(t, s, pool, key, id) + } + }) + } + } +} + +// Protection stored under canonical-case keys counts as the evaluator reads +// it, in the comparison of copies, in the merged version and in the set. +func TestPoolsLifecycleCanonicalProtection(t *testing.T) { + older := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano) + recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano) + until := UTCNow().AddDate(1, 0, 0).Format(time.RFC3339) + hold := func(status, stamp string) map[string]string { + return map[string]string{xhttp.AmzObjectLockLegalHold: status, ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: stamp} + } + retention := func(mode, stamp string) map[string]string { + m := map[string]string{ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp: stamp} + if mode != "" { + m[xhttp.AmzObjectLockMode], m[xhttp.AmzObjectLockRetainUntilDate] = mode, until + } + return m + } + for _, tc := range []struct { + name string + copies [2]map[string]string + expires bool + }{ + {"hold-equal-stamps", [2]map[string]string{hold("OFF", recent), hold("ON", recent)}, false}, + {"hold-newer", [2]map[string]string{hold("OFF", older), hold("ON", recent)}, false}, + {"retention-equal-stamps", [2]map[string]string{retention("", recent), retention("COMPLIANCE", recent)}, false}, + {"retention-newer", [2]map[string]string{retention("", older), retention("GOVERNANCE", recent)}, false}, + {"hold-released", [2]map[string]string{hold("ON", older), hold("OFF", recent)}, true}, // control + } { + for _, path := range []string{"single", "bulk"} { + t.Run(tc.name+"/"+path, func(t *testing.T) { + s := newLockedSplitILM(t, authorityNoncurrentRule) + const key = "ncv-canonical" + id, mtime := mustGetUUID(), UTCNow().AddDate(0, 0, -5) + for pool, metadata := range tc.copies { + putConsistencyObject(t, s.z, s.bucket, key, pool, "old", ObjectOptions{Versioned: true, VersionID: id, MTime: mtime, UserDefined: maps.Clone(metadata)}) + } + s.write(key, splitAt("new", 3, 0, 1)) + _, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key) + if !tc.expires && err == nil && events[1].Action != lifecycle.NoneAction { + t.Errorf("protected version approved: %s", events[1].Action) + } + err = authorityDeletePath(t, s, key, id, path) + if tc.expires { + if err != nil { + t.Fatalf("released version: %v", err) + } + if _, err := s.z.serverPools[0].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) { + t.Fatalf("released version remains: %v", err) + } + return + } + if !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("expiry of a protected version: %v", err) + } + for pool := range s.z.serverPools { + authorityPresent(t, s, pool, key, id) + } + }) + } + } +} + +// Control: a strictly newer release in another pool still expires the +// version by the ordinary bulk path too, beside the single and tiered bulk +// paths of TestPoolsLifecycleDuplicateLockMetadata. +func TestPoolsLifecycleProtectionReleaseBulk(t *testing.T) { + old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano) + recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano) + for _, tc := range []struct { + name string + copies [2]map[string]string + }{ + {"hold-released", [2]map[string]string{poolLockMetadata("", "ON", old, old), poolLockMetadata("", "OFF", old, recent)}}, + {"retention-released", [2]map[string]string{poolLockMetadata("GOVERNANCE", "OFF", old, old), poolLockMetadata("", "OFF", recent, old)}}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newLockedSplitILM(t, authorityNoncurrentRule) + const key = "ncv-released" + s.write(key, splitAt("old", 5, 0, 1)) + s.write(key, splitAt("new", 3, 0, 1)) + id := s.ids[key+" old"] + for pool, metadata := range tc.copies { + authorityPoolLock(t, s, key, id, pool, metadata) + } + if err := authorityDeletePath(t, s, key, id, "bulk"); err != nil { + t.Fatalf("released version: %v", err) + } + for pool := range s.z.serverPools { + if _, err := s.z.serverPools[pool].GetObjectInfo(t.Context(), s.bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) { + t.Errorf("pool %d still holds the released version: %v", pool, err) + } + } + }) + } +} + +// A set takes an all-pools approval only with the protection it was approved +// with. Protection that the set reads and the approval does not strictly +// supersede keeps the version. +func TestPoolsLifecycleCheckedExpirySafeguard(t *testing.T) { + old := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano) + recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano) + for _, tc := range []struct { + name string + local map[string]string + approved objectLockState + deleted bool + }{ + {"hold-newer-locally", poolLockMetadata("", "ON", old, recent), objectLockState{legalHold: "OFF", legalHoldTimestamp: old}, false}, + {"hold-missing-local-stamp", poolLockMetadata("", "ON", old, ""), objectLockState{legalHold: "OFF", legalHoldTimestamp: recent}, false}, + {"hold-equal-stamps", poolLockMetadata("", "ON", old, recent), objectLockState{legalHold: "OFF", legalHoldTimestamp: recent}, false}, + {"hold-missing-approved-stamp", poolLockMetadata("", "ON", old, old), objectLockState{legalHold: "OFF"}, false}, + {"retention-newer-locally", poolLockMetadata("GOVERNANCE", "OFF", recent, old), objectLockState{retentionTimestamp: old, legalHold: "OFF", legalHoldTimestamp: old}, false}, + // Controls: the approval strictly supersedes the local protection. + {"hold-released", poolLockMetadata("", "ON", old, old), objectLockState{legalHold: "OFF", legalHoldTimestamp: recent}, true}, + {"retention-released", poolLockMetadata("GOVERNANCE", "OFF", old, old), objectLockState{retentionTimestamp: recent, legalHold: "OFF", legalHoldTimestamp: old}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newLockedSplitILM(t, authorityNoncurrentRule) + ctx := t.Context() + const key = "ncv-safeguard" + s.write(key, splitAt("old", 5, 0)) + s.write(key, splitAt("new", 3, 0)) + id := s.ids[key+" old"] + authorityPoolLock(t, s, key, id, 0, tc.local) + opts := ObjectOptions{Versioned: true, VersionID: id, Expiration: ExpirationOptions{Expire: true, CheckedAllPools: true}} + opts.Expiration.ApprovedProtection = tc.approved + _, err := s.z.serverPools[0].DeleteObject(ctx, s.bucket, key, opts) + if !tc.deleted { + if !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("protected version: %v", err) + } + authorityPresent(t, s, 0, key, id) + return + } + if err != nil { + t.Fatalf("released version: %v", err) + } + if _, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}); !isErrVersionNotFound(err) { + t.Fatalf("released version remains: %v", err) + } + }) + } +} + +// A pending purge vetoes an approved expiry in the set, also when the read +// returns it with MethodNotAllowed, as for a delete marker. +func TestPoolsLifecycleCheckedExpiryPendingPurge(t *testing.T) { + for _, marker := range []bool{false, true} { + name := "version" + if marker { + name = "delete-marker" + } + t.Run(name, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "ncv-pending-purge" + label := "old" + s.write(key, splitAt("first", 6, 0)) + if marker { + label = "dm" + s.write(key, splitMarker(label, 5, 0)) + } else { + s.write(key, splitAt(label, 5, 0)) + } + s.write(key, splitAt("new", 3, 0)) + id := s.ids[key+" "+label] + target := newILMTestTarget(t) + setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket) + oi, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}) + if oi.DeleteMarker != marker || (err != nil) != marker || marker && !isErrMethodNotAllowed(err) { + t.Fatalf("fixture: marker=%t err=%v", oi.DeleteMarker, err) + } + _, err = s.z.serverPools[0].DeleteObject(ctx, s.bucket, key, ObjectOptions{ + VersionID: id, Versioned: true, MTime: oi.ModTime, + DeleteReplication: ReplicationState{ + VersionPurgeStatusInternal: target.arn + "=PENDING;", + PurgeTargets: map[string]VersionPurgeStatusType{target.arn: replication.VersionPurgePending}, + }, + }) + if err != nil { + t.Fatal(err) + } + pending, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}) + if !isErrMethodNotAllowed(err) || pending.VersionPurgeStatus != replication.VersionPurgePending { + t.Fatalf("fixture: purge=%q err=%v", pending.VersionPurgeStatus, err) + } + _, err = s.z.serverPools[0].DeleteObject(ctx, s.bucket, key, ObjectOptions{Versioned: true, VersionID: id, Expiration: ExpirationOptions{Expire: true, CheckedAllPools: true}}) + if !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("approved expiry of a pending purge: %v", err) + } + after, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}) + if !isErrMethodNotAllowed(err) || after.VersionPurgeStatus != replication.VersionPurgePending { + t.Fatalf("pending purge changed: purge=%q err=%v", after.VersionPurgeStatus, err) + } + }) + } +} + +// The copy of a version in another pool carries a pending purge that the +// merge would drop with the replication state of the first copy. +func TestPoolsLifecycleDuplicatePendingPurge(t *testing.T) { + for _, path := range []string{"single", "bulk"} { + t.Run(path, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "ncv-pending-copy" + s.write(key, splitAt("old", 5, 0, 1)) + s.write(key, splitAt("new", 3, 0, 1)) + id := s.ids[key+" old"] + target := newILMTestTarget(t) + setILMTestReplicationConfig(t, s.bucket, target.arn, target.bucket) + oi, err := s.z.serverPools[1].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}) + if err != nil { + t.Fatal(err) + } + _, err = s.z.serverPools[1].DeleteObject(ctx, s.bucket, key, ObjectOptions{ + VersionID: id, Versioned: true, MTime: oi.ModTime, + DeleteReplication: ReplicationState{ + VersionPurgeStatusInternal: target.arn + "=PENDING;", + PurgeTargets: map[string]VersionPurgeStatusType{target.arn: replication.VersionPurgePending}, + }, + }) + if err != nil { + t.Fatal(err) + } + files, err := s.z.serverPools[1].getHashedSet(key).objectVersions(ctx, s.bucket, key) + if err != nil || len(files) != 2 || files[1].ToObjectInfo(s.bucket, key, true).VersionPurgeStatus != replication.VersionPurgePending { + t.Fatalf("fixture lacks the pending copy: %d versions, err=%v", len(files), err) + } + if _, _, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key); !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("copies with a different purge state: %v", err) + } + if err := authorityDeletePath(t, s, key, id, path); !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("expiry beside a pending purge: %v", err) + } + stored, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}) + if err != nil || stored.VersionPurgeStatus != "" { + t.Fatalf("first copy changed: purge=%q err=%v", stored.VersionPurgeStatus, err) + } + if _, err := s.z.serverPools[1].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}); !isErrMethodNotAllowed(err) { + t.Fatalf("pending copy changed: %v", err) + } + }) + } +} + +// Copies of a version compare their per-target replication state as parsed: +// the order of the serialized targets does not count, a target does. +func TestPoolsLifecycleDuplicateReplicationTargets(t *testing.T) { + for _, tc := range []struct { + name string + statuses [2]string + refused bool + }{ + {"reordered", [2]string{"arn:a=COMPLETED;arn:b=COMPLETED;", "arn:b=COMPLETED;arn:a=COMPLETED;"}, false}, // control + {"other-targets", [2]string{"arn:a=COMPLETED;", "arn:a=COMPLETED;arn:b=COMPLETED;"}, true}, + {"other-status", [2]string{"arn:a=COMPLETED;arn:b=COMPLETED;", "arn:a=COMPLETED;arn:b=FAILED;"}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "ncv-replicated-copies" + id, mtime := mustGetUUID(), UTCNow().AddDate(0, 0, -5) + for pool, status := range tc.statuses { + putConsistencyObject(t, s.z, s.bucket, key, pool, "old", ObjectOptions{ + Versioned: true, VersionID: id, MTime: mtime, + UserDefined: map[string]string{ReservedMetadataPrefixLower + ReplicationStatus: status}, + }) + } + s.write(key, splitAt("new", 3, 0, 1)) + for pool, status := range tc.statuses { + oi, err := s.z.serverPools[pool].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}) + if err != nil || oi.ReplicationStatusInternal != status { + t.Fatalf("fixture pool %d: status=%q err=%v", pool, oi.ReplicationStatusInternal, err) + } + } + _, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key) + if tc.refused { + if !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("copies with other replication targets: %v", err) + } + if err := authorityExpireVersion(t, s, key, id); !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("expiry: %v", err) + } + authorityPresent(t, s, 0, key, id) + return + } + if err != nil || events[1].Action != lifecycle.DeleteVersionAction { + t.Fatalf("equivalent copies: events=%v err=%v", events, err) + } + if err := authorityExpireVersion(t, s, key, id); err != nil { + t.Fatalf("expiry of equivalent copies: %v", err) + } + }) + } +} + +// Null versions written in two pools of a suspended bucket share a version +// ID without being one write. Copies with another ModTime are older writes: +// the newest decides, unless an older one is still protected. Copies with +// the same ModTime are duplicates and must still agree. +func TestPoolsLifecycleNullGenerations(t *testing.T) { + hold := func(status, stamp string) map[string]string { + return map[string]string{authorityHoldKey: status, ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: stamp} + } + older := UTCNow().Add(-2 * time.Hour).Format(time.RFC3339Nano) + recent := UTCNow().Add(-time.Hour).Format(time.RFC3339Nano) + for _, tc := range []struct { + name string + lock bool + old, new map[string]string + sameModTime bool + wantNewest bool // evaluated on the newer write, else refused + replicateOld bool + }{ + {"older-unprotected", false, hold("OFF", ""), nil, false, true, true}, + {"older-held", true, hold("ON", older), hold("OFF", recent), false, false, false}, + {"same-modtime-duplicates", false, hold("OFF", ""), nil, true, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + s := &splitILM{migrationILM: newMigrationILM(t, versioning.Suspended, tc.lock, splitILMRules), ids: map[string]string{}, bodies: map[string]string{}} + ctx := t.Context() + const key = "exp-null-generations" + oldTime, newTime := UTCNow().AddDate(0, 0, -5), UTCNow().AddDate(0, 0, -3) + if tc.sameModTime { + newTime = oldTime + } + oldMeta := maps.Clone(tc.old) + if tc.replicateOld { + oldMeta[ReservedMetadataPrefixLower+ReplicationStatus] = "arn:a=COMPLETED;" + } + putConsistencyObject(t, s.z, s.bucket, key, 0, "old", ObjectOptions{VersionSuspended: true, MTime: oldTime, UserDefined: oldMeta}) + newest := putConsistencyObject(t, s.z, s.bucket, key, 1, "new", ObjectOptions{VersionSuspended: true, MTime: newTime, UserDefined: maps.Clone(tc.new)}) + + versions, events, err := s.z.lifecycleEventsAllPools(ctx, s.bucket, key) + opts := ObjectOptions{VersionSuspended: true, Expiration: ExpirationOptions{Expire: true}} + if !tc.wantNewest { + if !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("history: %d versions, events=%v, err=%v", len(versions), events, err) + } + if _, err := s.z.DeleteObject(ctx, s.bucket, key, opts); !errors.Is(err, errLifecycleNotApplicable) { + t.Errorf("expiry: %v", err) + } + for pool := range s.z.serverPools { + oi, err := s.z.serverPools[pool].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: nullVersionID}) + if err != nil || oi.DeleteMarker { + t.Fatalf("pool %d lost its null version: marker=%t err=%v", pool, oi.DeleteMarker, err) + } + } + return + } + if err != nil || len(versions) != 1 || !versions[0].ModTime.Equal(newest.ModTime) || events[0].Action != lifecycle.DeleteAction { + t.Fatalf("history: %d versions, events=%v, err=%v", len(versions), events, err) + } + if _, err := s.z.DeleteObject(ctx, s.bucket, key, opts); err != nil { + t.Fatalf("expiry of the newest null write: %v", err) + } + }) + } +} + +// authoritySideEffects collects the version IDs that removal notifications +// and ILM expiry audit traces report. +func authoritySideEffects(t *testing.T, run func()) (notified, traced []string) { + t.Helper() + done := make(chan struct{}) + defer close(done) + events := make(chan event.Event, 16) + if err := globalHTTPListen.Subscribe(pubsub.MaskFromMaskable(event.ObjectRemovedDelete), events, done, nil); err != nil { + t.Fatal(err) + } + traces := make(chan madmin.TraceInfo, 16) + if err := globalTrace.Subscribe(madmin.TraceILM, traces, done, nil); err != nil { + t.Fatal(err) + } + run() + // Both publishers deliver synchronously, so everything sent is buffered. + for { + select { + case ev := <-events: + notified = append(notified, ev.S3.Object.VersionID) + case tr := <-traces: + if tr.FuncName == ILMExpiry { + traced = append(traced, tr.Custom["version-id"]) + } + default: + slices.Sort(notified) + slices.Sort(traced) + return notified, traced + } + } +} + +// A batch expiry that another pool cannot read is not a removal. +func TestPoolsLifecycleBatchQuorumNotAnnounced(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + const key = "ncv-quorum-event" + s.write(key, splitAt("old", 5, 0)) + s.write(key, splitAt("new", 3, 0)) + id := s.ids[key+" old"] + lifecycleReadDisks(t, s, key, 1, errDiskNotFound) + result := &lifecycleBatchResult{ObjectLayer: s.z} + notified, traced := authoritySideEffects(t, func() { + deleteObjectVersions(t.Context(), result, s.bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: id}}}, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}}) + }) + if !isErrWriteQuorum(result.errs[0]) { + t.Fatalf("fixture result: %v", result.errs[0]) + } + authorityPresent(t, s, 0, key, id) + if len(notified) != 0 || len(traced) != 0 { + t.Fatalf("unapplied expiry announced: notifications=%v traces=%v", notified, traced) + } +} + +// Each entry of a batch gets the side effects of its own result: a completed +// or already absent delete is announced and audited, a refused or failed one +// is not. +func TestPoolsLifecycleBatchSideEffects(t *testing.T) { + for _, path := range []string{"single-pool", "bulk", "tiered-bulk"} { + t.Run(path, func(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + const key, failing = "ncv-side-effects", "ncv-side-effects-failing" + for _, k := range []string{key, failing} { + v := splitAt("old", 5, 0) + v.tiered = path == "tiered-bulk" + s.write(k, v) + s.write(k, splitAt("new", 3, 0)) + } + if path == "single-pool" { + prior := s.z.serverPools + t.Cleanup(func() { s.z.serverPools = prior }) + s.z.serverPools = s.z.serverPools[:1] + } + wrapAuthorityDisks(t, s, 0, failing, false, true, authorityDrives(0, 9)) + success, absent := s.ids[key+" old"], mustGetUUID() + entries := []ObjectToDelete{ + {ObjectV: ObjectV{ObjectName: key, VersionID: success}}, + {ObjectV: ObjectV{ObjectName: failing, VersionID: s.ids[failing+" old"]}}, + {ObjectV: ObjectV{ObjectName: key, VersionID: absent}}, + } + if path != "single-pool" { + // A single pool has no all-pools check to refuse a bulk entry. + entries = append(entries, ObjectToDelete{ObjectV: ObjectV{ObjectName: key, VersionID: s.ids[key+" new"]}}) + } + evts := make([]lifecycle.Event, len(entries)) + for i := range evts { + evts[i] = lifecycle.Event{Action: lifecycle.DeleteVersionAction} + } + result := &lifecycleBatchResult{ObjectLayer: s.z} + notified, traced := authoritySideEffects(t, func() { + deleteObjectVersions(t.Context(), result, s.bucket, entries, evts) + }) + if result.errs[0] != nil || !isErrWriteQuorum(result.errs[1]) || + result.errs[2] != nil && !isErrVersionNotFound(result.errs[2]) || + len(entries) > 3 && !errors.Is(result.errs[3], errLifecycleNotApplicable) { + t.Fatalf("fixture results: %v", result.errs) + } + want := []string{success, absent} + slices.Sort(want) + if !slices.Equal(notified, want) { + t.Errorf("removal notifications for %v, want %v", notified, want) + } + if !slices.Equal(traced, want) { + t.Errorf("ILM expiry audit for %v, want %v", traced, want) + } + }) + } +} + +// A single pool keeps the explicit refusal rather than a not-found result. +func TestPoolsLifecycleSinglePoolRefusal(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + s.write("exp-new", splitAt("fresh", 0, 0)) + prior := s.z.serverPools + t.Cleanup(func() { s.z.serverPools = prior }) + s.z.serverPools = s.z.serverPools[:1] + _, err := s.z.DeleteObject(t.Context(), s.bucket, "exp-new", ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}}) + if !errors.Is(err, errLifecycleNotApplicable) || isErrObjectNotFound(err) || isErrVersionNotFound(err) { + t.Fatalf("single-pool stale expiry: %v", err) + } +} + +// Migration leaves a version to the logical expiry. When that expiry is +// refused at retirement while the source still holds the version, the source +// is kept: a refusal is never absence. Decommission and rebalance both +// retire through removeMovedVersions. +func TestPoolsMigrationLifecycleRefusedExpiryKeepsSource(t *testing.T) { + s := newSplitILM(t, versioning.Enabled, splitILMRules) + ctx := t.Context() + const key = "exp-moved" + s.write(key, splitAt("old", 5, 0, 1)) + s.write(key, splitAt("new", 0.01, 1)) + id := s.ids[key+" old"] + source, err := s.z.serverPools[0].GetObjectInfo(ctx, s.bucket, key, ObjectOptions{VersionID: id}) + if err != nil { + t.Fatal(err) + } + _, updatedAt, err := globalBucketMetadataSys.GetLifecycleConfig(s.bucket) + if err != nil { + t.Fatal(err) + } + version := FileInfo{Name: key, VersionID: id, ModTime: source.ModTime} + err = s.z.removeMovedVersions(ctx, s.bucket, key, 0, []FileInfo{version}, []migrationExpiredVersion{{ + version: version, + event: lifecycle.Event{Action: lifecycle.DeleteAction}, + lifecycleUpdatedAt: updatedAt, + source: lcEventSrc_Decom, + }}) + if _, ok := err.(PreConditionFailed); !ok { + t.Fatalf("retirement after a refused expiry: %v", err) + } + authorityPresent(t, s, 0, key, id) + if views := s.views(); !slices.Equal(views[0][key], []string{"old"}) || !slices.Equal(views[1][key], []string{"new", "old"}) { + t.Fatalf("refused expiry changed the pools: %v", views) + } +} diff --git a/cmd/erasure-server-pool-lifecycle-safety_test.go b/cmd/erasure-server-pool-lifecycle-safety_test.go index 27791813f..0ee319350 100644 --- a/cmd/erasure-server-pool-lifecycle-safety_test.go +++ b/cmd/erasure-server-pool-lifecycle-safety_test.go @@ -94,23 +94,31 @@ func TestPoolsLifecycleDuplicateLockMetadata(t *testing.T) { setLifecycleCopyMetadata(t, s, key, id, pool, metadata, tc.newerPool1 && pool == 1) } versions, events, err := s.z.lifecycleEventsAllPools(t.Context(), s.bucket, key) - if err != nil { + switch { + case tc.newerPool1: + // Another ModTime makes the copies separate writes. The + // held older one refuses the object instead of merging. + if !errors.Is(err, errLifecycleNotApplicable) { + t.Fatalf("held older write: %v", err) + } + case err != nil: t.Fatal(err) - } - if len(versions) != 2 || versions[1].VersionID != id || versions[1].NumVersions != 2 { - t.Fatalf("wrong version history: %+v", versions) - } - state := storedObjectLockState(versions[1].UserDefined) - if state.mode != tc.mode || state.legalHold != tc.hold { - t.Errorf("merged state=%+v, want mode=%s hold=%s", state, tc.mode, tc.hold) - } - want := lifecycle.NoneAction - if tc.delete { - want = lifecycle.DeleteVersionAction - } - // Read-side expiry rechecks consume the same event producer. - if events[1].Action != want { - t.Errorf("expiry event=%s want=%s", events[1].Action, want) + default: + if len(versions) != 2 || versions[1].VersionID != id || versions[1].NumVersions != 2 { + t.Fatalf("wrong version history: %+v", versions) + } + state := storedObjectLockState(versions[1].UserDefined) + if state.mode != tc.mode || state.legalHold != tc.hold { + t.Errorf("merged state=%+v, want mode=%s hold=%s", state, tc.mode, tc.hold) + } + want := lifecycle.NoneAction + if tc.delete { + want = lifecycle.DeleteVersionAction + } + // Read-side expiry rechecks consume the same event producer. + if events[1].Action != want { + t.Errorf("expiry event=%s want=%s", events[1].Action, want) + } } opts := ObjectOptions{Versioned: true, Expiration: ExpirationOptions{Expire: true}} if batch { diff --git a/cmd/erasure-server-pool-lifecycle.go b/cmd/erasure-server-pool-lifecycle.go index 6df11bedb..597cdf249 100644 --- a/cmd/erasure-server-pool-lifecycle.go +++ b/cmd/erasure-server-pool-lifecycle.go @@ -19,12 +19,20 @@ package cmd import ( "context" + "crypto/sha256" + "encoding/binary" "errors" + "maps" + "slices" "sort" + "strings" "sync" "time" "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + xhttp "github.com/minio/minio/internal/http" ) // An ordinary ILM rejection is not evidence that the version is absent. In @@ -51,17 +59,25 @@ var errLifecycleNotApplicable = errors.New("lifecycle expiry no longer applies") // recheck of a set does it: the scanner queues it again once the rule // applies to the object. -// objectVersions returns the versions of object that a read quorum of the -// drives of the set agree on, newest first, without free versions. It returns -// no versions when the set does not hold the object, and an error when too -// few drives answer to tell. +// objectVersions returns the versions of object that the drives of the set +// hold authoritatively, newest first, without free versions. It returns no +// versions when the set does not hold the object, and an error when the +// drives cannot tell its complete history. +// +// Expiry deletes by this history, so a revision of a version, its header +// and its metadata (versionRevision), counts only when more than half of +// the configured drives and the read quorum of its metadata hold it. A +// version that some drive shows without such a revision, or too many +// unreadable drives, could hide what an expiry must not delete. Free +// versions are tier GC records and do not count. func (er erasureObjects) objectVersions(ctx context.Context, bucket, object string) ([]FileInfo, error) { disks := er.getDisks() rawFileInfos, errs := readAllRawFileInfo(ctx, disks, bucket, object, false) - readQuorum := (len(disks) + 1) / 2 + majority := len(disks)/2 + 1 var ( metas [][]xlMetaV2ShallowVersion + metaVs []uint8 metaV uint8 unknown int ) @@ -78,23 +94,23 @@ func (er erasureObjects) objectVersions(ctx context.Context, bucket, object stri continue } metas = append(metas, xl.versions) + metaVs = append(metaVs, xl.metaV) metaV = xl.metaV } - if len(metas) < readQuorum { - if len(metas)+unknown < readQuorum { - return nil, nil // not in this set - } + // Without an unknown drive the history is complete, also on one drive. + if unknown > 0 && unknown >= len(disks)/2 { return nil, toObjectErr(errErasureReadQuorum, bucket, object) } - merged := mergeXLV2Versions(readQuorum, false, 0, metas...) - if len(merged) == 0 { - for _, versions := range metas { - if len(versions) > 0 { - return nil, toObjectErr(errErasureReadQuorum, bucket, object) - } - } - return nil, nil + quorumMetas, quorumErrs := make([]FileInfo, len(disks)), make([]error, len(disks)) + merged, err := authoritativeHistory(ctx, metas, metaVs, metaV, majority, func(ver xlMetaV2ShallowVersion, metaV uint8) int { + return er.versionReadQuorum(ctx, bucket, object, ver, metaV, quorumMetas, quorumErrs) + }) + if err != nil { + return nil, toObjectErr(err, bucket, object) + } + if merged == nil { + return nil, nil // not in this set } all, err := xlMetaV2{versions: merged, metaV: metaV}.ListVersions(bucket, object, false) if err != nil { @@ -109,10 +125,253 @@ func (er erasureObjects) objectVersions(ctx context.Context, bucket, object stri return versions, nil } +// revisionKey identifies a revision of a version: its header and a strong +// digest of its metadata. The header Signature is a 32-bit fold of weak +// hashes; two revisions can share it. +type revisionKey struct { + header xlMetaV2VersionHeader + digest [sha256.Size]byte +} + +// versionRevision returns the revision of ver, whose metadata is encoded as +// metaV describes. As Signature does, it leaves out the fields that differ +// between the drives of one write: the erasure index, empty part lists and, +// for legacy versions, the release and the per-shard checksums. The digest +// input is canonical: each map, including the part checksums of a legacy +// version, is its own section with its entry count and sorted, +// length-prefixed entries, since msgp encodes maps in iteration order and an +// entry must not move between maps unnoticed. The header of a +// legacy version carries a signature over its per-shard checksums, so the +// key replaces it with one from the digest. +func versionRevision(ver xlMetaV2ShallowVersion, metaV uint8) revisionKey { + h := sha256.New() + writeUint := func(n int) { + var b [8]byte + binary.LittleEndian.PutUint64(b[:], uint64(n)) + h.Write(b[:]) + } + writeField := func(b []byte) { + writeUint(len(b)) + h.Write(b) + } + section := func(name string, entries int) { + writeField([]byte(name)) + writeUint(entries) + } + writeStrings := func(name string, m map[string]string) { + section(name, len(m)) + for _, k := range slices.Sorted(maps.Keys(m)) { + writeField([]byte(k)) + writeField([]byte(m[k])) + } + } + writeBytes := func(name string, m map[string][]byte) { + section(name, len(m)) + for _, k := range slices.Sorted(maps.Keys(m)) { + writeField([]byte(k)) + writeField(m[k]) + } + } + key := revisionKey{header: ver.header} + var v xlMetaV2Version + if _, err := v.unmarshalV(metaV, ver.meta); err == nil { + var ( + bts []byte + err error = errFileCorrupt + ) + section("type", 1) + writeField([]byte{byte(v.Type)}) + switch { + case v.Type == ObjectType && v.ObjectV2 != nil: + c := *v.ObjectV2 + c.ErasureIndex = 0 + if !slices.ContainsFunc(c.PartETags, func(tag string) bool { return tag != "" }) { + c.PartETags = nil + } + if len(c.PartActualSizes) == 0 { + c.PartActualSizes = nil + } + writeStrings("user", c.MetaUser) + writeBytes("sys", c.MetaSys) + c.MetaUser, c.MetaSys = nil, nil + bts, err = c.MarshalMsg(nil) + case v.Type == DeleteType && v.DeleteMarker != nil: + c := *v.DeleteMarker + writeBytes("sys", c.MetaSys) + c.MetaSys = nil + bts, err = c.MarshalMsg(nil) + case v.Type == LegacyType && v.ObjectV1 != nil: + c := *v.ObjectV1 + c.Erasure.Index = 0 + c.Erasure.Checksums = nil + c.Minio.Release = "" + writeStrings("meta", c.Meta) + c.Meta = nil + // Part checksums are the only other map: one section per part. + section("parts", len(c.Parts)) + c.Parts = slices.Clone(c.Parts) + for i := range c.Parts { + writeStrings("part-checksums", c.Parts[i].Checksums) + c.Parts[i].Checksums = nil + } + bts, err = c.MarshalMsg(nil) + } + if err == nil { + section("fields", 1) + writeField(bts) + h.Sum(key.digest[:0]) + if v.Type == LegacyType { + copy(key.header.Signature[:], key.digest[:]) + } + return key + } + h.Reset() + } + // Undecodable metadata only matches the same bytes. + section("raw", 1) + writeField(ver.meta) + h.Sum(key.digest[:0]) + return key +} + +// authoritativeHistory returns the versions of metas, the version lists of +// the drives that answered, as merged newest first, when every version +// that is not a free version has a revision that at least majority drives +// and its readQuorum hold. It returns no versions when none is live, +// errErasureReadQuorum when the drives cannot tell the history, and the +// error of ctx once it ends. metaVs are the metadata versions of metas, +// metaV the one the merged result is decoded with. +func authoritativeHistory(ctx context.Context, metas [][]xlMetaV2ShallowVersion, metaVs []uint8, metaV uint8, majority int, readQuorum func(xlMetaV2ShallowVersion, uint8) int) ([]xlMetaV2ShallowVersion, error) { + // Count the drives that hold each revision; a drive counts once for each + // version. + type revision struct { + ver xlMetaV2ShallowVersion + metaV uint8 + count int + } + keys := make([][]revisionKey, len(metas)) + revisions := make(map[[16]byte]map[revisionKey]*revision) + live := make(map[[16]byte]bool) + for i, versions := range metas { + keys[i] = make([]revisionKey, len(versions)) + seen := make(map[[16]byte]bool, len(versions)) + for j, ver := range versions { + // Decoding and hashing every history has a cost; stop with the + // request. + if err := ctx.Err(); err != nil { + return nil, err + } + id := ver.header.VersionID + if !ver.header.FreeVersion() { + live[id] = true + } + if seen[id] { + continue + } + seen[id] = true + key := versionRevision(ver, metaVs[i]) + keys[i][j] = key + if revisions[id] == nil { + revisions[id] = make(map[revisionKey]*revision) + } + r := revisions[id][key] + if r == nil { + r = &revision{ver: ver, metaV: metaVs[i]} + revisions[id][key] = r + } + r.count++ + } + } + accepted := make(map[[16]byte]revisionKey, len(live)) + for id := range live { + var best *revision + var bestKey revisionKey + for key, r := range revisions[id] { + if !key.header.FreeVersion() && (best == nil || r.count > best.count) { + best, bestKey = r, key + } + } + // Only one revision can hold more than half of the drives. + if best == nil || best.count < majority || best.count < readQuorum(best.ver, best.metaV) { + return nil, errErasureReadQuorum + } + accepted[id] = bestKey + } + + // The strict merge orders the history. It has to hold exactly the + // accepted revisions. A minority revision that sorts first would reset + // its count, and one that shares the header could stand in for the + // accepted one, so the merge only sees the accepted revision of each + // version and free versions as they are. It sees the headers of the + // keys, which do not differ between the drives of a legacy version. + filtered := make([][]xlMetaV2ShallowVersion, len(metas)) + for i, versions := range metas { + for j, ver := range versions { + key, ok := accepted[ver.header.VersionID] + switch { + case ver.header.FreeVersion(): + case ok && keys[i][j] == key: + ver.header = key.header + default: + continue + } + filtered[i] = append(filtered[i], ver) + } + } + if err := ctx.Err(); err != nil { + return nil, err + } + merged := mergeXLV2Versions(majority, true, 0, filtered...) + emitted := make(map[[16]byte]bool, len(accepted)) + for _, ver := range merged { + if err := ctx.Err(); err != nil { + return nil, err + } + if ver.header.FreeVersion() { + continue + } + id := ver.header.VersionID + if key, ok := accepted[id]; !ok || emitted[id] || versionRevision(ver, metaV) != key { + return nil, errErasureReadQuorum + } + emitted[id] = true + } + if len(emitted) != len(accepted) { + return nil, errErasureReadQuorum + } + if len(accepted) == 0 { + return nil, nil + } + return merged, nil +} + +// versionReadQuorum returns the read quorum of the metadata of ver, as +// objectQuorumFromMeta derives it when all drives of the set hold it, or +// more than the drives when its metadata cannot be read. metaArr and errs +// have one entry for each drive; errs stays nil. +func (er erasureObjects) versionReadQuorum(ctx context.Context, bucket, object string, ver xlMetaV2ShallowVersion, metaV uint8, metaArr []FileInfo, errs []error) int { + var v xlMetaV2Version + if _, err := v.unmarshalV(metaV, ver.meta); err != nil { + return len(metaArr) + 1 + } + fi, err := v.ToFileInfo(bucket, object, false) + if err != nil { + return len(metaArr) + 1 + } + for i := range metaArr { + metaArr[i] = fi + } + readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount) + if err != nil { + return len(metaArr) + 1 + } + return readQuorum +} + // lifecycleEventsAllPools returns the versions of object in all pools, // newest first, and the lifecycle event of each as the scanner evaluates it // on them. A version with copies in several pools, as an interrupted move -// leaves it, counts once. +// leaves it, counts once, and so do other writes of its version ID. func (z *erasureServerPools) lifecycleEventsAllPools(ctx context.Context, bucket, object string) ([]ObjectInfo, []lifecycle.Event, error) { lc, err := globalLifecycleSys.Get(bucket) if err != nil { @@ -169,14 +428,36 @@ func (z *erasureServerPools) lifecycleEventsAllPools(ctx context.Context, bucket versions = append(versions, []PoolObjInfo{copy}) } + e := lifecycle.NewEvaluator(*lc).WithLockRetention(&lr).WithReplicationConfig(rcfg) objInfos := make([]ObjectInfo, len(versions)) objOpts := make([]lifecycle.ObjectOpts, len(versions)) for i, copies := range versions { + // Copies with another ModTime are other writes of the version ID, + // such as null versions written in several pools, not duplicates of + // one write. The newest is the version. Data movement writes a new + // data dir, so only ModTime tells the duplicates of a write apart. + // An older write that is still protected keeps the whole object. + newest := 1 + for newest < len(copies) && copies[newest].ObjInfo.ModTime.Equal(copies[0].ObjInfo.ModTime) { + newest++ + } + for _, older := range copies[newest:] { + if opts := older.ObjInfo.ToLifecycleOpts(); e.IsObjectLocked(opts) || e.IsPendingReplication(opts) { + return nil, nil, errLifecycleNotApplicable + } + } + copies = copies[:newest] oi := copies[0].ObjInfo if len(copies) > 1 { + // The copies must tell which protection is the newest, and no + // copy may carry replication state that the merge would drop. + if !lockRevisionsOrdered(copies) || !replicationStatesAgree(copies) { + return nil, nil, errLifecycleNotApplicable + } // Count the version once, but retain independently ordered lock // and tag revisions, including explicit removals, from every copy. oi = mergedPoolObjectInfo(copies) + setLifecycleLockState(oi.UserDefined, newestLockState(copies)) } oi.IsLatest = i == 0 oi.NumVersions = len(versions) @@ -187,13 +468,160 @@ func (z *erasureServerPools) lifecycleEventsAllPools(ctx context.Context, bucket objInfos[i] = oi objOpts[i] = oi.ToLifecycleOpts() } - events, err := lifecycle.NewEvaluator(*lc).WithLockRetention(&lr).WithReplicationConfig(rcfg).Eval(objOpts) + events, err := e.Eval(objOpts) if err != nil { return nil, nil, err } return objInfos, events, nil } +// lockRevisionTime parses the timestamp of an Object Lock revision. Unlike +// olderThan, a missing, malformed or zero timestamp orders nothing. +func lockRevisionTime(stamp string) (time.Time, bool) { + t, err := time.Parse(time.RFC3339Nano, stamp) + return t, err == nil && !t.IsZero() +} + +// lockRevisionNewer reports whether the revision stamped newer strictly +// supersedes the one stamped older. +func lockRevisionNewer(newer, older string) bool { + n, ok := lockRevisionTime(newer) + o, ok2 := lockRevisionTime(older) + return ok && ok2 && n.After(o) +} + +// lifecycleLockState returns the Object Lock state of metadata as the +// lifecycle evaluator reads it (objectlock.GetObjectRetentionMeta and +// GetObjectLegalHoldMeta): a lowercase key before its canonical form. The +// stored state (storedObjectLockState) only has lowercase keys. +func lifecycleLockState(metadata map[string]string) objectLockState { + get := func(canonical string) string { + if v, ok := metadata[strings.ToLower(canonical)]; ok { + return v + } + return metadata[canonical] + } + return objectLockState{ + mode: get(objectlock.AmzObjectLockMode), + retainUntil: get(objectlock.AmzObjectLockRetainUntilDate), + retentionTimestamp: get(ReservedMetadataPrefix + ObjectLockRetentionTimestamp), + legalHold: get(objectlock.AmzObjectLockLegalHold), + legalHoldTimestamp: get(ReservedMetadataPrefix + ObjectLockLegalHoldTimestamp), + } +} + +// setLifecycleLockState sets state in metadata so that the evaluator reads +// it: lowercase keys take precedence over canonical ones. +func setLifecycleLockState(metadata map[string]string, state objectLockState) { + set := func(canonical, value string) { + lower := strings.ToLower(canonical) + _, hasLower := metadata[lower] + _, hasCanonical := metadata[canonical] + if value != "" || hasLower || hasCanonical { + metadata[lower] = value + } + } + set(objectlock.AmzObjectLockMode, state.mode) + set(objectlock.AmzObjectLockRetainUntilDate, state.retainUntil) + set(ReservedMetadataPrefix+ObjectLockRetentionTimestamp, state.retentionTimestamp) + set(objectlock.AmzObjectLockLegalHold, state.legalHold) + set(ReservedMetadataPrefix+ObjectLockLegalHoldTimestamp, state.legalHoldTimestamp) +} + +// newestLockState returns the newest hold and the newest retention of the +// copies of a version, which lockRevisionsOrdered found equal or ordered. +func newestLockState(copies []PoolObjInfo) objectLockState { + state := lifecycleLockState(copies[0].ObjInfo.UserDefined) + for _, copy := range copies[1:] { + next := lifecycleLockState(copy.ObjInfo.UserDefined) + if lockRevisionNewer(next.legalHoldTimestamp, state.legalHoldTimestamp) { + state.legalHold, state.legalHoldTimestamp = next.legalHold, next.legalHoldTimestamp + } + if lockRevisionNewer(next.retentionTimestamp, state.retentionTimestamp) { + state.mode, state.retainUntil, state.retentionTimestamp = next.mode, next.retainUntil, next.retentionTimestamp + } + } + return state +} + +// checkedExpiryRefused reports whether a set must keep a version whose ILM +// expiry the pools approved with the Object Lock state approved, by the +// metadata goi the set reads under the same lock. A pending purge always +// keeps it: its status orders nothing. The approved hold or retention only +// replaces the one the set reads when it is strictly newer; otherwise the +// protection the set reads stands, also when it is newer than the approval. +func checkedExpiryRefused(lc lifecycle.Lifecycle, lr objectlock.Retention, rcfg *replication.Config, goi ObjectInfo, approved objectLockState) bool { + e := lifecycle.NewEvaluator(lc).WithLockRetention(&lr).WithReplicationConfig(rcfg) + obj := goi.ToLifecycleOpts() + if e.IsPendingReplication(obj) { + return true + } + local := lifecycleLockState(goi.UserDefined) + obj.UserDefined = maps.Clone(goi.UserDefined) + if obj.UserDefined == nil { + obj.UserDefined = make(map[string]string) + } + if lockRevisionNewer(approved.legalHoldTimestamp, local.legalHoldTimestamp) { + obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = approved.legalHold + } + if lockRevisionNewer(approved.retentionTimestamp, local.retentionTimestamp) { + obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = approved.mode + obj.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = approved.retainUntil + } + return e.IsObjectLocked(obj) +} + +// lockRevisionsOrdered reports whether the copies of a version agree on the +// legal hold and on the retention, each on its own, or carry revisions with +// distinct valid timestamps, so that the newest one is known. +func lockRevisionsOrdered(copies []PoolObjInfo) bool { + ordered := func(revision func(objectLockState) (value, stamp string)) bool { + stamps := make(map[[2]string]string, len(copies)) + for _, copy := range copies { + value, stamp := revision(lifecycleLockState(copy.ObjInfo.UserDefined)) + stamps[[2]string{value, stamp}] = stamp + } + if len(stamps) == 1 { + return true + } + seen := make([]time.Time, 0, len(stamps)) + for _, stamp := range stamps { + t, ok := lockRevisionTime(stamp) + if !ok { + return false + } + for _, other := range seen { + if t.Equal(other) { + return false + } + } + seen = append(seen, t) + } + return true + } + return ordered(func(s objectLockState) (string, string) { + return s.legalHold, s.legalHoldTimestamp + }) && ordered(func(s objectLockState) (string, string) { + return s.mode + "\x00" + s.retainUntil, s.retentionTimestamp + }) +} + +// replicationStatesAgree reports whether the copies of a version carry the +// same replication and purge state. The per-target states are compared as +// parsed, regardless of their order. +func replicationStatesAgree(copies []PoolObjInfo) bool { + first := copies[0].ObjInfo + for _, copy := range copies[1:] { + oi := copy.ObjInfo + if oi.ReplicationStatus != first.ReplicationStatus || oi.VersionPurgeStatus != first.VersionPurgeStatus || + !maps.Equal(replicationStatusesMap(oi.ReplicationStatusInternal), replicationStatusesMap(first.ReplicationStatusInternal)) || + !maps.Equal(versionPurgeStatusesMap(oi.VersionPurgeStatusInternal), versionPurgeStatusesMap(first.VersionPurgeStatusInternal)) { + return false + } + } + return true +} + // lifecycleAllPools holds the versions of an object in all pools and their // lifecycle events, see lifecycleEventsAllPools. type lifecycleAllPools struct { @@ -204,11 +632,37 @@ type lifecycleAllPools struct { // checkExpiryAllPools returns nil when the ILM expiry that opts describe // still applies to object with the versions it has in all pools, see -// lifecycleAllPools.expiryApplies. -func (z *erasureServerPools) checkExpiryAllPools(ctx context.Context, bucket, object string, opts ObjectOptions) error { +// lifecycleAllPools.expiryApplies, and the Object Lock state with which it +// approved the expiry of a version. +func (z *erasureServerPools) checkExpiryAllPools(ctx context.Context, bucket, object string, opts ObjectOptions) (objectLockState, error) { var all lifecycleAllPools all.versions, all.events, all.err = z.lifecycleEventsAllPools(ctx, bucket, object) - return all.expiryApplies(bucket, object, opts) + if err := all.expiryApplies(bucket, object, opts); err != nil { + return objectLockState{}, err + } + return all.protection(opts.VersionID), nil +} + +// version returns the index of the version versionID, or -1. +func (all lifecycleAllPools) version(versionID string) int { + for i, oi := range all.versions { + if oi.VersionID == versionID || oi.VersionID == "" && versionID == nullVersionID { + return i + } + } + return -1 +} + +// protection returns the Object Lock state of the version versionID that +// the evaluator decided on, to be checked against the state the set reads. +func (all lifecycleAllPools) protection(versionID string) objectLockState { + if versionID == "" { + return objectLockState{} + } + if i := all.version(versionID); i >= 0 { + return lifecycleLockState(all.versions[i].UserDefined) + } + return objectLockState{} } // expiryApplies returns nil when the ILM expiry that opts describe applies @@ -237,13 +691,11 @@ func (all lifecycleAllPools) expiryApplies(bucket, object string, opts ObjectOpt return nil } default: - for i, oi := range all.versions { - if oi.VersionID == opts.VersionID || oi.VersionID == "" && opts.VersionID == nullVersionID { - if all.events[i].Action == lifecycle.DeleteVersionAction { - return nil - } - return errLifecycleNotApplicable + if i := all.version(opts.VersionID); i >= 0 { + if all.events[i].Action == lifecycle.DeleteVersionAction { + return nil } + return errLifecycleNotApplicable } return VersionNotFound{Bucket: bucket, Object: name, VersionID: opts.VersionID} } @@ -328,5 +780,6 @@ func (z *erasureServerPools) migrationExpiryApplies(ctx context.Context, oi Obje if !z.expiryNeedsAllPools(opts) { return true } - return z.checkExpiryAllPools(ctx, oi.Bucket, encodeDirObject(oi.Name), opts) == nil + _, err := z.checkExpiryAllPools(ctx, oi.Bucket, encodeDirObject(oi.Name), opts) + return err == nil } diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 2f9f0d7b4..849ad6c67 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1250,12 +1250,15 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob // An ILM expiry is decided on the versions in one pool. Under the lock, // check it with the versions in all pools; the sets then take it as - // decided, without a recheck by the versions each one holds. + // decided, without a recheck by the versions each one holds. They only + // keep a version whose protection they read differently. if z.expiryNeedsAllPools(opts) { - if err = z.checkExpiryAllPools(ctx, bucket, object, opts); err != nil { + var approved objectLockState + if approved, err = z.checkExpiryAllPools(ctx, bucket, object, opts); err != nil { return ObjectInfo{Name: decodeDirObject(object), VersionID: opts.VersionID}, err } opts.Expiration.CheckedAllPools = true + opts.Expiration.ApprovedProtection = approved } if opts.DeletePrefix { @@ -1527,6 +1530,7 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o batch = make([]ObjectToDelete, 0, len(objects)) batchIdx = make([]int, 0, len(objects)) data, dataIdx := make([]ObjectToDelete, 0, len(objects)), make([]int, 0, len(objects)) + approved := make([]objectLockState, 0, len(objects)) ilm := make(map[string]lifecycleAllPools) for i, obj := range objects { if derrs[i] != nil { @@ -1536,6 +1540,7 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o // its object in all pools, see checkExpiryAllPools. When the // expiry no longer applies, nothing is deleted and nothing is // left to replicate. + var protection objectLockState vopts := ObjectOptions{Expiration: opts.Expiration, VersionID: obj.VersionID} if obj.VersionID != "" && z.expiryNeedsAllPools(vopts) { all, ok := ilm[obj.ObjectName] @@ -1548,6 +1553,7 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o derrs[i] = err continue } + protection = all.protection(obj.VersionID) } if deletesData(obj, opts) { if z.copyWithoutDeleteQuorum(ctx, bucket, obj.ObjectName, obj.VersionID) { @@ -1556,13 +1562,14 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o } data = append(data, obj) dataIdx = append(dataIdx, i) + approved = append(approved, protection) continue } batch = append(batch, obj) batchIdx = append(batchIdx, i) } if len(data) > 0 { - deleted, errs := z.deleteObjectsReconciled(ctx, bucket, data, opts) + deleted, errs := z.deleteObjectsReconciled(ctx, bucket, data, approved, opts) for j, i := range dataIdx { dobjects[i], derrs[i] = deleted[j], errs[j] } diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 658e30f9b..d40ff3ce7 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -163,6 +163,10 @@ type ExpirationOptions struct { // object in all pools (checkExpiryAllPools). A set does not check it // again with the versions it holds alone. CheckedAllPools bool + // ApprovedProtection: the Object Lock state of the version with which + // the check approved its expiry. The set compares it with the state it + // reads (checkedExpiryRefused). In memory only. + ApprovedProtection objectLockState `msg:"-"` } // TransitionOptions represents object options for transition ObjectLayer operation diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go index 903e82731..41441882a 100644 --- a/cmd/object-handlers-common.go +++ b/cmd/object-handlers-common.go @@ -19,7 +19,6 @@ package cmd import ( "context" - "errors" "fmt" "net/http" "regexp" @@ -472,9 +471,11 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD }) for i, dobj := range deletedObjs { - if errors.Is(errs[i], errLifecycleNotApplicable) { - // The recheck kept this version. Do not audit or announce a - // removal that did not happen; the next scan re-evaluates it. + if errs[i] != nil && !isErrObjectNotFound(errs[i]) && !isErrVersionNotFound(errs[i]) { + // Only a completed delete, or one confirmed already absent, is + // a removal. After a refusal, a quorum, configuration or any + // other error the version may remain; do not audit or announce + // it. The next scan re-evaluates it. continue } oi := ObjectInfo{ From 7f38bf6282c11d6ad272313b7b1dff1d2ed81280 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 06:47:38 +0800 Subject: [PATCH 21/26] fix(restore): give each native restore an owner and a private upload A native restore (POST ?restore, issue #281) marks the version ongoing, and a background worker rehydrates it from the tier. The worker, its upload and every status write could be reached or confused by other callers. A public client could write into or complete the restore upload. POST copied a whole metadata snapshot over newer protection changes. An old or failed attempt could act on a newer one. Restore, expiry and heal decided on whichever drive's metadata they read. Each POST that starts a rehydration now writes an attempt ID into the version's internal metadata, with the identity it resolved: version ID, ETag, size and ModTime. The worker, its upload, its part writes and its completion carry the attempt. A worker whose attempt no longer owns the version stops without writing, completing or resetting. Every write uses the versioning state and explicit version ID captured by POST, so a null target stays null. The restore upload carries an internal marker, decided over all drives. Public multipart calls answer NoSuchUpload and listings hide it. An unknown marker fails closed. A completion holds a commit lock from staging its commit in the upload until the commit ends. Stale cleanup decides on every upload past the stale age under that lock, and keeps it until its rename ends, also past the drive deadline. Abort of a committing upload takes the lock and removes a failed commit's leftover only when every copy is staged and older than one hour. POST, the worker, the reset, the ExpireRestored recheck and heal all decide on the authoritative revision of the exact version. That is the revision held by max(n/2+1, read quorum) drives, identified by the version header and the normalized metadata digest. Status writes patch only the restore fields of that revision. They write only the drives that hold it, and put the old revision back on failure. Heal copies restore fields only when a quorum holds the same fields. A failed attempt resets itself with bounded in-memory retries. A later POST takes over a stranded marker when no attempt runs. Otherwise it answers RestoreAlreadyInProgress within a few seconds. Older nodes do not know the marker. A new grid capability handler lets the serving node ask every peer. POST answers 503 ServiceUnavailable and writes nothing until all peers support native restore, and a queued worker asks again before it creates its upload. The single-pool lock-domain split between DELETE/CopyObject and restore is closed by its own fix, which this change depends on. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/bucket-lifecycle-restore-boundary_test.go | 3 +- cmd/bucket-lifecycle-restore-safety_test.go | 2 +- cmd/bucket-lifecycle.go | 38 +- cmd/erasure-healing.go | 7 + cmd/erasure-multipart-cleanup_test.go | 26 +- cmd/erasure-multipart.go | 168 +- cmd/erasure-object.go | 136 +- cmd/erasure-server-pool.go | 17 +- cmd/object-api-interface.go | 6 + cmd/object-copy-metadata_test.go | 6 +- cmd/object-handlers.go | 77 +- cmd/peer-rest-server.go | 2 + cmd/restore-attempt.go | 862 ++++++++++ cmd/restore-capability.go | 184 +++ cmd/restore-capability_test.go | 271 ++++ cmd/restore-native-generation_test.go | 1418 +++++++++++++++++ cmd/restore-native-isolation_test.go | 1397 ++++++++++++++++ cmd/restore-native_test.go | 72 +- internal/grid/handlers.go | 2 + internal/grid/handlers_string.go | 11 +- 20 files changed, 4543 insertions(+), 162 deletions(-) create mode 100644 cmd/restore-attempt.go create mode 100644 cmd/restore-capability.go create mode 100644 cmd/restore-capability_test.go create mode 100644 cmd/restore-native-generation_test.go create mode 100644 cmd/restore-native-isolation_test.go diff --git a/cmd/bucket-lifecycle-restore-boundary_test.go b/cmd/bucket-lifecycle-restore-boundary_test.go index 14967cf6a..8f39342ba 100644 --- a/cmd/bucket-lifecycle-restore-boundary_test.go +++ b/cmd/bucket-lifecycle-restore-boundary_test.go @@ -137,8 +137,7 @@ func TestExpireRestoredMetadataReadKeepsMinorityVersion(t *testing.T) { case "restore-cleanup": // The initial read succeeded before remote download failed; // cleanup must not purge a now-incomplete target on reread. - opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1} - err = er.updateRestoreMetadata(ctx, r.bucket, r.object, fi.ToObjectInfo(r.bucket, r.object, true), opts) + err = er.clearRestoreStatus(ctx, r.bucket, r.object, oi.VersionID, nil) case "lookup-cleanup", "lookup-restore": // Version-sensitive lookup also covers native restore routing. opts.MetadataChg = true diff --git a/cmd/bucket-lifecycle-restore-safety_test.go b/cmd/bucket-lifecycle-restore-safety_test.go index 9facbfd0d..85a4be182 100644 --- a/cmd/bucket-lifecycle-restore-safety_test.go +++ b/cmd/bucket-lifecycle-restore-safety_test.go @@ -156,7 +156,7 @@ func TestExpireRestoredRenewedOrMissing(t *testing.T) { t.Fatal(err) } err = expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner) - if !isErrVersionNotFound(err) { + if !errors.Is(err, errLifecycleNotApplicable) { t.Errorf("stale task should be rejected, got %v", err) } if got := r.fileInfo(oi.VersionID); got.Metadata[xhttp.AmzRestore] != renewed || got.DataDir != before.DataDir { diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go index 5c5597587..5854fa64d 100644 --- a/cmd/bucket-lifecycle.go +++ b/cmd/bucket-lifecycle.go @@ -37,6 +37,7 @@ import ( "github.com/minio/minio/internal/amztime" sse "github.com/minio/minio/internal/bucket/encryption" "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/object/lock" "github.com/minio/minio/internal/bucket/replication" "github.com/minio/minio/internal/event" xhttp "github.com/minio/minio/internal/http" @@ -987,8 +988,10 @@ func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string }, nil } -// set ObjectOptions for PUT call to restore temporary copy of transitioned data -func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) { +// set ObjectOptions for PUT call to restore temporary copy of transitioned data. +// opts are the restore's options, with the versioning POST captured: the +// bucket's versioning may have changed since, and must not mint a new version. +func putRestoreOpts(rreq *RestoreObjectRequest, objInfo ObjectInfo, opts ObjectOptions) (putOpts ObjectOptions) { meta := make(map[string]string) sc := rreq.OutputLocation.S3.StorageClass if sc == "" { @@ -1011,8 +1014,8 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES } return ObjectOptions{ - Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, object), - VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object), + Versioned: opts.Versioned, + VersionSuspended: opts.VersionSuspended, UserDefined: meta, } } @@ -1020,19 +1023,38 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O if len(objInfo.UserTags) != 0 { meta[xhttp.AmzObjectTagging] = objInfo.UserTags } - // Set restore object status + // Set restore object status; a completed restore has no attempt. restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days) meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String() + delete(meta, restoreAttemptKey) + delete(meta, restoreUploadKey) return ObjectOptions{ - Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, object), - VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object), + Versioned: opts.Versioned, + VersionSuspended: opts.VersionSuspended, UserDefined: meta, - VersionID: objInfo.VersionID, + VersionID: restoreVersionID(objInfo.VersionID), MTime: objInfo.ModTime, Expires: objInfo.Expires, } } +// restoredCopyExpires reports whether the restored copy of oi, as read now, +// expires by its own reason: lifecycle expires the restored copy of this +// version, its restore deadline is valid and has passed, its transition is +// complete and no restore is running. Another rule falling due is no reason. +func restoredCopyExpires(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, oi ObjectInfo) bool { + if !evalActionFromLifecycle(ctx, lc, lr, rcfg, oi).Action.DeleteRestored() { + return false + } + status, err := parseRestoreObjStatus(oi.UserDefined[xhttp.AmzRestore]) + if err != nil { + return false + } + expiry, completed := status.Expiry() + return completed && !expiry.IsZero() && UTCNow().After(expiry) && + oi.TransitionedObject.Status == lifecycle.TransitionComplete && !oi.RestoreOngoing +} + var errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed") // IsRemote returns true if this object version's contents are in its remote diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go index 8fb371270..fad01b0cf 100644 --- a/cmd/erasure-healing.go +++ b/cmd/erasure-healing.go @@ -373,6 +373,13 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object if err != nil { return result, err } + // The restore fields fence native restores and restored-copy cleanup. + // Heal copies them only as a quorum of the drives holds them; while the + // drives disagree, it waits rather than create a quorum for one side. + latestMeta, err = restoreHealMeta(partsMetadata, latestMeta, max(len(storageDisks)/2+1, readQuorum)) + if err != nil { + return result, toObjectErr(err, bucket, object, versionID) + } // No modtime quorum filterDisksByETag := quorumETag != "" diff --git a/cmd/erasure-multipart-cleanup_test.go b/cmd/erasure-multipart-cleanup_test.go index 26a95db53..14c012968 100644 --- a/cmd/erasure-multipart-cleanup_test.go +++ b/cmd/erasure-multipart-cleanup_test.go @@ -123,7 +123,20 @@ func (d *multipartCleanupDisk) Endpoint() Endpoint { func (d *multipartCleanupDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) { d.reads++ - return FileInfo{ModTime: d.modTime}, d.readErr + if d.readErr != nil { + return FileInfo{}, d.readErr + } + fi := newFileInfo("obj", 1, 0) + fi.Erasure.Index, fi.ModTime = 1, d.modTime + return fi, nil +} + +// multipartCleanupSet is a set of one drive, which also answers the +// metadata reads that establish the restore marker of an expired upload. +// Its locks are local: stale cleanup decides a removal under the upload's +// commit lock. +func multipartCleanupSet(disk StorageAPI) erasureObjects { + return erasureObjects{getDisks: func() []StorageAPI { return []StorageAPI{disk} }, nsMutex: newNSLock(false)} } func (d *multipartCleanupDisk) StatVol(ctx context.Context, volume string) (VolInfo, error) { @@ -143,9 +156,10 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) { }{ {name: "recent", prefix: uuid, age: expiry / 2, readErr: errFileNotFound}, {name: "plaintext_base64_collision", prefix: "4eacd477-e733-4851-b821-6fd42785ff87", age: expiry / 2, readErr: errFileNotFound}, - {name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantDeleted: true}, + // Only an expired upload is read on every drive, for its restore marker. + {name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantReads: 1, wantDeleted: true}, {name: "legacy_recent", prefix: uuid, age: expiry / 2, legacy: true, wantReads: 1}, - {name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 1, wantDeleted: true}, + {name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 2, wantDeleted: true}, {name: "legacy_metadata_missing", prefix: uuid, age: 2 * expiry, legacy: true, readErr: errFileNotFound, wantReads: 1}, {name: "malformed_metadata_missing", prefix: "not-a-uuid", age: 2 * expiry, readErr: errFileNotFound, wantReads: 1}, } { @@ -166,7 +180,7 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) { if err := os.Chtimes(filepath.Dir(uploadDir), modTime, modTime); err != nil { t.Fatal(err) } - erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) + multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk) if disk.reads != tc.wantReads { t.Errorf("ReadVersion calls = %d, want %d", disk.reads, tc.wantReads) } @@ -199,7 +213,7 @@ func TestCleanupStaleUploadsEmptyHashDirectory(t *testing.T) { if err := os.Chtimes(shaDir, modTime, modTime); err != nil { t.Fatal(err) } - erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) + multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk) if _, err := os.Stat(shaDir); !os.IsNotExist(err) { t.Fatalf("empty hash directory must be removed regardless of age: %v", err) } @@ -224,7 +238,7 @@ func TestCleanupStaleUploadsSharedHashDirectory(t *testing.T) { t.Fatal(err) } - erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) + multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk) if _, err := os.Stat(filepath.Join(shaDir, expiredID)); !os.IsNotExist(err) { t.Errorf("expired upload must be removed: %v", err) } diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index c68fbc446..c4f5d97ff 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -34,7 +34,6 @@ import ( "github.com/google/uuid" "github.com/klauspost/readahead" "github.com/minio/minio-go/v7/pkg/set" - "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/config/storageclass" "github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/hash" @@ -72,6 +71,7 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string { } // checkUploadIDExists - verify if a given uploadID exists and is valid. +// The metadata read from each drive is returned also with an error. func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string, write bool) (fi FileInfo, metArr []FileInfo, err error) { defer func() { if errors.Is(err, errFileNotFound) { @@ -89,15 +89,15 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount) if err != nil { - return fi, nil, err + return fi, partsMetadata, err } if readQuorum < 0 { - return fi, nil, errErasureReadQuorum + return fi, partsMetadata, errErasureReadQuorum } if writeQuorum < 0 { - return fi, nil, errErasureWriteQuorum + return fi, partsMetadata, errErasureWriteQuorum } quorum := readQuorum @@ -114,7 +114,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum) } if err != nil { - return fi, nil, err + return fi, partsMetadata, err } // Pick one from the first valid metadata. @@ -217,12 +217,55 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto hasSkippedUploads = true return nil } + // A completion holds this lock from staging its commit in the + // upload until the commit ends: decide on a removal only while + // holding it, and skip an upload whose commit is running. Only + // uploads past the stale age pay for the lock and the read below. + lk := er.restoreCommitLock(uploadIDPath) + lkctx, err := lk.GetLock(ctx, newDynamicTimeout(time.Second, time.Second)) + if err != nil { + hasSkippedUploads = true + return nil + } + // The rename below releases the lock when it ends, also when it + // outlives its deadline; every other exit releases it here. + renaming := false + defer func() { + if !renaming { + lk.Unlock(lkctx) + } + }() + // A native restore can run past the stale age: keep its upload + // while its attempt still owns the version, and keep any upload + // whose marker the drives cannot establish. + view := er.readRestoreUpload(lkctx.Context(), "", uploadIDPath) + if view.committing { + // What a commit staged, with no commit running: only a staged + // copy itself shows when it staged, only an old one is a + // leftover of a failed commit, and a copy not staged yet + // tells nothing. + local, err := disk.ReadVersion(lkctx.Context(), "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{}) + st, serr := os.Stat(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath, xlStorageFormatFile)) + if err != nil || serr != nil || !restoreUploadStaged(local) || time.Since(st.ModTime()) < restoreCommitGrace { + hasSkippedUploads = true + return nil + } + } else if view.live && (!view.known || (view.marker != nil && er.restoreUploadActive(lkctx.Context(), view))) { + hasSkippedUploads = true + return nil + } + renaming = true w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) return w.Run(func() error { wait := deleteMultipartCleanupSleeper.Timer(ctx) pathUUID := mustGetUUID() targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID) - renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket)) + err := renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket)) + lk.Unlock(lkctx) + if err != nil { + // The next cleanup round tries again. + storageLogIf(ctx, fmt.Errorf("Unable to remove stale upload %s on %s: %w", uploadIDPath, drivePath, err)) + } wait() return nil }) @@ -463,6 +506,13 @@ func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, if populatedUploadIDs.Contains(uploadID) { continue } + // A native restore's upload is private to its attempt; list only an + // upload that the drives establish as an ordinary one. + metas, errs := readAllFileInfo(ctx, er.getDisks(), bucket, minioMetaMultipartBucket, + pathJoin(er.getMultipartSHADir(bucket, object), uploadID), "", false, false) + if !restoreUploadState(metas, errs).visible(ObjectOptions{}) { + continue + } var fallback time.Time if _, ok := multipartUploadTime(uploadID); !ok { fi, err := disk.ReadVersion(ctx, bucket, minioMetaMultipartBucket, @@ -562,6 +612,11 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string, if opts.PreserveETag != "" { userDefined["etag"] = opts.PreserveETag } + // Only a native restore marks its upload private, never caller metadata. + delete(userDefined, restoreUploadKey) + if a := opts.Transition.attempt; a != nil { + userDefined[restoreUploadKey] = a.String() + } onlineDisks := er.getDisks() // Get parity and data drive count based on storage class metadata @@ -743,7 +798,10 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) // Validates if upload ID exists. - fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) + fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) + if err == nil && !restoreUploadVisible(metas, opts) { + err = errUploadIDNotFound + } if err != nil { if errors.Is(err, errVolumeNotFound) { return pi, toObjectErr(err, bucket) @@ -980,7 +1038,10 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u UploadID: uploadID, } - fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + if err == nil && !restoreUploadVisible(metas, opts) { + err = errUploadIDNotFound + } if err != nil { if errors.Is(err, errVolumeNotFound) { return result, toObjectErr(err, bucket) @@ -1058,7 +1119,10 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up auditObjectErasureSet(ctx, "ListObjectParts", object, &er) } - fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + if err == nil && !restoreUploadVisible(metas, opts) { + err = errUploadIDNotFound + } if err != nil { return result, toObjectErr(err, bucket, object, uploadID) } @@ -1271,7 +1335,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str auditObjectErasureSet(ctx, "CompleteMultipartUpload", object, &er) } - if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile { + if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile || opts.Transition.attempt != nil { if !opts.NoLock { ns := er.NewNSLock(bucket, object) lkctx, err := ns.GetLock(ctx, globalOperationTimeout) @@ -1304,7 +1368,24 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str } } + if opts.Transition.attempt != nil { + // A native restore holds the upload write lock from validation + // through commit, after the object lock and in the lock domain of + // PutObjectPart, which takes a part lock and then this lock to read. + // NoLock only says that the caller holds the object lock. + uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID)) + ulkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + return oi, err + } + ctx = ulkctx.Context() + defer uploadIDLock.Unlock(ulkctx) + } + fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) + if err == nil && !restoreUploadVisible(partsMetadata, opts) { + err = errUploadIDNotFound + } if err != nil { if errors.Is(err, errVolumeNotFound) { return oi, toObjectErr(err, bucket) @@ -1618,6 +1699,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str delete(fi.Metadata, hash.MinIOMultipartChecksumType) delete(fi.Metadata, multipartMetaBucket) delete(fi.Metadata, multipartMetaObject) + delete(fi.Metadata, restoreUploadKey) // Save the final object size and modtime. fi.Size = objectSize @@ -1697,6 +1779,20 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo, opts.Transition); err != nil { return oi, toObjectErr(err, bucket, object) } + if a := opts.Transition.attempt; a != nil { + // A native restore writes back only the null version it targets. + current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{ + VersionID: nullVersionID, + Transition: opts.Transition, + NoLock: true, + }) + if gerr != nil { + return oi, toObjectErr(gerr, bucket, object) + } + if !a.rehydrates(current, fi) { + return oi, toObjectErr(errAuthentication, bucket, object) + } + } } else { current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{ VersionID: fi.VersionID, @@ -1706,11 +1802,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str NoLock: true, }) switch { - case gerr == nil && opts.Transition.RestoreRequest != nil && - current.TransitionedObject.Status == lifecycle.TransitionComplete && - current.ETag == fi.Metadata["etag"] && current.Size == fi.Size && current.ModTime.Equal(fi.ModTime): - // The internal restore path rehydrates this same tiered version. - // Ordinary multipart completion must still reject a version overwrite. + case gerr == nil && opts.Transition.attempt.rehydrates(current, fi): + // A native restore's own private upload rehydrates this same + // tiered version. Any other completion must still reject a + // version overwrite. case gerr == nil, errors.As(gerr, &MethodNotAllowed{}): return oi, toObjectErr(errAuthentication, bucket, object) case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr): @@ -1720,6 +1815,16 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str } } + // Stale cleanup and abort judge an upload whose commit staged object + // metadata in it by this lock: hold it from staging through the commit. + commitLock := er.restoreCommitLock(uploadIDPath) + clkctx, err := commitLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + return oi, err + } + ctx = clkctx.Context() + defer commitLock.Unlock(clkctx) + er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename(). defer func() { @@ -1791,7 +1896,38 @@ func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, objec if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") { return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID} } - defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID)) + // Authorize before the unconditional cleanup below. A native restore's + // upload is absent to every other caller, and an upload whose marker the + // drives cannot establish is not removed by anyone. + uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) + view := er.readRestoreUpload(ctx, bucket, uploadIDPath) + if view.committing { + // What a failed commit left behind, also after stale cleanup could + // not remove it: only with no commit running, and once every copy + // shows a staging older than the grace. + lk := er.restoreCommitLock(uploadIDPath) + lkctx, err := lk.GetLock(ctx, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait)) + if err != nil { + return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID) + } + defer lk.Unlock(lkctx) + if !er.restoreCommitAbandoned(lkctx.Context(), bucket, uploadIDPath) { + return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID) + } + er.deleteAll(lkctx.Context(), minioMetaMultipartBucket, uploadIDPath) + return true, nil + } + switch { + case view.visible(opts): + case view.known && view.marked: + return false, nil + case view.known && opts.Transition.attempt != nil: + // A native restore's own abort, of an upload no longer its own. + return false, nil + default: + return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID) + } + defer er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath) _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false) err = toObjectErr(err, bucket, object, uploadID) if _, absent := err.(InvalidUploadID); absent { diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 9307aae81..f3c8cfb42 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -2158,6 +2158,20 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string if opts.Transition.ExpireRestored && (gerr != nil || goi.DeleteMarker) { return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID} } + // A restored copy expires only by its own reason, decided on the + // version's authoritative generation: a drive that missed a renewal or a + // later POST must not authorize removing the cache, nor drives that + // disagree on the restore status. Another lifecycle deletion falling due, + // or the all-pools check of an ordinary expiry, does not stand in for it. + if opts.Transition.ExpireRestored { + gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID) + if err != nil || gen.split || gen.fi.Deleted || lc == nil { + return goi, errLifecycleNotApplicable + } + if !restoredCopyExpires(ctx, *lc, rcfg, replcfg, gen.fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)) { + return goi, errLifecycleNotApplicable + } + } if opts.EvalMetadataFn != nil { dsc, err := opts.EvalMetadataFn(&goi, gerr) @@ -2660,29 +2674,6 @@ func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket, return er.restoreTransitionedObject(ctx, bucket, object, opts) } -// update restore status header in the metadata -func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error { - oi := objInfo.Clone() - oi.metadataOnly = true // Perform only metadata updates. - - // allow retry in the case of failure to restore - delete(oi.UserDefined, xhttp.AmzRestore) - - if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{ - VersionID: oi.VersionID, - Transition: opts.Transition, - }, ObjectOptions{ - VersionID: oi.VersionID, - MTime: oi.ModTime, - NoLock: opts.NoLock, - Transition: opts.Transition, - }); err != nil { - storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err)) - return err - } - return nil -} - // getRestoreLock waits for a restore's object lock for as long as ctx lives. // The POST handler has already marked the version ongoing, and only the // restore clears that marker. A restore queued behind another version's @@ -2698,6 +2689,11 @@ func getRestoreLock(ctx context.Context, lk RWLocker) (LockContext, error) { // restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts // as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta +// +// A POST restore publishes its attempt on the version's ongoing marker. The +// restore rehydrates the version only while that attempt owns the marker, and +// on failure clears the marker only while it still does. A direct caller +// publishes no attempt; its attempt only names the private upload. func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error { if !opts.NoLock { // Single-pool and direct callers must protect the metadata snapshot @@ -2705,30 +2701,45 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s lk := er.NewNSLock(bucket, object) lkctx, err := getRestoreLock(ctx, lk) if err != nil { - return err + return restoreResetError{err} } ctx = lkctx.Context() defer lk.Unlock(lkctx) opts.NoLock = true } - setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error { - if rerr == nil { - return nil // nothing to do; restore object was successful - } - er.updateRestoreMetadata(ctx, bucket, object, oi, opts) - return rerr - } - var oi ObjectInfo - // get the file info on disk for transitioned object - actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false) + published := opts.Transition.attempt + // Restore the authoritative generation of the version, never a revision + // that only some drives hold. + gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID) if err != nil { - // No version was read, so there is no restore metadata to update. - return toObjectErr(err, bucket, object) + err = toObjectErr(err, bucket, object, opts.VersionID) + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + // No version was read, so there is no restore marker to clear. + return err + } + return restoreResetError{err} + } + actualfi := gen.fi + if actualfi.Deleted { + return toObjectErr(errMethodNotAllowed, bucket, object) } - oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) - ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi) + oi := actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) + attempt := published + if attempt == nil { + attempt = newRestoreAttempt(oi) + } else if !attempt.owns(oi) { + // A later POST took the version over, or it was replaced. + return errRestoreSuperseded + } + opts.Transition.attempt = attempt + versionID := restoreVersionID(actualfi.VersionID) + fail := func(cause error) error { + return er.restoreFailed(ctx, bucket, object, versionID, published, cause) + } + + ropts := putRestoreOpts(opts.Transition.RestoreRequest, oi, opts) ropts.NoLock = opts.NoLock ropts.Transition = opts.Transition // Rehydration copies encoded bytes without changing their content identity. @@ -2738,35 +2749,51 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s if len(oi.Parts) == 1 && !oi.isMultipart() { actualSize, err := oi.GetActualSize() if err != nil { - return setRestoreHeaderFn(oi, err) + return fail(err) } ropts.IndexCB = func() []byte { return oi.Parts[0].Index } var rs *HTTPRangeSpec gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts) if err != nil { - return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + return fail(toObjectErr(err, bucket, object)) } defer gr.Close() hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", actualSize) if err != nil { - return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + return fail(toObjectErr(err, bucket, object)) } pReader := NewPutObjReader(hashReader) - _, err = er.PutObject(ctx, bucket, object, pReader, ropts) - return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + if _, err = er.PutObject(ctx, bucket, object, pReader, ropts); err != nil { + return fail(toObjectErr(err, bucket, object)) + } + return nil } - res, err := er.NewMultipartUpload(ctx, bucket, object, ropts) - if err != nil { - return setRestoreHeaderFn(oi, err) + // Every server must keep the upload private, also a queued restore's. + if err := nativeRestoreAvailable(ctx); err != nil { + return fail(err) } + // The upload records when it starts, which the stale-upload cleanup ages; + // completion still commits the version's own ModTime. + uopts := ropts + uopts.MTime = time.Time{} + res, err := er.NewMultipartUpload(ctx, bucket, object, uopts) + if err != nil { + return fail(err) + } + completed := false + defer func() { + if !completed { + er.abortRestoreUpload(bucket, object, res.UploadID, attempt) + } + }() var uploadedParts []CompletePart var rs *HTTPRangeSpec // get reader from the warm backend - note that even in the case of encrypted objects, this stream is still encrypted. gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts) if err != nil { - return setRestoreHeaderFn(oi, err) + return fail(err) } defer gr.Close() @@ -2781,16 +2808,17 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s } hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", actualSize) if err != nil { - return setRestoreHeaderFn(oi, err) + return fail(err) } pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{ - IndexCB: func() []byte { return partInfo.Index }, + IndexCB: func() []byte { return partInfo.Index }, + Transition: TransitionOptions{attempt: attempt}, }) if err != nil { - return setRestoreHeaderFn(oi, err) + return fail(err) } if pInfo.Size != partInfo.Size { - return setRestoreHeaderFn(oi, InvalidObjectState{Bucket: bucket, Object: object}) + return fail(InvalidObjectState{Bucket: bucket, Object: object}) } uploadedParts = append(uploadedParts, CompletePart{ PartNumber: pInfo.PartNumber, @@ -2804,7 +2832,11 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s VersionSuspended: opts.VersionSuspended, Transition: opts.Transition, }) - return setRestoreHeaderFn(oi, err) + if err != nil { + return fail(err) + } + completed = true + return nil } // DecomTieredObject - moves tiered object to another pool during decommissioning. diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 849ad6c67..c168e22f9 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -3369,8 +3369,16 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec } // RestoreTransitionedObject - restore transitioned object content locally on this cluster. -func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { +func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) (err error) { object = encodeDirObject(object) + defer func() { + // The restore stopped before it could clear POST's ongoing marker. + var reset restoreResetError + if errors.As(err, &reset) { + z.recoverRestoreStatus(bucket, object, opts) + err = reset.error + } + }() if z.SinglePool() { return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts) } @@ -3380,7 +3388,7 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck lk := z.NewNSLock(bucket, object) lkctx, err := getRestoreLock(ctx, lk) if err != nil { - return err + return restoreResetError{err} } ctx = lkctx.Context() defer lk.Unlock(lkctx) @@ -3389,10 +3397,7 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck // Avoid restoring object from a pool being decommissioned. opts.SkipDecommissioned = true opts.NoLock = true - // Restore an existing version, not the latest version's write location. - lookupOpts := opts - lookupOpts.MetadataChg = true - idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts) + idx, err := z.restorePoolIdx(ctx, bucket, object, opts) if err != nil { return err } diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index d40ff3ce7..0f4a663ac 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -177,6 +177,10 @@ type TransitionOptions struct { RestoreRequest *RestoreObjectRequest RestoreExpiry time.Time ExpireRestored bool + + // attempt is the native restore a call belongs to. Only that restore + // sees its private upload, see restoreUploadVisible. + attempt *restoreAttempt } // MakeBucketOptions represents bucket options for ObjectLayer bucket operations @@ -295,6 +299,8 @@ type ObjectLayer interface { DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error + // restoreRequest records a POST restore on a version, see erasureServerPools. + restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error) // Multipart operations. ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) diff --git a/cmd/object-copy-metadata_test.go b/cmd/object-copy-metadata_test.go index 2cb05f3eb..72c729599 100644 --- a/cmd/object-copy-metadata_test.go +++ b/cmd/object-copy-metadata_test.go @@ -340,9 +340,9 @@ func TestCopyRewritesObjectData(t *testing.T) { name: "data copy always rewrites", want: true, }, - // PostRestoreObjectHandler, updateRestoreMetadata and batchKeyRotate all - // address the same version on both sides and never set Versioned, so they - // only ever reach these two cases. + // In-place metadata callers such as batchKeyRotate address the same + // version on both sides and never set Versioned, so they only ever + // reach these two cases. { name: "unversioned in-place metadata update", metadataOnly: true, diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index da9b772ac..81af341f5 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -3955,7 +3955,11 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * // Preserve a failed restore read even before the request body is parsed. opts.Transition.RestoreRequest = &RestoreObjectRequest{} - objInfo, err := getObjectInfo(ctx, bucket, object, opts) + // Resolve the target without the object lock: a running restore holds it + // for its download, and the status step bounds its own wait for it. + preOpts := opts + preOpts.NoLock = true + objInfo, err := getObjectInfo(ctx, bucket, object, preOpts) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return @@ -3982,48 +3986,47 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * } opts.Transition.RestoreRequest = rreq statusCode := http.StatusOK - alreadyRestored := false - if err == nil { - if objInfo.RestoreOngoing && rreq.Type != SelectRestoreRequest { + // set or upgrade restore expiry + restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days) + + // update self with restore metadata + var attempt *restoreAttempt + if rreq.Type != SelectRestoreRequest { + // Native restore needs every server to keep its upload private; + // refuse it, writing nothing, while that is not established. + if err := nativeRestoreAvailable(ctx); err != nil { + apiErr := errorCodes.ToAPIErr(ErrBusy) + apiErr.Code, apiErr.Description = "ServiceUnavailable", err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + // Decide on a fresh read under the object lock: renew a restored copy + // that is still there, or mark the version ongoing for a new attempt. + // Only the version resolved above, with its identity, is restored. + statusOpts := opts + statusOpts.Transition.RestoreExpiry = restoreExpiry + status, err := objectAPI.restoreRequest(GlobalContext, bucket, object, statusOpts, objInfo) + if errors.Is(err, errRestoreInProgress) { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectRestoreAlreadyInProgress), r.URL) return } - if !objInfo.RestoreOngoing && !objInfo.RestoreExpires.IsZero() { - statusCode = http.StatusAccepted - alreadyRestored = true - } - } - // set or upgrade restore expiry - restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days) - metadata := cloneMSS(objInfo.UserDefined) - - // update self with restore metadata - if rreq.Type != SelectRestoreRequest { - objInfo.metadataOnly = true // Perform only metadata updates. - metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days) - metadata[xhttp.AmzRestoreRequestDate] = time.Now().UTC().Format(http.TimeFormat) - if alreadyRestored { - metadata[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String() - } else { - metadata[xhttp.AmzRestore] = ongoingRestoreObj().String() - } - objInfo.UserDefined = metadata - if _, err := objectAPI.CopyObject(GlobalContext, bucket, object, bucket, object, objInfo, ObjectOptions{ - VersionID: objInfo.VersionID, - Transition: opts.Transition, - }, ObjectOptions{ - VersionID: objInfo.VersionID, - Transition: opts.Transition, - MTime: objInfo.ModTime, - MetadataChg: true, - }); err != nil { + if errors.As(err, &InvalidObjectState{}) || errors.As(err, &MethodNotAllowed{}) { writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL) return } - // for previously restored object, just update the restore expiry - if alreadyRestored { + if err != nil { + // Such as a read quorum error while telling whether it restores. + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } + // Events keep the version ID as this request resolved it. + status.objInfo.VersionID = objInfo.VersionID + objInfo = status.objInfo + // for previously restored object, just update the restore expiry + if status.attempt == nil { + return + } + attempt = status.attempt } restoreObject := mustGetUUID() @@ -4088,12 +4091,14 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * rreq.SelectParameters.Close() return } + // The versioning context and the explicit version POST resolved. restoreOpts := ObjectOptions{ Transition: TransitionOptions{ RestoreRequest: rreq, RestoreExpiry: restoreExpiry, + attempt: attempt, }, - VersionID: objInfo.VersionID, + VersionID: attempt.VersionID, Versioned: opts.Versioned, VersionSuspended: opts.VersionSuspended, } diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index b28c8db44..6661b5fba 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -116,6 +116,7 @@ var ( stopRebalanceRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerStopRebalance, grid.NewMSS, grid.NewNoPayload) updateMetacacheListingRPC = grid.NewSingleHandler[*metacache, *metacache](grid.HandlerUpdateMetacacheListing, func() *metacache { return &metacache{} }, func() *metacache { return &metacache{} }) cleanupUploadIDCacheMetaRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerClearUploadID, grid.NewMSS, grid.NewNoPayload) + restoreCapabilityRPC = grid.NewSingleHandler[*grid.MSS, *grid.MSS](grid.HandlerRestoreCapability, grid.NewMSS, grid.NewMSS) // STREAMS // Set an output capacity of 100 for consoleLog and listenRPC @@ -1440,6 +1441,7 @@ func registerPeerRESTHandlers(router *mux.Router, gm *grid.Manager) { logger.FatalIf(stopRebalanceRPC.Register(gm, server.StopRebalanceHandler), "unable to register handler") logger.FatalIf(updateMetacacheListingRPC.Register(gm, server.UpdateMetacacheListingHandler), "unable to register handler") logger.FatalIf(cleanupUploadIDCacheMetaRPC.Register(gm, server.HandlerClearUploadID), "unable to register handler") + logger.FatalIf(restoreCapabilityRPC.Register(gm, restoreCapabilityHandler), "unable to register handler") logger.FatalIf(gm.RegisterStreamingHandler(grid.HandlerTrace, grid.StreamHandler{ Handle: server.TraceHandler, diff --git a/cmd/restore-attempt.go b/cmd/restore-attempt.go new file mode 100644 index 000000000..0df3e36fd --- /dev/null +++ b/cmd/restore-attempt.go @@ -0,0 +1,862 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "net/http" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/minio/minio/internal/bucket/lifecycle" + xhttp "github.com/minio/minio/internal/http" +) + +// A native restore keeps two private records. The version that POST marks +// ongoing names the attempt that owns the marker; that attempt's multipart +// upload names the attempt and the version it rehydrates. Clients cannot +// set either: requests with reserved metadata are refused, it is never +// extracted from headers, and uploads only get the marker from the restore. +const ( + restoreAttemptKey = ReservedMetadataPrefixLower + "restore-attempt" + restoreUploadKey = ReservedMetadataPrefixLower + "restore-upload" +) + +const ( + // restoreTakeoverWait bounds how long a POST on an ongoing version waits + // for the object lock. A running restore holds it for its download. + restoreTakeoverWait = 2 * time.Second + // restoreCommitGrace is how long stale cleanup keeps an upload whose + // completion staged object metadata in it; a commit takes seconds. + restoreCommitGrace = time.Hour + // restoreCleanupTimeout bounds each status reset and upload abort. + restoreCleanupTimeout = time.Minute +) + +// restoreRecoveryDelays are the in-memory retries that clear the ongoing +// marker of a restore that failed before it could clear the marker itself. +// A restart loses them; a later POST then takes the marker over. +var restoreRecoveryDelays = []time.Duration{0, 100 * time.Millisecond, time.Second, 5 * time.Second, 30 * time.Second, 2 * time.Minute} + +var ( + // errRestoreSuperseded: the version no longer belongs to this attempt. + // The attempt must not rehydrate, complete or reset anything. + errRestoreSuperseded = errors.New("restore attempt superseded") + // errRestoreInProgress: a restore holds the object lock of the version. + errRestoreInProgress = errors.New("restore already in progress") + // errRestoreUnchanged: a restore status update found nothing to write. + errRestoreUnchanged = errors.New("restore status unchanged") +) + +// restoreResetError marks a failed restore that may have left its ongoing +// marker behind. The pools layer hands it to recoverRestoreStatus. +type restoreResetError struct{ error } + +func (e restoreResetError) Unwrap() error { return e.error } + +// restoreVersionID is the explicit version ID of a resolved restore target. +func restoreVersionID(versionID string) string { + if versionID == "" { + return nullVersionID + } + return versionID +} + +// restoreAttempt identifies one native restore of one stored version. +type restoreAttempt struct { + ID string `json:"id"` + VersionID string `json:"v"` // "null" for the null version + ETag string `json:"e"` + Size int64 `json:"s"` + ModTime time.Time `json:"m"` +} + +func newRestoreAttempt(oi ObjectInfo) *restoreAttempt { + return &restoreAttempt{ + ID: mustGetUUID(), + VersionID: restoreVersionID(oi.VersionID), + ETag: oi.ETag, + Size: oi.Size, + ModTime: oi.ModTime.UTC(), + } +} + +func (a *restoreAttempt) String() string { + b, _ := json.Marshal(a) + return string(b) +} + +func parseRestoreAttempt(s string) (*restoreAttempt, bool) { + var a restoreAttempt + if s == "" || json.Unmarshal([]byte(s), &a) != nil || a.ID == "" || a.VersionID == "" { + return nil, false + } + return &a, true +} + +func (a *restoreAttempt) equal(b *restoreAttempt) bool { + return a != nil && b != nil && a.ID == b.ID && a.VersionID == b.VersionID && + a.ETag == b.ETag && a.Size == b.Size && a.ModTime.Equal(b.ModTime) +} + +// targets reports whether oi is still the version the attempt was made for. +// A replacement under the same version ID differs in ETag, size or ModTime. +func (a *restoreAttempt) targets(oi ObjectInfo) bool { + return a != nil && !oi.DeleteMarker && restoreVersionID(oi.VersionID) == a.VersionID && + oi.ETag == a.ETag && oi.Size == a.Size && oi.ModTime.Equal(a.ModTime) && + oi.TransitionedObject.Status == lifecycle.TransitionComplete +} + +// owns reports whether the version is ongoing under this attempt. +func (a *restoreAttempt) owns(oi ObjectInfo) bool { + return a.targets(oi) && oi.RestoreOngoing && oi.UserDefined[restoreAttemptKey] == a.ID +} + +// rehydrates reports whether completing upload fi, the attempt's private +// upload, writes back the very version current. +func (a *restoreAttempt) rehydrates(current ObjectInfo, fi FileInfo) bool { + return a.targets(current) && fi.Metadata["etag"] == a.ETag && fi.Size == a.Size && fi.ModTime.Equal(a.ModTime) +} + +// restoreUploadView is the restore marker of an upload as the drives of +// its set establish it. A marked upload is written to a write quorum, at +// least a majority of the drives, and its copies never change their marker. +type restoreUploadView struct { + // marker is the attempt of a native restore's upload: one marker that a + // majority of the drives hold. + marker *restoreAttempt + // marked: some readable drive holds a marker. + marked bool + // known: the marker state is established. Without it, callers fail + // closed: nobody lists, uses, aborts or removes the upload. + known bool + // live: enough drives hold the upload or cannot tell that a written + // upload may exist; otherwise only remnants of one are left. + live bool + // committing: some drive holds object metadata in the upload's place. + // Completion stages the committed version's object metadata there, + // without the marker, before it renames the upload into the object. + committing bool + // bucket and object the upload of an established marker belongs to. + bucket, object string +} + +// restoreUploadState establishes the marker of an upload from the metadata of +// every drive. errs may be nil, then a drive without valid metadata is one +// that cannot tell. A marker is established on a majority of the drives; its +// absence only when no readable drive shows one and the drives that cannot +// tell are too few to hold a marked upload. +func restoreUploadState(metas []FileInfo, errs []error) restoreUploadView { + n := len(metas) + majority := n/2 + 1 + markers := make(map[string]int) + var unmarked, unknown, committing int + for i, fi := range metas { + if errs != nil && errs[i] != nil { + if !errors.Is(errs[i], errFileNotFound) && !errors.Is(errs[i], errFileVersionNotFound) && + !errors.Is(errs[i], errVolumeNotFound) && !errors.Is(errs[i], errUploadIDNotFound) { + unknown++ + } + continue + } + if !fi.IsValid() { + unknown++ + continue + } + if restoreUploadStaged(fi) { + committing++ + continue + } + if m := fi.Metadata[restoreUploadKey]; m != "" { + markers[m]++ + } else { + unmarked++ + } + } + present := unmarked + for _, c := range markers { + present += c + } + v := restoreUploadView{marked: len(markers) > 0, live: present+unknown+committing >= majority, committing: committing > 0} + if v.committing { + // Nobody but the completion may use or remove it now. + return v + } + for m, c := range markers { + if c >= majority { + v.marker, v.known = parseRestoreAttempt(m) + if v.known { + // The marker also names the upload's own target. + for _, fi := range metas { + if !fi.IsValid() || fi.Metadata[restoreUploadKey] != m { + continue + } + if restoreVersionID(fi.VersionID) != v.marker.VersionID || fi.Metadata["etag"] != v.marker.ETag { + v.marker, v.known = nil, false + break + } + v.bucket, v.object = fi.Metadata[multipartMetaBucket], fi.Metadata[multipartMetaObject] + } + } + return v + } + } + // A marker without a majority is established only as a remnant. + v.known = !v.live || (!v.marked && unknown < majority) + return v +} + +// restoreUploadStaged reports whether an upload's record is object metadata +// that a completion staged there: an upload record has no parts and no +// other versions. +func restoreUploadStaged(fi FileInfo) bool { + return fi.Deleted || len(fi.Parts) > 0 || fi.NumVersions > 1 +} + +// visible reports whether a caller sees the upload: a native restore its own +// upload, any other caller only an upload established as unmarked. +func (v restoreUploadView) visible(opts ObjectOptions) bool { + if a := opts.Transition.attempt; a != nil { + return v.marker.equal(a) + } + return v.known && !v.marked +} + +// restoreUploadVisible reports whether a caller sees an upload whose metadata +// is metas, one entry for each drive. +func restoreUploadVisible(metas []FileInfo, opts ObjectOptions) bool { + return restoreUploadState(metas, nil).visible(opts) +} + +// readRestoreUpload reads the metadata of an upload from every drive. +func (er erasureObjects) readRestoreUpload(ctx context.Context, bucket, uploadIDPath string) restoreUploadView { + metas, errs := readAllFileInfo(ctx, er.getDisks(), bucket, minioMetaMultipartBucket, uploadIDPath, "", false, false) + return restoreUploadState(metas, errs) +} + +// restoreCommitLock is held by a completion from staging through its commit; +// it is named by the upload's own path, which an upload whose copies all +// hold staged object metadata still has. +func (er erasureObjects) restoreCommitLock(uploadIDPath string) RWLocker { + return er.NewNSLock(minioMetaMultipartBucket, uploadIDPath) +} + +// restoreCommitAbandoned reports whether an upload holds only what a failed +// commit left: every drive answers, and every copy that is there is staged +// object metadata, see restoreUploadStaged, older than restoreCommitGrace by +// that drive's own metadata file. A copy that cannot be read, or that is not +// staged yet, may belong to a commit still running. The caller holds the +// upload's restoreCommitLock. +func (er erasureObjects) restoreCommitAbandoned(ctx context.Context, bucket, uploadIDPath string) bool { + disks := er.getDisks() + metas, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, uploadIDPath, "", false, false) + staged := 0 + for i, fi := range metas { + if errs[i] != nil { + if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errFileVersionNotFound) { + continue + } + return false + } + if !fi.IsValid() || !restoreUploadStaged(fi) { + return false + } + st, err := disks[i].StatInfoFile(ctx, minioMetaMultipartBucket, pathJoin(uploadIDPath, xlStorageFormatFile), false) + if err != nil || len(st) == 0 || time.Since(st[0].ModTime) < restoreCommitGrace { + return false + } + staged++ + } + return staged > 0 +} + +// restoreGeneration is the metadata generation of one version that the drives +// of a set hold authoritatively: the revision (header and normalized metadata +// digest, versionRevision) that at least max(n/2+1, its read quorum) drives +// hold. drives has each drive's own copy of it, with that drive's erasure +// index and inline data, and an invalid entry where a drive holds another or +// no revision; disks has nil there. +// +// A failed restore status write can leave drives that differ only in their +// restore fields, with no revision on a quorum. Then the generation is the +// restore-neutral revision that a quorum holds once those fields are left +// out: fi has no restore fields, and ongoing names the attempts whose +// ongoing marker some of its drives hold ("" for one without an attempt). +// Every other field, protection included, still needs the quorum. +type restoreGeneration struct { + fi FileInfo + drives []FileInfo + disks []StorageAPI + split bool + ongoing map[string]bool +} + +// restoreFieldKeys are the metadata keys a restore status write sets. +var restoreFieldKeys = []string{xhttp.AmzRestore, xhttp.AmzRestoreExpiryDays, xhttp.AmzRestoreRequestDate, restoreAttemptKey} + +// restoreHealMeta returns latest, the metadata heal copies to drives that +// lack the version, with the restore fields that at least quorum drives of +// that version hold. When some drive holds restore fields and no set of +// them has the quorum, healing the version waits. +func restoreHealMeta(metas []FileInfo, latest FileInfo, quorum int) (FileInfo, error) { + counts := make(map[string]int) + fields := make(map[string]map[string]string) + restoring := false + for _, m := range metas { + if !m.IsValid() || m.Deleted || !m.ModTime.Equal(latest.ModTime) { + continue + } + f := make(map[string]string) + var key strings.Builder + for _, k := range restoreFieldKeys { + if v, ok := m.Metadata[k]; ok { + f[k] = v + fmt.Fprintf(&key, "%d:%s=%d:%s;", len(k), k, len(v), v) + restoring = true + } + } + counts[key.String()]++ + fields[key.String()] = f + } + if !restoring { + return latest, nil + } + best, bestCount := "", 0 + for key, c := range counts { + if c > bestCount { + best, bestCount = key, c + } + } + if bestCount < quorum { + return latest, errErasureReadQuorum + } + latest.Metadata = maps.Clone(latest.Metadata) + for _, k := range restoreFieldKeys { + delete(latest.Metadata, k) + } + maps.Copy(latest.Metadata, fields[best]) + return latest, nil +} + +// restoreNeutralRevision is the revision of ver without its restore fields. +func restoreNeutralRevision(ver xlMetaV2ShallowVersion, metaV uint8) (revisionKey, bool) { + var v xlMetaV2Version + if _, err := v.unmarshalV(metaV, ver.meta); err != nil || v.Type != ObjectType || v.ObjectV2 == nil { + return revisionKey{}, false + } + o := *v.ObjectV2 + o.MetaUser, o.MetaSys = maps.Clone(o.MetaUser), maps.Clone(o.MetaSys) + for _, k := range restoreFieldKeys { + delete(o.MetaUser, k) + delete(o.MetaSys, k) + } + v.ObjectV2 = &o + meta, err := v.MarshalMsg(nil) + if err != nil { + return revisionKey{}, false + } + return versionRevision(xlMetaV2ShallowVersion{header: v.header(), meta: meta}, xlMetaVersion), true +} + +// readRestoreGeneration selects the authoritative generation of a version, +// or fails closed: a revision that only some drives hold, such as one a +// failed metadata update left behind, is never selected. An empty version ID +// addresses the latest version, as a direct restore may. +func (er erasureObjects) readRestoreGeneration(ctx context.Context, bucket, object, versionID string) (restoreGeneration, error) { + if versionID == "" { + fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, ObjectOptions{ + NoLock: true, + Transition: TransitionOptions{RestoreRequest: &RestoreObjectRequest{}}, + }, false) + if err != nil { + return restoreGeneration{}, err + } + versionID = restoreVersionID(fi.VersionID) + } + var uv [16]byte + if versionID != nullVersionID { + id, err := uuid.Parse(versionID) + if err != nil { + return restoreGeneration{}, errFileVersionNotFound + } + uv = id + } + disks := er.getDisks() + n := len(disks) + majority := n/2 + 1 + raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, true) + type revision struct { + ver xlMetaV2ShallowVersion + metaV uint8 + count int + } + keys := make([]*revisionKey, n) + neutral := make([]*revisionKey, n) + revisions := make(map[revisionKey]*revision) + neutrals := make(map[revisionKey]*revision) + absent := 0 + for i := range disks { + if errs[i] != nil { + if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errVolumeNotFound) { + absent++ + } + continue + } + var xl xlMetaV2 + if xl.LoadOrConvert(raws[i].Buf) != nil { + continue + } + found := false + for _, ver := range xl.versions { + if ver.header.VersionID != uv || ver.header.FreeVersion() { + continue + } + key := versionRevision(ver, xl.metaV) + keys[i], found = &key, true + r := revisions[key] + if r == nil { + r = &revision{ver: ver, metaV: xl.metaV} + revisions[key] = r + } + r.count++ + if nkey, ok := restoreNeutralRevision(ver, xl.metaV); ok { + neutral[i] = &nkey + if neutrals[nkey] == nil { + neutrals[nkey] = &revision{ver: ver, metaV: xl.metaV} + } + neutrals[nkey].count++ + } + break + } + if !found { + absent++ + } + } + accepted := func(revs map[revisionKey]*revision) (revisionKey, bool) { + var best *revision + var bestKey revisionKey + for key, r := range revs { + if best == nil || r.count > best.count { + best, bestKey = r, key + } + } + return bestKey, best != nil && best.count >= majority && + best.count >= er.versionReadQuorum(ctx, bucket, object, best.ver, best.metaV, make([]FileInfo, n), make([]error, n)) + } + gen := restoreGeneration{drives: make([]FileInfo, n), disks: make([]StorageAPI, n)} + bestKey, ok := accepted(revisions) + if !ok { + if bestKey, ok = accepted(neutrals); !ok { + if absent >= majority { + return restoreGeneration{}, errFileVersionNotFound + } + return restoreGeneration{}, errErasureReadQuorum + } + keys, gen.split, gen.ongoing = neutral, true, make(map[string]bool) + } + for i := range disks { + if keys[i] == nil || *keys[i] != bestKey { + continue + } + fi, err := getFileInfo(raws[i].Buf, bucket, object, versionID, fileInfoOpts{Data: true}) + if err != nil { + continue + } + gen.drives[i], gen.disks[i] = fi, disks[i] + if gen.split { + if st, err := parseRestoreObjStatus(fi.Metadata[xhttp.AmzRestore]); err == nil && st.Ongoing() { + gen.ongoing[fi.Metadata[restoreAttemptKey]] = true + } + } + if !gen.fi.IsValid() { + gen.fi = fi + } + } + if !gen.fi.IsValid() && !gen.fi.Deleted { + return restoreGeneration{}, errErasureReadQuorum + } + if gen.split { + gen.fi.Metadata = maps.Clone(gen.fi.Metadata) + for _, k := range restoreFieldKeys { + delete(gen.fi.Metadata, k) + } + } + return gen, nil +} + +// updateRestoreStatus rewrites the restore fields of one stored version. It +// selects the authoritative generation of the version, see +// readRestoreGeneration, patches only the restore fields onto exactly that +// generation, keeping every other field and its ModTime, and writes it back +// only to the drives that hold it. The caller holds the object lock. update +// edits the metadata in place, or returns an error, errRestoreUnchanged to +// write nothing; ongoing names the attempts whose ongoing marker the +// generation holds. A failed read never removes the version, and a failed +// write puts the generation it read back. +func (er erasureObjects) updateRestoreStatus(ctx context.Context, bucket, object, versionID string, update func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error) (ObjectInfo, error) { + gen, err := er.readRestoreGeneration(ctx, bucket, object, versionID) + if err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object, versionID) + } + fi := gen.fi + oi := fi.ToObjectInfo(bucket, object, false) + if fi.Deleted { + return oi, toObjectErr(errMethodNotAllowed, bucket, object) + } + ongoing := gen.ongoing + if !gen.split && oi.RestoreOngoing { + ongoing = map[string]bool{oi.UserDefined[restoreAttemptKey]: true} + } + meta := maps.Clone(fi.Metadata) + if err := update(oi, meta, ongoing); err != nil { + return oi, err + } + write := func(meta func(FileInfo) map[string]string) error { + metaArr := make([]FileInfo, len(gen.drives)) + for i, d := range gen.drives { + if gen.disks[i] == nil { + continue + } + if !d.InlineData() { + d.Data = nil + } + d.Metadata = meta(d) + metaArr[i] = d + } + // The write assigns each drive its slot as erasure index: order the + // drives by the version's distribution first. + disks, metaArr := shuffleDisksAndPartsMetadata(gen.disks, metaArr, fi) + _, err := writeUniqueFileInfo(ctx, disks, "", bucket, object, metaArr, fi.WriteQuorum(er.defaultWQuorum())) + return err + } + if err := write(func(FileInfo) map[string]string { return meta }); err != nil { + // Without the read generation back, its drives could hold two + // revisions and no quorum for either. A split that stays differs + // only in restore fields, which the next write or reset resolves. + if rerr := write(func(d FileInfo) map[string]string { return d.Metadata }); rerr != nil { + storageLogIf(ctx, fmt.Errorf("Unable to put back the restore status of %s/%s(%s): %w", bucket, object, versionID, rerr)) + } + return oi, toObjectErr(err, bucket, object) + } + fi.Metadata = meta + return fi.ToObjectInfo(bucket, object, false), nil +} + +// clearRestoreStatus makes a version retryable again while attempt a still +// owns its ongoing marker. It never brings back an earlier completed status: +// POST only starts a restore when that restored copy is gone. +func (er erasureObjects) clearRestoreStatus(ctx context.Context, bucket, object, versionID string, a *restoreAttempt) error { + _, err := er.updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, ongoing map[string]bool) error { + id := "" + if a != nil { + id = a.ID + if !a.targets(oi) { + return errRestoreUnchanged + } + } + // Clear only this attempt's marker. Where drives disagree, another + // attempt's marker may be the acknowledged one: leave that to it, + // or to a later POST, which takes it over under the lock. + if !ongoing[id] || len(ongoing) != 1 { + return errRestoreUnchanged + } + delete(meta, xhttp.AmzRestore) + delete(meta, xhttp.AmzRestoreExpiryDays) + delete(meta, xhttp.AmzRestoreRequestDate) + delete(meta, restoreAttemptKey) + return nil + }) + if errors.Is(err, errRestoreUnchanged) { + return nil + } + return err +} + +// restoreFailed clears the ongoing marker of a failed attempt while the +// object lock is held. A lost lock or an ended context leaves that to the +// bounded retries of the pools layer, which take the lock afresh. +func (er erasureObjects) restoreFailed(ctx context.Context, bucket, object, versionID string, published *restoreAttempt, cause error) error { + if ctx.Err() != nil { + return restoreResetError{cause} + } + if err := er.clearRestoreStatus(ctx, bucket, object, versionID, published); err != nil { + storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, versionID, err)) + return restoreResetError{cause} + } + return cause +} + +// abortRestoreUpload removes the private upload of a failed attempt, or +// leaves it to stale-upload cleanup. The restore's own context may have +// ended, so it uses a fresh bounded one, and the upload lock as any abort. +// It matches the upload to its own attempt rather than to the version, which +// a newer attempt may own by now, and it never touches the version. +func (er erasureObjects) abortRestoreUpload(bucket, object, uploadID string, a *restoreAttempt) { + ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout) + defer cancel() + lk := er.NewNSLock(bucket, pathJoin(object, uploadID)) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err == nil { + defer lk.Unlock(lkctx) + _, err = er.abortMultipartUpload(lkctx.Context(), bucket, object, uploadID, ObjectOptions{ + NoAuditLog: true, + Transition: TransitionOptions{attempt: a}, + }) + } + if err != nil { + storageLogIf(ctx, fmt.Errorf("Unable to abort the restore upload of %s/%s: %w", bucket, object, err)) + } +} + +// restoreUploadActive reports whether the restore that owns a stale-aged +// upload still runs: its attempt owns the version's ongoing marker. A long +// download keeps its upload; one whose attempt ended loses it, and an +// ongoing marker alone keeps nothing. +func (er erasureObjects) restoreUploadActive(ctx context.Context, v restoreUploadView) bool { + gen, err := er.readRestoreGeneration(ctx, v.bucket, v.object, v.marker.VersionID) + if err != nil { + // An unreadable version may still be restoring; retry next round. + return !errors.Is(err, errFileVersionNotFound) && !errors.Is(err, errFileNotFound) + } + if gen.split { + // Drives disagree on the restore status: that the attempt ended + // is not established. + return gen.ongoing[v.marker.ID] + } + return v.marker.owns(gen.fi.ToObjectInfo(v.bucket, v.object, false)) +} + +// restoreLock is the object lock that restores and their status updates +// share: with one pool the set's own lock, as single-pool writes use. +func (z *erasureServerPools) restoreLock(bucket, object string) RWLocker { + if z.SinglePool() { + return z.serverPools[0].getHashedSet(object).NewNSLock(bucket, object) + } + return z.NewNSLock(bucket, object) +} + +// restoreRequestStatus is what a POST restore recorded on its version. +type restoreRequestStatus struct { + objInfo ObjectInfo + // attempt is the restore that must rehydrate the version; nil when a + // restored copy was still there and its expiry was renewed. + attempt *restoreAttempt +} + +// restoreRequest records a POST restore on target, the version that POST +// resolved, on a fresh read under the object lock. The version must still +// be target: a replacement under the same version ID is refused. A copy that +// still holds its restored data is renewed. Otherwise the version is marked +// ongoing for a new attempt; that also takes over a marker no running +// restore holds, as a running restore holds the object lock for its whole +// download. Only the copy that is to be restored is written, so no copy +// claims a restored copy it does not hold. +func (z *erasureServerPools) restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error) { + object = encodeDirObject(object) + versionID := restoreVersionID(target.VersionID) + lk := z.restoreLock(bucket, object) + // Wait only briefly first, whatever the lockless read before saw: a + // restore of this version holds the lock for its whole download. Then + // wait as usual only if the drives show no restore of it running. + lkctx, err := lk.GetLock(ctx, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait)) + if errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil { + running, rerr := z.restoreRunning(ctx, bucket, object, versionID) + switch { + case rerr != nil: + // Whether it is restoring is unknown: answer now, not after + // the lock frees. + return restoreRequestStatus{}, rerr + case running: + return restoreRequestStatus{}, errRestoreInProgress + } + lkctx, err = lk.GetLock(ctx, globalOperationTimeout) + } + if err != nil { + return restoreRequestStatus{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + + expected := newRestoreAttempt(target) + copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{ + VersionID: versionID, + Versioned: opts.Versioned, + VersionSuspended: opts.VersionSuspended, + Transition: opts.Transition, + }) + if err != nil { + return restoreRequestStatus{}, err + } + // Restore the copy reads serve first, outside pools being decommissioned, + // as RestoreTransitionedObject does. + pool := -1 + for _, c := range copies { + if !z.IsSuspended(c.Index) { + pool = c.Index + break + } + } + if pool < 0 { + return restoreRequestStatus{}, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID} + } + + rreq := opts.Transition.RestoreRequest + date := UTCNow().Format(http.TimeFormat) + var attempt *restoreAttempt + oi, err := z.serverPools[pool].getHashedSet(object).updateRestoreStatus(ctx, bucket, object, versionID, func(oi ObjectInfo, meta map[string]string, _ map[string]bool) error { + if !expected.targets(oi) { + // Not the version POST resolved, or no longer transitioned. + return InvalidObjectState{Bucket: bucket, Object: decodeDirObject(object)} + } + attempt = nil + meta[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days) + meta[xhttp.AmzRestoreRequestDate] = date + if !oi.RestoreOngoing && !oi.RestoreExpires.IsZero() { + // Cleanup removes a restored copy together with its header. + meta[xhttp.AmzRestore] = completedRestoreObj(opts.Transition.RestoreExpiry).String() + delete(meta, restoreAttemptKey) + return nil + } + attempt = newRestoreAttempt(oi) + meta[xhttp.AmzRestore] = ongoingRestoreObj().String() + meta[restoreAttemptKey] = attempt.ID + return nil + }) + if err != nil { + if attempt != nil { + // The marker may have reached some drives, or all of them. + z.recoverRestoreStatus(bucket, object, ObjectOptions{ + VersionID: attempt.VersionID, + Transition: TransitionOptions{attempt: attempt}, + }) + } + return restoreRequestStatus{}, err + } + return restoreRequestStatus{objInfo: oi, attempt: attempt}, nil +} + +// restoreRunning reports, without the object lock and within the takeover +// wait, whether the drives show a restore of the version ongoing in a pool. +// An error means that this cannot be told. +func (z *erasureServerPools) restoreRunning(ctx context.Context, bucket, object, versionID string) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, restoreTakeoverWait) + defer cancel() + for _, pool := range z.serverPools { + gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, versionID) + if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) { + continue + } + if err == nil && ctx.Err() != nil { + err = ctx.Err() + } + if err != nil { + return false, toObjectErr(err, bucket, object, versionID) + } + if len(gen.ongoing) > 0 || !gen.split && gen.fi.ToObjectInfo(bucket, object, false).RestoreOngoing { + return true, nil + } + } + return false, nil +} + +// restorePoolIdx returns the pool whose copy the restore rehydrates: the one +// POST marked for its attempt, or for a direct caller the existing version. +func (z *erasureServerPools) restorePoolIdx(ctx context.Context, bucket, object string, opts ObjectOptions) (int, error) { + a := opts.Transition.attempt + if a == nil { + // Restore an existing version, not the latest version's write location. + lookupOpts := opts + lookupOpts.MetadataChg = true + idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, lookupOpts) + if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + return idx, restoreResetError{err} + } + return idx, err + } + found := false + for idx, pool := range z.serverPools { + gen, err := pool.getHashedSet(object).readRestoreGeneration(ctx, bucket, object, a.VersionID) + if errors.Is(err, errFileVersionNotFound) || errors.Is(err, errFileNotFound) { + continue + } + if err != nil { + return -1, restoreResetError{toObjectErr(err, bucket, object, a.VersionID)} + } + found = true + if !z.IsSuspended(idx) && a.owns(gen.fi.ToObjectInfo(bucket, object, false)) { + return idx, nil + } + } + if !found { + return -1, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: a.VersionID} + } + return -1, errRestoreSuperseded +} + +// recoverRestoreStatus retries, in memory and for a bounded time, to clear the +// ongoing marker that a failed restore left behind, so a later POST is not +// refused. Each try takes the object lock afresh and clears the marker only +// while it still belongs to this attempt. +func (z *erasureServerPools) recoverRestoreStatus(bucket, object string, opts ObjectOptions) { + go func() { + var err error + for _, delay := range restoreRecoveryDelays { + select { + case <-GlobalContext.Done(): + return + case <-time.After(delay): + } + if err = z.resetRestoreStatus(bucket, object, opts); err == nil { + return + } + } + storageLogIf(GlobalContext, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, err)) + }() +} + +// resetRestoreStatus is one try of recoverRestoreStatus. +func (z *erasureServerPools) resetRestoreStatus(bucket, object string, opts ObjectOptions) error { + ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout) + defer cancel() + lk := z.restoreLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return err + } + defer lk.Unlock(lkctx) + ctx = lkctx.Context() + + a := opts.Transition.attempt + versionID := opts.VersionID + if a != nil { + versionID = a.VersionID + } + for _, pool := range z.serverPools { + err := pool.getHashedSet(object).clearRestoreStatus(ctx, bucket, object, versionID, a) + if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !isErrMethodNotAllowed(err) && !isErrBucketNotFound(err) { + return err + } + } + return nil +} diff --git a/cmd/restore-capability.go b/cmd/restore-capability.go new file mode 100644 index 000000000..3594a6dc9 --- /dev/null +++ b/cmd/restore-capability.go @@ -0,0 +1,184 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" + "time" + + "github.com/minio/minio/internal/grid" +) + +// A native restore keeps its multipart upload private and marks the version +// it rehydrates (restore-attempt.go). A server that predates this ignores +// the marker: it would list, change, abort or complete the upload. Native +// restore therefore runs only while every server of the cluster positively +// reports that it understands the marker. A server that does not answer, +// or does not know the question, as an older server does not, blocks it. + +// nativeRestoreCapability is the native restore protocol this server speaks. +const nativeRestoreCapability = "1" + +const ( + peerRESTRestoreCapability = "restore" + // restoreCapabilityTTL bounds how long a positive answer is reused while + // no server reconnected. + restoreCapabilityTTL = 10 * time.Second + // restoreCapabilityTimeout bounds the question to each server. + restoreCapabilityTimeout = 5 * time.Second +) + +var errNativeRestoreUnavailable = errors.New("native restore is unavailable during a rolling upgrade: not every server of the cluster supports it yet") + +// restoreCapabilityHandler answers the question on this server. +func restoreCapabilityHandler(*grid.MSS) (*grid.MSS, *grid.RemoteErr) { + return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: nativeRestoreCapability}), nil +} + +// restoreCapabilityPeer is another server of the cluster. +type restoreCapabilityPeer interface { + // restoreConnection identifies the current connection to the server; it + // changes when the server reconnects, as a restarted one does. ok is + // false when the server is not connected. + restoreConnection() (key string, ok bool) + restoreCapability(ctx context.Context) (string, error) +} + +type restorePeerClient struct{ *peerRESTClient } + +func (c restorePeerClient) restoreConnection() (string, bool) { + if c.peerRESTClient == nil || c.gridConn == nil { + return "", false + } + conn := c.gridConn() + if conn == nil { + return "", false + } + st := conn.Stats() + if st.Connected == 0 { + return "", false + } + // Keys are compared by the position of their server. + return fmt.Sprintf("%d/%d", st.LastConnectTime.UnixNano(), st.ReconnectCount), true +} + +func (c restorePeerClient) restoreCapability(ctx context.Context) (string, error) { + conn := c.gridConn() + if conn == nil { + return "", errPeerOffline + } + resp, err := restoreCapabilityRPC.Call(ctx, conn, grid.NewMSS()) + if err != nil { + return "", err + } + return resp.Get(peerRESTRestoreCapability), nil +} + +// restoreCapabilityGate decides whether native restore may run, caching a +// positive answer for restoreCapabilityTTL while the connections to all +// servers stay the same. A negative answer is not cached. +type restoreCapabilityGate struct { + peers func() []restoreCapabilityPeer + + once sync.Once + turn chan struct{} // one check at a time + checked time.Time + keys []string +} + +var globalRestoreCapability = &restoreCapabilityGate{peers: notificationRestorePeers} + +func notificationRestorePeers() []restoreCapabilityPeer { + if globalNotificationSys == nil { + return nil + } + peers := make([]restoreCapabilityPeer, 0, len(globalNotificationSys.peerClients)) + for _, client := range globalNotificationSys.peerClients { + peers = append(peers, restorePeerClient{client}) + } + return peers +} + +// nativeRestoreAvailable returns errNativeRestoreUnavailable unless every +// other server of the cluster supports native restore. +func nativeRestoreAvailable(ctx context.Context) error { + return globalRestoreCapability.check(ctx) +} + +func (g *restoreCapabilityGate) check(ctx context.Context) error { + g.once.Do(func() { g.turn = make(chan struct{}, 1) }) + // One deadline covers the wait for another check and the questions. + ctx, cancel := context.WithTimeout(ctx, restoreCapabilityTimeout) + defer cancel() + select { + case g.turn <- struct{}{}: + case <-ctx.Done(): + return errNativeRestoreUnavailable + } + defer func() { <-g.turn }() + + // Read the connections only now: one may have changed while this check + // waited for another. + peers := g.peers() + connections := func() ([]string, bool) { + keys := make([]string, len(peers)) + for i, p := range peers { + key, ok := p.restoreConnection() + if !ok { + return nil, false + } + keys[i] = key + } + return keys, true + } + keys, ok := connections() + if !ok { + g.checked, g.keys = time.Time{}, nil + return errNativeRestoreUnavailable + } + if ctx.Err() != nil { + return errNativeRestoreUnavailable + } + if !g.checked.IsZero() && time.Since(g.checked) < restoreCapabilityTTL && slices.Equal(keys, g.keys) { + return nil + } + g.checked, g.keys = time.Time{}, nil + answers := make([]bool, len(peers)) + var wg sync.WaitGroup + for i, p := range peers { + wg.Add(1) + go func() { + defer wg.Done() + version, err := p.restoreCapability(ctx) + answers[i] = err == nil && version == nativeRestoreCapability + }() + } + wg.Wait() + // An answer counts only for the connection it was asked on: a server + // that reconnected meanwhile may be another binary. + after, ok := connections() + if !ok || !slices.Equal(keys, after) || slices.Contains(answers, false) || ctx.Err() != nil { + return errNativeRestoreUnavailable + } + g.checked, g.keys = time.Now(), keys + return nil +} diff --git a/cmd/restore-capability_test.go b/cmd/restore-capability_test.go new file mode 100644 index 000000000..306764d12 --- /dev/null +++ b/cmd/restore-capability_test.go @@ -0,0 +1,271 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "net/http" + "reflect" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/grid" + xhttp "github.com/minio/minio/internal/http" +) + +// A server answers the capability question over the grid. One that does not +// know it, as an older server, or answers something else, blocks native +// restore, and so does one that is not connected. +func TestNativeRestoreCapabilityGrid(t *testing.T) { + tg, err := grid.SetupTestGrid(4) + if err != nil { + t.Fatal(err) + } + t.Cleanup(tg.Cleanup) + if err := restoreCapabilityRPC.Register(tg.Managers[1], restoreCapabilityHandler); err != nil { + t.Fatal(err) + } + // Managers[2] never registers the handler. + if err := restoreCapabilityRPC.Register(tg.Managers[3], func(*grid.MSS) (*grid.MSS, *grid.RemoteErr) { + return grid.NewMSSWith(map[string]string{peerRESTRestoreCapability: "0"}), nil + }); err != nil { + t.Fatal(err) + } + peer := func(i int) restoreCapabilityPeer { + return restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[i]) }}} + } + offline := restorePeerClient{&peerRESTClient{gridConn: func() *grid.Connection { return nil }}} + for _, tc := range []struct { + name string + peers []restoreCapabilityPeer + ok bool + }{ + {"single-server", nil, true}, + {"all-capable", []restoreCapabilityPeer{peer(1)}, true}, + {"older-server", []restoreCapabilityPeer{peer(1), peer(2)}, false}, + {"other-answer", []restoreCapabilityPeer{peer(1), peer(3)}, false}, + {"offline", []restoreCapabilityPeer{peer(1), offline}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return tc.peers }} + if err := g.check(t.Context()); (err == nil) != tc.ok { + t.Errorf("capability: %v, want available=%v", err, tc.ok) + } + }) + } +} + +// restoreCapabilityFake is a server whose connection and answer a test sets. +type restoreCapabilityFake struct { + capable atomic.Bool + connected atomic.Bool + conn atomic.Int64 + asked atomic.Int64 +} + +func newRestoreCapabilityFake() *restoreCapabilityFake { + p := &restoreCapabilityFake{} + p.capable.Store(true) + p.connected.Store(true) + return p +} + +func (p *restoreCapabilityFake) restoreConnection() (string, bool) { + return strings.Repeat("c", int(p.conn.Load())), p.connected.Load() +} + +func (p *restoreCapabilityFake) restoreCapability(context.Context) (string, error) { + p.asked.Add(1) + if !p.capable.Load() { + return "", errors.New("Invalid Handler for type: RestoreCapability") + } + return nativeRestoreCapability, nil +} + +// A positive answer is reused while the connections stay the same; a +// reconnect asks again, a disconnected server blocks without being asked, +// and a negative answer is not reused. +func TestNativeRestoreCapabilityCache(t *testing.T) { + p := newRestoreCapabilityFake() + g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }} + for range 3 { + if err := g.check(t.Context()); err != nil { + t.Fatal(err) + } + } + if n := p.asked.Load(); n != 1 { + t.Errorf("asked %d times while connected and capable, want 1", n) + } + // The server restarts with an older binary and reconnects. + p.capable.Store(false) + p.conn.Add(1) + if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) { + t.Errorf("after a reconnect to an older server: %v", err) + } + p.capable.Store(true) + if err := g.check(t.Context()); err != nil { + t.Errorf("a negative answer was reused: %v", err) + } + asked := p.asked.Load() + p.connected.Store(false) + if err := g.check(t.Context()); !errors.Is(err, errNativeRestoreUnavailable) || p.asked.Load() != asked { + t.Errorf("disconnected server: %v, asked %d -> %d", err, asked, p.asked.Load()) + } +} + +// During a rolling upgrade POST restore answers 503 and writes nothing, and +// a restore queued before it creates no upload and leaves the version +// retryable. +func TestNativeRestoreRollingUpgradeGate(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, false, true) + p := newRestoreCapabilityFake() + previous := globalRestoreCapability + globalRestoreCapability = &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }} + t.Cleanup(func() { globalRestoreCapability = previous }) + + p.capable.Store(false) + before := r.fileInfo(oi.VersionID) + layer := newRestoreIsolationLayer(r) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + if rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "rolling upgrade") { + t.Errorf("POST during a rolling upgrade: %d %s", rec.Code, rec.Body.String()) + } + if after := r.fileInfo(oi.VersionID); !reflect.DeepEqual(before.Metadata, after.Metadata) { + t.Error("refused POST changed the version") + } + select { + case err := <-layer.done: + t.Fatalf("refused POST started a restore: %v", err) + default: + } + + p.capable.Store(true) + release := make(chan struct{}) + layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + <-release + return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) + } + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + close(release) + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + // A server is replaced by an older binary while the restore is queued. + p.capable.Store(false) + p.conn.Add(1) + close(release) + if err := layer.wait(t); !errors.Is(err, errNativeRestoreUnavailable) { + t.Fatalf("queued restore during a rolling upgrade: %v", err) + } + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("queued restore created an upload during a rolling upgrade: %v", uploads) + } + info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil || info.RestoreOngoing || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" { + t.Errorf("version after a refused queued restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err) + } +} + +// restoreCapabilityHeld is a fake server whose answer a test holds. +type restoreCapabilityHeld struct { + restoreCapabilityFake + asking chan struct{} + release chan struct{} +} + +func (p *restoreCapabilityHeld) restoreCapability(ctx context.Context) (string, error) { + v, err := p.restoreCapabilityFake.restoreCapability(ctx) + p.asking <- struct{}{} + select { + case <-p.release: + return v, err + case <-ctx.Done(): + return "", ctx.Err() + } +} + +// A check that waits for another reads the connections once its turn +// comes, an answer counts only if its server did not reconnect meanwhile, +// and a waiting check ends with its context. +func TestNativeRestoreCapabilityRevalidates(t *testing.T) { + t.Run("reconnect-during-check", func(t *testing.T) { + held := func() *restoreCapabilityHeld { + p := &restoreCapabilityHeld{asking: make(chan struct{}, 4), release: make(chan struct{})} + p.capable.Store(true) + p.connected.Store(true) + return p + } + fast, slow := held(), held() + g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{fast, slow} }} + first := make(chan error, 1) + go func() { first <- g.check(t.Context()) }() + <-fast.asking + <-slow.asking + waiting := make(chan error, 1) + go func() { waiting <- g.check(t.Context()) }() + // Both answered "capable"; then fast reconnects as an older server. + fast.capable.Store(false) + fast.conn.Add(1) + close(fast.release) + close(slow.release) + if err := <-first; !errors.Is(err, errNativeRestoreUnavailable) { + t.Errorf("check whose server reconnected while it asked: %v", err) + } + if err := <-waiting; !errors.Is(err, errNativeRestoreUnavailable) { + t.Errorf("waiting check accepted a reconnected older server: %v", err) + } + }) + t.Run("canceled-cached", func(t *testing.T) { + p := newRestoreCapabilityFake() + g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{p} }} + if err := g.check(t.Context()); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := g.check(ctx); err == nil { + t.Error("a cached answer accepted an ended context") + } + }) + t.Run("canceled-wait", func(t *testing.T) { + slow := &restoreCapabilityHeld{restoreCapabilityFake: restoreCapabilityFake{}, asking: make(chan struct{}, 4), release: make(chan struct{})} + slow.capable.Store(true) + slow.connected.Store(true) + defer close(slow.release) + g := &restoreCapabilityGate{peers: func() []restoreCapabilityPeer { return []restoreCapabilityPeer{slow} }} + go g.check(t.Context()) + <-slow.asking + ctx, cancel := context.WithCancel(t.Context()) + waiting := make(chan error, 1) + go func() { waiting <- g.check(ctx) }() + cancel() + select { + case err := <-waiting: + if err == nil { + t.Error("canceled check succeeded") + } + case <-time.After(time.Second): + t.Error("canceled check kept waiting for another check") + } + }) +} diff --git a/cmd/restore-native-generation_test.go b/cmd/restore-native-generation_test.go new file mode 100644 index 000000000..d79bc1030 --- /dev/null +++ b/cmd/restore-native-generation_test.go @@ -0,0 +1,1418 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "errors" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "os" + "reflect" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/config/drive" + xhttp "github.com/minio/minio/internal/http" +) + +// restoreGenerationDisk fails metadata writes or updates of the object, or +// metadata reads of uploads, while its toggles are set. +type restoreGenerationDisk struct { + StorageAPI + bucket, object string + write, update, uploadRead *atomic.Bool +} + +func (d restoreGenerationDisk) WriteMetadata(ctx context.Context, orig, vol, path string, fi FileInfo) error { + if d.write != nil && d.write.Load() && vol == d.bucket && path == d.object { + return errFaultyDisk + } + return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi) +} + +func (d restoreGenerationDisk) UpdateMetadata(ctx context.Context, vol, path string, fi FileInfo, opts UpdateMetadataOpts) error { + if d.update != nil && d.update.Load() && vol == d.bucket && path == d.object { + return errFaultyDisk + } + return d.StorageAPI.UpdateMetadata(ctx, vol, path, fi, opts) +} + +func (d restoreGenerationDisk) ReadVersion(ctx context.Context, orig, vol, path, version string, opts ReadOptions) (FileInfo, error) { + if d.uploadRead != nil && d.uploadRead.Load() && vol == minioMetaMultipartBucket { + return FileInfo{}, errFaultyDisk + } + return d.StorageAPI.ReadVersion(ctx, orig, vol, path, version, opts) +} + +// restoreGenerationOffline makes a drive's object metadata unreadable and +// unwritable while set, its uploads still readable. +type restoreGenerationOffline struct { + StorageAPI + bucket, object string + fail *atomic.Bool +} + +func (d restoreGenerationOffline) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) { + if d.fail.Load() && bucket == d.bucket && object == d.object { + return RawFileInfo{}, errFaultyDisk + } + return d.StorageAPI.ReadXL(ctx, bucket, object, data) +} + +func (d restoreGenerationOffline) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) { + if d.fail.Load() && bucket == d.bucket && object == d.object { + return FileInfo{}, errFaultyDisk + } + return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts) +} + +func (d restoreGenerationOffline) WriteMetadata(ctx context.Context, orig, bucket, object string, fi FileInfo) error { + if d.fail.Load() && bucket == d.bucket && object == d.object { + return errFaultyDisk + } + return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi) +} + +// restoreGenerationMetadata returns the version's metadata on each drive. +func restoreGenerationMetadata(t *testing.T, disks []StorageAPI, r *restoreTest, versionID string) []map[string]string { + t.Helper() + metas := make([]map[string]string, len(disks)) + for i, d := range disks { + fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, versionID, ReadOptions{}) + if err != nil { + t.Fatal(err) + } + metas[i] = fi.Metadata + } + return metas +} + +// An Object Lock update acknowledged on a quorum, which one drive missed, +// survives POST restore: POST patches the restore fields onto the quorum's +// generation and never writes the stale drive's metadata anywhere. +func TestRestoreStatusUsesQuorumGeneration(t *testing.T) { + for _, kind := range []string{"legal-hold", "retention"} { + t.Run(kind, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + enableBucketObjectLock(t, r.bucket) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + initial := []byte(`OFF`) + updated := []byte(`ON`) + key := strings.ToLower(xhttp.AmzObjectLockLegalHold) + if kind == "retention" { + initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour)) + updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour)) + key = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate) + } + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var fail atomic.Bool + wrapped := append([]StorageAPI(nil), disks...) + wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, update: &fail} + er.getDisks = func() []StorageAPI { return wrapped } + fail.Store(true) + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK { + t.Fatalf("protection update: %d %s", rec.Code, rec.Body.String()) + } + fail.Store(false) + before := restoreGenerationMetadata(t, disks, r, oi.VersionID) + acknowledged := before[1][key] + if before[0][key] == acknowledged { + t.Fatal("fixture: drive 0 did not miss the update") + } + for i := 1; i < len(disks); i++ { + if before[i][key] != acknowledged { + t.Fatal("fixture: the update did not reach the other drives") + } + } + if rec := restoreIsolationPOST(t, r, r.z, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + after := restoreGenerationMetadata(t, disks, r, oi.VersionID) + for i := 1; i < len(disks); i++ { + if after[i][key] != acknowledged { + t.Errorf("drive %d: POST wrote %q over the acknowledged %q", i, after[i][key], acknowledged) + } + if after[i][xhttp.AmzRestoreRequestDate] == before[i][xhttp.AmzRestoreRequestDate] { + t.Errorf("drive %d: POST did not record its request", i) + } + } + // The stale drive is left to heal; reads that pick it are not + // changed here. + if after[0][key] != before[0][key] || after[0][xhttp.AmzRestoreRequestDate] != before[0][xhttp.AmzRestoreRequestDate] { + t.Error("POST wrote the stale drive instead of leaving it to heal") + } + }) + } +} + +// The attempt is part of a version's metadata generation. A POST whose +// marker reached all drives but one is acknowledged: its restore owns the +// version, and the reset of an older attempt leaves the newer marker alone. +func TestRestoreAttemptGenerationFencing(t *testing.T) { + for _, mode := range []string{"worker", "old-reset", "degraded-old-reset"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var fail, offline atomic.Bool + wrapped := append([]StorageAPI(nil), disks...) + wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, write: &fail} + wrapped[1] = restoreGenerationOffline{StorageAPI: disks[1], bucket: r.bucket, object: r.object, fail: &offline} + er.getDisks = func() []StorageAPI { return wrapped } + captured := make(chan ObjectOptions, 2) + layer := newRestoreIsolationLayer(r) + layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error { + captured <- opts + return nil + } + var older ObjectOptions + if mode != "worker" { + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + older = <-captured + layer.wait(t) + } + fail.Store(true) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + fail.Store(false) + if rec.Code != http.StatusOK { + t.Fatalf("POST on all drives but one: %d %s", rec.Code, rec.Body.String()) + } + newer := <-captured + layer.wait(t) + id := newer.Transition.attempt.ID + if mode == "worker" { + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, newer); err != nil { + t.Fatalf("restore of the acknowledged attempt: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } + return + } + if mode == "degraded-old-reset" { + // One drive of the newer attempt is unreadable: the drives + // that answer no longer show which attempt is acknowledged. + offline.Store(true) + } + if err := r.z.resetRestoreStatus(r.bucket, r.object, older); err != nil { + t.Fatal(err) + } + offline.Store(false) + after := restoreGenerationMetadata(t, disks, r, oi.VersionID) + for i := 1; i < len(disks); i++ { + if after[i][restoreAttemptKey] != id || !strings.Contains(after[i][xhttp.AmzRestore], "true") { + t.Errorf("drive %d: the older attempt's reset cleared the newer marker: %q %q", i, after[i][restoreAttemptKey], after[i][xhttp.AmzRestore]) + } + } + if after[0][restoreAttemptKey] != older.Transition.attempt.ID { + t.Error("the reset wrote the stale drive") + } + // The newer attempt still owns the version and restores it. + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, newer); err != nil { + t.Fatalf("restore of the newer attempt after the older reset: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } + }) + } +} + +// restoreGenerationUploadPath is the stored path of an upload. +func restoreGenerationUploadPath(t *testing.T, r *restoreTest, id string) string { + t.Helper() + b, err := base64.RawURLEncoding.DecodeString(id) + if err != nil { + t.Fatal(err) + } + _, uploadUUID, ok := strings.Cut(string(b), ".") + if !ok { + t.Fatalf("upload ID without UUID: %q", id) + } + return pathJoin(r.z.serverPools[0].getHashedSet(r.object).getMultipartSHADir(r.bucket, r.object), uploadUUID) +} + +// The restore marker is established on a majority of the drives. One drive +// whose copy of the upload lacks it does not open the upload to clients, +// and the restore still completes. +func TestRestoreUploadMarkerQuorum(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + tier := newRestoreIsolationGateTier(5<<20, nil) + globalTierConfigMgr.drivercache["WARM"] = tier + defer tier.unblock() + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + restoreIsolationEntered(t, tier) + id, _ := restoreIsolationUpload(t, r, 0) + disk := r.z.serverPools[0].getHashedSet(r.object).getDisks()[0] + path := restoreGenerationUploadPath(t, r, id) + fi, err := disk.ReadVersion(t.Context(), "", minioMetaMultipartBucket, path, "", ReadOptions{}) + if err != nil { + t.Fatal(err) + } + delete(fi.Metadata, restoreUploadKey) + if err := disk.WriteMetadata(t.Context(), "", minioMetaMultipartBucket, path, fi); err != nil { + t.Fatal(err) + } + poison := bytes.Repeat([]byte("p"), 5<<20) + if _, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, id, 1, + mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", ""), ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) { + t.Errorf("public part with one unmarked copy of the upload: %v, want NoSuchUpload", err) + } + for _, op := range []func() error{ + func() error { + _, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{}) + return err + }, + func() error { + _, err := r.z.GetMultipartInfo(t.Context(), r.bucket, r.object, id, ObjectOptions{}) + return err + }, + } { + if err := op(); !errors.As(err, &InvalidUploadID{}) { + t.Errorf("public read with one unmarked copy of the upload: %v, want NoSuchUpload", err) + } + } + if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10); err != nil || len(uploads.Uploads) != 0 { + t.Errorf("listing with one unmarked copy of the upload: %+v, %v", uploads.Uploads, err) + } + tier.unblock() + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } +} + +// When the drives cannot tell an upload's marker, no client lists it, no +// abort removes it and stale cleanup keeps it; the restore completes once +// the drives answer again. +func TestRestoreUploadUnreadableFailsClosed(t *testing.T) { + for _, mode := range []string{"list", "abort", "stale"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var fail atomic.Bool + // Stale cleanup with a short age also removes in-flight temporary + // parts: hold that restore in completion, after its last part and + // before its commit, and the others in their download, outside + // the upload lock. + gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} + defer gate.unblock() + tier := newRestoreIsolationGateTier(5<<20, nil) + defer tier.unblock() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + disk := d + if mode == "stale" { + disk = restoreIsolationValidationDisk{d, gate} + } + wrapped[i] = restoreGenerationDisk{StorageAPI: disk, uploadRead: &fail} + } + er.getDisks = func() []StorageAPI { return wrapped } + if mode != "stale" { + globalTierConfigMgr.drivercache["WARM"] = tier + } + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + if mode == "stale" { + select { + case <-gate.entered: + case <-time.After(10 * time.Second): + t.Fatal("restore never reached completion") + } + } else { + restoreIsolationEntered(t, tier) + } + stored := restoreIsolationUploads(t, r, 0) + if len(stored) != 1 { + t.Fatalf("restore upload on disk: %v", stored) + } + var id string + for uploadUUID := range stored { + id = base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)) + } + fail.Store(true) + switch mode { + case "list": + if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 100); len(uploads.Uploads) != 0 { + t.Errorf("an upload with an unknown marker is listed: %+v, %v", uploads.Uploads, err) + } + case "abort": + if err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); err == nil { + t.Error("abort of an upload with an unknown marker succeeded") + } + case "stale": + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + for _, disk := range wrapped { + er.cleanupStaleUploadsOnDisk(t.Context(), disk) + } + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + } + fail.Store(false) + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 1 { + t.Errorf("%s removed an upload with an unknown marker: %v", mode, uploads) + } + gate.unblock() + tier.unblock() + if err := layer.wait(t); err != nil { + t.Fatalf("restore after %s: %v", mode, err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } + }) + } +} + +// POST restores the version it resolved with that version's identity. A +// replacement under the same version ID between its read and its status +// write is refused, and nothing restores it. +func TestRestorePOSTRejectsReplacedTarget(t *testing.T) { + for _, state := range []versioning.State{"", versioning.Suspended} { + t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, state) + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, true, false) + layer := newRestoreIsolationLayer(r) + var replacement ObjectInfo + layer.afterRead = func() { + data := []byte("replacement after POST resolved its target") + var err error + replacement, err = r.z.PutObject(t.Context(), r.bucket, r.object, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{VersionSuspended: state == versioning.Suspended}) + if err != nil { + t.Fatal(err) + } + opts := r.opts(nullVersionID) + opts.MTime = replacement.ModTime + opts.Transition = TransitionOptions{Tier: "WARM", ETag: replacement.ETag} + if err := r.z.serverPools[0].TransitionObject(t.Context(), r.bucket, r.object, opts); err != nil { + t.Fatal(err) + } + } + if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code < 400 { + layer.wait(t) + t.Errorf("POST accepted a replacement of its target: %d", rec.Code) + } + after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(nullVersionID)) + if err != nil { + t.Fatal(err) + } + if after.ETag != replacement.ETag || after.UserDefined[xhttp.AmzRestore] != "" || after.ETag == oi.ETag { + t.Errorf("replacement %q changed by POST: etag %q restore %q", replacement.ETag, after.ETag, after.UserDefined[xhttp.AmzRestore]) + } + }) + } +} + +// A POST on a version whose restore holds the object lock answers within +// the two-second takeover bound: its first read does not wait for the lock. +func TestRestorePOSTOngoingBound(t *testing.T) { + for _, mode := range []string{"pools=1", "pools=2", "stale-minority", "failed-activity-read"} { + t.Run(mode, func(t *testing.T) { + pools := 1 + if mode == "pools=2" { + pools = 2 + } + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, false, false) + tier := newRestoreIsolationGateTier(0, nil) + globalTierConfigMgr.drivercache["WARM"] = tier + defer tier.unblock() + var fail atomic.Bool + if mode == "stale-minority" { + // The first read may pick a drive that missed the marker. + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + wrapped := append([]StorageAPI(nil), disks...) + wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, write: &fail} + er.getDisks = func() []StorageAPI { return wrapped } + fail.Store(true) + } + layer := newRestoreIsolationLayer(r) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + fail.Store(false) + if rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + restoreIsolationEntered(t, tier) + var unreadable atomic.Bool + second := newRestoreIsolationLayer(r) + if mode == "failed-activity-read" { + // Once POST resolved its target, nothing tells whether it + // is restoring. + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreGenerationOffline{StorageAPI: d, bucket: r.bucket, object: r.object, fail: &unreadable} + } + er.getDisks = func() []StorageAPI { return wrapped } + second.afterRead = func() { unreadable.Store(true) } + } + done := make(chan *httptest.ResponseRecorder, 1) + start := time.Now() + go func() { done <- restoreIsolationPOST(t, r, second, oi.VersionID) }() + select { + case rec := <-done: + want := rec.Code == http.StatusConflict + if mode == "failed-activity-read" { + want = rec.Code >= 500 + } + if !want || time.Since(start) > restoreTakeoverWait+time.Second { + t.Errorf("POST during a running restore: %d after %s, want a bounded refusal", rec.Code, time.Since(start)) + } + case <-time.After(restoreTakeoverWait + 2*time.Second): + t.Error("POST during a running restore waited for the object lock") + unreadable.Store(false) + tier.unblock() + <-done + } + unreadable.Store(false) + tier.unblock() + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + }) + } +} + +// A POST whose status write fails on drives that stay unwritten puts the +// generation it read back, and a split that a failed put-back leaves +// differs only in restore fields: the attempt's reset and a later POST both +// resolve it, while every other field still needs the quorum. +func TestRestorePOSTStatusWriteSplit(t *testing.T) { + for _, mode := range []string{"put-back", "split-reset", "split-post"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var fail atomic.Bool + wrapped := append([]StorageAPI(nil), disks...) + for i := range 2 { + wrapped[i] = restoreGenerationDisk{StorageAPI: disks[i], bucket: r.bucket, object: r.object, write: &fail} + } + er.getDisks = func() []StorageAPI { return wrapped } + before := restoreGenerationMetadata(t, disks, r, oi.VersionID) + var attempt *restoreAttempt + if mode == "put-back" { + fail.Store(true) + rec := restoreIsolationPOST(t, r, r.z, oi.VersionID) + fail.Store(false) + if rec.Code < 400 { + t.Fatalf("POST acknowledged a status write on two of four drives: %d", rec.Code) + } + if after := restoreGenerationMetadata(t, disks, r, oi.VersionID); !reflect.DeepEqual(before, after) { + t.Error("failed status write was not put back") + } + } else { + // A status write and its put-back both failed on two drives. + attempt = newRestoreAttempt(oi) + for _, d := range disks[2:] { + fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true}) + if err != nil { + t.Fatal(err) + } + fi.Metadata[xhttp.AmzRestore] = ongoingRestoreObj().String() + fi.Metadata[restoreAttemptKey] = attempt.ID + if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + if _, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); err != nil { + t.Fatalf("a split in restore fields only has no generation: %v", err) + } + } + if mode == "split-reset" { + opts := nativeRestoreOptions(r, oi.VersionID) + opts.Transition.attempt = attempt + if err := r.z.resetRestoreStatus(r.bucket, r.object, opts); err != nil { + t.Fatalf("reset of the split attempt: %v", err) + } + for i, meta := range restoreGenerationMetadata(t, disks, r, oi.VersionID) { + if meta[xhttp.AmzRestore] != "" || meta[restoreAttemptKey] != "" { + t.Errorf("drive %d keeps the split attempt: %q %q", i, meta[xhttp.AmzRestore], meta[restoreAttemptKey]) + } + } + } + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST after the failed status write: %d %s", rec.Code, rec.Body.String()) + } + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } + }) + } +} + +// A split in any other field is not resolved: the generation fails closed. +func TestRestoreGenerationSplitNeedsQuorum(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, _ := restoreIsolationSeed(t, r, 0, false, false) + er := r.z.serverPools[0].getHashedSet(r.object) + for i, d := range er.getDisks()[2:] { + fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true}) + if err != nil { + t.Fatal(err) + } + fi.Metadata[xhttp.AmzRestore] = ongoingRestoreObj().String() + fi.Metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = fmt.Sprint("ON", i) + if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + if _, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); !errors.Is(err, errErasureReadQuorum) { + t.Fatalf("split outside the restore fields: %v, want read quorum", err) + } + if rec := restoreIsolationPOST(t, r, r.z, oi.VersionID); rec.Code < 400 { + t.Errorf("POST over a split outside the restore fields: %d", rec.Code) + } +} + +// restoreGenerationStageDisk holds a completion that is staging: as +// xlStorage.RenameData does, a drive that stages writes the committed +// object metadata into the upload before it renames the upload into the +// object. +type restoreGenerationStageDisk struct { + StorageAPI + bucket, object string + stage bool + arrived *atomic.Int32 + entered chan struct{} + release chan struct{} + disks int32 +} + +func (d restoreGenerationStageDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { + if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { + if d.stage { + if err := d.StorageAPI.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil { + return RenameDataResp{}, err + } + } + if d.arrived.Add(1) == d.disks { + close(d.entered) + } + select { + case <-d.release: + case <-ctx.Done(): + return RenameDataResp{}, ctx.Err() + } + } + return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) +} + +// While a completion stages its commit, the upload holds object metadata +// without the marker on the drives that staged. Nobody lists, aborts or +// removes it then, also when the copies not staged yet are old. +func TestRestoreUploadStagingFailsClosed(t *testing.T) { + for _, mode := range []string{"staged", "partly-staged", "stalled-commit"} { + t.Run(mode, func(t *testing.T) { testRestoreUploadStaging(t, mode) }) + } +} + +func testRestoreUploadStaging(t *testing.T, mode string) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var arrived atomic.Int32 + entered, release := make(chan struct{}), make(chan struct{}) + released := false + defer func() { + if !released { + close(release) + } + }() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: mode != "partly-staged" || i == 0, + arrived: &arrived, entered: entered, release: release, disks: int32(len(disks))} + } + er.getDisks = func() []StorageAPI { return wrapped } + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("completion never staged its commit") + } + stored := restoreIsolationUploads(t, r, 0) + var id string + for uploadUUID := range stored { + id = base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)) + if mode != "staged" { + // partly-staged: a long restore, the copies not staged yet are + // old. stalled-commit: drives that stall the running commit + // past the grace make its staged copies look abandoned. + old := UTCNow().Add(-48 * time.Hour) + for _, d := range disks[1:] { + if err := os.Chtimes(pathJoin(d.Endpoint().Path, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), uploadUUID, xlStorageFormatFile), old, old); err != nil { + t.Fatal(err) + } + } + } + } + if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10); err != nil || len(uploads.Uploads) != 0 { + t.Errorf("a staged restore upload is listed: %+v, %v", uploads.Uploads, err) + } + // The set's own abort: the pools layer would wait for the upload lock + // the completion holds. + if err := er.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); err == nil { + t.Error("abort of a staged restore upload succeeded") + } + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + for _, disk := range wrapped { + er.cleanupStaleUploadsOnDisk(t.Context(), disk) + } + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) { + t.Errorf("a staged restore upload was removed: %v -> %v", stored, after) + } + released = true + close(release) + if err := layer.wait(t); err != nil { + t.Fatalf("restore after its staging was attacked: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } +} + +// When a drive of an acknowledged attempt becomes unreadable, the drives that +// answer disagree on the restore status. That the attempt ended is then not +// established, and stale cleanup keeps its upload. +func TestRestoreSplitKeepsRunningUpload(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var missed, offline atomic.Bool + gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} + defer gate.unblock() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + var disk StorageAPI = restoreIsolationValidationDisk{d, gate} + switch i { + case 0: + disk = restoreGenerationDisk{StorageAPI: disk, bucket: r.bucket, object: r.object, write: &missed} + case 1: + disk = restoreGenerationOffline{StorageAPI: disk, bucket: r.bucket, object: r.object, fail: &offline} + } + wrapped[i] = disk + } + er.getDisks = func() []StorageAPI { return wrapped } + layer := newRestoreIsolationLayer(r) + missed.Store(true) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + missed.Store(false) + if rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + select { + case <-gate.entered: + case <-time.After(10 * time.Second): + t.Fatal("restore never reached completion") + } + stored := restoreIsolationUploads(t, r, 0) + offline.Store(true) + if gen, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); err != nil || !gen.split { + t.Fatalf("fixture: no split generation: %v", err) + } + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + for _, disk := range wrapped { + er.cleanupStaleUploadsOnDisk(t.Context(), disk) + } + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + offline.Store(false) + if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) { + t.Errorf("stale cleanup removed a running restore's upload: %v -> %v", stored, after) + } + gate.unblock() + if err := layer.wait(t); err != nil { + t.Fatalf("restore after the split: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } +} + +// restoreGenerationCleanupDisk misses restored-copy cleanups and metadata +// writes of the object while its toggles are set. +type restoreGenerationCleanupDisk struct { + StorageAPI + bucket, object string + cleanup, metaWrites *atomic.Bool +} + +func (d restoreGenerationCleanupDisk) DeleteVersion(ctx context.Context, bucket, object string, fi FileInfo, force bool, opts DeleteOptions) error { + if bucket == d.bucket && object == d.object && fi.ExpireRestored && d.cleanup.Load() { + return errFaultyDisk + } + return d.StorageAPI.DeleteVersion(ctx, bucket, object, fi, force, opts) +} + +func (d restoreGenerationCleanupDisk) WriteMetadata(ctx context.Context, orig, bucket, object string, fi FileInfo) error { + if bucket == d.bucket && object == d.object && d.metaWrites.Load() { + return errFaultyDisk + } + return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi) +} + +// Restored-copy expiry decides on the version's authoritative generation. A +// drive that missed a cleanup and the POST after it keeps an expired header, +// which must not clear the quorum's ongoing restore; drives split between +// an expired and a renewed header authorize nothing. +func TestExpireRestoredUsesQuorumGeneration(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, mode := range []string{"after-partial-cleanup", "split-renewal"} { + t.Run(fmt.Sprintf("pools=%d/%s", pools, mode), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi := r.put("cache", false, true) + r.restore(oi) + stale, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil { + t.Fatal(err) + } + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + expire := func() error { + return expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner) + } + if mode == "split-renewal" { + // A renewal reached two drives; the other two keep the + // expired header. + dataDir := r.fileInfo(oi.VersionID).DataDir + for _, d := range disks[2:] { + fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, oi.VersionID, ReadOptions{ReadData: true}) + if err != nil { + t.Fatal(err) + } + fi.Metadata[xhttp.AmzRestore] = completedRestoreObj(UTCNow().Add(48 * time.Hour)).String() + if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + if err := expire(); err == nil || r.dataDirs(dataDir) != len(disks) { + t.Errorf("expiry on drives split between expired and renewed: %v, %d/%d cache dirs", err, r.dataDirs(dataDir), len(disks)) + } + return + } + var cleanup, metaWrites atomic.Bool + wrapped := append([]StorageAPI(nil), disks...) + wrapped[0] = restoreGenerationCleanupDisk{StorageAPI: disks[0], bucket: r.bucket, object: r.object, cleanup: &cleanup, metaWrites: &metaWrites} + er.getDisks = func() []StorageAPI { return wrapped } + cleanup.Store(true) + err = expire() + cleanup.Store(false) + if err != nil { + t.Fatalf("cleanup on all drives but one: %v", err) + } + captured := make(chan ObjectOptions, 1) + layer := newRestoreIsolationLayer(r) + layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error { + captured <- opts + return nil + } + metaWrites.Store(true) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + metaWrites.Store(false) + if rec.Code != http.StatusOK { + t.Fatalf("POST after the cleanup: %d %s", rec.Code, rec.Body.String()) + } + opts := <-captured + layer.wait(t) + // The same cleanup, queued twice, runs again. + if err := expire(); err == nil { + t.Error("a drive's expired header cleared the quorum's ongoing restore") + } + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, opts); err != nil { + t.Fatalf("restore of the acknowledged attempt: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, r.bodies["cache"]) { + t.Fatalf("restored contents: %v", err) + } + }) + } + } +} + +// restoreGenerationHidden hides a drive's copy of the object version while +// set, as a drive that lost it and that heal writes again. +type restoreGenerationHidden struct { + StorageAPI + bucket, object string + hidden *atomic.Bool +} + +func (d restoreGenerationHidden) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) { + if d.hidden.Load() && bucket == d.bucket && object == d.object { + return FileInfo{}, errFileNotFound + } + return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts) +} + +// Heal copies the restore fields to a drive only as a quorum of the drives +// holds them. While they disagree, healing the version waits: it must not +// create a quorum for an expired header over a renewal, nor for a +// superseded attempt. Fields a quorum agrees on are copied. +func TestRestoreHealNeedsRestoreQuorum(t *testing.T) { + for _, mode := range []string{"expired-over-renewal", "superseded-attempt", "agreed"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + setRestore := func(versionID string, drives []StorageAPI, fields map[string]string) { + for _, d := range drives { + fi, err := d.ReadVersion(t.Context(), "", r.bucket, r.object, versionID, ReadOptions{ReadData: true}) + if err != nil { + t.Fatal(err) + } + maps.Copy(fi.Metadata, fields) + if err := d.WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + } + var versionID string + var stale ObjectInfo + var older ObjectOptions + hide := 3 + switch mode { + case "expired-over-renewal", "agreed": + oi := r.put("cache", false, true) + r.restore(oi) + versionID = oi.VersionID + var err error + if stale, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(versionID)); err != nil { + t.Fatal(err) + } + renewed := map[string]string{xhttp.AmzRestore: completedRestoreObj(UTCNow().Add(48 * time.Hour)).String()} + if mode == "agreed" { + // Three drives renewed, the hidden one expired. + setRestore(versionID, disks[:3], renewed) + } else { + // No quorum: two drives expired an hour ago, one renewed, + // and the hidden one expired a day ago. + setRestore(versionID, disks[:2], map[string]string{xhttp.AmzRestore: completedRestoreObj(UTCNow().Add(-time.Hour)).String()}) + setRestore(versionID, disks[2:3], renewed) + } + case "superseded-attempt": + oi, _ := restoreIsolationSeed(t, r, 0, false, false) + versionID = oi.VersionID + captured := make(chan ObjectOptions, 1) + layer := newRestoreIsolationLayer(r) + layer.restore = func(_ context.Context, _, _ string, opts ObjectOptions) error { + captured <- opts + return nil + } + if rec := restoreIsolationPOST(t, r, layer, versionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + older = <-captured + layer.wait(t) + // A newer attempt reached two drives. + setRestore(versionID, disks[2:], map[string]string{xhttp.AmzRestore: ongoingRestoreObj().String(), restoreAttemptKey: newRestoreAttempt(oi).ID}) + hide = 2 + } + before := restoreGenerationMetadata(t, disks, r, versionID) + var hidden atomic.Bool + wrapped := append([]StorageAPI(nil), disks...) + wrapped[hide] = restoreGenerationHidden{StorageAPI: disks[hide], bucket: r.bucket, object: r.object, hidden: &hidden} + er.getDisks = func() []StorageAPI { return wrapped } + hidden.Store(true) + _, healErr := er.HealObject(t.Context(), r.bucket, r.object, versionID, madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + hidden.Store(false) + er.getDisks = func() []StorageAPI { return disks } + after := restoreGenerationMetadata(t, disks, r, versionID) + if mode == "agreed" { + if healErr != nil || after[hide][xhttp.AmzRestore] != after[0][xhttp.AmzRestore] { + t.Errorf("heal did not copy the restore fields a quorum holds: %v, %q", healErr, after[hide][xhttp.AmzRestore]) + } + return + } + if healErr == nil { + t.Error("heal proceeded on drives that disagree on the restore fields") + } + if after[hide][xhttp.AmzRestore] != before[hide][xhttp.AmzRestore] || after[hide][restoreAttemptKey] != before[hide][restoreAttemptKey] { + t.Errorf("heal wrote a minority restore state: %q -> %q", before[hide][xhttp.AmzRestore], after[hide][xhttp.AmzRestore]) + } + if mode == "expired-over-renewal" { + dataDir := r.fileInfo(versionID).DataDir + if err := expireTransitionedObject(t.Context(), r.z, &stale, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err == nil || r.dataDirs(dataDir) != len(disks) { + t.Errorf("expiry after heal: %v, %d/%d cache dirs", err, r.dataDirs(dataDir), len(disks)) + } + return + } + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, older); !errors.Is(err, errRestoreSuperseded) { + t.Errorf("superseded attempt after heal: %v", err) + } + }) + } +} + +// restoreGenerationStageFails fails the commit before it stages. +type restoreGenerationStageFails struct { + StorageAPI + bucket, object string +} + +func (d restoreGenerationStageFails) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { + if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { + return RenameDataResp{}, errFaultyDisk + } + return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) +} + +// restoreGenerationFailedCommit stages the committed metadata into the +// upload, as RenameData does, and then fails the commit. +type restoreGenerationFailedCommit struct { + StorageAPI + bucket, object string +} + +func (d restoreGenerationFailedCommit) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { + if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { + if err := d.StorageAPI.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil { + return RenameDataResp{}, err + } + return RenameDataResp{}, errFaultyDisk + } + return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) +} + +// restoreGenerationLeftover runs a restore whose commit fails after it +// staged on every drive, or on every drive but drive 0, which fails before +// it stages. It returns the upload left behind and its on-disk UUID. +func restoreGenerationLeftover(t *testing.T, r *restoreTest, unstaged bool) (map[string]int, string) { + t.Helper() + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, false, true) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + if unstaged && i == 0 { + wrapped[i] = restoreGenerationStageFails{StorageAPI: d, bucket: r.bucket, object: r.object} + continue + } + wrapped[i] = restoreGenerationFailedCommit{StorageAPI: d, bucket: r.bucket, object: r.object} + } + er.getDisks = func() []StorageAPI { return wrapped } + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatal(rec.Body.String()) + } + if err := layer.wait(t); err == nil { + t.Fatal("fixture: the failed commit succeeded") + } + er.getDisks = func() []StorageAPI { return disks } + stored := restoreIsolationUploads(t, r, 0) + if len(stored) != 1 { + t.Fatalf("fixture: no leftover of the failed commit: %v", stored) + } + for uploadUUID := range stored { + return stored, uploadUUID + } + return nil, "" +} + +// restoreGenerationAgeUpload makes every drive's copy of an upload's +// metadata look written d ago. +func restoreGenerationAgeUpload(t *testing.T, r *restoreTest, uploadUUID string, d time.Duration) { + t.Helper() + er := r.z.serverPools[0].getHashedSet(r.object) + old := UTCNow().Add(-d) + for _, disk := range er.getDisks() { + if err := os.Chtimes(pathJoin(disk.Endpoint().Path, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), uploadUUID, xlStorageFormatFile), old, old); err != nil { + t.Fatal(err) + } + } +} + +// What a failed commit leaves in an upload can always be removed once every +// copy is staged and older than the commit grace: abort removes it, also +// when stale cleanup could not. A recent staging, a copy that cannot be read +// and a copy not staged yet are refused, and nothing is removed then. +func TestRestoreAbandonedCommitAbort(t *testing.T) { + for _, age := range []string{"recent", "abandoned", "unreadable-copy", "unstaged-copy"} { + t.Run(age, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + stored, uploadUUID := restoreGenerationLeftover(t, r, age == "unstaged-copy") + if age != "recent" { + restoreGenerationAgeUpload(t, r, uploadUUID, 2*restoreCommitGrace) + } + id := base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)) + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + var unreadable atomic.Bool + if age == "unreadable-copy" { + wrapped := append([]StorageAPI(nil), disks...) + wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], uploadRead: &unreadable} + er.getDisks = func() []StorageAPI { return wrapped } + unreadable.Store(true) + } + err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}) + unreadable.Store(false) + left := restoreIsolationUploads(t, r, 0) + if age != "abandoned" { + if err == nil || !reflect.DeepEqual(left, stored) { + t.Errorf("abort of an upload that may still commit (%s): %v, left %v", age, err, left) + } + return + } + if err != nil || len(left) != 0 { + t.Errorf("abort of an abandoned commit: %v, left %v", err, left) + } + }) + } +} + +// With no commit running, stale cleanup removes what a failed commit left +// drive by drive: a staged copy older than the commit grace. A recent +// staging and a copy that never staged stay. +func TestRestoreFailedCommitStaleCleanup(t *testing.T) { + for _, mode := range []string{"recent", "abandoned", "unstaged-copy"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + stored, uploadUUID := restoreGenerationLeftover(t, r, mode == "unstaged-copy") + if mode != "recent" { + // The copy that never staged is as old as the others. + restoreGenerationAgeUpload(t, r, uploadUUID, 2*restoreCommitGrace) + } + er := r.z.serverPools[0].getHashedSet(r.object) + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + for _, disk := range er.getDisks() { + er.cleanupStaleUploadsOnDisk(t.Context(), disk) + } + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + want := map[string]int{} + switch mode { + case "recent": + want = stored + case "unstaged-copy": + want = map[string]int{uploadUUID: 1} + } + if left := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(left, want) { + t.Errorf("stale cleanup of a failed commit's leftover: %v -> %v, want %v", stored, left, want) + } + }) + } +} + +// restoreGenerationSnapshotKey marks the context of a stale cleanup whose +// reads of upload metadata restoreGenerationSnapshotDisk holds. +type restoreGenerationSnapshotKey struct{} + +// restoreGenerationSnapshotDisk reads upload metadata at once, and hands the +// reply to a marked stale cleanup only on release. +type restoreGenerationSnapshotDisk struct { + StorageAPI + reads *atomic.Int32 + release chan struct{} +} + +func (d restoreGenerationSnapshotDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) { + fi, err := d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts) + if volume == minioMetaMultipartBucket && ctx.Value(restoreGenerationSnapshotKey{}) != nil { + d.reads.Add(1) + select { + case <-d.release: + case <-ctx.Done(): + return FileInfo{}, ctx.Err() + } + } + return fi, err +} + +// Stale cleanup decides on a removal under the commit lock. A completion +// that starts while cleanup reads an upload past the stale age waits for +// it, and a cleanup that comes later skips the running commit: the upload, +// an ordinary one, still commits with its contents. +func TestCleanupStaleUploadsSkipsRunningCommit(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + er := r.z.serverPools[0].getHashedSet(r.object) + mp, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true, MTime: UTCNow().Add(-48 * time.Hour)}) + if err != nil { + t.Fatal(err) + } + body := []byte("an ordinary upload past the stale age") + part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + disks := er.getDisks() + var reads, staged atomic.Int32 + readRelease := make(chan struct{}) + stageEntered, stageRelease := make(chan struct{}), make(chan struct{}) + var readOnce, stageOnce sync.Once + defer readOnce.Do(func() { close(readRelease) }) + defer stageOnce.Do(func() { close(stageRelease) }) + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + stage := restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: true, + arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks))} + wrapped[i] = restoreGenerationSnapshotDisk{StorageAPI: stage, reads: &reads, release: readRelease} + } + er.getDisks = func() []StorageAPI { return wrapped } + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + defer func() { + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + }() + // Every drive's cleanup at once, each reading the upload on every drive. + ctx := context.WithValue(t.Context(), restoreGenerationSnapshotKey{}, true) + var cleanups sync.WaitGroup + for _, disk := range wrapped { + cleanups.Go(func() { er.cleanupStaleUploadsOnDisk(ctx, disk) }) + } + cleanupDone := make(chan struct{}) + go func() { cleanups.Wait(); close(cleanupDone) }() + // The completion starts once every cleanup that may decide holds its + // read of the upload from before the commit. + for deadline := time.Now().Add(2 * time.Second); reads.Load() < int32(len(disks)*len(disks)) && time.Now().Before(deadline); { + time.Sleep(10 * time.Millisecond) + } + completion := make(chan ObjectInfo, 1) + completionErr := make(chan error, 1) + go func() { + oi, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true}) + completion <- oi + completionErr <- err + }() + select { + case <-stageEntered: + case <-time.After(3 * time.Second): + } + readOnce.Do(func() { close(readRelease) }) + select { + case <-cleanupDone: + case <-time.After(10 * time.Second): + t.Fatal("stale cleanup never returned") + } + stageOnce.Do(func() { close(stageRelease) }) + oi := <-completion + if err := <-completionErr; err != nil { + t.Fatalf("completion of an upload stale cleanup read: %v", err) + } + er.getDisks = func() []StorageAPI { return disks } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("completed contents: %q, %v", got, err) + } +} + +// restoreGenerationRand draws UUIDs from crypto/rand for the whole test +// binary, installed before any goroutine runs. While a pause is set, it holds +// the UUID that stale cleanup draws in its deadline-bound worker, for the +// trash path right before its rename, until the pause is released. +var restoreGenerationRand = &restoreGenerationRandReader{} + +func init() { uuid.SetRand(restoreGenerationRand) } + +type restoreGenerationRandReader struct { + pause atomic.Pointer[restoreGenerationPause] +} + +type restoreGenerationPause struct { + entered, release chan struct{} +} + +func (r *restoreGenerationRandReader) Read(b []byte) (int, error) { + if p := r.pause.Load(); p != nil && restoreGenerationInCleanupRename() && r.pause.CompareAndSwap(p, nil) { + close(p.entered) + <-p.release + } + return rand.Read(b) +} + +// restoreGenerationInCleanupRename reports whether the caller runs in stale +// cleanup's deadline-bound worker. +func restoreGenerationInCleanupRename() bool { + pcs := make([]uintptr, 32) + frames := runtime.CallersFrames(pcs[:runtime.Callers(3, pcs)]) + cleanup, worker := false, false + for { + f, more := frames.Next() + cleanup = cleanup || strings.Contains(f.Function, "cleanupStaleUploadsOnDisk") + worker = worker || strings.Contains(f.Function, "ioutil.WithDeadline") + if !more { + return cleanup && worker + } + } +} + +// Stale cleanup holds the commit lock until its rename ends, also when the +// rename outlives the drive deadline and cleanup has moved on: a completion +// does not stage while that rename can still remove its upload. +func TestCleanupStaleUploadsHoldsLockThroughRename(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + er := r.z.serverPools[0].getHashedSet(r.object) + mp, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true, MTime: UTCNow().Add(-48 * time.Hour)}) + if err != nil { + t.Fatal(err) + } + body := []byte("an upload whose stale cleanup outlives the drive deadline") + part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + disks := er.getDisks() + stored := restoreIsolationUploads(t, r, 0) + prev := globalDriveConfig.GetMaxTimeout() + // The shortest drive deadline the configuration takes. + globalDriveConfig.Update(drive.Config{MaxTimeout: 1100 * time.Millisecond}) + defer globalDriveConfig.Update(drive.Config{MaxTimeout: prev}) + pause := &restoreGenerationPause{entered: make(chan struct{}), release: make(chan struct{})} + var paused sync.Once + defer paused.Do(func() { close(pause.release) }) + restoreGenerationRand.pause.Store(pause) + defer restoreGenerationRand.pause.CompareAndSwap(pause, nil) + + cleanupDone := make(chan struct{}) + go func() { + er.cleanupStaleUploadsOnDisk(t.Context(), disks[0]) + close(cleanupDone) + }() + select { + case <-pause.entered: + case <-time.After(5 * time.Second): + t.Fatal("stale cleanup never reached its rename") + } + select { + case <-cleanupDone: + case <-time.After(5 * time.Second): + t.Fatal("stale cleanup did not move on after the drive deadline") + } + if left := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(left, stored) { + t.Fatalf("fixture: the held rename already ran: %v -> %v", stored, left) + } + + var staged atomic.Int32 + stageEntered, stageRelease := make(chan struct{}), make(chan struct{}) + var stageOnce sync.Once + defer stageOnce.Do(func() { close(stageRelease) }) + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: true, + arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks))} + } + er.getDisks = func() []StorageAPI { return wrapped } + completion := make(chan ObjectInfo, 1) + completionErr := make(chan error, 1) + go func() { + oi, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true}) + completion <- oi + completionErr <- err + }() + select { + case <-stageEntered: + t.Error("a completion staged while stale cleanup's rename could still run") + case <-time.After(2 * time.Second): + } + paused.Do(func() { close(pause.release) }) + select { + case <-stageEntered: + case <-time.After(10 * time.Second): + t.Fatal("completion never staged after the rename ended") + } + stageOnce.Do(func() { close(stageRelease) }) + oi := <-completion + if err := <-completionErr; err != nil { + t.Fatalf("completion after the rename ended: %v", err) + } + er.getDisks = func() []StorageAPI { return disks } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("completed contents: %q, %v", got, err) + } +} diff --git a/cmd/restore-native-isolation_test.go b/cmd/restore-native-isolation_test.go new file mode 100644 index 000000000..232eb4323 --- /dev/null +++ b/cmd/restore-native-isolation_test.go @@ -0,0 +1,1397 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/versioning" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/mux" +) + +// The private metadata of a native restore, as stored. Literal names keep +// these regressions meaningful on a tree without the keys. +const ( + restoreIsolationAttemptKey = "x-minio-internal-restore-attempt" + restoreIsolationUploadKey = "x-minio-internal-restore-upload" +) + +// restoreIsolationLayer hooks the object layer a POST restore handler uses. +// afterRead runs once POST has read the version, before POST records its +// status. restore, when set, replaces the background restore; every +// background restore result is sent to done. +type restoreIsolationLayer struct { + ObjectLayer + afterRead func() + once sync.Once + restore func(ctx context.Context, bucket, object string, opts ObjectOptions) error + done chan error +} + +func newRestoreIsolationLayer(r *restoreTest) *restoreIsolationLayer { + return &restoreIsolationLayer{ObjectLayer: r.z, done: make(chan error, 4)} +} + +func (l *restoreIsolationLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + oi, err := l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts) + if err == nil && l.afterRead != nil { + l.once.Do(l.afterRead) + } + return oi, err +} + +func (l *restoreIsolationLayer) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + var err error + if l.restore != nil { + err = l.restore(ctx, bucket, object, opts) + } else { + err = l.ObjectLayer.RestoreTransitionedObject(ctx, bucket, object, opts) + } + l.done <- err + return err +} + +// wait returns the result of the next background restore. +func (l *restoreIsolationLayer) wait(t *testing.T) error { + t.Helper() + select { + case err := <-l.done: + return err + case <-time.After(10 * time.Second): + t.Fatal("background restore did not finish") + return nil + } +} + +func restoreIsolationPOST(t *testing.T, r *restoreTest, layer ObjectLayer, versionID string) *httptest.ResponseRecorder { + t.Helper() + body := []byte(`1`) + path := "/" + r.bucket + "/" + r.object + "?restore" + if versionID != "" { + path += "&versionId=" + url.QueryEscape(versionID) + } + req, err := newTestSignedRequestV4(http.MethodPost, path, int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + if err := req.ParseForm(); err != nil { + t.Fatal(err) + } + req = mux.SetURLVars(req, map[string]string{"bucket": r.bucket, "object": r.object}) + rec := httptest.NewRecorder() + api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return layer }} + api.PostRestoreObjectHandler(rec, req) + return rec +} + +// restoreIsolationSeed is a transitioned version without the ongoing marker +// that nativeRestoreSeed leaves for direct restores: POST sets its own. +func restoreIsolationSeed(t *testing.T, r *restoreTest, pool int, null, multipart bool) (ObjectInfo, []byte) { + t.Helper() + oi, body := nativeRestoreSeed(t, r, pool, null, multipart) + opts := r.opts(oi.VersionID) + opts.MTime = oi.ModTime + opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[xhttp.AmzRestore] = "" + return ReplicateDecision{}, nil + } + if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil { + t.Fatal(err) + } + return oi, body +} + +// restoreIsolationSettled waits until the version is no longer ongoing. +func restoreIsolationSettled(t *testing.T, r *restoreTest, versionID string) ObjectInfo { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + oi, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(versionID)) + if err == nil && !oi.RestoreOngoing { + return oi + } + if time.Now().After(deadline) { + t.Fatalf("version stays ongoing, refusing every later POST: %q, %v", oi.UserDefined[xhttp.AmzRestore], err) + } + time.Sleep(20 * time.Millisecond) + } +} + +// restoreIsolationUploads returns the upload directories of the object on +// each drive of a pool, as stored, whatever listings show. +func restoreIsolationUploads(t *testing.T, r *restoreTest, pool int) map[string]int { + t.Helper() + er := r.z.serverPools[pool].getHashedSet(r.object) + found := make(map[string]int) + for _, disk := range er.getDisks() { + entries, err := disk.ListDir(t.Context(), r.bucket, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), -1) + if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) { + continue + } + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + found[strings.TrimSuffix(entry, SlashSeparator)]++ + } + } + return found +} + +// restoreIsolationUpload finds the single upload of the object in a pool on +// disk, with its first part, and returns its public upload ID. +func restoreIsolationUpload(t *testing.T, r *restoreTest, pool int) (string, ObjectPartInfo) { + t.Helper() + uploads := restoreIsolationUploads(t, r, pool) + if len(uploads) != 1 { + t.Fatalf("want one upload on disk, found %v", uploads) + } + var uploadUUID string + for uploadUUID = range uploads { + } + er := r.z.serverPools[pool].getHashedSet(r.object) + uploadPath := pathJoin(er.getMultipartSHADir(r.bucket, r.object), uploadUUID) + fi, err := er.getDisks()[0].ReadVersion(t.Context(), "", minioMetaMultipartBucket, uploadPath, "", ReadOptions{}) + if err != nil { + t.Fatal(err) + } + parts, err := readParts(t.Context(), er.getDisks(), minioMetaMultipartBucket, []string{pathJoin(uploadPath, fi.DataDir, "part.1.meta")}, []int{1}, 1) + if err != nil || len(parts) != 1 || parts[0].Error != "" { + t.Fatalf("restore must have published its first part: %+v, %v", parts, err) + } + return base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)), parts[0] +} + +// restoreIsolationGateTier holds a download after its first bytes and can +// fail it once released. +type restoreIsolationGateTier struct { + WarmBackend + first int + entered chan struct{} + release chan struct{} + failure error + enter sync.Once + open sync.Once +} + +func newRestoreIsolationGateTier(first int, failure error) *restoreIsolationGateTier { + return &restoreIsolationGateTier{ + WarmBackend: globalTierConfigMgr.drivercache["WARM"], + first: first, failure: failure, + entered: make(chan struct{}), release: make(chan struct{}), + } +} + +func (b *restoreIsolationGateTier) unblock() { b.open.Do(func() { close(b.release) }) } + +func (b *restoreIsolationGateTier) Get(ctx context.Context, object string, version remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) { + rd, err := b.WarmBackend.Get(ctx, object, version, opts) + if err != nil { + return nil, err + } + return &restoreIsolationGateReader{ReadCloser: rd, tier: b, ctx: ctx, remaining: b.first}, nil +} + +type restoreIsolationGateReader struct { + io.ReadCloser + tier *restoreIsolationGateTier + ctx context.Context + remaining int +} + +func (r *restoreIsolationGateReader) Read(p []byte) (int, error) { + if r.remaining > 0 { + if len(p) > r.remaining { + p = p[:r.remaining] + } + n, err := r.ReadCloser.Read(p) + r.remaining -= n + return n, err + } + r.tier.enter.Do(func() { close(r.tier.entered) }) + select { + case <-r.tier.release: + case <-r.ctx.Done(): + return 0, r.ctx.Err() + } + if r.tier.failure != nil { + return 0, r.tier.failure + } + return r.ReadCloser.Read(p) +} + +func restoreIsolationEntered(t *testing.T, tier *restoreIsolationGateTier) { + t.Helper() + select { + case <-tier.entered: + case <-time.After(10 * time.Second): + t.Fatal("restore never reached its download gate") + } +} + +// restoreIsolationFaultDisk fails metadata reads of the object while +// failReads is set, and reports writes of its metadata as failed, after +// they were made, while failWrites is set. Toggles keep the wrapper in +// place, so no restore retry races a swap of the drives. +type restoreIsolationFaultDisk struct { + StorageAPI + bucket, object string + failReads *atomic.Bool + failWrites *atomic.Bool +} + +func (d restoreIsolationFaultDisk) ReadVersion(ctx context.Context, origvolume, volume, path, version string, opts ReadOptions) (FileInfo, error) { + if d.failReads.Load() && volume == d.bucket && path == d.object { + return FileInfo{}, errDiskNotFound + } + return d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, version, opts) +} + +func (d restoreIsolationFaultDisk) ReadXL(ctx context.Context, volume, path string, readData bool) (RawFileInfo, error) { + if d.failReads.Load() && volume == d.bucket && path == d.object { + return RawFileInfo{}, errDiskNotFound + } + return d.StorageAPI.ReadXL(ctx, volume, path, readData) +} + +func (d restoreIsolationFaultDisk) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error { + err := d.StorageAPI.WriteMetadata(ctx, origvolume, volume, path, fi) + if err == nil && d.failWrites != nil && d.failWrites.Load() && volume == d.bucket && path == d.object { + return errFaultyDisk + } + return err +} + +// installRestoreIsolationFaults wraps the drives of the object's set in a +// pool for the rest of the test; failing marks the drives whose writes fail. +func installRestoreIsolationFaults(r *restoreTest, pool int, failing int) (reads, writes *atomic.Bool) { + reads, writes = new(atomic.Bool), new(atomic.Bool) + er := r.z.serverPools[pool].getHashedSet(r.object) + disks := er.getDisks() + wrapped := make([]StorageAPI, len(disks)) + for i, disk := range disks { + d := restoreIsolationFaultDisk{StorageAPI: disk, bucket: r.bucket, object: r.object, failReads: reads} + if i < failing { + d.failWrites = writes + } + wrapped[i] = d + } + er.getDisks = func() []StorageAPI { return wrapped } + return reads, writes +} + +// A public client must not replace a part of a native restore's upload, +// before or after the restore validated it: the bytes would become the +// COMPLIANCE-protected version under its original ETag. +func TestRestoreUploadIsolatedFromPublicParts(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + enableBucketObjectLock(t, r.bucket) + previous := newObjectLayerFn() + t.Cleanup(func() { setObjectLayer(previous) }) + router := initTestAPIEndPoints(r.z, []string{"PutObjectPart", "GetObject"}) + oi, body := nativeRestoreSeed(t, r, 0, false, true) + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, "retention", retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))); rec.Code != http.StatusOK { + t.Fatalf("lock: %d %s", rec.Code, rec.Body.String()) + } + // Hold completion after it validated the parts, before it commits. + er := r.z.serverPools[0].getHashedSet(r.object) + original := er.getDisks + gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} + defer gate.unblock() + disks := original() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreIsolationCompletionDisk{d, gate} + } + er.getDisks = func() []StorageAPI { return wrapped } + defer func() { er.getDisks = original }() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + restored := make(chan error, 1) + go func() { + restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + select { + case <-gate.entered: + case err := <-restored: + t.Fatalf("restore stopped before validation: %v", err) + case <-ctx.Done(): + t.Fatal("no completion gate") + } + id, _ := restoreIsolationUpload(t, r, 0) + poison := bytes.Repeat([]byte("z"), 5<<20) + // The request may wait for the upload lock that completion holds. + uploaded := make(chan *httptest.ResponseRecorder, 1) + go func() { + uploaded <- attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodPut, "/"+r.bucket+"/"+r.object+"?uploadId="+url.QueryEscape(id)+"&partNumber=1", poison, nil) + }() + var rec *httptest.ResponseRecorder + select { + case rec = <-uploaded: + case <-time.After(500 * time.Millisecond): + } + gate.unblock() + if err := <-restored; err != nil { + t.Fatalf("restore: %v", err) + } + if rec == nil { + rec = <-uploaded + } + if rec.Code != http.StatusNotFound || !strings.Contains(rec.Body.String(), "NoSuchUpload") { + t.Errorf("public UploadPart on a native restore upload: %d %s, want NoSuchUpload", rec.Code, rec.Body.String()) + } + got := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, "/"+r.bucket+"/"+r.object+"?versionId="+oi.VersionID, nil, nil) + if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), body) { + t.Errorf("COMPLIANCE version holds other bytes after restore: GET=%d poisoned=%v", got.Code, bytes.HasPrefix(got.Body.Bytes(), poison)) + } +} + +type restoreIsolationCompletionGate struct { + entered, release chan struct{} + enter, open sync.Once +} + +func (g *restoreIsolationCompletionGate) unblock() { g.open.Do(func() { close(g.release) }) } + +type restoreIsolationCompletionDisk struct { + StorageAPI + gate *restoreIsolationCompletionGate +} + +// Completion removes the part metadata it validated just before it renames +// the upload into place. +func (d restoreIsolationCompletionDisk) DeleteBulk(ctx context.Context, bucket string, paths ...string) error { + if bucket == minioMetaMultipartBucket && len(paths) == 2 && strings.HasSuffix(paths[0], ".meta") && strings.HasSuffix(paths[1], ".meta") { + d.gate.enter.Do(func() { close(d.gate.entered) }) + select { + case <-d.gate.release: + case <-ctx.Done(): + return ctx.Err() + } + } + return d.StorageAPI.DeleteBulk(ctx, bucket, paths...) +} + +// restoreIsolationValidationDisk holds a completion where it reads the parts +// it validates: after its object and upload locks, before its commit lock. +// Stale cleanup then decides on the upload itself, not on a running commit. +type restoreIsolationValidationDisk struct { + StorageAPI + gate *restoreIsolationCompletionGate +} + +func (d restoreIsolationValidationDisk) ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error) { + if bucket == minioMetaMultipartBucket { + d.gate.enter.Do(func() { close(d.gate.entered) }) + select { + case <-d.gate.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return d.StorageAPI.ReadParts(ctx, bucket, partMetaPaths...) +} + +// Every public multipart entry point finds no native restore upload, and +// none of them can change it: the restore still completes with its own data. +func TestRestoreUploadHiddenFromPublicEntryPoints(t *testing.T) { + for _, pools := range []int{1, 2} { + t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, 0, false, true) + tier := newRestoreIsolationGateTier(5<<20, nil) + globalTierConfigMgr.drivercache["WARM"] = tier + defer tier.unblock() + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + restored := make(chan error, 1) + go func() { + restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + restoreIsolationEntered(t, tier) + id, part := restoreIsolationUpload(t, r, 0) + stored := restoreIsolationUploads(t, r, 0) + + for _, prefix := range []string{r.object, ""} { + uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, prefix, "", "", "", 100) + if err != nil { + t.Fatal(err) + } + for _, u := range uploads.Uploads { + if u.UploadID == id { + t.Errorf("ListMultipartUploads(prefix=%q) shows the native restore upload", prefix) + } + } + } + poison := bytes.Repeat([]byte("z"), 5<<20) + noUpload := func(op string, err error) { + t.Helper() + if !errors.As(err, &InvalidUploadID{}) { + t.Errorf("%s on a native restore upload: %v, want NoSuchUpload", op, err) + } + } + // With one pool, part writes first wait for the object lock the + // restore holds; with more they reach the upload at once. + written := make(chan error, 2) + go func() { + _, err := r.z.PutObjectPart(ctx, r.bucket, r.object, id, 1, mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", ""), ObjectOptions{}) + written <- err + }() + go func() { + src := ObjectInfo{PutObjReader: mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", "")} + _, err := r.z.CopyObjectPart(ctx, r.bucket, r.object, r.bucket, r.object, id, 1, 0, int64(len(poison)), src, ObjectOptions{}, ObjectOptions{}) + written <- err + }() + var writes []error + if pools > 1 { + writes = append(writes, <-written, <-written) + } + _, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{}) + noUpload("ListObjectParts", err) + _, err = r.z.GetMultipartInfo(t.Context(), r.bucket, r.object, id, ObjectOptions{}) + noUpload("GetMultipartInfo", err) + err = r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}) + noUpload("AbortMultipartUpload", err) + if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) { + t.Errorf("public calls changed the restore upload on disk: %v -> %v", stored, after) + } + // Completion waits for the object lock the restore holds. + completed := make(chan error, 1) + go func() { + _, err := r.z.CompleteMultipartUpload(ctx, r.bucket, r.object, id, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, r.opts(oi.VersionID)) + completed <- err + }() + tier.unblock() + if err := <-restored; err != nil { + t.Fatalf("restore after public calls: %v", err) + } + for len(writes) < 2 { + writes = append(writes, <-written) + } + for _, err := range writes { + noUpload("PutObjectPart/CopyObjectPart", err) + } + noUpload("CompleteMultipartUpload", <-completed) + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents after public calls: %v", err) + } + }) + } +} + +// POST records its status on a fresh read under the object lock, so an +// Object Lock update it did not see survives. +func TestRestorePOSTKeepsConcurrentProtection(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, kind := range []string{"legal-hold", "retention"} { + t.Run(fmt.Sprintf("pools=%d/%s", pools, kind), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + enableBucketObjectLock(t, r.bucket) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + initial := []byte(`OFF`) + updated := []byte(`ON`) + key := strings.ToLower(xhttp.AmzObjectLockLegalHold) + if kind == "retention" { + initial = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour)) + updated = retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour)) + key = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate) + } + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, initial); rec.Code != http.StatusOK { + t.Fatalf("initial protection: %d %s", rec.Code, rec.Body.String()) + } + var accepted ObjectInfo + layer := newRestoreIsolationLayer(r) + layer.afterRead = func() { + if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK { + t.Fatalf("update: %d %s", rec.Code, rec.Body.String()) + } + var err error + if accepted, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err != nil { + t.Fatal(err) + } + } + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code >= 400 { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + after, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil { + t.Fatal(err) + } + if accepted.UserDefined[key] == "" || after.UserDefined[key] != accepted.UserDefined[key] { + t.Errorf("POST restore overwrote acknowledged Object Lock metadata: %q -> %q", accepted.UserDefined[key], after.UserDefined[key]) + } + if !after.ModTime.Equal(oi.ModTime) || after.ETag != oi.ETag { + t.Error("POST restore changed the version identity") + } + }) + } + } +} + +// A restored copy removed between POST's read and its status write leads to +// a new restore, never to a completed header without the restored data. +func TestRestorePOSTRacesCacheExpiry(t *testing.T) { + for _, pools := range []int{1, 2} { + t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi := r.put("restored", false, true) + r.restore(oi) + layer := newRestoreIsolationLayer(r) + layer.afterRead = func() { + if err := expireTransitionedObject(t.Context(), r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner); err != nil { + t.Fatal(err) + } + } + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code >= 400 { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + select { + case err := <-layer.done: + if err != nil { + t.Fatalf("rehydration: %v", err) + } + case <-time.After(5 * time.Second): + t.Error("POST renewed a removed restored copy instead of restoring it") + } + _, body, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(body, r.bodies["restored"]) { + t.Errorf("version unreadable after POST raced cache expiry: %v", err) + } + }) + } +} + +// A restore queued with the versioning POST captured writes back the null +// version it resolved, even after versioning was enabled in between. +func TestRestoreKeepsCapturedNullIdentity(t *testing.T) { + enable := func(t *testing.T, r *restoreTest) { + meta, err := globalBucketMetadataSys.Get(r.bucket) + if err != nil { + t.Fatal(err) + } + meta.VersioningConfigXML = []byte(`Enabled`) + if err := meta.Save(t.Context(), r.z); err != nil { + t.Fatal(err) + } + globalBucketMetadataSys.Set(r.bucket, meta) + } + for _, mode := range []string{"direct", "post"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, "") + initConfigSubsystem(t.Context(), r.z) + var oi ObjectInfo + var body []byte + if mode == "direct" { + oi, body = nativeRestoreSeed(t, r, 0, true, false) + // These are the options that POST captured while versioning was off. + queued := nativeRestoreOptions(r, "") + enable(t, r) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, queued); err != nil { + t.Fatal(err) + } + } else { + oi, body = restoreIsolationSeed(t, r, 0, true, false) + release := make(chan struct{}) + layer := newRestoreIsolationLayer(r) + layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + <-release + return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) + } + if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + enable(t, r) + close(release) + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + } + versions := nativeRestoreSnapshot(t, r)[0] + if len(versions) != 1 || versions[0].VersionID != "" { + t.Errorf("restore created a UUID alias of the queued null version: %d versions", len(versions)) + } + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("null original: %v", err) + } + }) + } +} + +// A restore that fails before it reads the version, on its lock wait or on +// its first read, must not leave POST's ongoing marker for every later POST +// to refuse: bounded retries clear it once the version is readable again. +func TestRestoreRecoversStrandedMarker(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, stage := range []string{"lock-wait", "initial-read"} { + t.Run(fmt.Sprintf("pools=%d/%s", pools, stage), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + var err error + if stage == "lock-wait" { + lk := r.z.NewNSLock(r.bucket, r.object) + if pools == 1 { + lk = r.z.serverPools[0].getHashedSet(r.object).NewNSLock(r.bucket, r.object) + } + lkctx, lerr := lk.GetLock(t.Context(), globalOperationTimeout) + if lerr != nil { + t.Fatal(lerr) + } + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + err = r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + cancel() + lk.Unlock(lkctx) + } else { + reads, _ := installRestoreIsolationFaults(r, 0, 0) + reads.Store(true) + err = r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + reads.Store(false) + } + if err == nil { + t.Fatal("restore succeeded without its lock or its version") + } + after := restoreIsolationSettled(t, r, oi.VersionID) + if !after.IsRemote() || !after.ModTime.Equal(oi.ModTime) { + t.Error("recovery changed more than the restore status") + } + }) + } + } +} + +// A later POST takes over a marker that no running restore holds, and the +// superseded attempt then neither downloads nor clears the newer status, +// also when both POSTs share a second, the precision of the request date. +func TestRestoreSupersededAttempt(t *testing.T) { + for _, mode := range []string{"newer-completes", "newer-fails"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + release := make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(unblock) + older := newRestoreIsolationLayer(r) + older.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + <-release + return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) + } + newer := newRestoreIsolationLayer(r) + time.Sleep(time.Until(time.Now().Truncate(time.Second).Add(time.Second + 10*time.Millisecond))) + if rec := restoreIsolationPOST(t, r, older, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("first POST: %d %s", rec.Code, rec.Body.String()) + } + first := r.fileInfo(oi.VersionID).Metadata[xhttp.AmzRestoreRequestDate] + if mode == "newer-fails" { + globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, errors.New("injected remote failure")} + } + if rec := restoreIsolationPOST(t, r, newer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST over a marker no restore holds: %d %s", rec.Code, rec.Body.String()) + } + second := r.fileInfo(oi.VersionID).Metadata[xhttp.AmzRestoreRequestDate] + t.Logf("request dates %q %q, same second: %v", first, second, first == second) + newerErr := newer.wait(t) + globalTierConfigMgr.drivercache["WARM"] = tier + before := r.fileInfo(oi.VersionID) + gets := tier.gets.Load() + unblock() + if err := older.wait(t); err == nil { + t.Error("superseded restore reported success") + } + if tier.gets.Load() != gets { + t.Error("superseded restore downloaded the version again") + } + after := r.fileInfo(oi.VersionID) + if !reflect.DeepEqual(before.Metadata, after.Metadata) || before.DataDir != after.DataDir { + t.Errorf("superseded restore changed the version: %q -> %q", before.Metadata[xhttp.AmzRestore], after.Metadata[xhttp.AmzRestore]) + } + info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil || info.RestoreOngoing { + t.Fatalf("version left ongoing: %q, %v", info.UserDefined[xhttp.AmzRestore], err) + } + if mode == "newer-completes" { + if newerErr != nil || info.IsRemote() { + t.Fatalf("newer restore: %v remote=%v", newerErr, info.IsRemote()) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) || tier.gets.Load() != gets { + t.Errorf("restored contents: %v", err) + } + return + } + if newerErr == nil || !info.IsRemote() || info.UserDefined[xhttp.AmzRestore] != "" { + t.Fatalf("failed newer restore is not retryable: %v %q", newerErr, info.UserDefined[xhttp.AmzRestore]) + } + if rec := restoreIsolationPOST(t, r, newer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("retry POST: %d %s", rec.Code, rec.Body.String()) + } + if err := newer.wait(t); err != nil { + t.Fatal(err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Errorf("retried restore contents: %v", err) + } + }) + } +} + +// A restart loses the restore that POST started; a POST to the restarted +// server takes the stranded marker over and restores the version. +func TestRestoreRestartTakeover(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + lost := newRestoreIsolationLayer(r) + lost.restore = func(context.Context, string, string, ObjectOptions) error { return nil } + if rec := restoreIsolationPOST(t, r, lost, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST before restart: %d %s", rec.Code, rec.Body.String()) + } + lost.wait(t) + if info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err != nil || !info.RestoreOngoing { + t.Fatalf("fixture: %q %v", info.UserDefined[xhttp.AmzRestore], err) + } + + endpoints := make(EndpointServerPools, len(r.z.serverPools)) + for i, pool := range r.z.serverPools { + endpoints[i] = pool.endpoints + } + obj, err := newErasureServerPools(t.Context(), endpoints) + if err != nil { + t.Fatal(err) + } + restarted := obj.(*erasureServerPools) + t.Cleanup(func() { restarted.Shutdown(context.Background()) }) + r.z = restarted + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST after restart: %d %s", rec.Code, rec.Body.String()) + } + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + info := restoreIsolationSettled(t, r, oi.VersionID) + if info.IsRemote() || info.UserDefined[restoreIsolationAttemptKey] != "" { + t.Errorf("restore after restart incomplete: remote=%v attempt=%q", info.IsRemote(), info.UserDefined[restoreIsolationAttemptKey]) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Errorf("contents after restart: %v", err) + } +} + +// A null version replaced while its restore was queued is another version: +// the restore must not rehydrate it, even if it is transitioned as well. +func TestRestoreReplacementNullObject(t *testing.T) { + for _, state := range []versioning.State{"", versioning.Suspended} { + t.Run(fmt.Sprintf("versioning=%q", state), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, state) + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, true, false) + release := make(chan struct{}) + layer := newRestoreIsolationLayer(r) + layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + <-release + return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) + } + if rec := restoreIsolationPOST(t, r, layer, ""); rec.Code != http.StatusOK { + close(release) + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + replacement := []byte("replacement null object") + repl, err := r.z.serverPools[0].PutObject(t.Context(), r.bucket, r.object, + mustGetPutObjReader(t, bytes.NewReader(replacement), int64(len(replacement)), "", ""), + ObjectOptions{VersionSuspended: state == versioning.Suspended}) + if err != nil { + close(release) + t.Fatal(err) + } + topts := r.opts(nullVersionID) + topts.MTime = repl.ModTime + topts.Transition = TransitionOptions{Tier: "WARM", ETag: repl.ETag} + if err := r.z.serverPools[0].TransitionObject(t.Context(), r.bucket, r.object, topts); err != nil { + close(release) + t.Fatal(err) + } + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + before := r.fileInfo(nullVersionID) + gets := tier.gets.Load() + close(release) + layer.wait(t) + after := r.fileInfo(nullVersionID) + if tier.gets.Load() != gets || after.Metadata[xhttp.AmzRestore] != "" || !reflect.DeepEqual(before.Metadata, after.Metadata) { + t.Errorf("queued restore rehydrated the replacement null object: %q", after.Metadata[xhttp.AmzRestore]) + } + if after.ModTime.Equal(oi.ModTime) || !after.ModTime.Equal(repl.ModTime) { + t.Fatal("fixture did not replace the null version") + } + if _, got, err := r.read(nullVersionID); err != nil || !bytes.Equal(got, replacement) { + t.Errorf("replacement contents: %v", err) + } + }) + } +} + +// POST's own status write can fail after its ongoing marker reached the +// drives; the marker must not stay for every later POST to refuse. +func TestRestorePOSTStatusWritePartialFailure(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + _, writes := installRestoreIsolationFaults(r, 0, 2) + layer := newRestoreIsolationLayer(r) + writes.Store(true) + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + writes.Store(false) + if rec.Code < 400 { + t.Fatalf("POST acknowledged a failed status write: %d %s", rec.Code, rec.Body.String()) + } + if info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); err == nil && !info.RestoreOngoing { + t.Log("the failed status write left no marker in quorum") + } + restoreIsolationSettled(t, r, oi.VersionID) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST after a failed status write: %d %s", rec.Code, rec.Body.String()) + } + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Errorf("restore after a failed status write: %v", err) + } +} + +// Duplicate copies of a version in two pools can differ in whether they +// hold a restored copy. A completed restore header must never reach a copy +// without one; POST renews or restores only the copy that restores serve. +func TestRestoreDuplicatePoolCacheStates(t *testing.T) { + for _, mode := range []string{"suspended-owner", "unrestored-first", "restored-first"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + // The duplicate is the same stored version, metadata only. + src := r.z.serverPools[0].getHashedSet(r.object).getDisks() + dst := r.z.serverPools[1].getHashedSet(r.object).getDisks() + for i := range src { + raw, err := src[i].ReadAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile)) + if err != nil { + t.Fatal(err) + } + if err := dst[i].WriteAll(t.Context(), r.bucket, pathJoin(r.object, xlStorageFormatFile), raw); err != nil { + t.Fatal(err) + } + } + owner := 0 + if mode == "unrestored-first" { + owner = 1 + } + if err := r.z.serverPools[owner].RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + if mode == "suspended-owner" { + r.z.poolMetaMutex.Lock() + r.z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{} + r.z.poolMetaMutex.Unlock() + t.Cleanup(func() { + r.z.poolMetaMutex.Lock() + r.z.poolMeta.Pools[0].Decommission = nil + r.z.poolMetaMutex.Unlock() + }) + } + before := nativeRestoreSnapshot(t, r) + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + if mode != "restored-first" { + select { + case err := <-layer.done: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Error("POST did not restore the copy without restored data") + } + } + after := nativeRestoreSnapshot(t, r) + for pool := range r.z.serverPools { + gr, err := r.z.serverPools[pool].GetObjectNInfo(t.Context(), r.bucket, r.object, nil, nil, ObjectOptions{VersionID: oi.VersionID}) + if err != nil { + t.Errorf("pool %d copy unreadable, header %q: %v", pool, after[pool][0].Metadata[xhttp.AmzRestore], err) + continue + } + got, err := io.ReadAll(gr) + gr.Close() + if err != nil || !bytes.Equal(got, body) { + t.Errorf("pool %d copy contents: %v", pool, err) + } + } + // The copy that is neither renewed nor restored stays as it was. + untouched := 1 + if mode == "suspended-owner" { + untouched = 0 + } + if !reflect.DeepEqual(before[untouched], after[untouched]) { + t.Errorf("POST changed the pool %d copy: %q -> %q", untouched, before[untouched][0].Metadata[xhttp.AmzRestore], after[untouched][0].Metadata[xhttp.AmzRestore]) + } + }) + } +} + +// Stale-upload cleanup ages an upload by when it started, not by the +// version's ModTime, keeps the upload of a running restore beyond the stale +// age, and still removes ordinary stale uploads and those of ended attempts. +func TestRestoreUploadOutlivesStaleAge(t *testing.T) { + cleanup := func(t *testing.T, r *restoreTest) { + er := r.z.serverPools[0].getHashedSet(r.object) + for _, disk := range er.getDisks() { + er.cleanupStaleUploadsOnDisk(t.Context(), disk) + } + } + shortExpiry := func(t *testing.T) { + globalAPIConfig.mu.Lock() + prev := globalAPIConfig.staleUploadsExpiry + globalAPIConfig.staleUploadsExpiry = time.Nanosecond + globalAPIConfig.mu.Unlock() + t.Cleanup(func() { + globalAPIConfig.mu.Lock() + globalAPIConfig.staleUploadsExpiry = prev + globalAPIConfig.mu.Unlock() + }) + } + for _, mode := range []string{"old-version", "past-stale-age", "public-abort", "ordinary-stale", "unowned-marker"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + if mode == "ordinary-stale" { + // Control: an abandoned client upload, old by its upload ID. + if _, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{MTime: UTCNow().Add(-72 * time.Hour)}); err != nil { + t.Fatal(err) + } + cleanup(t, r) + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("ordinary stale upload survived cleanup: %v", uploads) + } + return + } + var oi ObjectInfo + var body []byte + if mode == "past-stale-age" { + oi, body = restoreIsolationSeed(t, r, 0, false, true) + } else { + oi, body = nativeRestoreSeed(t, r, 0, false, true) + } + old := r.opts(oi.VersionID) + old.MTime = UTCNow().Add(-72 * time.Hour) + old.EvalMetadataFn = func(*ObjectInfo, error) (ReplicateDecision, error) { return ReplicateDecision{}, nil } + if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, old); err != nil { + t.Fatal(err) + } + // The cleanup with a short stale age also removes in-flight + // temporary parts; hold such restores in completion instead, + // before their commit. + gated := mode == "past-stale-age" || mode == "unowned-marker" + tier := newRestoreIsolationGateTier(5<<20, nil) + if !gated { + globalTierConfigMgr.drivercache["WARM"] = tier + } + defer tier.unblock() + gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} + defer gate.unblock() + if gated { + er := r.z.serverPools[0].getHashedSet(r.object) + original := er.getDisks + disks := original() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreIsolationValidationDisk{d, gate} + } + er.getDisks = func() []StorageAPI { return wrapped } + defer func() { er.getDisks = original }() + } + result := make(chan error, 1) + layer := newRestoreIsolationLayer(r) + if mode == "past-stale-age" { + // A POST publishes the attempt that a running restore owns. + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + } else { + go func() { + result <- r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + } + if gated { + select { + case <-gate.entered: + case <-time.After(10 * time.Second): + t.Fatal("restore never reached completion") + } + } else { + restoreIsolationEntered(t, tier) + } + switch mode { + case "past-stale-age", "unowned-marker": + shortExpiry(t) + cleanup(t, r) + kept := len(restoreIsolationUploads(t, r, 0)) == 1 + if mode == "unowned-marker" && kept { + // The version is ongoing, but not for this direct restore's + // upload: an ongoing marker alone does not keep an upload. + t.Error("stale cleanup kept an upload no attempt owns") + } + if mode == "past-stale-age" && !kept { + t.Error("stale cleanup removed the upload of a running restore") + } + case "old-version": + cleanup(t, r) + case "public-abort": + id, _ := restoreIsolationUpload(t, r, 0) + if err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) { + t.Errorf("public abort of a native restore upload: %v, want NoSuchUpload", err) + } + } + tier.unblock() + gate.unblock() + var err error + if mode == "past-stale-age" { + err = layer.wait(t) + } else { + err = <-result + } + if mode == "unowned-marker" { + if err == nil { + t.Error("restore completed without its upload") + } + return + } + if err != nil { + t.Fatalf("restore lost its upload: %v", err) + } + _, got, err := r.read(oi.VersionID) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("restore contents: %v", err) + } + }) + } +} + +// A failed multipart restore removes its own upload from the drives, also +// when a newer attempt owns the version by then, and leaves that newer +// attempt's marker alone. +func TestRestoreFailureAbortsUpload(t *testing.T) { + for _, mode := range []string{"download-failure", "superseded-mid-download"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + failure := errors.New("injected remote failure") + if mode == "download-failure" { + oi, _ := nativeRestoreSeed(t, r, 0, false, true) + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, failure} + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); !errors.Is(err, failure) { + t.Fatalf("restore error=%v, want %v", err, failure) + } + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("failed native restore left its upload on disk: %v", uploads) + } + return + } + oi, _ := restoreIsolationSeed(t, r, 0, false, true) + tier := newRestoreIsolationGateTier(5<<20, failure) + globalTierConfigMgr.drivercache["WARM"] = tier + defer tier.unblock() + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + restoreIsolationEntered(t, tier) + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 1 { + t.Fatalf("restore upload on disk: %v", uploads) + } + // Model a newer generation that owns the version by now, such as + // one that took over after this restore lost its lock. + newer := r.opts(oi.VersionID) + newer.MTime = oi.ModTime + newer.NoLock = true + newer.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[restoreIsolationAttemptKey] = "newer-generation" + return ReplicateDecision{}, nil + } + if _, err := r.z.serverPools[0].getHashedSet(r.object).PutObjectMetadata(t.Context(), r.bucket, r.object, newer); err != nil { + t.Fatal(err) + } + tier.unblock() + if err := layer.wait(t); !errors.Is(err, failure) { + t.Fatalf("restore error=%v, want %v", err, failure) + } + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("superseded restore left its upload on disk: %v", uploads) + } + after := r.fileInfo(oi.VersionID) + if !strings.Contains(after.Metadata[xhttp.AmzRestore], "true") || after.Metadata[restoreIsolationAttemptKey] != "newer-generation" { + t.Errorf("failed restore cleared a newer attempt's marker: %q %q", after.Metadata[xhttp.AmzRestore], after.Metadata[restoreIsolationAttemptKey]) + } + }) + } +} + +// A restored copy expires only by its own reason, on a fresh read: its +// restore deadline passed and nothing else. A renewed deadline survives +// another rule falling due and an expiry checked across all pools. +func TestExpireRestoredNeedsOwnReason(t *testing.T) { + for _, mode := range []string{"other-expiry-due", "checked-all-pools", "ongoing", "renewed", "expired"} { + t.Run(mode, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi := r.put("restored", false, true) + r.restore(oi) + before := r.fileInfo(oi.VersionID) + if mode == "other-expiry-due" { + meta, err := globalBucketMetadataSys.Get(r.bucket) + if err != nil { + t.Fatal(err) + } + meta.LifecycleConfigXML = []byte(`deleteEnabled2020-01-01T00:00:00Z`) + if err := meta.Save(t.Context(), r.z); err != nil { + t.Fatal(err) + } + globalBucketMetadataSys.Set(r.bucket, meta) + } + header := completedRestoreObj(UTCNow().Add(48 * time.Hour)).String() + switch mode { + case "ongoing": + header = ongoingRestoreObj().String() + case "expired": + header = "" + } + if header != "" { + opts := r.opts(oi.VersionID) + opts.MTime = oi.ModTime + opts.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { + o.UserDefined[xhttp.AmzRestore] = header + return ReplicateDecision{}, nil + } + if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, opts); err != nil { + t.Fatal(err) + } + } + var err error + if mode == "checked-all-pools" { + opts := r.opts(oi.VersionID) + opts.Expiration = ExpirationOptions{Expire: true, CheckedAllPools: true} + opts.Transition.ExpireRestored = true + _, err = r.z.serverPools[0].getHashedSet(r.object).DeleteObject(t.Context(), r.bucket, r.object, opts) + } else { + err = expireTransitionedObject(t.Context(), r.z, &oi, lifecycle.Event{Action: lifecycle.DeleteRestoredAction}, lcEventSrc_Scanner) + } + after := r.fileInfo(oi.VersionID) + if mode == "expired" { + // Control: an elapsed deadline is the reason itself. + if err != nil || after.Metadata[xhttp.AmzRestore] != "" || r.dataDirs(before.DataDir) != 0 { + t.Errorf("expired restored copy not removed: %v", err) + } + return + } + t.Logf("cleanup=%v header=%q", err, after.Metadata[xhttp.AmzRestore]) + if err == nil || after.Metadata[xhttp.AmzRestore] != header || r.dataDirs(before.DataDir) != len(r.z.serverPools[0].getHashedSet(r.object).getDisks()) { + t.Error("cache cleanup removed a restored copy whose own deadline has not passed") + } + }) + } +} + +// Caller metadata cannot make an upload a native restore upload: the marker +// is dropped, the upload stays an ordinary one and nothing commits it. +func TestRestoreUploadMarkerSpoofing(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, _ := nativeRestoreSeed(t, r, 0, false, true) + forged := fmt.Sprintf(`{"id":%q,"v":%q,"e":%q,"s":%d,"m":%q}`, mustGetUUID(), oi.VersionID, oi.ETag, oi.Size, oi.ModTime.UTC().Format(time.RFC3339Nano)) + for _, target := range []string{"existing-version", "new-object"} { + t.Run(target, func(t *testing.T) { + opts := ObjectOptions{Versioned: true, UserDefined: map[string]string{restoreIsolationUploadKey: forged}} + object := r.object + if target == "existing-version" { + opts.VersionID = oi.VersionID + opts.MTime = oi.ModTime + opts.PreserveETag = oi.ETag + } else { + object = "spoofed-" + r.object + } + upload, err := r.z.NewMultipartUpload(t.Context(), r.bucket, object, opts) + if err != nil { + t.Fatal(err) + } + uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, object, "", "", "", 100) + if err != nil || len(uploads.Uploads) != 1 || uploads.Uploads[0].UploadID != upload.UploadID { + t.Fatalf("a caller's upload must stay listed: %+v, %v", uploads.Uploads, err) + } + data := []byte("spoofed part") + part, err := r.z.PutObjectPart(t.Context(), r.bucket, object, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + before := nativeRestoreSnapshot(t, r) + done, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, object, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true, MTime: opts.MTime}) + if target == "existing-version" { + if !errors.Is(err, errAuthentication) { + t.Errorf("spoofed upload overwrote the transitioned version: %v", err) + } + if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) { + t.Error("rejected spoofed completion changed the version") + } + return + } + if err != nil { + t.Fatal(err) + } + info, err := r.z.GetObjectInfo(t.Context(), r.bucket, object, ObjectOptions{VersionID: done.VersionID}) + if err != nil || info.UserDefined[restoreIsolationUploadKey] != "" { + t.Errorf("caller metadata committed a restore upload marker: %q, %v", info.UserDefined[restoreIsolationUploadKey], err) + } + }) + } +} + +// Controls: an ordinary POST restore, a renewal, and a client's multipart +// upload of the same key while a restore runs. +func TestRestorePOSTControls(t *testing.T) { + t.Run("restore", func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil || info.IsRemote() || info.RestoreOngoing || info.UserDefined[restoreIsolationAttemptKey] != "" { + t.Fatalf("restored status: %q %q, %v", info.UserDefined[xhttp.AmzRestore], info.UserDefined[restoreIsolationAttemptKey], err) + } + if !info.ModTime.Equal(oi.ModTime) || info.ETag != oi.ETag || info.UserDefined["x-amz-meta-preserve"] != "keep-me" { + t.Error("restore changed the version identity or metadata") + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Errorf("restored contents: %v", err) + } + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("completed restore left its upload: %v", uploads) + } + }) + t.Run("renewal", func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := nativeRestoreSeed(t, r, 0, false, false) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) + before := r.fileInfo(oi.VersionID) + gets := tier.gets.Load() + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + select { + case err := <-layer.done: + t.Fatalf("renewal restored again: %v", err) + case <-time.After(200 * time.Millisecond): + } + after := r.fileInfo(oi.VersionID) + if after.DataDir != before.DataDir || !after.ModTime.Equal(before.ModTime) || after.Metadata[xhttp.AmzRestore] == "" { + t.Error("renewal changed the restored copy") + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) || tier.gets.Load() != gets { + t.Errorf("renewed copy not served locally: %v", err) + } + }) + t.Run("public-multipart-same-key", func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + tier := newRestoreIsolationGateTier(5<<20, nil) + globalTierConfigMgr.drivercache["WARM"] = tier + defer tier.unblock() + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + restoreIsolationEntered(t, tier) + upload, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true}) + if err != nil { + t.Fatal(err) + } + uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 100) + listed := false + for _, u := range uploads.Uploads { + listed = listed || u.UploadID == upload.UploadID + } + if err != nil || !listed { + t.Fatalf("client upload not listed: %+v, %v", uploads.Uploads, err) + } + tier.unblock() + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + // Single-pool part writes wait for the object lock of a running restore. + data := []byte("client data") + part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + parts, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, upload.UploadID, 0, 10, ObjectOptions{}) + if err != nil || len(parts.Parts) != 1 { + t.Fatalf("client parts: %+v, %v", parts, err) + } + newer, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true}) + if err != nil { + t.Fatal(err) + } + if _, got, err := r.read(newer.VersionID); err != nil || !bytes.Equal(got, data) { + t.Errorf("client version: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Errorf("restored version: %v", err) + } + }) +} diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go index 884b88d24..44676ac7c 100644 --- a/cmd/restore-native_test.go +++ b/cmd/restore-native_test.go @@ -375,8 +375,10 @@ func TestNativeRestorePOSTVersionIdentity(t *testing.T) { // A restore request is internal context, not an exception for normal S3 // multipart overwrites or for a version that has never completed transition. +// Only the upload of a native restore attempt, whose marker names the very +// version being completed, may write that version back. func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { - for _, mode := range []string{"matching-restore", "ordinary-complete", "not-tiered", "changed-time", "changed-size", "changed-etag"} { + for _, mode := range []string{"matching-restore", "ordinary-complete", "restore-without-attempt", "other-attempt", "not-tiered", "changed-time", "changed-size", "changed-etag"} { t.Run(mode, func(t *testing.T) { r := newRestoreSafetyTest(t, 1, versioning.Enabled) oi, body := nativeRestoreSeed(t, r, 0, false, true) @@ -398,8 +400,9 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { if err != nil { t.Fatal(err) } - opts := putRestoreOpts(r.bucket, r.object, &RestoreObjectRequest{Days: 1}, current) + opts := putRestoreOpts(&RestoreObjectRequest{Days: 1}, current, r.opts(oi.VersionID)) opts.Transition.RestoreRequest = &RestoreObjectRequest{Days: 1} + opts.PreserveETag = current.ETag if mode == "ordinary-complete" { opts.Transition.RestoreRequest = nil } @@ -408,20 +411,26 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { } // Hold the ETag constant for the size case, so only size can reject it. if mode == "changed-size" { - opts.UserDefined["etag"] = current.ETag body = append(body, 'x') } if mode == "changed-etag" { - opts.UserDefined["etag"] = "different-etag" + opts.PreserveETag = "different-etag" } - upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, opts) + // The attempt claims what its upload claims; only the matching + // mode claims exactly the stored version. + if mode != "ordinary-complete" && mode != "restore-without-attempt" { + opts.Transition.attempt = &restoreAttempt{ID: mustGetUUID(), VersionID: oi.VersionID, ETag: opts.PreserveETag, Size: current.Size, ModTime: opts.MTime} + } + uopts := opts + uopts.MTime = time.Time{} + upload, err := pool.NewMultipartUpload(t.Context(), r.bucket, r.object, uopts) if err != nil { t.Fatal(err) } var complete []CompletePart for i, data := range [][]byte{body[:5<<20], body[5<<20:]} { part, err := pool.PutObjectPart(t.Context(), r.bucket, r.object, upload.UploadID, i+1, - mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Transition: opts.Transition}) if err != nil { t.Fatal(err) } @@ -430,6 +439,11 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { if mode != "changed-size" && getCompleteMultipartMD5(complete) != current.ETag { t.Fatal("fixture multipart ETag does not match original") } + if mode == "other-attempt" { + other := *opts.Transition.attempt + other.ID = mustGetUUID() + opts.Transition.attempt = &other + } before := nativeRestoreSnapshot(t, r) _, err = pool.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, complete, opts) if mode == "matching-restore" { @@ -440,9 +454,16 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { if err != nil || !bytes.Equal(got, body) { t.Fatalf("matching restore contents: %v", err) } + if after := r.fileInfo(oi.VersionID); after.Metadata[restoreUploadKey] != "" || after.Metadata[restoreAttemptKey] != "" { + t.Error("restore committed its private upload or attempt marker") + } return } - if !errors.Is(err, errAuthentication) { + if mode == "other-attempt" { + if !errors.As(err, &InvalidUploadID{}) { + t.Fatalf("another attempt completed a private upload: %v", err) + } + } else if !errors.Is(err, errAuthentication) { t.Fatalf("existing version overwrite: %v", err) } if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) { @@ -721,18 +742,10 @@ func TestNativeRestoreConcurrentCompletion(t *testing.T) { case <-ctx.Done(): t.Fatal("restore never started") } - // A client's exact-prefix listing can discover the internal upload. It can - // try to complete the first part while native restore owns the object. - uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10) - if err != nil || len(uploads.Uploads) != 1 { - t.Fatalf("restore upload discovery: %+v, %v", uploads, err) - } - id := uploads.Uploads[0].UploadID - parts, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{}) - if err != nil || len(parts.Parts) != 1 { - t.Fatalf("restore must have published its first part: %+v, %v", parts, err) - } - part := parts.Parts[0] + // A client that learns the internal upload ID, which listings no + // longer show, can try to complete the first part while native + // restore owns the object. + id, part := restoreIsolationUpload(t, r, 0) completed := make(chan struct{}) var completeErr error go func() { @@ -942,17 +955,22 @@ func TestNativeRestoreCompletionReadSafety(t *testing.T) { } } +// nativeRestoreStatusHook runs afterRead once POST has read the version and +// before POST records its status on it. type nativeRestoreStatusHook struct { ObjectLayer - beforeCopy func() - copyCalled bool + afterRead func() + readCalled bool backgroundCalled atomic.Bool } -func (h *nativeRestoreStatusHook) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, info ObjectInfo, srcOpts, dstOpts ObjectOptions) (ObjectInfo, error) { - h.copyCalled = true - h.beforeCopy() - return h.ObjectLayer.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, info, srcOpts, dstOpts) +func (h *nativeRestoreStatusHook) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + oi, err := h.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts) + if err == nil && !h.readCalled { + h.readCalled = true + h.afterRead() + } + return oi, err } func (h *nativeRestoreStatusHook) RestoreTransitionedObject(context.Context, string, string, ObjectOptions) error { @@ -985,7 +1003,7 @@ func TestNativeRestorePOSTReadSafety(t *testing.T) { } before = restoreBoundarySnapshot(t, r) } - hook := &nativeRestoreStatusHook{ObjectLayer: r.z, beforeCopy: mutate} + hook := &nativeRestoreStatusHook{ObjectLayer: r.z, afterRead: mutate} if stage == "initial" { mutate() } @@ -1004,7 +1022,7 @@ func TestNativeRestorePOSTReadSafety(t *testing.T) { if rec.Code < 400 { t.Fatalf("POST acknowledged failed read: %d %s", rec.Code, rec.Body.String()) } - if hook.copyCalled != (stage == "status-copy") || hook.backgroundCalled.Load() { + if hook.readCalled != (stage == "status-copy") || hook.backgroundCalled.Load() { t.Fatal("request followed the wrong failure path") } if !reflect.DeepEqual(before, restoreBoundarySnapshot(t, r)) { diff --git a/internal/grid/handlers.go b/internal/grid/handlers.go index 1d20fa9b9..0f2c39d7e 100644 --- a/internal/grid/handlers.go +++ b/internal/grid/handlers.go @@ -116,6 +116,7 @@ const ( HandlerRenamePart HandlerClearUploadID HandlerCheckParts3 + HandlerRestoreCapability // Add more above here ^^^ // If all handlers are used, the type of Handler can be changed. @@ -200,6 +201,7 @@ var handlerPrefixes = [handlerLast]string{ HandlerCheckParts3: storagePrefix, HandlerRenamePart: storagePrefix, HandlerClearUploadID: peerPrefix, + HandlerRestoreCapability: peerPrefix, } const ( diff --git a/internal/grid/handlers_string.go b/internal/grid/handlers_string.go index d6c6b4c38..aea3f45fb 100644 --- a/internal/grid/handlers_string.go +++ b/internal/grid/handlers_string.go @@ -86,14 +86,15 @@ func _() { _ = x[HandlerRenamePart-75] _ = x[HandlerClearUploadID-76] _ = x[HandlerCheckParts3-77] - _ = x[handlerTest-78] - _ = x[handlerTest2-79] - _ = x[handlerLast-80] + _ = x[HandlerRestoreCapability-78] + _ = x[handlerTest-79] + _ = x[handlerTest2-80] + _ = x[handlerLast-81] } -const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3handlerTesthandlerTest2handlerLast" +const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3RestoreCapabilityhandlerTesthandlerTest2handlerLast" -var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 971, 983, 994} +var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 977, 988, 1000, 1011} func (i HandlerID) String() string { idx := int(i) - 0 From 32ab506924a9500eee9fa108e79947a74e3a0fc6 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 03:31:03 +0800 Subject: [PATCH 22/26] fix(pools): lock single-pool object writes in the object's own erasure set In a single pool, DELETE in every form, bulk delete, the CopyObject destination, SelectObjectContent, batch key rotation and the upload-ID guards of complete, abort, get-info and list-parts locked through the first erasure set, while PUT, multipart commit, PutObjectPart, tag, retention, legal-hold and metadata updates, restore, transition and heal lock in the set the object hashes to. When the sets of a pool use different hosts, as every automatic layout of 18 or more hosts does, the two locks ask different lockers and do not exclude each other, and concurrent read-modify-writes of xl.meta can lose acknowledged versions and delete markers or leave an object on a minority of drives. Route these locks through NewObjectNSLock, which returns the lock of the object's own set in a single pool and the pools lock otherwise. Upload guards choose the set by their object and keep the name object/uploadID. True prefix deletion and the locks of names that are not objects keep their domain. Bulk delete groups its names by set and takes one set's lock at a time, in ascending set index; a failed group fails alone. A single node, whose sets share one local lock table, keeps its one bulk lock. The COPY handler takes its source and destination locks before it opens the source, sorted, and transactionally: bounded waits while holding a lock, release in reverse order and retry with new lock instances, one deadline. It reads the source under them, passes NoLock to every nested call, and runs that work in the locks' context. This also removes a deadlock of copies in opposite directions wherever both locks were in one domain, such as a single node or several pools. Lock names and RPCs are unchanged, but a pool is protected only when every node runs this change: affected pools must be upgraded and rolled back with a drained restart. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/batch-rotate.go | 2 +- cmd/erasure-server-pool-lock-domain_test.go | 1199 +++++++++++++++++++ cmd/erasure-server-pool-locks.go | 322 +++++ cmd/erasure-server-pool-locks_test.go | 555 +++++++++ cmd/erasure-server-pool.go | 62 +- cmd/metacache-server-pool.go | 9 +- cmd/object-api-interface.go | 6 + cmd/object-handlers.go | 61 +- cmd/object-retention-authz_test.go | 12 + 9 files changed, 2194 insertions(+), 34 deletions(-) create mode 100644 cmd/erasure-server-pool-lock-domain_test.go create mode 100644 cmd/erasure-server-pool-locks.go create mode 100644 cmd/erasure-server-pool-locks_test.go diff --git a/cmd/batch-rotate.go b/cmd/batch-rotate.go index a238af08b..3ebbc844d 100644 --- a/cmd/batch-rotate.go +++ b/cmd/batch-rotate.go @@ -180,7 +180,7 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject) - lock := api.NewNSLock(r.Bucket, objInfo.Name) + lock := api.NewObjectNSLock(r.Bucket, objInfo.Name) lkctx, err := lock.GetLock(ctx, globalOperationTimeout) if err != nil { return err diff --git a/cmd/erasure-server-pool-lock-domain_test.go b/cmd/erasure-server-pool-lock-domain_test.go new file mode 100644 index 000000000..f2a305cb3 --- /dev/null +++ b/cmd/erasure-server-pool-lock-domain_test.go @@ -0,0 +1,1199 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/amztime" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/dsync" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" +) + +// A single pool whose two erasure sets share the test drives, but lock as the +// sets of a distributed pool do whose sets use different hosts. With one +// shared local table the sets are one lock domain, as on a single node. +const ( + ldShared = "shared" // one local lock table for both sets + ldLocal = "local-split" // a local lock table per set + ldDsync = "dsync-disjoint" // in-process dsync, no locker in common + ldOverlap = "dsync-overlap" // in-process dsync, one of three lockers in common +) + +var ldModes = []string{ldShared, ldLocal, ldDsync, ldOverlap} + +// ldWait is how long an operation must stay blocked behind a held commit. +const ldWait = 400 * time.Millisecond + +type ldGateKey struct{} + +// ldGate holds the storage commit of the operation whose context carries it. +type ldGate struct { + entered, release chan struct{} + once, open sync.Once +} + +func newLDGate() *ldGate { + return &ldGate{entered: make(chan struct{}), release: make(chan struct{})} +} + +func (g *ldGate) let() { g.open.Do(func() { close(g.release) }) } + +func ldHold(ctx context.Context) { + g, _ := ctx.Value(ldGateKey{}).(*ldGate) + if g == nil { + return + } + g.once.Do(func() { close(g.entered) }) + select { + case <-g.release: + case <-ctx.Done(): + } +} + +// ldGateDisk holds every metadata commit of a gated operation. +type ldGateDisk struct{ StorageAPI } + +func (d ldGateDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { + ldHold(ctx) + return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) +} + +func (d ldGateDisk) RenamePart(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string, meta []byte, skipParent string) error { + ldHold(ctx) + return d.StorageAPI.RenamePart(ctx, srcVolume, srcPath, dstVolume, dstPath, meta, skipParent) +} + +func (d ldGateDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error { + ldHold(ctx) + return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts) +} + +func (d ldGateDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error { + ldHold(ctx) + return d.StorageAPI.DeleteVersions(ctx, volume, versions, opts) +} + +func (d ldGateDisk) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error { + ldHold(ctx) + return d.StorageAPI.WriteMetadata(ctx, origvolume, volume, path, fi) +} + +func (d ldGateDisk) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error { + ldHold(ctx) + return d.StorageAPI.UpdateMetadata(ctx, volume, path, fi, opts) +} + +// ldSplit turns the single pool of z into two sets that share its drives and +// lock as mode says. Their drives hold gated commits. +func ldSplit(t *testing.T, z *erasureServerPools, mode string) { + t.Helper() + pool := z.serverPools[0] + sets, count := pool.sets, pool.setCount + t.Cleanup(func() { pool.sets, pool.setCount = sets, count }) + first, second := *sets[0], *sets[0] + second.setIndex = 1 + switch mode { + case ldShared: + case ldLocal: + first.nsMutex, second.nsMutex = newNSLock(false), newNSLock(false) + case ldDsync, ldOverlap: + l := make([]dsync.NetLocker, 8) + for i := range l { + l[i] = newLocker() + } + a, b := l[:4], l[4:] + if mode == ldOverlap { + // Each set can grant its quorum of two without the shared one. + a, b = l[:3], l[2:5] + } + first.nsMutex, second.nsMutex = newNSLock(true), newNSLock(true) + first.getLockers = func() ([]dsync.NetLocker, string) { return slices.Clone(a), "lock-domain-test" } + second.getLockers = func() ([]dsync.NetLocker, string) { return slices.Clone(b), "lock-domain-test" } + default: + t.Fatalf("unknown mode %q", mode) + } + disks := sets[0].getDisks + gated := func() []StorageAPI { + ds := disks() + for i, d := range ds { + if d != nil { + ds[i] = ldGateDisk{d} + } + } + return ds + } + first.getDisks, second.getDisks = gated, gated + pool.sets, pool.setCount = []*erasureObjects{&first, &second}, 2 +} + +// ldPool returns a split single pool and its unversioned test bucket. +func ldPool(t *testing.T, mode string) (*erasureServerPools, string) { + t.Helper() + z, bucket := bulkDeletePools(t, 1) + ldSplit(t, z, mode) + return z, bucket +} + +// ldKey returns a name with prefix that hashes to set. +func ldKey(t *testing.T, z *erasureServerPools, prefix string, set int) string { + t.Helper() + for i := range 1000 { + key := fmt.Sprintf("%s-%d", prefix, i) + if z.serverPools[0].getHashedSetIndex(key) == set { + return key + } + } + t.Fatalf("no name with prefix %q hashes to set %d", prefix, set) + return "" +} + +// ldOverlap runs hold until it is inside its storage commit, then runs other. +// It reports whether other was still waiting after ldWait, then lets hold +// commit and returns both results. +func ldOverlapRun(t *testing.T, hold, other func(context.Context) error) (holdErr, otherErr error, waited bool) { + t.Helper() + g := newLDGate() + defer g.let() + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) + defer cancel() + held := make(chan error, 1) + go func() { held <- hold(context.WithValue(ctx, ldGateKey{}, g)) }() + select { + case <-g.entered: + case err := <-held: + t.Fatalf("held operation finished before its commit: %v", err) + case <-ctx.Done(): + t.Fatal("held operation never reached its commit") + } + done := make(chan error, 1) + go func() { done <- other(ctx) }() + select { + case otherErr = <-done: + case <-time.After(ldWait): + waited = true + } + g.let() + holdErr = <-held + if waited { + otherErr = <-done + } + return holdErr, otherErr, waited +} + +func ldReader(r io.Reader, size int64) *PutObjReader { + hr, err := hash.NewReader(context.Background(), r, size, "", "", size) + if err != nil { + panic(err) + } + return NewPutObjReader(hr) +} + +func ldPut(ctx context.Context, z *erasureServerPools, bucket, key, body string) error { + _, err := z.PutObject(ctx, bucket, key, ldReader(strings.NewReader(body), int64(len(body))), ObjectOptions{}) + return err +} + +func ldMustPut(t *testing.T, z *erasureServerPools, bucket, key, body string) { + t.Helper() + if err := ldPut(t.Context(), z, bucket, key, body); err != nil { + t.Fatal(err) + } +} + +func ldBody(t *testing.T, z *erasureServerPools, bucket, key string) (string, ObjectInfo) { + t.Helper() + gr, err := z.GetObjectNInfo(t.Context(), bucket, key, nil, nil, ObjectOptions{}) + if err != nil { + t.Fatalf("read %s: %v", key, err) + } + defer gr.Close() + body, err := io.ReadAll(gr) + if err != nil { + t.Fatal(err) + } + return string(body), gr.ObjInfo +} + +func ldAbsent(t *testing.T, z *erasureServerPools, bucket, key string) { + t.Helper() + if _, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{}); !isErrObjectNotFound(err) { + t.Errorf("%s: %v, want not found", key, err) + } +} + +// ldMetadataCopy is a metadata-only self copy of key from a snapshot read +// without its lock, as the COPY handler read a copy onto itself. +func ldMetadataCopy(ctx context.Context, z *erasureServerPools, bucket, key, meta string) error { + oi, err := z.GetObjectInfo(ctx, bucket, key, ObjectOptions{NoLock: true}) + if err != nil { + return err + } + oi.metadataOnly = true + oi.UserDefined = cloneMSS(oi.UserDefined) + oi.UserDefined["x-amz-meta-copied"] = meta + _, err = z.CopyObject(ctx, bucket, key, bucket, key, oi, ObjectOptions{}, ObjectOptions{}) + return err +} + +// In one pool, a DELETE, bulk DELETE, COPY or tag update of a key must wait for +// a write of the same key that is inside its commit, and the reverse, whatever +// the hosts of the key's set are. +func TestLockDomainSinglePoolWritersExclude(t *testing.T) { + type tc struct { + name string + setup func(t *testing.T, z *erasureServerPools, bucket, key string) + hold, other func(ctx context.Context, z *erasureServerPools, bucket, key string) error + check func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) + } + seed := func(t *testing.T, z *erasureServerPools, bucket, key string) { ldMustPut(t, z, bucket, key, "seed") } + put := func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + return ldPut(ctx, z, bucket, key, "written") + } + noErrs := func(t *testing.T, holdErr, otherErr error) { + t.Helper() + if holdErr != nil || otherErr != nil { + t.Fatalf("held: %v, other: %v", holdErr, otherErr) + } + } + cases := []tc{{ + name: "put/delete", + setup: seed, + hold: put, + other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + _, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{}) + return err + }, + check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) { + noErrs(t, holdErr, otherErr) + ldAbsent(t, z, bucket, key) + }, + }, { + // The If-Match check and the delete are atomic with writers: the + // condition is checked against the PUT's version. + name: "put/delete-if-match", + setup: seed, + hold: put, + other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + seedETag := getMD5Hash([]byte("seed")) + _, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{CheckPrecondFn: func(oi ObjectInfo) bool { return oi.ETag != seedETag }}) + return err + }, + check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) { + if holdErr != nil || !isErrPreconditionFailed(otherErr) { + t.Fatalf("put: %v; If-Match delete of the replaced version: %v, want precondition failed", holdErr, otherErr) + } + if body, _ := ldBody(t, z, bucket, key); body != "written" { + t.Errorf("body %q, want the PUT's", body) + } + }, + }, { + // ILM's delete-all deletes one key through prefix deletion. + name: "put/delete-all", + setup: seed, + hold: put, + other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + _, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{DeletePrefix: true, DeletePrefixObject: true}) + return err + }, + check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) { + noErrs(t, holdErr, otherErr) + ldAbsent(t, z, bucket, key) + }, + }, { + name: "put/bulk-delete", + setup: seed, + hold: put, + other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + _, errs := z.DeleteObjects(ctx, bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key}}}, ObjectOptions{}) + return errs[0] + }, + check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) { + noErrs(t, holdErr, otherErr) + ldAbsent(t, z, bucket, key) + }, + }, { + name: "put/copy-metadata", + setup: seed, + hold: put, + other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + return ldMetadataCopy(ctx, z, bucket, key, "1") + }, + check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) { + noErrs(t, holdErr, otherErr) + if body, _ := ldBody(t, z, bucket, key); body != "written" { + t.Errorf("body %q after copy, want the PUT's", body) + } + }, + }, { + name: "put/copy-rewrite", + setup: func(t *testing.T, z *erasureServerPools, bucket, key string) { + ldMustPut(t, z, bucket, key, "seed") + ldMustPut(t, z, bucket, key+"-src", "copied") + }, + hold: put, + other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + gr, err := z.GetObjectNInfo(ctx, bucket, key+"-src", nil, nil, ObjectOptions{}) + if err != nil { + return err + } + defer gr.Close() + src := gr.ObjInfo + src.PutObjReader = ldReader(gr, src.Size) + _, err = z.CopyObject(ctx, bucket, key+"-src", bucket, key, src, ObjectOptions{}, ObjectOptions{}) + return err + }, + check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) { + noErrs(t, holdErr, otherErr) + if body, _ := ldBody(t, z, bucket, key); body != "copied" { + t.Errorf("body %q, want the copy's, which ran after the PUT", body) + } + }, + }, { + name: "delete/tags", + setup: seed, + hold: func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + _, err := z.DeleteObject(ctx, bucket, key, ObjectOptions{}) + return err + }, + other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + _, err := z.PutObjectTags(ctx, bucket, key, "k=v", ObjectOptions{}) + return err + }, + check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) { + if holdErr != nil || !isErrObjectNotFound(otherErr) { + t.Fatalf("delete: %v; tags: %v, want not found", holdErr, otherErr) + } + ldAbsent(t, z, bucket, key) + }, + }, { + name: "copy/tags", + setup: seed, + hold: func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + return ldMetadataCopy(ctx, z, bucket, key, "1") + }, + other: func(ctx context.Context, z *erasureServerPools, bucket, key string) error { + _, err := z.PutObjectTags(ctx, bucket, key, "k=v", ObjectOptions{}) + return err + }, + check: func(t *testing.T, z *erasureServerPools, bucket, key string, holdErr, otherErr error) { + noErrs(t, holdErr, otherErr) + oi, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{}) + if err != nil || oi.UserTags != "k=v" || oi.UserDefined["x-amz-meta-copied"] != "1" { + t.Errorf("after copy then tags: tags %q, meta %q, %v", oi.UserTags, oi.UserDefined["x-amz-meta-copied"], err) + } + }, + }} + for _, mode := range ldModes { + t.Run(mode, func(t *testing.T) { + z, bucket := ldPool(t, mode) + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + key := ldKey(t, z, strings.ReplaceAll(c.name, "/", "-"), 1) + c.setup(t, z, bucket, key) + holdErr, otherErr, waited := ldOverlapRun(t, + func(ctx context.Context) error { return c.hold(ctx, z, bucket, key) }, + func(ctx context.Context) error { return c.other(ctx, z, bucket, key) }) + if !waited { + t.Errorf("%s ran while %s held its commit (other: %v)", c.name[strings.Index(c.name, "/")+1:], c.name[:strings.Index(c.name, "/")], otherErr) + } + c.check(t, z, bucket, key, holdErr, otherErr) + }) + } + }) + } +} + +// ldUpload starts an upload of key whose upload resource, and that of its +// first part, hash to the set of key when sameSet is set. +func ldUpload(t *testing.T, z *erasureServerPools, bucket, key string, sameSet bool) string { + t.Helper() + pool := z.serverPools[0] + for range 64 { + res, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + set := pool.getHashedSetIndex(key) + if !sameSet || (pool.getHashedSetIndex(pathJoin(key, res.UploadID)) == set && + pool.getHashedSetIndex(pathJoin(key, res.UploadID, "1")) == set) { + return res.UploadID + } + if err := z.AbortMultipartUpload(t.Context(), bucket, key, res.UploadID, ObjectOptions{}); err != nil { + t.Fatal(err) + } + } + t.Fatal("no upload ID with the wanted hashes") + return "" +} + +// The upload-ID guards of complete and abort exclude a PutObjectPart of the +// same upload. Get-info and list-parts read, as PutObjectPart's own guard +// does, and do not wait for it. +func TestLockDomainUploadGuards(t *testing.T) { + part := []byte("the only part") + partETag := getMD5Hash(part) + putPart := func(ctx context.Context, z *erasureServerPools, bucket, key, id string) error { + _, err := z.PutObjectPart(ctx, bucket, key, id, 1, ldReader(bytes.NewReader(part), int64(len(part))), ObjectOptions{}) + return err + } + for _, mode := range ldModes { + t.Run(mode, func(t *testing.T) { + z, bucket := ldPool(t, mode) + for _, op := range []string{"complete", "abort", "info", "list"} { + t.Run(op, func(t *testing.T) { + key := ldKey(t, z, "upload-"+op, 1) + id := ldUpload(t, z, bucket, key, false) + holdErr, otherErr, waited := ldOverlapRun(t, + func(ctx context.Context) error { return putPart(ctx, z, bucket, key, id) }, + func(ctx context.Context) error { + var err error + switch op { + case "complete": + _, err = z.CompleteMultipartUpload(ctx, bucket, key, id, []CompletePart{{PartNumber: 1, ETag: partETag}}, ObjectOptions{}) + case "abort": + err = z.AbortMultipartUpload(ctx, bucket, key, id, ObjectOptions{}) + case "info": + _, err = z.GetMultipartInfo(ctx, bucket, key, id, ObjectOptions{}) + case "list": + _, err = z.ListObjectParts(ctx, bucket, key, id, 0, 10, ObjectOptions{}) + } + return err + }) + exclusive := op == "complete" || op == "abort" + if waited != exclusive { + t.Errorf("%s waited=%v for a part upload inside its commit, want %v (err %v)", op, waited, exclusive, otherErr) + } + if holdErr != nil || otherErr != nil { + t.Fatalf("part: %v, %s: %v", holdErr, op, otherErr) + } + switch op { + case "complete": + if body, _ := ldBody(t, z, bucket, key); body != string(part) { + t.Errorf("completed object %q", body) + } + case "abort": + if _, err := z.GetMultipartInfo(t.Context(), bucket, key, id, ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) { + t.Errorf("upload after abort: %v", err) + } + } + }) + } + }) + } +} + +// Bulk delete groups its names by set and must still wait for a completion +// of one of them, also when the batch names the completion's upload +// resource itself; neither may hold the other. +func TestLockDomainBulkDeleteAgainstUploads(t *testing.T) { + part := []byte("the only part") + partETag := getMD5Hash(part) + for _, mode := range ldModes { + t.Run(mode, func(t *testing.T) { + z, bucket := ldPool(t, mode) + for _, alias := range []bool{false, true} { + t.Run(fmt.Sprintf("complete/alias=%v", alias), func(t *testing.T) { + key := ldKey(t, z, fmt.Sprintf("bulk-complete-%v", alias), 1) + id := ldUpload(t, z, bucket, key, false) + if _, err := z.PutObjectPart(t.Context(), bucket, key, id, 1, mustGetPutObjReader(t, bytes.NewReader(part), int64(len(part)), "", ""), ObjectOptions{}); err != nil { + t.Fatal(err) + } + names := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key}}} + if alias { + names = append(names, ObjectToDelete{ObjectV: ObjectV{ObjectName: pathJoin(key, id)}}) + } + holdErr, otherErr, waited := ldOverlapRun(t, + func(ctx context.Context) error { + _, err := z.CompleteMultipartUpload(ctx, bucket, key, id, []CompletePart{{PartNumber: 1, ETag: partETag}}, ObjectOptions{}) + return err + }, + func(ctx context.Context) error { + _, errs := z.DeleteObjects(ctx, bucket, names, ObjectOptions{}) + return errors.Join(errs...) + }) + if !waited { + t.Errorf("bulk delete ran while the completion held its commit (%v)", otherErr) + } + if holdErr != nil || otherErr != nil { + t.Fatalf("complete: %v, bulk: %v", holdErr, otherErr) + } + ldAbsent(t, z, bucket, key) + }) + } + // The names alias the locks of a part upload; they are other keys + // and need not wait, but neither side may block the other. + t.Run("put-part/alias", func(t *testing.T) { + key := ldKey(t, z, "bulk-part", 1) + id := ldUpload(t, z, bucket, key, true) + names := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: pathJoin(key, id)}}, {ObjectV: ObjectV{ObjectName: pathJoin(key, id, "1")}}} + holdErr, otherErr, _ := ldOverlapRun(t, + func(ctx context.Context) error { + _, err := z.PutObjectPart(ctx, bucket, key, id, 1, ldReader(bytes.NewReader(part), int64(len(part))), ObjectOptions{}) + return err + }, + func(ctx context.Context) error { + _, errs := z.DeleteObjects(ctx, bucket, names, ObjectOptions{}) + return errors.Join(errs...) + }) + if holdErr != nil || otherErr != nil { + t.Fatalf("part: %v, bulk: %v", holdErr, otherErr) + } + if _, err := z.CompleteMultipartUpload(t.Context(), bucket, key, id, []CompletePart{{PartNumber: 1, ETag: partETag}}, ObjectOptions{}); err != nil { + t.Fatalf("upload after the bulk delete: %v", err) + } + }) + }) + } +} + +// A group of a bulk delete whose lock is held by a writer of its set fails +// alone; the other set's group commits. A canceled request fails the groups it +// has not run, keeps the ones it has, and leaves no lock behind. +func TestLockDomainBulkDeleteGroups(t *testing.T) { + for _, mode := range ldModes { + t.Run(mode, func(t *testing.T) { + z, bucket := ldPool(t, mode) + split := mode != ldShared + for _, cancelIt := range []bool{false, true} { + t.Run(fmt.Sprintf("cancel=%v", cancelIt), func(t *testing.T) { + k0 := ldKey(t, z, fmt.Sprintf("group-%v", cancelIt), 0) + k1 := ldKey(t, z, fmt.Sprintf("group-%v", cancelIt), 1) + ldMustPut(t, z, bucket, k0, "zero") + ldMustPut(t, z, bucket, k1, "one") + // A writer of k1 in the lock domain of its writes. + writer := z.serverPools[0].getHashedSet(k1).NewNSLock(bucket, k1) + wctx, err := writer.GetLock(t.Context(), globalOperationTimeout) + if err != nil { + t.Fatal(err) + } + var once sync.Once + unlock := func() { once.Do(func() { writer.Unlock(wctx) }) } + defer unlock() + + ctx, cancel := context.WithTimeout(t.Context(), 1500*time.Millisecond) + defer cancel() + names := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: k1}}, {ObjectV: ObjectV{ObjectName: k0}}} + type result struct { + objs []DeletedObject + errs []error + } + done := make(chan result, 1) + go func() { + objs, errs := z.DeleteObjects(ctx, bucket, names, ObjectOptions{}) + done <- result{objs, errs} + }() + if cancelIt { + if split { + // Cancel once the first group has committed. + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := z.GetObjectInfo(t.Context(), bucket, k0, ObjectOptions{}); isErrObjectNotFound(err) { + break + } + if time.Now().After(deadline) { + t.Fatal("set 0's group did not commit") + } + time.Sleep(5 * time.Millisecond) + } + } else { + time.Sleep(200 * time.Millisecond) + } + cancel() + } + res := <-done + if res.objs[0].ObjectName != k1 || res.objs[1].ObjectName != k0 { + t.Fatalf("results out of order: %+v", res.objs) + } + if res.errs[0] == nil { + t.Errorf("k1 deleted while its writer held the lock") + } + if cancelIt && !errors.Is(res.errs[0], context.Canceled) { + t.Errorf("k1 after cancellation: %v", res.errs[0]) + } + if split != (res.errs[1] == nil) { + t.Errorf("k0 (other set): %v, want committed=%v", res.errs[1], split) + } + unlock() + if body, _ := ldBody(t, z, bucket, k1); body != "one" { + t.Errorf("k1 = %q", body) + } + if split { + ldAbsent(t, z, bucket, k0) + } + // No lock is left behind. + lk := z.serverPools[0].getHashedSet(k1).NewNSLock(bucket, k1) + lctx, err := lk.GetLock(t.Context(), newDynamicTimeout(5*time.Second, 5*time.Second)) + if err != nil { + t.Fatalf("lock left behind: %v", err) + } + lk.Unlock(lctx) + }) + } + }) + } +} + +// ldBulkScenario runs one versioned bulk delete over names of both sets and +// describes its results and the versions left, without generated version IDs. +func ldBulkScenario(t *testing.T, mode string) []string { + t.Helper() + z, _ := ldPool(t, mode) + bucket := "lock-domain-versioned" + if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil { + t.Fatal(err) + } + vput := func(key, body string) ObjectInfo { + oi, err := z.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true}) + if err != nil { + t.Fatal(err) + } + return oi + } + a, b, c := ldKey(t, z, "map-a", 1), ldKey(t, z, "map-b", 0), ldKey(t, z, "map-c", 1) + dir := ldKey(t, z, "map-dir", 0) + "/" + va1, va2 := vput(a, "a1"), vput(a, "a2") + vb := vput(b, "b1") + vput(c, "c1") + if _, err := z.PutObject(t.Context(), bucket, dir, mustGetPutObjReader(t, strings.NewReader(""), 0, "", ""), ObjectOptions{Versioned: true}); err != nil { + t.Fatal(err) + } + absent := mustGetUUID() + invalid := strings.Repeat("x", 1100) + names := []ObjectToDelete{ + {ObjectV: ObjectV{ObjectName: a}}, // marker + {ObjectV: ObjectV{ObjectName: b, VersionID: vb.VersionID}}, // purge + {ObjectV: ObjectV{ObjectName: invalid}}, // invalid + {ObjectV: ObjectV{ObjectName: a, VersionID: va1.VersionID}}, // purge, same key + {ObjectV: ObjectV{ObjectName: c}}, // marker + {ObjectV: ObjectV{ObjectName: dir}}, // directory marker + {ObjectV: ObjectV{ObjectName: "map-missing"}}, // absent + {ObjectV: ObjectV{ObjectName: a, VersionID: va1.VersionID}}, // duplicate + {ObjectV: ObjectV{ObjectName: b, VersionID: absent}}, // absent version + } + // Names hash with the deployment ID: compare them by role. + label := strings.NewReplacer(dir, "DIR/", a, "A", b, "B", c, "C", invalid, "INVALID", + va1.VersionID, "a1", va2.VersionID, "a2", vb.VersionID, "b1", absent, "absent") + objs, errs := z.DeleteObjects(t.Context(), bucket, names, ObjectOptions{Versioned: true}) + var out []string + for i := range names { + o := objs[i] + vid := label.Replace(o.VersionID) + if len(vid) > 10 { + vid = "generated" + } + out = append(out, label.Replace(fmt.Sprintf("%d %s vid=%q marker=%v markerVid=%v found=%v err=%v", + i, o.ObjectName, vid, o.DeleteMarker, o.DeleteMarkerVersionID != "", o.found, errs[i]))) + } + for _, key := range []string{a, b, c, dir} { + info, err := z.ListObjectVersions(t.Context(), bucket, key, "", "", "", 100) + if err != nil { + t.Fatal(err) + } + var vs []string + for _, v := range info.Objects { + if v.Name != key { + continue + } + id := "other" + switch v.VersionID { + case va2.VersionID: + id = "a2" + case va1.VersionID: + id = "a1" + case vb.VersionID: + id = "b1" + } + vs = append(vs, fmt.Sprintf("%s:marker=%v:latest=%v", id, v.DeleteMarker, v.IsLatest)) + } + out = append(out, label.Replace(key)+" "+strings.Join(vs, ",")) + } + return out +} + +// Grouping by set keeps every result where its name was and gives the same +// results and the same versions as the one bulk lock of a single node. +func TestLockDomainBulkDeleteMapping(t *testing.T) { + want := ldBulkScenario(t, ldShared) + for _, mode := range ldModes[1:] { + t.Run(mode, func(t *testing.T) { + got := ldBulkScenario(t, mode) + if !slices.Equal(got, want) { + t.Errorf("results differ from the single bulk lock:\n got %q\nwant %q", got, want) + } + }) + } +} + +// A restore holds the object lock of its set for its download. DELETE and COPY +// of the object must wait for it. +func TestLockDomainRestoreExcludesDeleteAndCopy(t *testing.T) { + for _, mode := range ldModes { + for _, op := range []string{"delete", "copy"} { + t.Run(mode+"/"+op, func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + ldSplit(t, r.z, mode) + r.object = ldKey(t, r.z, "restore-"+op, 1) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + blocked := &nativeRestoreBlockedTier{WarmBackend: globalTierConfigMgr.drivercache["WARM"], entered: make(chan struct{}), release: make(chan struct{})} + globalTierConfigMgr.drivercache["WARM"] = blocked + var once sync.Once + unblock := func() { once.Do(func() { close(blocked.release) }) } + defer unblock() + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + restored := make(chan error, 1) + go func() { + restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + select { + case <-blocked.entered: + case <-ctx.Done(): + t.Fatal("restore never started its download") + } + done := make(chan error, 1) + go func() { + switch op { + case "delete": + _, err := r.z.DeleteObject(ctx, r.bucket, r.object, r.opts("")) + done <- err + case "copy": + ropts := r.opts(oi.VersionID) + ropts.NoLock = true + cur, err := r.z.GetObjectInfo(ctx, r.bucket, r.object, ropts) + if err != nil { + done <- err + return + } + cur.metadataOnly = true + cur.UserDefined = cloneMSS(cur.UserDefined) + cur.UserDefined["x-amz-meta-copied"] = "1" + vopts := r.opts(oi.VersionID) + _, err = r.z.CopyObject(ctx, r.bucket, r.object, r.bucket, r.object, cur, vopts, vopts) + done <- err + } + }() + var opErr error + select { + case opErr = <-done: + t.Errorf("%s ran during the restore (%v)", op, opErr) + case <-time.After(ldWait): + unblock() + opErr = <-done + } + unblock() + if err := <-restored; err != nil { + t.Fatalf("restore: %v", err) + } + if opErr != nil { + t.Fatalf("%s after the restore: %v", op, opErr) + } + }) + } + } +} + +// ldAPI is a split single pool served by the S3 API router. +func ldAPI(t *testing.T, mode string) (*erasureServerPools, string, http.Handler) { + t.Helper() + z, bucket := ldPool(t, mode) + if err := newTestConfig(globalMinioDefaultRegion, z); err != nil { + t.Fatal(err) + } + return z, bucket, initTestAPIEndPoints(z, nil) +} + +func ldRequest(ctx context.Context, t *testing.T, router http.Handler, method, path, body string, headers map[string]string) *httptest.ResponseRecorder { + t.Helper() + req, err := newTestSignedRequestV4(method, path, int64(len(body)), strings.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, headers) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req.WithContext(ctx)) + return rec +} + +func ldCopyRequest(ctx context.Context, t *testing.T, router http.Handler, bucket, src, dst string, headers map[string]string) *httptest.ResponseRecorder { + t.Helper() + h := map[string]string{xhttp.AmzCopySource: "/" + bucket + "/" + src} + for k, v := range headers { + h[k] = v + } + return ldRequest(ctx, t, router, http.MethodPut, "/"+bucket+"/"+dst, "", h) +} + +// The COPY handler locks its destination, and a copy onto itself reads its +// source, in the lock domain of the object's writes. A copy onto itself must +// read the version that the PUT before it committed. +func TestLockDomainCopyHandler(t *testing.T) { + for _, mode := range ldModes { + t.Run(mode, func(t *testing.T) { + z, bucket, router := ldAPI(t, mode) + for _, self := range []bool{false, true} { + t.Run(fmt.Sprintf("self=%v", self), func(t *testing.T) { + key := ldKey(t, z, fmt.Sprintf("copy-handler-%v", self), 1) + src := key + "-src" + ldMustPut(t, z, bucket, key, "seed") + ldMustPut(t, z, bucket, src, "source") + var rec *httptest.ResponseRecorder + holdErr, _, waited := ldOverlapRun(t, + func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written") }, + func(ctx context.Context) error { + if self { + rec = ldCopyRequest(ctx, t, router, bucket, key, key, map[string]string{ + xhttp.AmzMetadataDirective: "REPLACE", "x-amz-meta-copied": "1", + }) + } else { + rec = ldCopyRequest(ctx, t, router, bucket, src, key, nil) + } + return nil + }) + if !waited { + t.Errorf("COPY ran while a PUT of its destination held its commit") + } + if holdErr != nil || rec.Code != http.StatusOK { + t.Fatalf("put: %v; copy: %d %s", holdErr, rec.Code, rec.Body.String()) + } + body, oi := ldBody(t, z, bucket, key) + want := "source" + if self { + want = "written" + if oi.UserDefined["X-Amz-Meta-Copied"] != "1" { + t.Errorf("copied metadata missing: %v", oi.UserDefined) + } + } + if body != want || oi.ETag != getMD5Hash([]byte(want)) { + t.Errorf("after the copy: body %q etag %q, want %q with its own ETag", body, oi.ETag, want) + } + }) + } + }) + } +} + +// Copies in opposite directions always make progress: between keys of +// different sets, and where both locks share one domain, as on a single node +// or with several pools. +func TestLockDomainCopyHandlerReciprocal(t *testing.T) { + for _, mode := range append(slices.Clone(ldModes), "pools") { + t.Run(mode, func(t *testing.T) { + var z *erasureServerPools + var bucket string + var router http.Handler + keys := [2]string{"reciprocal-a", "reciprocal-b"} + if mode == "pools" { + z, bucket = bulkDeletePools(t, 2) + if err := newTestConfig(globalMinioDefaultRegion, z); err != nil { + t.Fatal(err) + } + router = initTestAPIEndPoints(z, nil) + } else { + z, bucket, router = ldAPI(t, mode) + keys = [2]string{ldKey(t, z, "reciprocal", 0), ldKey(t, z, "reciprocal", 1)} + } + // Not inlined: the source is read while the destination is written. + body := strings.Repeat("reciprocal copy source ", 1<<14) + for _, k := range keys { + ldMustPut(t, z, bucket, k, body) + } + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + for round := range 8 { + var wg sync.WaitGroup + codes := make([]int, 2) + start := make(chan struct{}) + for i := range keys { + wg.Add(1) + go func() { + defer wg.Done() + <-start + codes[i] = ldCopyRequest(ctx, t, router, bucket, keys[i], keys[1-i], nil).Code + }() + } + close(start) + wg.Wait() + if codes[0] != http.StatusOK || codes[1] != http.StatusOK { + t.Fatalf("round %d: reciprocal copies returned %v", round, codes) + } + } + }) + } +} + +// The COPY handler looks up its destination while it holds the destination's +// lock: Object Lock's retention check and the tier sweeper's lookup must not +// take that lock again. +func TestLockDomainCopyHandlerDestinationLookups(t *testing.T) { + prevTier := globalTierConfigMgr + t.Cleanup(func() { globalTierConfigMgr = prevTier }) + for _, mode := range ldModes { + t.Run(mode, func(t *testing.T) { + z, _, router := ldAPI(t, mode) + globalTierConfigMgr = NewTierConfigMgr() + globalTierConfigMgr.Tiers["WARM"] = madmin.TierConfig{Version: madmin.TierConfigVer, Type: madmin.S3, Name: "WARM", S3: &madmin.TierS3{Bucket: "warm"}} + bucket := "lock-domain-worm" + if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{LockEnabled: true, VersioningEnabled: true}); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + src := ldKey(t, z, "lookup-src", 0) + if _, err := z.PutObject(t.Context(), bucket, src, mustGetPutObjReader(t, strings.NewReader(""), 0, "", ""), ObjectOptions{Versioned: true}); err != nil { + t.Fatal(err) + } + for _, dst := range []string{ldKey(t, z, "lookup-dst", 1), ldKey(t, z, "lookup-dir", 1) + "/"} { + if rec := ldCopyRequest(ctx, t, router, bucket, src, dst, nil); rec.Code != http.StatusOK { + t.Fatalf("copy to %s: %d %s", dst, rec.Code, rec.Body.String()) + } + // Self copy, metadata only. + if rec := ldCopyRequest(ctx, t, router, bucket, dst, dst, map[string]string{ + xhttp.AmzMetadataDirective: "REPLACE", "x-amz-meta-copied": "1", + }); rec.Code != http.StatusOK { + t.Fatalf("self copy of %s: %d %s", dst, rec.Code, rec.Body.String()) + } + } + }) + } +} + +// SelectObjectContent reads under the lock of the object's writes. +func TestLockDomainSelectWaitsForWriter(t *testing.T) { + const request = `SELECT * FROM S3ObjectSQL` + + `NONE` + + `` + for _, mode := range ldModes { + t.Run(mode, func(t *testing.T) { + z, bucket, router := ldAPI(t, mode) + key := ldKey(t, z, "select", 1) + ldMustPut(t, z, bucket, key, "seed,1\n") + var rec *httptest.ResponseRecorder + holdErr, _, waited := ldOverlapRun(t, + func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written,2\n") }, + func(ctx context.Context) error { + rec = ldRequest(ctx, t, router, http.MethodPost, "/"+bucket+"/"+key+"?select&select-type=2", request, nil) + return nil + }) + if !waited { + t.Errorf("SELECT ran while a PUT held its commit") + } + if holdErr != nil || rec.Code != http.StatusOK || !bytes.Contains(rec.Body.Bytes(), []byte("written,2")) { + t.Fatalf("put: %v; select: %d %q", holdErr, rec.Code, rec.Body.String()) + } + }) + } +} + +// Batch key rotation locks the object in the lock domain of its writes. +func TestLockDomainKeyRotationWaitsForWriter(t *testing.T) { + for _, mode := range ldModes { + t.Run(mode, func(t *testing.T) { + z, bucket := ldPool(t, mode) + key := ldKey(t, z, "rotate", 1) + ldMustPut(t, z, bucket, key, "seed") + job := &BatchJobKeyRotateV1{Bucket: bucket, Encryption: BatchJobKeyRotateEncryption{Type: sses3}} + // The rotation's listing found an SSE-S3 object; it reads it again + // under the lock. + listed := ObjectInfo{Bucket: bucket, Name: key, UserDefined: map[string]string{crypto.MetaSealedKeyS3: "listed"}} + _, _, waited := ldOverlapRun(t, + func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written") }, + func(ctx context.Context) error { return job.KeyRotate(ctx, z, listed) }) + if !waited { + t.Errorf("key rotation ran while a PUT held its commit") + } + }) + } +} + +// ldCopyGate holds a COPY of one destination inside the object layer's +// CopyObject, after the handler's Object Lock check, and reports when a +// metadata update of that destination reaches the object layer. +type ldCopyGate struct { + ObjectLayer + bucket, object string + copying, updating chan struct{} + release chan struct{} + copyOnce, updatingOnce sync.Once +} + +func (g *ldCopyGate) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, info ObjectInfo, srcOpts, dstOpts ObjectOptions) (ObjectInfo, error) { + if dstBucket == g.bucket && dstObject == g.object { + g.copyOnce.Do(func() { close(g.copying) }) + select { + case <-g.release: + case <-ctx.Done(): + return ObjectInfo{}, ctx.Err() + } + } + return g.ObjectLayer.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, info, srcOpts, dstOpts) +} + +func (g *ldCopyGate) PutObjectMetadata(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + if bucket == g.bucket && object == g.object { + g.updatingOnce.Do(func() { close(g.updating) }) + } + return g.ObjectLayer.PutObjectMetadata(ctx, bucket, object, opts) +} + +// A retention or legal-hold update of a COPY destination cannot land between +// the handler's Object Lock check and the copy: it waits for the copy, which +// then succeeds, and is applied after it. +func TestLockDomainCopyExcludesObjectLockUpdates(t *testing.T) { + for _, mode := range append(slices.Clone(ldModes), "pools") { + t.Run(mode, func(t *testing.T) { + var z *erasureServerPools + if mode == "pools" { + z, _ = bulkDeletePools(t, 2) + } else { + z, _ = ldPool(t, mode) + } + if err := newTestConfig(globalMinioDefaultRegion, z); err != nil { + t.Fatal(err) + } + bucket := "lock-domain-worm-copy" + if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{LockEnabled: true, VersioningEnabled: true}); err != nil { + t.Fatal(err) + } + plain := initTestAPIEndPoints(z, nil) + put := func(key string) { + t.Helper() + if rec := ldRequest(t.Context(), t, plain, http.MethodPut, "/"+bucket+"/"+key, "", nil); rec.Code != http.StatusOK { + t.Fatalf("PUT %s: %d %s", key, rec.Code, rec.Body.String()) + } + } + put("copy-source/") + for _, self := range []bool{false, true} { + for _, state := range []string{"GOVERNANCE", "COMPLIANCE", "LEGAL-HOLD"} { + t.Run(fmt.Sprintf("self=%v/%s", self, state), func(t *testing.T) { + // Directory objects: Object Lock checks their null version. + key := fmt.Sprintf("copy-destination-%v-%s/", self, state) + if mode != "pools" { + for z.serverPools[0].getHashedSetIndex(encodeDirObject(key)) != 1 { + key = "x" + key + } + } + put(key) + g := &ldCopyGate{ObjectLayer: z, bucket: bucket, object: key, + copying: make(chan struct{}), updating: make(chan struct{}), release: make(chan struct{})} + setObjectLayer(g) + defer setObjectLayer(z) + router := initTestAPIEndPoints(g, nil) + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + release := sync.OnceFunc(func() { close(g.release) }) + var wg sync.WaitGroup + defer func() { release(); cancel(); wg.Wait() }() + + src := "copy-source/" + if self { + src = key + } + copied, updated := make(chan *httptest.ResponseRecorder, 1), make(chan *httptest.ResponseRecorder, 1) + wg.Add(1) + go func() { + defer wg.Done() + copied <- ldCopyRequest(ctx, t, router, bucket, src, key, map[string]string{ + xhttp.AmzMetadataDirective: "REPLACE", "x-amz-meta-copied": "1", + }) + }() + select { + case <-g.copying: + case rec := <-copied: + t.Fatalf("COPY returned before its write: %d %s", rec.Code, rec.Body.String()) + case <-ctx.Done(): + t.Fatal("COPY never reached its write") + } + + query := url.Values{"versionId": {nullVersionID}} + body := `ON` + if state == "LEGAL-HOLD" { + query.Set("legal-hold", "") + } else { + query.Set("retention", "") + body = fmt.Sprintf(`%s%s`, + state, amztime.ISO8601Format(UTCNow().Add(365*24*time.Hour))) + } + sum := md5.Sum([]byte(body)) + wg.Add(1) + go func() { + defer wg.Done() + updated <- ldRequest(ctx, t, router, http.MethodPut, makeTestTargetURL("", bucket, key, query), body, + map[string]string{xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:])}) + }() + select { + case <-g.updating: + case rec := <-updated: + t.Fatalf("update returned before reaching the object layer: %d %s", rec.Code, rec.Body.String()) + case <-ctx.Done(): + t.Fatal("update never reached the object layer") + } + select { + case rec := <-updated: + t.Errorf("%s update ran between the COPY's Object Lock check and its write: %d", state, rec.Code) + updated <- rec + case <-time.After(ldWait): + } + release() + for name, result := range map[string]chan *httptest.ResponseRecorder{"COPY": copied, "update": updated} { + select { + case rec := <-result: + if rec.Code != http.StatusOK { + t.Errorf("%s: %d %s", name, rec.Code, rec.Body.String()) + } + case <-ctx.Done(): + t.Fatalf("%s did not finish", name) + } + } + after, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID}) + if err != nil { + t.Fatal(err) + } + if after.UserDefined["X-Amz-Meta-Copied"] != "1" { + t.Errorf("copied metadata missing: %v", after.UserDefined) + } + if state == "LEGAL-HOLD" { + if objectlock.GetObjectLegalHoldMeta(after.UserDefined).Status != objectlock.LegalHoldOn { + t.Error("legal hold applied before the copy was lost") + } + } else if string(objectlock.GetObjectRetentionMeta(after.UserDefined).Mode) != state { + t.Error("retention applied before the copy was lost") + } + }) + } + } + }) + } +} diff --git a/cmd/erasure-server-pool-locks.go b/cmd/erasure-server-pool-locks.go new file mode 100644 index 000000000..b577bf566 --- /dev/null +++ b/cmd/erasure-server-pool-locks.go @@ -0,0 +1,322 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "math/rand/v2" + "slices" + "strings" + "time" + + "github.com/minio/minio-go/v7/pkg/set" +) + +// Object lock domains. +// +// Every erasure set has its own lockers, one per host among its drives, and a +// distributed lock is granted by a majority of them. Two locks of one name +// exclude each other only when they ask the same lockers. In a single pool the +// writers (PutObject, multipart commit, PutObjectPart, tags, retention, legal +// hold, metadata, restore, transition, heal) lock in the set an object hashes +// to. The pools layer locks an object, and its upload IDs, in the same set, so +// that a pool whose sets use different hosts still has one lock per object. +// With several pools every object write takes the pools lock instead, which +// these helpers then return unchanged. +// +// Generic NewNSLock keeps one stable domain for names that are not objects: +// bucket and IAM metadata locks, format and leader coordination, and the +// /[replicate]/ serializer. + +// NewObjectNSLock returns the lock of one object in the lock domain that its +// writes use: in a single pool the object's own erasure set, otherwise the +// pools lock. Directory objects are named as storage names them. +func (z *erasureServerPools) NewObjectNSLock(bucket, object string) RWLocker { + if z.SinglePool() { + object = encodeDirObject(object) + } + return z.objectNSLock(bucket, object, object) +} + +// uploadIDNSLock returns the guard of one upload ID. It is chosen by the +// upload's object, as PutObjectPart locks it in that object's set; the name +// locked stays object/uploadID. +func (z *erasureServerPools) uploadIDNSLock(bucket, object, uploadID string) RWLocker { + return z.objectNSLock(bucket, object, pathJoin(object, uploadID)) +} + +// objectNSLock returns the lock of resource, a name owned by object, in the +// lock domain of object. +func (z *erasureServerPools) objectNSLock(bucket, object, resource string) RWLocker { + if z.SinglePool() { + return z.serverPools[0].getHashedSet(encodeDirObject(object)).NewNSLock(bucket, resource) + } + return z.NewNSLock(bucket, resource) +} + +// objectLockRequest is one object lock that lockObjects takes. +type objectLockRequest struct { + bucket, object string + write bool +} + +// lockObjects takes the locks of several objects, such as the source and +// destination of a copy, before any of them is read. A name that appears twice +// gets one lock, a write lock if either asks for one. The locks are taken in +// one order, by lock domain and then by bucket and object, and transactionally +// (see getNSLocks), so that two copies in opposite directions, or an operation +// that orders its locks differently, cannot hold each other forever. The +// returned context ends when any of the locks is lost; the returned function +// releases them all. +func (z *erasureServerPools) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) { + type entry struct { + objectLockRequest + domain int + } + locks := make([]entry, 0, len(reqs)) + for _, r := range reqs { + r.object = encodeDirObject(r.object) + i := slices.IndexFunc(locks, func(e entry) bool { return e.bucket == r.bucket && e.object == r.object }) + if i >= 0 { + locks[i].write = locks[i].write || r.write + continue + } + domain := 0 + if z.SinglePool() { + domain = z.serverPools[0].getHashedSetIndex(r.object) + } + locks = append(locks, entry{objectLockRequest: r, domain: domain}) + } + slices.SortFunc(locks, func(a, b entry) int { + if a.domain != b.domain { + return a.domain - b.domain + } + return strings.Compare(pathJoin(a.bucket, a.object), pathJoin(b.bucket, b.object)) + }) + nsLocks := make([]nsLockRequest, len(locks)) + for i, l := range locks { + nsLocks[i] = nsLockRequest{ + newLock: func() RWLocker { return z.NewObjectNSLock(l.bucket, l.object) }, + write: l.write, + atomic: true, + } + } + return getNSLocks(ctx, nsLocks, globalOperationTimeout) +} + +// nsLockRequest is one lock of a transactional acquisition. +type nsLockRequest struct { + // newLock returns a new instance of the lock for each attempt. An + // instance names its holder, and a distributed unlock retries in the + // background; a later attempt with the same instance could be released + // by such a retry of the attempt before. + newLock func() RWLocker + write bool + // atomic: the lock is granted as a whole or not at all. A distributed + // lock is, also over several names. The local backend takes several + // names one at a time, holding the first while it waits for the next; + // its lock context is the context it was given. + atomic bool +} + +const ( + // nsLockTryTimeout bounds each wait for a lock while holding another. + nsLockTryTimeout = 2 * time.Second + // nsLockRetryWait is the longest random pause before a new attempt. + nsLockRetryWait = 250 * time.Millisecond +) + +// getNSLocks acquires every lock of locks, in their order, or none. While it +// holds a lock it waits for the next one only up to nsLockTryTimeout; then it +// releases what it holds in reverse order, pauses, and tries again with a +// fresh context, until the timeout of all attempts. A wait that holds nothing +// can last for the rest of the timeout. Each lock is taken in the context of +// the one before, so the returned context ends when any lease is lost. +func getNSLocks(ctx context.Context, locks []nsLockRequest, timeout *dynamicTimeout) (LockContext, func(), error) { + type held struct { + lock RWLocker + write bool + lkctx LockContext + } + release := func(hs []held) { + for i := len(hs) - 1; i >= 0; i-- { + if hs[i].write { + hs[i].lock.Unlock(hs[i].lkctx) + } else { + hs[i].lock.RUnlock(hs[i].lkctx) + } + } + } + // Fail as a single lock does: a canceled request with its own error, + // anything else as a timeout. + failed := func() error { + if err := ctx.Err(); errors.Is(err, context.Canceled) { + return err + } + return OperationTimedOut{} + } + start := time.Now() + deadline := start.Add(timeout.Timeout()) + for { + hs := make([]held, 0, len(locks)) + lctx := ctx + var err error + for _, l := range locks { + wait := time.Until(deadline) + if len(hs) > 0 || !l.atomic { + wait = min(wait, nsLockTryTimeout) + } + if wait <= 0 { + err = OperationTimedOut{} + break + } + lk := l.newLock() + try := newDynamicTimeout(wait, wait) + // The local backend waits up to the timeout for each name: bound + // the whole attempt by its context instead. + tctx, cancel := lctx, context.CancelFunc(func() {}) + if !l.atomic { + tctx, cancel = context.WithTimeout(lctx, wait) + } + var lkctx LockContext + if l.write { + lkctx, err = lk.GetLock(tctx, try) + } else { + lkctx, err = lk.GetRLock(tctx, try) + } + cancel() + if err != nil { + break + } + if !l.atomic { + lkctx.ctx = lctx + } + hs = append(hs, held{lock: lk, write: l.write, lkctx: lkctx}) + lctx = lkctx.Context() + } + if err == nil { + timeout.LogSuccess(time.Since(start)) + if len(hs) == 0 { + return LockContext{ctx: ctx, cancel: func() {}}, func() {}, nil + } + return LockContext{ctx: lctx, cancel: func() {}}, func() { release(hs) }, nil + } + release(hs) + // A lost lease cancels the context of a later wait; only the + // request's own context ends the attempts. + if ctx.Err() != nil || (!errors.As(err, &OperationTimedOut{}) && !errors.Is(err, context.Canceled)) { + return LockContext{}, nil, err + } + pause := rand.N(nsLockRetryWait) + time.Millisecond + if time.Until(deadline) <= pause { + timeout.LogFailure() + return LockContext{}, nil, OperationTimedOut{} + } + t := time.NewTimer(pause) + select { + case <-ctx.Done(): + t.Stop() + return LockContext{}, nil, failed() + case <-t.C: + } + } +} + +// sharesLocalNSLock reports whether every set of the pool uses one local lock +// table, as a single node does: there all sets are one lock domain. +func (s *erasureSets) sharesLocalNSLock() bool { + first := s.sets[0].nsMutex + if first.isDistErasure { + return false + } + for _, set := range s.sets[1:] { + if set.nsMutex != first { + return false + } + } + return true +} + +// deleteObjectsInSets is the bulk delete of a single pool whose sets do not +// share one lock table. The names are grouped by the set they hash to, the +// lock domain of their writers. The groups run one after another in ascending +// set index; each takes its set's lock over all of its names in one +// acquisition, deletes its sub-batch, and releases the lock before the next +// group starts. No two sets' locks are held at once. +// +// Each group's lock is taken in the request's context. A group whose lock +// cannot be taken fails as a whole and the next group still runs; once the +// request is canceled the remaining groups fail without being attempted. The +// results keep their positions in objects. Replication, events and auditing +// stay with the callers, on the merged results. +func (z *erasureServerPools) deleteObjectsInSets(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions, dobjects []DeletedObject, derrs []error, timeout *dynamicTimeout) { + pool := z.serverPools[0] + groups := make(map[int][]int) + for i, obj := range objects { + idx := pool.getHashedSetIndex(obj.ObjectName) + groups[idx] = append(groups[idx], i) + } + order := make([]int, 0, len(groups)) + for idx := range groups { + order = append(order, idx) + } + slices.Sort(order) + + for _, idx := range order { + members := groups[idx] + fail := func(err error) { + for _, i := range members { + derrs[i] = err + } + } + if err := ctx.Err(); err != nil { + fail(err) + continue + } + er := pool.sets[idx] + names := set.NewStringSet() + batch := make([]ObjectToDelete, len(members)) + for j, i := range members { + names.Add(objects[i].ObjectName) + batch[j] = objects[i] + } + resources := names.ToSlice() + lkctx, unlock, err := getNSLocks(ctx, []nsLockRequest{{ + newLock: func() RWLocker { return er.NewNSLock(bucket, resources...) }, + write: true, + atomic: er.nsMutex.isDistErasure || len(resources) == 1, + }}, timeout) + if err != nil { + fail(err) + continue + } + deleted, errs := er.DeleteObjects(lkctx.Context(), bucket, batch, opts) + unlock() + // Merge as the pools loop of DeleteObjects does for one pool. + for j, i := range members { + if errs[j] == nil && deleted[j].found { + dobjects[i], derrs[i] = deleted[j], nil + continue + } + if derrs[i] == nil { + dobjects[i], derrs[i] = deleted[j], errs[j] + } + } + } +} diff --git a/cmd/erasure-server-pool-locks_test.go b/cmd/erasure-server-pool-locks_test.go new file mode 100644 index 000000000..696c22771 --- /dev/null +++ b/cmd/erasure-server-pool-locks_test.go @@ -0,0 +1,555 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "net/http" + "sync/atomic" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/dsync" +) + +// ldTry reports whether lk can be taken now, and releases it again. +func ldTry(t *testing.T, lk RWLocker, write bool) bool { + t.Helper() + dt := newDynamicTimeout(100*time.Millisecond, 100*time.Millisecond) + if write { + lc, err := lk.GetLock(t.Context(), dt) + if err == nil { + lk.Unlock(lc) + } + return err == nil + } + lc, err := lk.GetRLock(t.Context(), dt) + if err == nil { + lk.RUnlock(lc) + } + return err == nil +} + +// ldFree reports whether lk can be taken within a few seconds: a distributed +// unlock completes in the background. +func ldFree(t *testing.T, lk RWLocker, write bool) bool { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !ldTry(t, lk, write) { + if time.Now().After(deadline) { + return false + } + } + return true +} + +// ldTaken waits until someone else holds lk. +func ldTaken(t *testing.T, lk RWLocker) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for ldTry(t, lk, false) { + if time.Now().After(deadline) { + t.Fatal("lock never taken") + } + time.Sleep(time.Millisecond) + } +} + +func ldHoldLock(t *testing.T, lk RWLocker) func() { + t.Helper() + lc, err := lk.GetLock(t.Context(), globalOperationTimeout) + if err != nil { + t.Fatal(err) + } + done := false + return func() { + if !done { + done = true + lk.Unlock(lc) + } + } +} + +// Object locks, and upload guards by their object, are in the set the +// object's writes lock in; with several pools they stay the pools lock. +func TestObjectNSLockRouting(t *testing.T) { + for _, mode := range ldModes { + t.Run(mode, func(t *testing.T) { + z, bucket := ldPool(t, mode) + pool := z.serverPools[0] + key := ldKey(t, z, "route", 1) + dir := ldKey(t, z, "route-dir", 0) + "/" + for _, name := range []string{key, dir} { + release := ldHoldLock(t, z.NewObjectNSLock(bucket, name)) + encoded := encodeDirObject(name) + if ldTry(t, pool.getHashedSet(encoded).NewNSLock(bucket, encoded), false) { + t.Errorf("%s: a writer of the object's set ignores the object lock", name) + } + release() + } + id := mustGetUUID() + release := ldHoldLock(t, z.uploadIDNSLock(bucket, key, id)) + if ldTry(t, pool.getHashedSet(key).NewNSLock(bucket, pathJoin(key, id)), false) { + t.Error("PutObjectPart's guard ignores the upload guard") + } + release() + if mode != ldShared && pool.sharesLocalNSLock() { + t.Error("split sets reported as one local table") + } + if mode == ldShared && !pool.sharesLocalNSLock() { + t.Error("one local table not recognized") + } + }) + } + t.Run("pools", func(t *testing.T) { + z, bucket := bulkDeletePools(t, 2) + release := ldHoldLock(t, z.NewNSLock(bucket, "pooled")) + defer release() + if ldTry(t, z.NewObjectNSLock(bucket, "pooled"), false) || ldTry(t, z.uploadIDNSLock(bucket, "pooled", ""), false) { + t.Error("multi-pool object locks left the pools lock") + } + }) +} + +// lockObjects merges a name asked twice into one write lock and takes source +// and destination in their own sets. +func TestLockObjects(t *testing.T) { + for _, mode := range ldModes { + t.Run(mode, func(t *testing.T) { + z, bucket := ldPool(t, mode) + pool := z.serverPools[0] + src, dst := ldKey(t, z, "lock-src", 1), ldKey(t, z, "lock-dst", 0) + lkctx, unlock, err := z.lockObjects(t.Context(), objectLockRequest{bucket: bucket, object: src}, objectLockRequest{bucket: bucket, object: dst, write: true}) + if err != nil { + t.Fatal(err) + } + if lkctx.Context().Err() != nil { + t.Fatal("lock context ended") + } + if !ldTry(t, pool.getHashedSet(src).NewNSLock(bucket, src), false) { + t.Error("source not shared with readers") + } + if ldTry(t, pool.getHashedSet(src).NewNSLock(bucket, src), true) { + t.Error("source not locked against writers") + } + if ldTry(t, pool.getHashedSet(dst).NewNSLock(bucket, dst), false) { + t.Error("destination not locked") + } + unlock() + if !ldFree(t, pool.getHashedSet(dst).NewNSLock(bucket, dst), true) || !ldFree(t, pool.getHashedSet(src).NewNSLock(bucket, src), true) { + t.Error("locks left behind") + } + + // A copy onto itself takes one write lock; a directory object is + // locked by its stored name. + self := ldKey(t, z, "lock-self", 1) + "/" + _, unlock, err = z.lockObjects(t.Context(), objectLockRequest{bucket: bucket, object: self}, objectLockRequest{bucket: bucket, object: self, write: true}) + if err != nil { + t.Fatal(err) + } + encoded := encodeDirObject(self) + if ldTry(t, pool.getHashedSet(encoded).NewNSLock(bucket, encoded), false) { + t.Error("self copy not write locked") + } + unlock() + }) + } +} + +// ldLeaseLock is a lock whose lease the test ends. +type ldLeaseLock struct { + RWLocker + lost chan context.CancelFunc +} + +func (l ldLeaseLock) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) { + lc, err := l.RWLocker.GetLock(ctx, timeout) + if err != nil { + return lc, err + } + lease, cancel := context.WithCancel(lc.Context()) + select { + case l.lost <- cancel: + default: + panic("too many lock attempts") + } + return LockContext{ctx: lease, cancel: cancel}, nil +} + +// getNSLocks never holds one lock while it waits for another for long: it +// releases what it holds and retries, also after a lost lease, until its +// timeout or the request ends, and leaves nothing behind. +func TestGetNSLocksTransactional(t *testing.T) { + ns := newNSLock(false) + lock := func(name string) RWLocker { return ns.NewNSLock(nil, "bucket", name) } + newLock := func(name string) func() RWLocker { return func() RWLocker { return lock(name) } } + reqs := func() []nsLockRequest { + return []nsLockRequest{{newLock: newLock("a"), write: true, atomic: true}, {newLock: newLock("b"), write: true, atomic: true}} + } + + t.Run("partial", func(t *testing.T) { + releaseB := ldHoldLock(t, lock("b")) + defer releaseB() + type result struct { + unlock func() + err error + } + done := make(chan result, 1) + go func() { + _, unlock, err := getNSLocks(t.Context(), reqs(), globalOperationTimeout) + done <- result{unlock, err} + }() + // "a" is taken while "b" is waited for, and free again between + // attempts. + ldTaken(t, lock("a")) + deadline := time.Now().Add(3 * nsLockTryTimeout) + for !ldTry(t, lock("a"), true) { + if time.Now().After(deadline) { + t.Fatal("a stays held while b is waited for") + } + } + releaseB() + res := <-done + if res.err != nil { + t.Fatal(res.err) + } + if ldTry(t, lock("a"), false) || ldTry(t, lock("b"), false) { + t.Error("not both locks held") + } + res.unlock() + if !ldTry(t, lock("a"), true) || !ldTry(t, lock("b"), true) { + t.Error("locks left behind") + } + }) + + t.Run("timeout", func(t *testing.T) { + releaseB := ldHoldLock(t, lock("b")) + defer releaseB() + start := time.Now() + _, _, err := getNSLocks(t.Context(), reqs(), newDynamicTimeout(3*time.Second, 3*time.Second)) + if !errors.As(err, &OperationTimedOut{}) { + t.Fatalf("got %v, want a timeout", err) + } + if took := time.Since(start); took > 6*time.Second { + t.Errorf("timed out after %v", took) + } + if !ldTry(t, lock("a"), true) { + t.Error("a left behind") + } + }) + + t.Run("cancel", func(t *testing.T) { + releaseB := ldHoldLock(t, lock("b")) + defer releaseB() + ctx, cancel := context.WithCancel(t.Context()) + time.AfterFunc(300*time.Millisecond, cancel) + _, _, err := getNSLocks(ctx, reqs(), globalOperationTimeout) + if !errors.Is(err, context.Canceled) { + t.Fatalf("got %v, want canceled", err) + } + if !ldTry(t, lock("a"), true) { + t.Error("a left behind") + } + }) + + t.Run("lease", func(t *testing.T) { + lost := make(chan context.CancelFunc, 64) + leased := []nsLockRequest{{newLock: func() RWLocker { return ldLeaseLock{lock("a"), lost} }, write: true, atomic: true}, {newLock: newLock("b"), write: true, atomic: true}} + // Losing the first lease while the second is waited for retries + // with a fresh context. + releaseB := ldHoldLock(t, lock("b")) + type result struct { + lkctx LockContext + unlock func() + err error + } + done := make(chan result, 1) + go func() { + lkctx, unlock, err := getNSLocks(t.Context(), leased, globalOperationTimeout) + done <- result{lkctx, unlock, err} + }() + (<-lost)() + releaseB() + res := <-done + if res.err != nil { + t.Fatal(res.err) + } + if res.lkctx.Context().Err() != nil { + t.Fatal("returned context ended before any lease was lost") + } + // Losing a held lease ends the returned context. + var cancel context.CancelFunc + for len(lost) > 0 { + cancel = <-lost + } + cancel() + select { + case <-res.lkctx.Context().Done(): + case <-time.After(time.Second): + t.Error("lost lease did not end the locks' context") + } + res.unlock() + if !ldTry(t, lock("a"), true) || !ldTry(t, lock("b"), true) { + t.Error("locks left behind") + } + }) +} + +// A group whose lock times out fails alone and the later groups still run. +func TestDeleteObjectsInSetsLaterGroupsRun(t *testing.T) { + for _, mode := range ldModes[1:] { + t.Run(mode, func(t *testing.T) { + z, bucket := ldPool(t, mode) + k0, k1 := ldKey(t, z, "later", 0), ldKey(t, z, "later", 1) + ldMustPut(t, z, bucket, k0, "zero") + ldMustPut(t, z, bucket, k1, "one") + release := ldHoldLock(t, z.serverPools[0].getHashedSet(k0).NewNSLock(bucket, k0)) + defer release() + objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: k0}}, {ObjectV: ObjectV{ObjectName: k1}}} + dobjects := []DeletedObject{{ObjectName: k0}, {ObjectName: k1}} + derrs := make([]error, 2) + z.deleteObjectsInSets(t.Context(), bucket, objects, ObjectOptions{}, dobjects, derrs, newDynamicTimeout(time.Second, time.Second)) + if !errors.As(derrs[0], &OperationTimedOut{}) || derrs[1] != nil { + t.Fatalf("results %v, want a timeout for k0 only", derrs) + } + release() + if body, _ := ldBody(t, z, bucket, k0); body != "zero" { + t.Errorf("k0 = %q", body) + } + ldAbsent(t, z, bucket, k1) + }) + } +} + +// The local lock takes the names of a group one by one. A group that waits +// for a name must not keep the names it holds: a part upload holds its part +// lock and then reads its upload ID, which the batch names too. +func TestDeleteObjectsInSetsLocalGroupReleases(t *testing.T) { + z, bucket := ldPool(t, ldLocal) + pool := z.serverPools[0] + var upload, part string + for i := range 1000 { + upload = ldKey(t, z, "local-upload", 1) + fmt.Sprintf("/u%d", i) + part = upload + "/1" + if pool.getHashedSetIndex(upload) == 1 && pool.getHashedSetIndex(part) == 1 { + break + } + } + er := pool.getHashedSet(upload) + releasePart := ldHoldLock(t, er.NewNSLock(bucket, part)) + defer releasePart() + done := make(chan []error, 1) + go func() { + _, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: upload}}, {ObjectV: ObjectV{ObjectName: part}}}, ObjectOptions{}) + done <- errs + }() + // The batch takes the upload name, waits for the part, and lets the + // part upload's read of its upload ID through. + ldTaken(t, er.NewNSLock(bucket, upload)) + read := er.NewNSLock(bucket, upload) + rctx, err := read.GetRLock(t.Context(), newDynamicTimeout(4*nsLockTryTimeout, 4*nsLockTryTimeout)) + if err != nil { + t.Fatalf("batch kept a name while waiting: %v", err) + } + read.RUnlock(rctx) + releasePart() + select { + case errs := <-done: + if errors.Join(errs...) != nil { + t.Fatal(errs) + } + case <-time.After(30 * time.Second): + t.Fatal("bulk delete did not finish") + } +} + +// ldLostUnlockReply applies its first unlock but loses the reply until gate +// opens, so dsync retries that unlock in the background. +type ldLostUnlockReply struct { + dsync.NetLocker + gate <-chan struct{} + applied, retried chan<- struct{} + calls atomic.Int32 +} + +func (l *ldLostUnlockReply) Unlock(ctx context.Context, args dsync.LockArgs) (bool, error) { + n := l.calls.Add(1) + ok, err := l.NetLocker.Unlock(ctx, args) + switch n { + case 1: + l.applied <- struct{}{} + <-l.gate + return false, errors.New("unlock reply lost") + case 2: + l.retried <- struct{}{} + } + return ok, err +} + +// ldFailFirst fails the first acquisition of any of its instances. +type ldFailFirst struct { + RWLocker + calls *atomic.Int32 +} + +func (l ldFailFirst) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) { + if l.calls.Add(1) == 1 { + return LockContext{}, OperationTimedOut{} + } + return l.RWLocker.GetLock(ctx, timeout) +} + +// A distributed unlock of a failed attempt that is retried late must not +// release the lock the next attempt took. +func TestGetNSLocksFreshLockPerAttempt(t *testing.T) { + gate := make(chan struct{}) + applied, retried := make(chan struct{}, 3), make(chan struct{}, 3) + lockers := make([]dsync.NetLocker, 3) + for i := range lockers { + lockers[i] = &ldLostUnlockReply{NetLocker: newLocker(), gate: gate, applied: applied, retried: retried} + } + ns := newNSLock(true) + dst := func() RWLocker { + return ns.NewNSLock(func() ([]dsync.NetLocker, string) { return lockers, "test" }, "bucket", "destination") + } + local := newNSLock(false) + var calls atomic.Int32 + src := func() RWLocker { return ldFailFirst{local.NewNSLock(nil, "bucket", "source"), &calls} } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + opened := false + open := func() { + if !opened { + opened = true + close(gate) + } + } + defer open() + _, unlock, err := getNSLocks(ctx, []nsLockRequest{{newLock: dst, write: true, atomic: true}, {newLock: src, write: true, atomic: true}}, newDynamicTimeout(10*time.Second, 10*time.Second)) + if err != nil { + t.Fatal(err) + } + defer unlock() + for range lockers { + select { + case <-applied: + case <-ctx.Done(): + t.Fatal("the first attempt's unlock never reached the lockers") + } + } + open() + for range lockers { + select { + case <-retried: + case <-ctx.Done(): + t.Fatal("the lost unlock was not retried") + } + } + competitor := dst() + lc, err := competitor.GetLock(ctx, newDynamicTimeout(300*time.Millisecond, 300*time.Millisecond)) + if err == nil { + competitor.Unlock(lc) + t.Fatal("a late unlock retry released the held lock") + } +} + +// One attempt at a local lock over several names is bounded as a whole, and so +// is the acquisition. +func TestGetNSLocksLocalAttemptBound(t *testing.T) { + ns := newNSLock(false) + a, b := ns.NewNSLock(nil, "bucket", "a"), ns.NewNSLock(nil, "bucket", "b") + releaseA, releaseB := ldHoldLock(t, a), ldHoldLock(t, b) + defer releaseA() + defer releaseB() + time.AfterFunc(900*time.Millisecond, releaseA) + time.AfterFunc(1800*time.Millisecond, releaseB) + start := time.Now() + _, unlock, err := getNSLocks(t.Context(), []nsLockRequest{{ + newLock: func() RWLocker { return ns.NewNSLock(nil, "bucket", "a", "b") }, write: true, + }}, newDynamicTimeout(1500*time.Millisecond, 1500*time.Millisecond)) + if err == nil { + unlock() + t.Fatalf("acquired after %v with a deadline of 1.5s", time.Since(start)) + } + if !errors.As(err, &OperationTimedOut{}) { + t.Fatalf("got %v, want a timeout", err) + } +} + +// ldLeaseLayer gives the COPY handler locks whose lease the test ends, and +// holds its destination lookup until the lookup's context ends. +type ldLeaseLayer struct { + ObjectLayer + destination string + lost chan context.CancelFunc + lookup chan context.Context +} + +func (l *ldLeaseLayer) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) { + lc, unlock, err := l.ObjectLayer.lockObjects(ctx, reqs...) + if err != nil { + return lc, unlock, err + } + lease, cancel := context.WithCancel(lc.Context()) + l.lost <- cancel + return LockContext{ctx: lease, cancel: cancel}, func() { cancel(); unlock() }, nil +} + +func (l *ldLeaseLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + if object == l.destination && opts.NoLock { + l.lookup <- ctx + <-ctx.Done() + return ObjectInfo{}, ctx.Err() + } + return l.ObjectLayer.GetObjectInfo(ctx, bucket, object, opts) +} + +// The COPY handler's work under its locks ends when a lease is lost, the +// destination lookups included. +func TestLockDomainCopyLookupEndsWithLease(t *testing.T) { + z, bucket, _ := ldAPI(t, ldShared) + ldMustPut(t, z, bucket, "source", "") + prevTier := globalTierConfigMgr + t.Cleanup(func() { globalTierConfigMgr = prevTier }) + globalTierConfigMgr = NewTierConfigMgr() + globalTierConfigMgr.Tiers["WARM"] = madmin.TierConfig{Version: madmin.TierConfigVer, Type: madmin.S3, Name: "WARM", S3: &madmin.TierS3{Bucket: "warm"}} + layer := &ldLeaseLayer{ObjectLayer: z, destination: "destination", lost: make(chan context.CancelFunc, 1), lookup: make(chan context.Context, 1)} + router := initTestAPIEndPoints(layer, []string{"CopyObject"}) + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + code := make(chan int, 1) + go func() { code <- ldCopyRequest(ctx, t, router, bucket, "source", "destination", nil).Code }() + var lookup context.Context + select { + case lookup = <-layer.lookup: + case <-ctx.Done(): + t.Fatal("COPY did not look up its destination") + } + (<-layer.lost)() + select { + case <-lookup.Done(): + case <-time.After(5 * time.Second): + t.Fatal("the destination lookup outlived the lost lease") + } + if c := <-code; c == http.StatusOK { + t.Error("COPY succeeded after losing its lock") + } +} diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index c168e22f9..221156480 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1239,8 +1239,26 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob object = encodeDirObject(object) } - // Serialize logical deletion with pooled writes and metadata updates. - lk := z.NewNSLock(bucket, object) + // Serialize logical deletion with the writes and metadata updates of the + // object: take the lock they take (see NewObjectNSLock). With one pool + // that is the object's own erasure set, which is where PutObject, + // multipart commit and the metadata updates lock, so a pool whose sets + // use different hosts still excludes them. With several pools it is the + // pools lock, as before. Exact-key deletes that use prefix deletion + // (DeletePrefixObject) are deletes of one object. A true prefix deletion + // keeps the pools lock on the prefix name, which never covered the + // objects below it. + // + // Nodes that still run a release before this routing lock single-pool + // deletes and copies in the first set. In a pool whose sets use different + // hosts they do not exclude the deletes and copies of updated nodes, nor + // their writes, so such a pool must not run mixed versions. + var lk RWLocker + if opts.DeletePrefix && !opts.DeletePrefixObject { + lk = z.NewNSLock(bucket, object) + } else { + lk = z.NewObjectNSLock(bucket, object) + } lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) if err != nil { return ObjectInfo{}, err @@ -1322,10 +1340,10 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob // Evaluate the conditional (If-Match) precondition while the write lock // acquired above is held, before the delete-marker short-circuit and before - // any version is removed, so the object cannot change between the check and - // the delete. This is scoped to a single erasure set (see the note at the - // lock above): only there do the delete lock and the write path share the - // same lock namespace, making the check-then-delete atomic. + // any version is removed. The object's writers take the same lock (see the + // note at the lock above), so the object cannot change between the check + // and the delete. Multi-pool conditional deletes return above, through + // deleteObjectReconciled, under the same pools lock. if opts.CheckPrecondFn != nil { // pinfo.ObjInfo is the current latest version. getPoolInfoExistingWithOpts // intentionally strips VersionID, so for a version-scoped delete read the @@ -1509,6 +1527,18 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o objSets.Add(objects[i].ObjectName) } + // A single pool locks each name in the set its writers lock it in, one + // set at a time. A single node keeps one lock table for all sets, so it + // keeps the one bulk lock below: its local lock takes the names one by + // one and holds the earlier names while it waits. + if z.SinglePool() && !z.serverPools[0].sharesLocalNSLock() { + z.deleteObjectsInSets(ctx, bucket, objects, opts, dobjects, derrs, globalOperationTimeout) + for i := range dobjects { + dobjects[i].ObjectName = decodeDirObject(dobjects[i].ObjectName) + } + return dobjects, derrs + } + // Acquire a bulk write lock across 'objects' multiDeleteLock := z.NewNSLock(bucket, objSets.ToSlice()...) lkctx, err := multiDeleteLock.GetLock(ctx, globalOperationTimeout) @@ -1633,8 +1663,10 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) + // A caller that also locks the source takes both with lockObjects, before + // it reads the source, and passes NoLock. if !dstOpts.NoLock { - ns := z.NewNSLock(dstBucket, dstObject) + ns := z.NewObjectNSLock(dstBucket, dstObject) lkctx, err := ns.GetLock(ctx, globalOperationTimeout) if err != nil { return ObjectInfo{}, err @@ -2231,7 +2263,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec return MultipartInfo{}, err } - uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID)) + uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID) lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout) if err != nil { return MultipartInfo{}, err @@ -2272,7 +2304,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object return ListPartsInfo{}, err } - uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID)) + uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID) lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout) if err != nil { return ListPartsInfo{}, err @@ -2323,7 +2355,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o } }(ctx) - lk := z.NewNSLock(bucket, pathJoin(object, uploadID)) + lk := z.uploadIDNSLock(bucket, object, uploadID) lkctx, err := lk.GetLock(ctx, globalOperationTimeout) if err != nil { return err @@ -2363,12 +2395,10 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket // Keep object -> upload lock ordering for single-pool completion too: // native restore holds the object lock while publishing upload parts. + // Both locks are in the object's lock domain, the set-local one of + // single-pool writes. if !opts.NoLock { - lk := z.NewNSLock(bucket, encodeDirObject(object)) - if z.SinglePool() { - // Preserve the set-local lock domain used by single-pool writes. - lk = z.serverPools[0].getHashedSet(object).NewNSLock(bucket, encodeDirObject(object)) - } + lk := z.NewObjectNSLock(bucket, encodeDirObject(object)) lkctx, err := lk.GetLock(ctx, globalOperationTimeout) if err != nil { return objInfo, err @@ -2433,7 +2463,7 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket // An empty upload ID aliases the object lock already held above. // Let the storage layer return its normal missing bucket/upload error. if uploadID != "" { - uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID)) + uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID) wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout) if err != nil { return objInfo, err diff --git a/cmd/metacache-server-pool.go b/cmd/metacache-server-pool.go index 2889e9a96..276cef0fb 100644 --- a/cmd/metacache-server-pool.go +++ b/cmd/metacache-server-pool.go @@ -265,9 +265,12 @@ func (z *erasureServerPools) listPathShortcutEOF(ctx context.Context, bucket str } // listingRLock covers the lock domains used by normal same-key mutations. -// Multi-pool writes use the pools lock. In a single pool, DELETE/COPY use the -// first set while PUT and metadata updates use the destination set, so recovery -// takes both in that order. No writer routing or wire protocol changes here. +// Multi-pool writes use the pools lock. In a single pool every object write, +// DELETE and COPY included, locks the object's own set (see NewObjectNSLock), +// which is the source set here. Releases before that routing locked +// single-pool DELETE and COPY in the first set, so recovery still takes the +// first set too, in that order, while nodes of those releases may run. It can +// drop that lock only behind a version boundary that keeps such nodes out. func (z *erasureServerPools) listingRLock(ctx context.Context, bucket, object string, source *erasureObjects) (LockContext, func(), error) { firstSet := z.serverPools[0].sets[0] first := z.NewNSLock(bucket, object) diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 0f4a663ac..11e867171 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -262,7 +262,13 @@ func (o *ObjectOptions) SetEvalRetentionBypassFn(f EvalRetentionBypassFn) { // ObjectLayer implements primitives for object API layer. type ObjectLayer interface { // Locking operations on object. + // NewNSLock locks names in one stable domain. It is for names that are + // not objects, such as bucket and IAM metadata locks. NewNSLock(bucket string, objects ...string) RWLocker + // NewObjectNSLock locks one object in the lock domain of its writes. + NewObjectNSLock(bucket, object string) RWLocker + // lockObjects locks several objects together, see erasureServerPools. + lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) // Storage operations. Shutdown(context.Context) error diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 81af341f5..a1133be7e 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -34,6 +34,7 @@ import ( "sort" "strconv" "strings" + "sync" "sync/atomic" "time" @@ -183,7 +184,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r // Take read lock on object, here so subsequent lower-level // calls do not need to. - lock := objectAPI.NewNSLock(bucket, object) + lock := objectAPI.NewObjectNSLock(bucket, object) lkctx, err := lock.GetRLock(ctx, globalOperationTimeout) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) @@ -1470,12 +1471,35 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re return checkCopyObjectPreconditions(ctx, w, r, o) } getOpts.CheckPrecondFn = checkCopyPrecondFn - if cpSrcDstSame { - getOpts.NoLock = true + + // Lock the source for reading and a local destination for writing before + // the source is opened, both in the lock domain of their writers. Taking + // them together in one order keeps copies in opposite directions from + // holding each other; a copy onto itself takes one write lock. The source + // is then read, preconditions and inline data included, and the + // destination looked up and written, all under these locks and without + // taking them again. That work runs in lockCtx, which ends when either + // lock is lost and so aborts the copy; errors are still reported in the + // request's context. Both locks are released once the copy has been + // written: a synchronous replication below reads the destination with its + // own lock. + locks := []objectLockRequest{{bucket: srcBucket, object: srcObject}} + if !remoteCallRequired { + locks = append(locks, objectLockRequest{bucket: dstBucket, object: dstObject, write: true}) } + lkctx, unlock, err := objectAPI.lockObjects(ctx, locks...) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + unlockCopy := sync.OnceFunc(unlock) + defer unlockCopy() + lockCtx := lkctx.Context() + getOpts.NoLock = true + dstOpts.NoLock = true var rs *HTTPRangeSpec - gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, getOpts) + gr, err := getObjectNInfo(lockCtx, srcBucket, srcObject, rs, r.Header, getOpts) if err != nil { if isErrPreconditionFailed(err) { return @@ -1542,7 +1566,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re length := actualSize if !cpSrcDstSame { - if err := enforceBucketQuotaHard(ctx, dstBucket, actualSize); err != nil { + if err := enforceBucketQuotaHard(lockCtx, dstBucket, actualSize); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } @@ -1560,12 +1584,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2 compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10) - reader = etag.NewReader(ctx, reader, nil, nil) + reader = etag.NewReader(lockCtx, reader, nil, nil) } else { reader = gr } - srcInfo.Reader, err = hash.NewReader(ctx, reader, length, "", "", actualSize) + srcInfo.Reader, err = hash.NewReader(lockCtx, reader, length, "", "", actualSize) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return @@ -1660,7 +1684,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } } - if err = rotateKey(ctx, oldKey, newKeyID, newKey, srcBucket, srcObject, encMetadata, kmsCtx); err != nil { + if err = rotateKey(lockCtx, oldKey, newKeyID, newKey, srcBucket, srcObject, encMetadata, kmsCtx); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return } @@ -1745,7 +1769,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re dstOpts.IndexCB = cb defer s2c.Close() reader = etag.Wrap(s2c, checksumReader) - srcInfo.Reader, err = hash.NewReader(ctx, reader, -1, "", "", actualSize) + srcInfo.Reader, err = hash.NewReader(lockCtx, reader, -1, "", "", actualSize) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return @@ -1758,7 +1782,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re if isTargetEncrypted && !remoteCallRequired { var encReader io.Reader kind, _ := crypto.IsRequested(r.Header) - encReader, objEncKey, err = newEncryptReader(ctx, srcInfo.Reader, kind, newKeyID, newKey, dstBucket, dstObject, encMetadata, kmsCtx) + encReader, objEncKey, err = newEncryptReader(lockCtx, srcInfo.Reader, kind, newKeyID, newKey, dstBucket, dstObject, encMetadata, kmsCtx) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return @@ -1773,7 +1797,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } // do not try to verify encrypted content - srcInfo.Reader, err = hash.NewReader(ctx, reader, targetSize, "", "", actualSize) + srcInfo.Reader, err = hash.NewReader(lockCtx, reader, targetSize, "", "", actualSize) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return @@ -1845,7 +1869,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true) retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectRetentionAction) holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectLegalHoldAction) - getObjectInfo := objectAPI.GetObjectInfo + // The destination is locked above: look it up without its lock, in the + // locks' context. + getObjectInfo := func(_ context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + opts.NoLock = true + return objectAPI.GetObjectInfo(lockCtx, bucket, object, opts) + } // apply default bucket configuration/governance headers for dest side. retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms, replicaTrusted) @@ -2031,9 +2060,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // For an ordinary federated copy, actualSize is the logical plaintext // length to declare. srcInfo.Size is not a reliable wire length because // the read path may already have adjusted it. - writeCtx, modified := withFederatedWriteTime(ctx) + writeCtx, modified := withFederatedWriteTime(lockCtx) remoteObjInfo, rerr := core.PutObject(writeCtx, dstBucket, dstObject, srcInfo.Reader, actualSize, "", "", opts) + gr.Close() + unlockCopy() if rerr != nil { writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL) return @@ -2080,7 +2111,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // Copy source object to destination, if source and destination // object is same then only metadata is updated. - objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts) + objInfo, err = copyObjectFn(lockCtx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts) + gr.Close() + unlockCopy() if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) return diff --git a/cmd/object-retention-authz_test.go b/cmd/object-retention-authz_test.go index af44aac37..c62477dc0 100644 --- a/cmd/object-retention-authz_test.go +++ b/cmd/object-retention-authz_test.go @@ -544,6 +544,18 @@ func (p *hookBeforeWrite) CompleteMultipartUpload(ctx context.Context, bucket, k return p.ObjectLayer.CompleteMultipartUpload(ctx, bucket, key, uploadID, parts, opts) } +// The COPY handler checks Object Lock under its destination's write lock, so +// no writer can land between that check and the copy. The latest a writer can +// land is just before the handler takes its locks. +func (p *hookBeforeWrite) lockObjects(ctx context.Context, reqs ...objectLockRequest) (LockContext, func(), error) { + for _, r := range reqs { + if r.write { + p.fire(r.object) + } + } + return p.ObjectLayer.lockObjects(ctx, reqs...) +} + func TestDirectoryObjectLockAfterPreflight(t *testing.T) { ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ t: t, From 7fac646923aae833fed2e1fea7e9f881cb9e9604 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 06:50:32 +0800 Subject: [PATCH 23/26] refactor(restore): take the restore lock from the object lock domain Native restore chose its own single-pool lock in the object's erasure set before the pools layer did. Now that every object write locks in that domain through NewObjectNSLock, the restore lock is that lock. Behavior is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/restore-attempt.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/restore-attempt.go b/cmd/restore-attempt.go index 0df3e36fd..e95903dd3 100644 --- a/cmd/restore-attempt.go +++ b/cmd/restore-attempt.go @@ -648,12 +648,9 @@ func (er erasureObjects) restoreUploadActive(ctx context.Context, v restoreUploa } // restoreLock is the object lock that restores and their status updates -// share: with one pool the set's own lock, as single-pool writes use. +// share: the lock every write of the object takes. func (z *erasureServerPools) restoreLock(bucket, object string) RWLocker { - if z.SinglePool() { - return z.serverPools[0].getHashedSet(object).NewNSLock(bucket, object) - } - return z.NewNSLock(bucket, object) + return z.NewObjectNSLock(bucket, object) } // restoreRequestStatus is what a POST restore recorded on its version. From 67a6532deccf03cfead09038bc3057e9ed3a1fe7 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 11:34:34 +0800 Subject: [PATCH 24/26] test(restore): give native-restore liveness waits one minute The native-restore tests waited 10 seconds for steps that do real drive work: metadata writes with synchronous persistence and renames on every drive. A full suite run on a machine whose SSD was busy with another decommission run failed TestRestoreUploadStagingFailsClosed/staged once with "background restore did not finish"; about 430 reruns, under CPU and fsync stress as well, passed. The test uses local locks, and after its release the completion takes no new namespace lock, so the bound is a liveness bound, not a latency contract. All eight such waits in the native-restore tests now use one restoreTestWait of one minute, and the background-restore helper prints the goroutine stacks when it times out. Assertions that something must not happen keep their own bounds. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/restore-native-generation_test.go | 10 +++++----- cmd/restore-native-isolation_test.go | 16 ++++++++++++---- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/cmd/restore-native-generation_test.go b/cmd/restore-native-generation_test.go index d79bc1030..48b5d19ea 100644 --- a/cmd/restore-native-generation_test.go +++ b/cmd/restore-native-generation_test.go @@ -369,7 +369,7 @@ func TestRestoreUploadUnreadableFailsClosed(t *testing.T) { if mode == "stale" { select { case <-gate.entered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("restore never reached completion") } } else { @@ -701,7 +701,7 @@ func testRestoreUploadStaging(t *testing.T, mode string) { } select { case <-entered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("completion never staged its commit") } stored := restoreIsolationUploads(t, r, 0) @@ -784,7 +784,7 @@ func TestRestoreSplitKeepsRunningUpload(t *testing.T) { } select { case <-gate.entered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("restore never reached completion") } stored := restoreIsolationUploads(t, r, 0) @@ -1277,7 +1277,7 @@ func TestCleanupStaleUploadsSkipsRunningCommit(t *testing.T) { readOnce.Do(func() { close(readRelease) }) select { case <-cleanupDone: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("stale cleanup never returned") } stageOnce.Do(func() { close(stageRelease) }) @@ -1403,7 +1403,7 @@ func TestCleanupStaleUploadsHoldsLockThroughRename(t *testing.T) { paused.Do(func() { close(pause.release) }) select { case <-stageEntered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("completion never staged after the rename ended") } stageOnce.Do(func() { close(stageRelease) }) diff --git a/cmd/restore-native-isolation_test.go b/cmd/restore-native-isolation_test.go index 232eb4323..6e5e412da 100644 --- a/cmd/restore-native-isolation_test.go +++ b/cmd/restore-native-isolation_test.go @@ -28,6 +28,7 @@ import ( "net/http/httptest" "net/url" "reflect" + "runtime" "strings" "sync" "sync/atomic" @@ -82,14 +83,21 @@ func (l *restoreIsolationLayer) RestoreTransitionedObject(ctx context.Context, b return err } +// restoreTestWait bounds how long a restore test waits for a step that does +// real drive work: metadata writes with synchronous persistence and renames +// on every drive. It is a liveness bound, not a latency contract; a loaded +// machine has stalled such a step past ten seconds. +const restoreTestWait = time.Minute + // wait returns the result of the next background restore. func (l *restoreIsolationLayer) wait(t *testing.T) error { t.Helper() select { case err := <-l.done: return err - case <-time.After(10 * time.Second): - t.Fatal("background restore did not finish") + case <-time.After(restoreTestWait): + buf := make([]byte, 1<<20) + t.Fatalf("background restore did not finish in %v; goroutines:\n%s", restoreTestWait, buf[:runtime.Stack(buf, true)]) return nil } } @@ -255,7 +263,7 @@ func restoreIsolationEntered(t *testing.T, tier *restoreIsolationGateTier) { t.Helper() select { case <-tier.entered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("restore never reached its download gate") } } @@ -1062,7 +1070,7 @@ func TestRestoreUploadOutlivesStaleAge(t *testing.T) { if gated { select { case <-gate.entered: - case <-time.After(10 * time.Second): + case <-time.After(restoreTestWait): t.Fatal("restore never reached completion") } } else { From f85d4a5fa1b08e96fa8d5ea5ded12955adc9f60e Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Thu, 5 Nov 2026 16:04:56 +0800 Subject: [PATCH 25/26] test: satisfy acceptance lint and compatibility checks Signed-off-by: Feng Ruohang --- .../rebrand-guard/compat-baseline.json | 4 +++ cmd/erasure-server-pool-lock-domain_test.go | 27 ++++++++++-------- cmd/restore-native-generation_test.go | 28 +++++++++++-------- 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/buildscripts/rebrand-guard/compat-baseline.json b/buildscripts/rebrand-guard/compat-baseline.json index 2ff232340..620cbe847 100644 --- a/buildscripts/rebrand-guard/compat-baseline.json +++ b/buildscripts/rebrand-guard/compat-baseline.json @@ -594,11 +594,13 @@ "x-minio-from-peer", "x-minio-healing-drives", "x-minio-internal", + "x-minio-internal-a", "x-minio-internal-actual-object-size", "x-minio-internal-actual-size", "x-minio-internal-archive-info", "x-minio-internal-archive-type", "x-minio-internal-b", + "x-minio-internal-c", "x-minio-internal-compression", "x-minio-internal-encrypted-multipart", "x-minio-internal-encryptedmultipart", @@ -611,6 +613,8 @@ "x-minio-internal-replication-reset-arn", "x-minio-internal-replication-status", "x-minio-internal-replication-timestamp", + "x-minio-internal-restore-attempt", + "x-minio-internal-restore-upload", "x-minio-internal-server-side-encryption", "x-minio-internal-server-side-encryption-context", "x-minio-internal-server-side-encryption-iv", diff --git a/cmd/erasure-server-pool-lock-domain_test.go b/cmd/erasure-server-pool-lock-domain_test.go index f2a305cb3..85b0360e1 100644 --- a/cmd/erasure-server-pool-lock-domain_test.go +++ b/cmd/erasure-server-pool-lock-domain_test.go @@ -185,7 +185,7 @@ func ldKey(t *testing.T, z *erasureServerPools, prefix string, set int) string { // ldOverlap runs hold until it is inside its storage commit, then runs other. // It reports whether other was still waiting after ldWait, then lets hold // commit and returns both results. -func ldOverlapRun(t *testing.T, hold, other func(context.Context) error) (holdErr, otherErr error, waited bool) { +func ldOverlapRun(t *testing.T, hold, other func(context.Context) error) (waited bool, holdErr, otherErr error) { t.Helper() g := newLDGate() defer g.let() @@ -212,7 +212,7 @@ func ldOverlapRun(t *testing.T, hold, other func(context.Context) error) (holdEr if waited { otherErr = <-done } - return holdErr, otherErr, waited + return waited, holdErr, otherErr } func ldReader(r io.Reader, size int64) *PutObjReader { @@ -425,11 +425,12 @@ func TestLockDomainSinglePoolWritersExclude(t *testing.T) { t.Run(c.name, func(t *testing.T) { key := ldKey(t, z, strings.ReplaceAll(c.name, "/", "-"), 1) c.setup(t, z, bucket, key) - holdErr, otherErr, waited := ldOverlapRun(t, + waited, holdErr, otherErr := ldOverlapRun(t, func(ctx context.Context) error { return c.hold(ctx, z, bucket, key) }, func(ctx context.Context) error { return c.other(ctx, z, bucket, key) }) if !waited { - t.Errorf("%s ran while %s held its commit (other: %v)", c.name[strings.Index(c.name, "/")+1:], c.name[:strings.Index(c.name, "/")], otherErr) + holder, contender, _ := strings.Cut(c.name, "/") + t.Errorf("%s ran while %s held its commit (other: %v)", contender, holder, otherErr) } c.check(t, z, bucket, key, holdErr, otherErr) }) @@ -478,7 +479,7 @@ func TestLockDomainUploadGuards(t *testing.T) { t.Run(op, func(t *testing.T) { key := ldKey(t, z, "upload-"+op, 1) id := ldUpload(t, z, bucket, key, false) - holdErr, otherErr, waited := ldOverlapRun(t, + waited, holdErr, otherErr := ldOverlapRun(t, func(ctx context.Context) error { return putPart(ctx, z, bucket, key, id) }, func(ctx context.Context) error { var err error @@ -537,7 +538,7 @@ func TestLockDomainBulkDeleteAgainstUploads(t *testing.T) { if alias { names = append(names, ObjectToDelete{ObjectV: ObjectV{ObjectName: pathJoin(key, id)}}) } - holdErr, otherErr, waited := ldOverlapRun(t, + waited, holdErr, otherErr := ldOverlapRun(t, func(ctx context.Context) error { _, err := z.CompleteMultipartUpload(ctx, bucket, key, id, []CompletePart{{PartNumber: 1, ETag: partETag}}, ObjectOptions{}) return err @@ -561,7 +562,7 @@ func TestLockDomainBulkDeleteAgainstUploads(t *testing.T) { key := ldKey(t, z, "bulk-part", 1) id := ldUpload(t, z, bucket, key, true) names := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: pathJoin(key, id)}}, {ObjectV: ObjectV{ObjectName: pathJoin(key, id, "1")}}} - holdErr, otherErr, _ := ldOverlapRun(t, + _, holdErr, otherErr := ldOverlapRun(t, func(ctx context.Context) error { _, err := z.PutObjectPart(ctx, bucket, key, id, 1, ldReader(bytes.NewReader(part), int64(len(part))), ObjectOptions{}) return err @@ -871,7 +872,7 @@ func TestLockDomainCopyHandler(t *testing.T) { ldMustPut(t, z, bucket, key, "seed") ldMustPut(t, z, bucket, src, "source") var rec *httptest.ResponseRecorder - holdErr, _, waited := ldOverlapRun(t, + waited, holdErr, _ := ldOverlapRun(t, func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written") }, func(ctx context.Context) error { if self { @@ -1002,7 +1003,7 @@ func TestLockDomainSelectWaitsForWriter(t *testing.T) { key := ldKey(t, z, "select", 1) ldMustPut(t, z, bucket, key, "seed,1\n") var rec *httptest.ResponseRecorder - holdErr, _, waited := ldOverlapRun(t, + waited, holdErr, _ := ldOverlapRun(t, func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written,2\n") }, func(ctx context.Context) error { rec = ldRequest(ctx, t, router, http.MethodPost, "/"+bucket+"/"+key+"?select&select-type=2", request, nil) @@ -1029,7 +1030,7 @@ func TestLockDomainKeyRotationWaitsForWriter(t *testing.T) { // The rotation's listing found an SSE-S3 object; it reads it again // under the lock. listed := ObjectInfo{Bucket: bucket, Name: key, UserDefined: map[string]string{crypto.MetaSealedKeyS3: "listed"}} - _, _, waited := ldOverlapRun(t, + waited, _, _ := ldOverlapRun(t, func(ctx context.Context) error { return ldPut(ctx, z, bucket, key, "written") }, func(ctx context.Context) error { return job.KeyRotate(ctx, z, listed) }) if !waited { @@ -1107,8 +1108,10 @@ func TestLockDomainCopyExcludesObjectLockUpdates(t *testing.T) { } } put(key) - g := &ldCopyGate{ObjectLayer: z, bucket: bucket, object: key, - copying: make(chan struct{}), updating: make(chan struct{}), release: make(chan struct{})} + g := &ldCopyGate{ + ObjectLayer: z, bucket: bucket, object: key, + copying: make(chan struct{}), updating: make(chan struct{}), release: make(chan struct{}), + } setObjectLayer(g) defer setObjectLayer(z) router := initTestAPIEndPoints(g, nil) diff --git a/cmd/restore-native-generation_test.go b/cmd/restore-native-generation_test.go index 48b5d19ea..1cea3faf7 100644 --- a/cmd/restore-native-generation_test.go +++ b/cmd/restore-native-generation_test.go @@ -56,7 +56,7 @@ func (d restoreGenerationDisk) WriteMetadata(ctx context.Context, orig, vol, pat if d.write != nil && d.write.Load() && vol == d.bucket && path == d.object { return errFaultyDisk } - return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi) + return d.WriteMetadata(ctx, orig, vol, path, fi) } func (d restoreGenerationDisk) UpdateMetadata(ctx context.Context, vol, path string, fi FileInfo, opts UpdateMetadataOpts) error { @@ -99,7 +99,7 @@ func (d restoreGenerationOffline) WriteMetadata(ctx context.Context, orig, bucke if d.fail.Load() && bucket == d.bucket && object == d.object { return errFaultyDisk } - return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi) + return d.WriteMetadata(ctx, orig, bucket, object, fi) } // restoreGenerationMetadata returns the version's metadata on each drive. @@ -650,7 +650,7 @@ type restoreGenerationStageDisk struct { func (d restoreGenerationStageDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { if d.stage { - if err := d.StorageAPI.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil { + if err := d.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil { return RenameDataResp{}, err } } @@ -691,8 +691,10 @@ func testRestoreUploadStaging(t *testing.T, mode string) { }() wrapped := make([]StorageAPI, len(disks)) for i, d := range disks { - wrapped[i] = restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: mode != "partly-staged" || i == 0, - arrived: &arrived, entered: entered, release: release, disks: int32(len(disks))} + wrapped[i] = restoreGenerationStageDisk{ + StorageAPI: d, bucket: r.bucket, object: r.object, stage: mode != "partly-staged" || i == 0, + arrived: &arrived, entered: entered, release: release, disks: int32(len(disks)), + } } er.getDisks = func() []StorageAPI { return wrapped } layer := newRestoreIsolationLayer(r) @@ -834,7 +836,7 @@ func (d restoreGenerationCleanupDisk) WriteMetadata(ctx context.Context, orig, b if bucket == d.bucket && object == d.object && d.metaWrites.Load() { return errFaultyDisk } - return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi) + return d.WriteMetadata(ctx, orig, bucket, object, fi) } // Restored-copy expiry decides on the version's authoritative generation. A @@ -1053,7 +1055,7 @@ type restoreGenerationFailedCommit struct { func (d restoreGenerationFailedCommit) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { - if err := d.StorageAPI.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil { + if err := d.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil { return RenameDataResp{}, err } return RenameDataResp{}, errFaultyDisk @@ -1236,8 +1238,10 @@ func TestCleanupStaleUploadsSkipsRunningCommit(t *testing.T) { defer stageOnce.Do(func() { close(stageRelease) }) wrapped := make([]StorageAPI, len(disks)) for i, d := range disks { - stage := restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: true, - arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks))} + stage := restoreGenerationStageDisk{ + StorageAPI: d, bucket: r.bucket, object: r.object, stage: true, + arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks)), + } wrapped[i] = restoreGenerationSnapshotDisk{StorageAPI: stage, reads: &reads, release: readRelease} } er.getDisks = func() []StorageAPI { return wrapped } @@ -1384,8 +1388,10 @@ func TestCleanupStaleUploadsHoldsLockThroughRename(t *testing.T) { defer stageOnce.Do(func() { close(stageRelease) }) wrapped := make([]StorageAPI, len(disks)) for i, d := range disks { - wrapped[i] = restoreGenerationStageDisk{StorageAPI: d, bucket: r.bucket, object: r.object, stage: true, - arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks))} + wrapped[i] = restoreGenerationStageDisk{ + StorageAPI: d, bucket: r.bucket, object: r.object, stage: true, + arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks)), + } } er.getDisks = func() []StorageAPI { return wrapped } completion := make(chan ObjectInfo, 1) From 45b1e4bc00e5a7251b2776c9ba0d5eb01200641f Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Fri, 6 Nov 2026 11:30:29 +0800 Subject: [PATCH 26/26] fix(restore): stage native restores outside the object lock The unreleased native-restore fixes after RELEASE.2026-10-26T00-00-00Z let the lock decide who owns a restore. Since fa42a0982 a single-pool worker holds the object lock for the whole tier download, and since 7f38bf628 a busy lock means "a restore runs". 7f38bf628 also keeps the restore's data in a marked multipart upload. Two regressions followed: - While an old version restored, every GET, HEAD and write of the same key waited for the download. On a single pool this is new; with several pools the released code already held the lock that long. - An ordinary upload whose completion failed after staging its metadata became InvalidUploadID for every caller, and could not be aborted for an hour. The marker that hid restore uploads could not tell them apart once a completion had rewritten the upload's xl.meta. A restore now runs in two phases: - The worker erasure-codes the tier copy into the tmp volume of the version's set, without the object lock, as heal does. A drive that fails any part, or misses a part when CheckParts runs before the commit, takes no part in the commit. - A restore lock per version tells whether a restore runs. POST takes it before the object lock and hands it to the worker. It is a liveness hint: only the checks under the object lock decide what is written. - The commit takes the object lock briefly, on one pool or several. It reads the version again and checks the attempt's ownership, the version identity, the tier copy, the part layout and the erasure layout. It then publishes with RenameData drive by drive, never through erasure renameData's undo, and needs the write quorum. The replaced data directory goes only when every published drive reports the same one. - A publication that reached some drives but not the write quorum converges under the same lock hold, in bounded retries after it: back to the unrestored version on every drive that holds it, or to a committed restore when the copy turns out to be on a write quorum. Each pass first checks that every drive holds what the version held before the commit, the commit wrote, or a convergence wrote, and stops on anything else. It deletes no directory. Multipart goes back to main: erasure-multipart.go and bucket-object-lock.go are identical to 69fcf50cd. The commit lock, the marker and the restore exceptions of multipart completion and of the null-version Object Lock check are gone, and single-pool completion again takes the upload lock first and the object lock right before it commits. With several pools, abandoned-data cleanup now takes the pools lock that writes take, as heal does, so it cannot remove a directory that a commit moved in place and has not yet referenced. The restore capability becomes protocol 2, and a queued worker checks it again. Recovery after a coordinator crash during publication, and storage RPCs that complete after their caller returned, stay excluded. Main has the same exposure, and they are tracked separately. Tests: - New regression tests cover both problems. They fail on f85d4a5fa, and the multipart one also passes on main. - Fault-injection tests cover a publication below the write quorum, including seven of twelve drives on 8+4; lost answers; convergence retries and stops; the old data directory; a stale inline marker; a recreated tmp volume; a published revision read across its expiry; a second POST; a target deleted during the download; a tier 404; and pools-locked abandoned cleanup. - Thirteen tests of the upload marker from 7f38bf628 are dropped, and so is TestNativeRestoreConcurrentCompletion from fa42a0982, which completed the restore's upload from a client. Four tests are reframed for the new semantics, including the lock-domain restore test, which now gates the commit. - Three self-recursive WriteMetadata test doubles are fixed, and the helpers of the dropped tests are removed. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Feng Ruohang --- cmd/bucket-lifecycle.go | 1 - cmd/bucket-object-lock.go | 6 +- cmd/erasure-multipart-cleanup_test.go | 26 +- cmd/erasure-multipart.go | 167 +--- cmd/erasure-object.go | 179 +--- cmd/erasure-server-pool-lock-domain_test.go | 57 +- cmd/erasure-server-pool.go | 86 +- cmd/object-api-interface.go | 9 +- cmd/object-handlers.go | 22 +- cmd/object-retention-overwrite_test.go | 2 +- cmd/restore-attempt.go | 323 ++----- cmd/restore-capability.go | 2 +- cmd/restore-native-generation_test.go | 706 +-------------- cmd/restore-native-isolation_test.go | 489 +---------- cmd/restore-native_test.go | 149 +--- cmd/restore-stage.go | 900 ++++++++++++++++++++ cmd/restore-two-phase_test.go | 858 +++++++++++++++++++ 17 files changed, 1956 insertions(+), 2026 deletions(-) create mode 100644 cmd/restore-stage.go create mode 100644 cmd/restore-two-phase_test.go diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go index 5854fa64d..54511b851 100644 --- a/cmd/bucket-lifecycle.go +++ b/cmd/bucket-lifecycle.go @@ -1027,7 +1027,6 @@ func putRestoreOpts(rreq *RestoreObjectRequest, objInfo ObjectInfo, opts ObjectO restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days) meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String() delete(meta, restoreAttemptKey) - delete(meta, restoreUploadKey) return ObjectOptions{ Versioned: opts.Versioned, VersionSuspended: opts.VersionSuspended, diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go index 293042fc3..397b039ef 100644 --- a/cmd/bucket-object-lock.go +++ b/cmd/bucket-object-lock.go @@ -251,9 +251,8 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec // checkObjectLockedForOverwrite re-checks under the object write lock whether an // in-place write to the null version (such as a directory object) would -// overwrite a locked version in an Object Lock bucket. Restore context keeps -// failed reads from deleting the version being rehydrated. -func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, getObjectInfoFn GetObjectInfoFn, transition TransitionOptions) error { +// overwrite a locked version in an Object Lock bucket. +func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, getObjectInfoFn GetObjectInfoFn) error { if isMinioMetaBucketName(bucket) { return nil } @@ -266,7 +265,6 @@ func checkObjectLockedForOverwrite(ctx context.Context, bucket, object string, g } objInfo, err := getObjectInfoFn(ctx, bucket, object, ObjectOptions{ VersionID: nullVersionID, - Transition: transition, NoLock: true, NoAuditLog: true, }) diff --git a/cmd/erasure-multipart-cleanup_test.go b/cmd/erasure-multipart-cleanup_test.go index 14c012968..26a95db53 100644 --- a/cmd/erasure-multipart-cleanup_test.go +++ b/cmd/erasure-multipart-cleanup_test.go @@ -123,20 +123,7 @@ func (d *multipartCleanupDisk) Endpoint() Endpoint { func (d *multipartCleanupDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) { d.reads++ - if d.readErr != nil { - return FileInfo{}, d.readErr - } - fi := newFileInfo("obj", 1, 0) - fi.Erasure.Index, fi.ModTime = 1, d.modTime - return fi, nil -} - -// multipartCleanupSet is a set of one drive, which also answers the -// metadata reads that establish the restore marker of an expired upload. -// Its locks are local: stale cleanup decides a removal under the upload's -// commit lock. -func multipartCleanupSet(disk StorageAPI) erasureObjects { - return erasureObjects{getDisks: func() []StorageAPI { return []StorageAPI{disk} }, nsMutex: newNSLock(false)} + return FileInfo{ModTime: d.modTime}, d.readErr } func (d *multipartCleanupDisk) StatVol(ctx context.Context, volume string) (VolInfo, error) { @@ -156,10 +143,9 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) { }{ {name: "recent", prefix: uuid, age: expiry / 2, readErr: errFileNotFound}, {name: "plaintext_base64_collision", prefix: "4eacd477-e733-4851-b821-6fd42785ff87", age: expiry / 2, readErr: errFileNotFound}, - // Only an expired upload is read on every drive, for its restore marker. - {name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantReads: 1, wantDeleted: true}, + {name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantDeleted: true}, {name: "legacy_recent", prefix: uuid, age: expiry / 2, legacy: true, wantReads: 1}, - {name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 2, wantDeleted: true}, + {name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 1, wantDeleted: true}, {name: "legacy_metadata_missing", prefix: uuid, age: 2 * expiry, legacy: true, readErr: errFileNotFound, wantReads: 1}, {name: "malformed_metadata_missing", prefix: "not-a-uuid", age: 2 * expiry, readErr: errFileNotFound, wantReads: 1}, } { @@ -180,7 +166,7 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) { if err := os.Chtimes(filepath.Dir(uploadDir), modTime, modTime); err != nil { t.Fatal(err) } - multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk) + erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) if disk.reads != tc.wantReads { t.Errorf("ReadVersion calls = %d, want %d", disk.reads, tc.wantReads) } @@ -213,7 +199,7 @@ func TestCleanupStaleUploadsEmptyHashDirectory(t *testing.T) { if err := os.Chtimes(shaDir, modTime, modTime); err != nil { t.Fatal(err) } - multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk) + erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) if _, err := os.Stat(shaDir); !os.IsNotExist(err) { t.Fatalf("empty hash directory must be removed regardless of age: %v", err) } @@ -238,7 +224,7 @@ func TestCleanupStaleUploadsSharedHashDirectory(t *testing.T) { t.Fatal(err) } - multipartCleanupSet(disk).cleanupStaleUploadsOnDisk(t.Context(), disk) + erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) if _, err := os.Stat(filepath.Join(shaDir, expiredID)); !os.IsNotExist(err) { t.Errorf("expired upload must be removed: %v", err) } diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index c4f5d97ff..6d717df26 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -71,7 +71,6 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string { } // checkUploadIDExists - verify if a given uploadID exists and is valid. -// The metadata read from each drive is returned also with an error. func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string, write bool) (fi FileInfo, metArr []FileInfo, err error) { defer func() { if errors.Is(err, errFileNotFound) { @@ -89,15 +88,15 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount) if err != nil { - return fi, partsMetadata, err + return fi, nil, err } if readQuorum < 0 { - return fi, partsMetadata, errErasureReadQuorum + return fi, nil, errErasureReadQuorum } if writeQuorum < 0 { - return fi, partsMetadata, errErasureWriteQuorum + return fi, nil, errErasureWriteQuorum } quorum := readQuorum @@ -114,7 +113,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum) } if err != nil { - return fi, partsMetadata, err + return fi, nil, err } // Pick one from the first valid metadata. @@ -217,55 +216,12 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto hasSkippedUploads = true return nil } - // A completion holds this lock from staging its commit in the - // upload until the commit ends: decide on a removal only while - // holding it, and skip an upload whose commit is running. Only - // uploads past the stale age pay for the lock and the read below. - lk := er.restoreCommitLock(uploadIDPath) - lkctx, err := lk.GetLock(ctx, newDynamicTimeout(time.Second, time.Second)) - if err != nil { - hasSkippedUploads = true - return nil - } - // The rename below releases the lock when it ends, also when it - // outlives its deadline; every other exit releases it here. - renaming := false - defer func() { - if !renaming { - lk.Unlock(lkctx) - } - }() - // A native restore can run past the stale age: keep its upload - // while its attempt still owns the version, and keep any upload - // whose marker the drives cannot establish. - view := er.readRestoreUpload(lkctx.Context(), "", uploadIDPath) - if view.committing { - // What a commit staged, with no commit running: only a staged - // copy itself shows when it staged, only an old one is a - // leftover of a failed commit, and a copy not staged yet - // tells nothing. - local, err := disk.ReadVersion(lkctx.Context(), "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{}) - st, serr := os.Stat(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath, xlStorageFormatFile)) - if err != nil || serr != nil || !restoreUploadStaged(local) || time.Since(st.ModTime()) < restoreCommitGrace { - hasSkippedUploads = true - return nil - } - } else if view.live && (!view.known || (view.marker != nil && er.restoreUploadActive(lkctx.Context(), view))) { - hasSkippedUploads = true - return nil - } - renaming = true w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) return w.Run(func() error { wait := deleteMultipartCleanupSleeper.Timer(ctx) pathUUID := mustGetUUID() targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID) - err := renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket)) - lk.Unlock(lkctx) - if err != nil { - // The next cleanup round tries again. - storageLogIf(ctx, fmt.Errorf("Unable to remove stale upload %s on %s: %w", uploadIDPath, drivePath, err)) - } + renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket)) wait() return nil }) @@ -506,13 +462,6 @@ func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, if populatedUploadIDs.Contains(uploadID) { continue } - // A native restore's upload is private to its attempt; list only an - // upload that the drives establish as an ordinary one. - metas, errs := readAllFileInfo(ctx, er.getDisks(), bucket, minioMetaMultipartBucket, - pathJoin(er.getMultipartSHADir(bucket, object), uploadID), "", false, false) - if !restoreUploadState(metas, errs).visible(ObjectOptions{}) { - continue - } var fallback time.Time if _, ok := multipartUploadTime(uploadID); !ok { fi, err := disk.ReadVersion(ctx, bucket, minioMetaMultipartBucket, @@ -612,11 +561,6 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string, if opts.PreserveETag != "" { userDefined["etag"] = opts.PreserveETag } - // Only a native restore marks its upload private, never caller metadata. - delete(userDefined, restoreUploadKey) - if a := opts.Transition.attempt; a != nil { - userDefined[restoreUploadKey] = a.String() - } onlineDisks := er.getDisks() // Get parity and data drive count based on storage class metadata @@ -798,10 +742,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) // Validates if upload ID exists. - fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) - if err == nil && !restoreUploadVisible(metas, opts) { - err = errUploadIDNotFound - } + fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) if err != nil { if errors.Is(err, errVolumeNotFound) { return pi, toObjectErr(err, bucket) @@ -1038,10 +979,7 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u UploadID: uploadID, } - fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) - if err == nil && !restoreUploadVisible(metas, opts) { - err = errUploadIDNotFound - } + fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) if err != nil { if errors.Is(err, errVolumeNotFound) { return result, toObjectErr(err, bucket) @@ -1119,10 +1057,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up auditObjectErasureSet(ctx, "ListObjectParts", object, &er) } - fi, metas, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) - if err == nil && !restoreUploadVisible(metas, opts) { - err = errUploadIDNotFound - } + fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) if err != nil { return result, toObjectErr(err, bucket, object, uploadID) } @@ -1335,7 +1270,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str auditObjectErasureSet(ctx, "CompleteMultipartUpload", object, &er) } - if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile || opts.Transition.attempt != nil { + if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile { if !opts.NoLock { ns := er.NewNSLock(bucket, object) lkctx, err := ns.GetLock(ctx, globalOperationTimeout) @@ -1368,24 +1303,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str } } - if opts.Transition.attempt != nil { - // A native restore holds the upload write lock from validation - // through commit, after the object lock and in the lock domain of - // PutObjectPart, which takes a part lock and then this lock to read. - // NoLock only says that the caller holds the object lock. - uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID)) - ulkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout) - if err != nil { - return oi, err - } - ctx = ulkctx.Context() - defer uploadIDLock.Unlock(ulkctx) - } - fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) - if err == nil && !restoreUploadVisible(partsMetadata, opts) { - err = errUploadIDNotFound - } if err != nil { if errors.Is(err, errVolumeNotFound) { return oi, toObjectErr(err, bucket) @@ -1699,7 +1617,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str delete(fi.Metadata, hash.MinIOMultipartChecksumType) delete(fi.Metadata, multipartMetaBucket) delete(fi.Metadata, multipartMetaObject) - delete(fi.Metadata, restoreUploadKey) // Save the final object size and modtime. fi.Size = objectSize @@ -1776,36 +1693,17 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str getObjectInfo = opts.replicaObjectInfo } if fi.VersionID == "" || fi.VersionID == nullVersionID { - if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo, opts.Transition); err != nil { + if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo); err != nil { return oi, toObjectErr(err, bucket, object) } - if a := opts.Transition.attempt; a != nil { - // A native restore writes back only the null version it targets. - current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{ - VersionID: nullVersionID, - Transition: opts.Transition, - NoLock: true, - }) - if gerr != nil { - return oi, toObjectErr(gerr, bucket, object) - } - if !a.rehydrates(current, fi) { - return oi, toObjectErr(errAuthentication, bucket, object) - } - } } else { - current, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{ + _, gerr := getObjectInfo(ctx, bucket, object, ObjectOptions{ VersionID: fi.VersionID, Versioned: opts.Versioned, VersionSuspended: opts.VersionSuspended, - Transition: opts.Transition, NoLock: true, }) switch { - case gerr == nil && opts.Transition.attempt.rehydrates(current, fi): - // A native restore's own private upload rehydrates this same - // tiered version. Any other completion must still reject a - // version overwrite. case gerr == nil, errors.As(gerr, &MethodNotAllowed{}): return oi, toObjectErr(errAuthentication, bucket, object) case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr): @@ -1815,16 +1713,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str } } - // Stale cleanup and abort judge an upload whose commit staged object - // metadata in it by this lock: hold it from staging through the commit. - commitLock := er.restoreCommitLock(uploadIDPath) - clkctx, err := commitLock.GetLock(ctx, globalOperationTimeout) - if err != nil { - return oi, err - } - ctx = clkctx.Context() - defer commitLock.Unlock(clkctx) - er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename(). defer func() { @@ -1896,38 +1784,7 @@ func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, objec if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") { return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID} } - // Authorize before the unconditional cleanup below. A native restore's - // upload is absent to every other caller, and an upload whose marker the - // drives cannot establish is not removed by anyone. - uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) - view := er.readRestoreUpload(ctx, bucket, uploadIDPath) - if view.committing { - // What a failed commit left behind, also after stale cleanup could - // not remove it: only with no commit running, and once every copy - // shows a staging older than the grace. - lk := er.restoreCommitLock(uploadIDPath) - lkctx, err := lk.GetLock(ctx, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait)) - if err != nil { - return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID) - } - defer lk.Unlock(lkctx) - if !er.restoreCommitAbandoned(lkctx.Context(), bucket, uploadIDPath) { - return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID) - } - er.deleteAll(lkctx.Context(), minioMetaMultipartBucket, uploadIDPath) - return true, nil - } - switch { - case view.visible(opts): - case view.known && view.marked: - return false, nil - case view.known && opts.Transition.attempt != nil: - // A native restore's own abort, of an upload no longer its own. - return false, nil - default: - return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID) - } - defer er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath) + defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID)) _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false) err = toObjectErr(err, bucket, object, uploadID) if _, absent := err.(InvalidUploadID); absent { diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index f3c8cfb42..8a0de02ad 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -45,7 +45,6 @@ import ( "github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/event" "github.com/minio/minio/internal/grid" - "github.com/minio/minio/internal/hash" xhttp "github.com/minio/minio/internal/http" xioutil "github.com/minio/minio/internal/ioutil" "github.com/minio/minio/internal/logger" @@ -156,7 +155,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d if dstOpts.replicaObjectInfo != nil { getObjectInfo = dstOpts.replicaObjectInfo } - if err := checkObjectLockedForOverwrite(ctx, dstBucket, dstObject, getObjectInfo, dstOpts.Transition); err != nil { + if err := checkObjectLockedForOverwrite(ctx, dstBucket, dstObject, getObjectInfo); err != nil { return oi, toObjectErr(err, dstBucket, dstObject) } } @@ -1673,7 +1672,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st if opts.replicaObjectInfo != nil { getObjectInfo = opts.replicaObjectInfo } - if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo, opts.Transition); err != nil { + if err := checkObjectLockedForOverwrite(ctx, bucket, object, getObjectInfo); err != nil { return ObjectInfo{}, toObjectErr(err, bucket, object) } } @@ -2665,180 +2664,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st return err } -// RestoreTransitionedObject - restore transitioned object content locally on this cluster. -// This is similar to PostObjectRestore from AWS GLACIER -// storage class. When PostObjectRestore API is called, a temporary copy of the object -// is restored locally to the bucket on source cluster until the restore expiry date. -// The copy that was transitioned continues to reside in the transitioned tier. -func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { - return er.restoreTransitionedObject(ctx, bucket, object, opts) -} - -// getRestoreLock waits for a restore's object lock for as long as ctx lives. -// The POST handler has already marked the version ongoing, and only the -// restore clears that marker. A restore queued behind another version's -// download must therefore keep waiting rather than give up after one timeout. -func getRestoreLock(ctx context.Context, lk RWLocker) (LockContext, error) { - for { - lkctx, err := lk.GetLock(ctx, globalRestoreLockTimeout) - if !errors.As(err, &OperationTimedOut{}) || ctx.Err() != nil { - return lkctx, err - } - } -} - -// restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts -// as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta -// -// A POST restore publishes its attempt on the version's ongoing marker. The -// restore rehydrates the version only while that attempt owns the marker, and -// on failure clears the marker only while it still does. A direct caller -// publishes no attempt; its attempt only names the private upload. -func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error { - if !opts.NoLock { - // Single-pool and direct callers must protect the metadata snapshot - // through download, rehydration and failure cleanup, just like pools. - lk := er.NewNSLock(bucket, object) - lkctx, err := getRestoreLock(ctx, lk) - if err != nil { - return restoreResetError{err} - } - ctx = lkctx.Context() - defer lk.Unlock(lkctx) - opts.NoLock = true - } - - published := opts.Transition.attempt - // Restore the authoritative generation of the version, never a revision - // that only some drives hold. - gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID) - if err != nil { - err = toObjectErr(err, bucket, object, opts.VersionID) - if isErrObjectNotFound(err) || isErrVersionNotFound(err) { - // No version was read, so there is no restore marker to clear. - return err - } - return restoreResetError{err} - } - actualfi := gen.fi - if actualfi.Deleted { - return toObjectErr(errMethodNotAllowed, bucket, object) - } - - oi := actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) - attempt := published - if attempt == nil { - attempt = newRestoreAttempt(oi) - } else if !attempt.owns(oi) { - // A later POST took the version over, or it was replaced. - return errRestoreSuperseded - } - opts.Transition.attempt = attempt - versionID := restoreVersionID(actualfi.VersionID) - fail := func(cause error) error { - return er.restoreFailed(ctx, bucket, object, versionID, published, cause) - } - - ropts := putRestoreOpts(opts.Transition.RestoreRequest, oi, opts) - ropts.NoLock = opts.NoLock - ropts.Transition = opts.Transition - // Rehydration copies encoded bytes without changing their content identity. - ropts.PreserveETag = oi.ETag - // Even a one-part multipart upload must retain its original part number: - // encrypted multipart data uses that number to derive its decryption key. - if len(oi.Parts) == 1 && !oi.isMultipart() { - actualSize, err := oi.GetActualSize() - if err != nil { - return fail(err) - } - ropts.IndexCB = func() []byte { return oi.Parts[0].Index } - var rs *HTTPRangeSpec - gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts) - if err != nil { - return fail(toObjectErr(err, bucket, object)) - } - defer gr.Close() - hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", actualSize) - if err != nil { - return fail(toObjectErr(err, bucket, object)) - } - pReader := NewPutObjReader(hashReader) - if _, err = er.PutObject(ctx, bucket, object, pReader, ropts); err != nil { - return fail(toObjectErr(err, bucket, object)) - } - return nil - } - - // Every server must keep the upload private, also a queued restore's. - if err := nativeRestoreAvailable(ctx); err != nil { - return fail(err) - } - // The upload records when it starts, which the stale-upload cleanup ages; - // completion still commits the version's own ModTime. - uopts := ropts - uopts.MTime = time.Time{} - res, err := er.NewMultipartUpload(ctx, bucket, object, uopts) - if err != nil { - return fail(err) - } - completed := false - defer func() { - if !completed { - er.abortRestoreUpload(bucket, object, res.UploadID, attempt) - } - }() - - var uploadedParts []CompletePart - var rs *HTTPRangeSpec - // get reader from the warm backend - note that even in the case of encrypted objects, this stream is still encrypted. - gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts) - if err != nil { - return fail(err) - } - defer gr.Close() - - // rehydrate the parts back on disk as per the original xl.meta prior to transition - for _, partInfo := range oi.Parts { - // Old uncompressed parts may not record ActualSize. Encrypted parts - // derive it from the written ciphertext in PutObjectPart; compressed - // parts need their recorded logical length, including valid zero sizes. - actualSize := partInfo.Size - if oi.IsCompressed() { - actualSize = partInfo.ActualSize - } - hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", actualSize) - if err != nil { - return fail(err) - } - pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{ - IndexCB: func() []byte { return partInfo.Index }, - Transition: TransitionOptions{attempt: attempt}, - }) - if err != nil { - return fail(err) - } - if pInfo.Size != partInfo.Size { - return fail(InvalidObjectState{Bucket: bucket, Object: object}) - } - uploadedParts = append(uploadedParts, CompletePart{ - PartNumber: pInfo.PartNumber, - ETag: pInfo.ETag, - }) - } - _, err = er.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{ - MTime: oi.ModTime, - NoLock: opts.NoLock, - Versioned: opts.Versioned, - VersionSuspended: opts.VersionSuspended, - Transition: opts.Transition, - }) - if err != nil { - return fail(err) - } - completed = true - return nil -} - // DecomTieredObject - moves tiered object to another pool during decommissioning. func (er erasureObjects) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error { if opts.UserDefined == nil { diff --git a/cmd/erasure-server-pool-lock-domain_test.go b/cmd/erasure-server-pool-lock-domain_test.go index 85b0360e1..13249b213 100644 --- a/cmd/erasure-server-pool-lock-domain_test.go +++ b/cmd/erasure-server-pool-lock-domain_test.go @@ -760,8 +760,10 @@ func TestLockDomainBulkDeleteMapping(t *testing.T) { } } -// A restore holds the object lock of its set for its download. DELETE and COPY -// of the object must wait for it. +// A restore takes the object lock of its set only to commit, not for its +// download. DELETE and COPY of the object wait for that commit, also where +// the sets of the pool lock with lockers of their own, and the restore still +// completes. func TestLockDomainRestoreExcludesDeleteAndCopy(t *testing.T) { for _, mode := range ldModes { for _, op := range []string{"delete", "copy"} { @@ -769,60 +771,41 @@ func TestLockDomainRestoreExcludesDeleteAndCopy(t *testing.T) { r := newRestoreSafetyTest(t, 1, versioning.Enabled) ldSplit(t, r.z, mode) r.object = ldKey(t, r.z, "restore-"+op, 1) - oi, _ := nativeRestoreSeed(t, r, 0, false, false) - blocked := &nativeRestoreBlockedTier{WarmBackend: globalTierConfigMgr.drivercache["WARM"], entered: make(chan struct{}), release: make(chan struct{})} - globalTierConfigMgr.drivercache["WARM"] = blocked - var once sync.Once - unblock := func() { once.Do(func() { close(blocked.release) }) } - defer unblock() - ctx, cancel := context.WithTimeout(t.Context(), time.Minute) - defer cancel() - restored := make(chan error, 1) - go func() { - restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) - }() - select { - case <-blocked.entered: - case <-ctx.Done(): - t.Fatal("restore never started its download") - } - done := make(chan error, 1) - go func() { + oi, body := nativeRestoreSeed(t, r, 0, false, false) + waited, restoreErr, opErr := ldOverlapRun(t, func(ctx context.Context) error { + return r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }, func(ctx context.Context) error { switch op { case "delete": _, err := r.z.DeleteObject(ctx, r.bucket, r.object, r.opts("")) - done <- err - case "copy": + return err + default: ropts := r.opts(oi.VersionID) ropts.NoLock = true cur, err := r.z.GetObjectInfo(ctx, r.bucket, r.object, ropts) if err != nil { - done <- err - return + return err } cur.metadataOnly = true cur.UserDefined = cloneMSS(cur.UserDefined) cur.UserDefined["x-amz-meta-copied"] = "1" vopts := r.opts(oi.VersionID) _, err = r.z.CopyObject(ctx, r.bucket, r.object, r.bucket, r.object, cur, vopts, vopts) - done <- err + return err } - }() - var opErr error - select { - case opErr = <-done: - t.Errorf("%s ran during the restore (%v)", op, opErr) - case <-time.After(ldWait): - unblock() - opErr = <-done + }) + if !waited { + t.Errorf("%s ran during the restore's commit (%v)", op, opErr) } - unblock() - if err := <-restored; err != nil { - t.Fatalf("restore: %v", err) + if restoreErr != nil { + t.Fatalf("restore: %v", restoreErr) } if opErr != nil { t.Fatalf("%s after the restore: %v", op, opErr) } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } }) } } diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 221156480..0811edebc 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -2393,22 +2393,21 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket } }() - // Keep object -> upload lock ordering for single-pool completion too: - // native restore holds the object lock while publishing upload parts. - // Both locks are in the object's lock domain, the set-local one of - // single-pool writes. - if !opts.NoLock { - lk := z.NewObjectNSLock(bucket, encodeDirObject(object)) - lkctx, err := lk.GetLock(ctx, globalOperationTimeout) - if err != nil { - return objInfo, err - } - ctx = lkctx.Context() - defer lk.Unlock(lkctx) - } - opts.NoLock = true - + // Several pools take the object lock first, then the upload lock below. + // A single pool takes the upload lock here and the set takes the object + // lock, in the object's own erasure set, only right before it commits. if !z.SinglePool() { + if !opts.NoLock { + lk := z.NewObjectNSLock(bucket, encodeDirObject(object)) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return objInfo, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + opts.NoLock = true + // A conditional completion must be evaluated against the logical // latest object across pools, under the object write lock held for // this operation. The pool hosting the upload may only hold a stale @@ -2460,8 +2459,9 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket // Hold write locks to verify uploaded parts, also disallows any // parallel PutObjectPart() requests. - // An empty upload ID aliases the object lock already held above. - // Let the storage layer return its normal missing bucket/upload error. + // An empty upload ID would name the object lock itself, which this + // completion holds or takes later: let the storage layer return its + // normal missing bucket/upload error instead. if uploadID != "" { uploadIDLock := z.uploadIDNSLock(bucket, object, uploadID) wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout) @@ -3398,48 +3398,26 @@ func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, objec return z.serverPools[idx].TransitionObject(ctx, bucket, object, opts) } -// RestoreTransitionedObject - restore transitioned object content locally on this cluster. -func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) (err error) { - object = encodeDirObject(object) - defer func() { - // The restore stopped before it could clear POST's ongoing marker. - var reset restoreResetError - if errors.As(err, &reset) { - z.recoverRestoreStatus(bucket, object, opts) - err = reset.error - } - }() - if z.SinglePool() { - return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts) - } - - if !opts.NoLock { - // Acquire write lock before restoring transitioned object - lk := z.NewNSLock(bucket, object) - lkctx, err := getRestoreLock(ctx, lk) - if err != nil { - return restoreResetError{err} - } - ctx = lkctx.Context() - defer lk.Unlock(lkctx) - } - - // Avoid restoring object from a pool being decommissioned. - opts.SkipDecommissioned = true - opts.NoLock = true - idx, err := z.restorePoolIdx(ctx, bucket, object, opts) - if err != nil { - return err - } - - return z.serverPools[idx].RestoreTransitionedObject(ctx, bucket, object, opts) -} - func (z *erasureServerPools) CheckAbandonedParts(ctx context.Context, bucket, object string, opts madmin.HealOpts) error { object = encodeDirObject(object) if z.SinglePool() { return z.serverPools[0].CheckAbandonedParts(ctx, bucket, object, opts) } + // Several pools: remove abandoned data under the pools lock, which every + // write of the object takes, as healObjectInPool does. The lock of one + // pool's set would not exclude a write that has moved its data directory + // in place but not yet published the metadata that references it, such + // as a native restore's commit. + if opts.Remove && !opts.DryRun && !opts.NoLock { + lk := z.NewObjectNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + opts.NoLock = true + } errs := make([]error, len(z.serverPools)) var wg sync.WaitGroup for idx, pool := range z.serverPools { diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 11e867171..307268adb 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -178,9 +178,14 @@ type TransitionOptions struct { RestoreExpiry time.Time ExpireRestored bool - // attempt is the native restore a call belongs to. Only that restore - // sees its private upload, see restoreUploadVisible. + // attempt is the native restore a call belongs to: the attempt that + // POST marked on the version, see restoreRequest. attempt *restoreAttempt + // owner is the restore lock of the version, which POST hands to the + // worker that runs attempt, see erasureServerPools.RestoreTransitionedObject. + owner *restoreOwner + // restoreDate is the request date POST recorded with attempt. + restoreDate string } // MakeBucketOptions represents bucket options for ObjectLayer bucket operations diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index a1133be7e..4f9100df2 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -3988,8 +3988,8 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * // Preserve a failed restore read even before the request body is parsed. opts.Transition.RestoreRequest = &RestoreObjectRequest{} - // Resolve the target without the object lock: a running restore holds it - // for its download, and the status step bounds its own wait for it. + // Resolve the target without the object lock; the status step below + // takes the restore lock and then the object lock. preOpts := opts preOpts.NoLock = true objInfo, err := getObjectInfo(ctx, bucket, object, preOpts) @@ -4023,10 +4023,15 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days) // update self with restore metadata - var attempt *restoreAttempt + var ( + attempt *restoreAttempt + owner *restoreOwner + restoreDate string + ) if rreq.Type != SelectRestoreRequest { - // Native restore needs every server to keep its upload private; - // refuse it, writing nothing, while that is not established. + // Native restore needs every server to follow the restore lock and + // the staged commit; refuse it, writing nothing, while that is not + // established. if err := nativeRestoreAvailable(ctx); err != nil { apiErr := errorCodes.ToAPIErr(ErrBusy) apiErr.Code, apiErr.Description = "ServiceUnavailable", err.Error() @@ -4060,6 +4065,8 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * return } attempt = status.attempt + // The restore lock goes to the worker below, which releases it. + owner, restoreDate = status.owner, status.date } restoreObject := mustGetUUID() @@ -4079,6 +4086,9 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * // now process the restore in background go func() { rctx := GlobalContext + // The restore releases the lock it was handed; this covers a path + // that does not reach it. Releasing twice is a no-op. + defer owner.release() if !rreq.SelectParameters.IsEmpty() { actualSize, err := objInfo.GetActualSize() if err != nil { @@ -4130,6 +4140,8 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r * RestoreRequest: rreq, RestoreExpiry: restoreExpiry, attempt: attempt, + owner: owner, + restoreDate: restoreDate, }, VersionID: attempt.VersionID, Versioned: opts.Versioned, diff --git a/cmd/object-retention-overwrite_test.go b/cmd/object-retention-overwrite_test.go index 7c864e733..83ce43ebe 100644 --- a/cmd/object-retention-overwrite_test.go +++ b/cmd/object-retention-overwrite_test.go @@ -57,7 +57,7 @@ func TestObjectLockOverwriteConfigUnavailable(t *testing.T) { err := checkObjectLockedForOverwrite(t.Context(), "lock-config-unavailable", "object", func(context.Context, string, string, ObjectOptions) (ObjectInfo, error) { t.Fatal("object must not be read without a bucket configuration") return ObjectInfo{}, nil - }, TransitionOptions{}) + }) if !errors.Is(err, errServerNotInitialized) { t.Fatalf("unavailable lock configuration must fail closed, got %v", err) } diff --git a/cmd/restore-attempt.go b/cmd/restore-attempt.go index e95903dd3..ec2d01ecb 100644 --- a/cmd/restore-attempt.go +++ b/cmd/restore-attempt.go @@ -24,6 +24,7 @@ import ( "fmt" "maps" "net/http" + "slices" "strconv" "strings" "time" @@ -33,24 +34,16 @@ import ( xhttp "github.com/minio/minio/internal/http" ) -// A native restore keeps two private records. The version that POST marks -// ongoing names the attempt that owns the marker; that attempt's multipart -// upload names the attempt and the version it rehydrates. Clients cannot -// set either: requests with reserved metadata are refused, it is never -// extracted from headers, and uploads only get the marker from the restore. -const ( - restoreAttemptKey = ReservedMetadataPrefixLower + "restore-attempt" - restoreUploadKey = ReservedMetadataPrefixLower + "restore-upload" -) +// The version that POST marks ongoing names the native restore attempt that +// owns the marker. Clients cannot set it: requests with reserved metadata are +// refused, and it is never extracted from headers. +const restoreAttemptKey = ReservedMetadataPrefixLower + "restore-attempt" const ( - // restoreTakeoverWait bounds how long a POST on an ongoing version waits - // for the object lock. A running restore holds it for its download. + // restoreTakeoverWait bounds how long a POST waits for the restore lock + // of its version. A running restore holds it for its whole run. restoreTakeoverWait = 2 * time.Second - // restoreCommitGrace is how long stale cleanup keeps an upload whose - // completion staged object metadata in it; a commit takes seconds. - restoreCommitGrace = time.Hour - // restoreCleanupTimeout bounds each status reset and upload abort. + // restoreCleanupTimeout bounds each status reset and convergence pass. restoreCleanupTimeout = time.Minute ) @@ -63,7 +56,7 @@ var ( // errRestoreSuperseded: the version no longer belongs to this attempt. // The attempt must not rehydrate, complete or reset anything. errRestoreSuperseded = errors.New("restore attempt superseded") - // errRestoreInProgress: a restore holds the object lock of the version. + // errRestoreInProgress: a restore holds the restore lock of the version. errRestoreInProgress = errors.New("restore already in progress") // errRestoreUnchanged: a restore status update found nothing to write. errRestoreUnchanged = errors.New("restore status unchanged") @@ -107,19 +100,6 @@ func (a *restoreAttempt) String() string { return string(b) } -func parseRestoreAttempt(s string) (*restoreAttempt, bool) { - var a restoreAttempt - if s == "" || json.Unmarshal([]byte(s), &a) != nil || a.ID == "" || a.VersionID == "" { - return nil, false - } - return &a, true -} - -func (a *restoreAttempt) equal(b *restoreAttempt) bool { - return a != nil && b != nil && a.ID == b.ID && a.VersionID == b.VersionID && - a.ETag == b.ETag && a.Size == b.Size && a.ModTime.Equal(b.ModTime) -} - // targets reports whether oi is still the version the attempt was made for. // A replacement under the same version ID differs in ETag, size or ModTime. func (a *restoreAttempt) targets(oi ObjectInfo) bool { @@ -133,162 +113,16 @@ func (a *restoreAttempt) owns(oi ObjectInfo) bool { return a.targets(oi) && oi.RestoreOngoing && oi.UserDefined[restoreAttemptKey] == a.ID } -// rehydrates reports whether completing upload fi, the attempt's private -// upload, writes back the very version current. -func (a *restoreAttempt) rehydrates(current ObjectInfo, fi FileInfo) bool { - return a.targets(current) && fi.Metadata["etag"] == a.ETag && fi.Size == a.Size && fi.ModTime.Equal(a.ModTime) -} - -// restoreUploadView is the restore marker of an upload as the drives of -// its set establish it. A marked upload is written to a write quorum, at -// least a majority of the drives, and its copies never change their marker. -type restoreUploadView struct { - // marker is the attempt of a native restore's upload: one marker that a - // majority of the drives hold. - marker *restoreAttempt - // marked: some readable drive holds a marker. - marked bool - // known: the marker state is established. Without it, callers fail - // closed: nobody lists, uses, aborts or removes the upload. - known bool - // live: enough drives hold the upload or cannot tell that a written - // upload may exist; otherwise only remnants of one are left. - live bool - // committing: some drive holds object metadata in the upload's place. - // Completion stages the committed version's object metadata there, - // without the marker, before it renames the upload into the object. - committing bool - // bucket and object the upload of an established marker belongs to. - bucket, object string -} - -// restoreUploadState establishes the marker of an upload from the metadata of -// every drive. errs may be nil, then a drive without valid metadata is one -// that cannot tell. A marker is established on a majority of the drives; its -// absence only when no readable drive shows one and the drives that cannot -// tell are too few to hold a marked upload. -func restoreUploadState(metas []FileInfo, errs []error) restoreUploadView { - n := len(metas) - majority := n/2 + 1 - markers := make(map[string]int) - var unmarked, unknown, committing int - for i, fi := range metas { - if errs != nil && errs[i] != nil { - if !errors.Is(errs[i], errFileNotFound) && !errors.Is(errs[i], errFileVersionNotFound) && - !errors.Is(errs[i], errVolumeNotFound) && !errors.Is(errs[i], errUploadIDNotFound) { - unknown++ - } - continue - } - if !fi.IsValid() { - unknown++ - continue - } - if restoreUploadStaged(fi) { - committing++ - continue - } - if m := fi.Metadata[restoreUploadKey]; m != "" { - markers[m]++ - } else { - unmarked++ - } +// ownsGeneration reports whether the attempt owns the authoritative +// generation of the version. Where the drives disagree only in their restore +// fields, see restoreGeneration, it owns the version only when its marker is +// the only ongoing marker there: holding one of several is not enough. +func (a *restoreAttempt) ownsGeneration(gen restoreGeneration, bucket, object string) bool { + oi := gen.fi.ToObjectInfo(bucket, object, false) + if !gen.split { + return a.owns(oi) } - present := unmarked - for _, c := range markers { - present += c - } - v := restoreUploadView{marked: len(markers) > 0, live: present+unknown+committing >= majority, committing: committing > 0} - if v.committing { - // Nobody but the completion may use or remove it now. - return v - } - for m, c := range markers { - if c >= majority { - v.marker, v.known = parseRestoreAttempt(m) - if v.known { - // The marker also names the upload's own target. - for _, fi := range metas { - if !fi.IsValid() || fi.Metadata[restoreUploadKey] != m { - continue - } - if restoreVersionID(fi.VersionID) != v.marker.VersionID || fi.Metadata["etag"] != v.marker.ETag { - v.marker, v.known = nil, false - break - } - v.bucket, v.object = fi.Metadata[multipartMetaBucket], fi.Metadata[multipartMetaObject] - } - } - return v - } - } - // A marker without a majority is established only as a remnant. - v.known = !v.live || (!v.marked && unknown < majority) - return v -} - -// restoreUploadStaged reports whether an upload's record is object metadata -// that a completion staged there: an upload record has no parts and no -// other versions. -func restoreUploadStaged(fi FileInfo) bool { - return fi.Deleted || len(fi.Parts) > 0 || fi.NumVersions > 1 -} - -// visible reports whether a caller sees the upload: a native restore its own -// upload, any other caller only an upload established as unmarked. -func (v restoreUploadView) visible(opts ObjectOptions) bool { - if a := opts.Transition.attempt; a != nil { - return v.marker.equal(a) - } - return v.known && !v.marked -} - -// restoreUploadVisible reports whether a caller sees an upload whose metadata -// is metas, one entry for each drive. -func restoreUploadVisible(metas []FileInfo, opts ObjectOptions) bool { - return restoreUploadState(metas, nil).visible(opts) -} - -// readRestoreUpload reads the metadata of an upload from every drive. -func (er erasureObjects) readRestoreUpload(ctx context.Context, bucket, uploadIDPath string) restoreUploadView { - metas, errs := readAllFileInfo(ctx, er.getDisks(), bucket, minioMetaMultipartBucket, uploadIDPath, "", false, false) - return restoreUploadState(metas, errs) -} - -// restoreCommitLock is held by a completion from staging through its commit; -// it is named by the upload's own path, which an upload whose copies all -// hold staged object metadata still has. -func (er erasureObjects) restoreCommitLock(uploadIDPath string) RWLocker { - return er.NewNSLock(minioMetaMultipartBucket, uploadIDPath) -} - -// restoreCommitAbandoned reports whether an upload holds only what a failed -// commit left: every drive answers, and every copy that is there is staged -// object metadata, see restoreUploadStaged, older than restoreCommitGrace by -// that drive's own metadata file. A copy that cannot be read, or that is not -// staged yet, may belong to a commit still running. The caller holds the -// upload's restoreCommitLock. -func (er erasureObjects) restoreCommitAbandoned(ctx context.Context, bucket, uploadIDPath string) bool { - disks := er.getDisks() - metas, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, uploadIDPath, "", false, false) - staged := 0 - for i, fi := range metas { - if errs[i] != nil { - if errors.Is(errs[i], errFileNotFound) || errors.Is(errs[i], errFileVersionNotFound) { - continue - } - return false - } - if !fi.IsValid() || !restoreUploadStaged(fi) { - return false - } - st, err := disks[i].StatInfoFile(ctx, minioMetaMultipartBucket, pathJoin(uploadIDPath, xlStorageFormatFile), false) - if err != nil || len(st) == 0 || time.Since(st[0].ModTime) < restoreCommitGrace { - return false - } - staged++ - } - return staged > 0 + return a.targets(oi) && len(gen.ongoing) == 1 && gen.ongoing[a.ID] } // restoreGeneration is the metadata generation of one version that the drives @@ -304,12 +138,18 @@ func (er erasureObjects) restoreCommitAbandoned(ctx context.Context, bucket, upl // out: fi has no restore fields, and ongoing names the attempts whose // ongoing marker some of its drives hold ("" for one without an attempt). // Every other field, protection included, still needs the quorum. +// +// revs has, for every drive that holds the version, the full revision of the +// entry it holds, whether or not that revision is the generation; nil where a +// drive does not hold the version or cannot be read. A native restore's +// commit compares the drives against it later, see restorePublication. type restoreGeneration struct { fi FileInfo drives []FileInfo disks []StorageAPI split bool ongoing map[string]bool + revs []*revisionKey } // restoreFieldKeys are the metadata keys a restore status write sets. @@ -464,7 +304,7 @@ func (er erasureObjects) readRestoreGeneration(ctx context.Context, bucket, obje return bestKey, best != nil && best.count >= majority && best.count >= er.versionReadQuorum(ctx, bucket, object, best.ver, best.metaV, make([]FileInfo, n), make([]error, n)) } - gen := restoreGeneration{drives: make([]FileInfo, n), disks: make([]StorageAPI, n)} + gen := restoreGeneration{drives: make([]FileInfo, n), disks: make([]StorageAPI, n), revs: slices.Clone(keys)} bestKey, ok := accepted(revisions) if !ok { if bestKey, ok = accepted(neutrals); !ok { @@ -593,60 +433,6 @@ func (er erasureObjects) clearRestoreStatus(ctx context.Context, bucket, object, return err } -// restoreFailed clears the ongoing marker of a failed attempt while the -// object lock is held. A lost lock or an ended context leaves that to the -// bounded retries of the pools layer, which take the lock afresh. -func (er erasureObjects) restoreFailed(ctx context.Context, bucket, object, versionID string, published *restoreAttempt, cause error) error { - if ctx.Err() != nil { - return restoreResetError{cause} - } - if err := er.clearRestoreStatus(ctx, bucket, object, versionID, published); err != nil { - storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, versionID, err)) - return restoreResetError{cause} - } - return cause -} - -// abortRestoreUpload removes the private upload of a failed attempt, or -// leaves it to stale-upload cleanup. The restore's own context may have -// ended, so it uses a fresh bounded one, and the upload lock as any abort. -// It matches the upload to its own attempt rather than to the version, which -// a newer attempt may own by now, and it never touches the version. -func (er erasureObjects) abortRestoreUpload(bucket, object, uploadID string, a *restoreAttempt) { - ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout) - defer cancel() - lk := er.NewNSLock(bucket, pathJoin(object, uploadID)) - lkctx, err := lk.GetLock(ctx, globalOperationTimeout) - if err == nil { - defer lk.Unlock(lkctx) - _, err = er.abortMultipartUpload(lkctx.Context(), bucket, object, uploadID, ObjectOptions{ - NoAuditLog: true, - Transition: TransitionOptions{attempt: a}, - }) - } - if err != nil { - storageLogIf(ctx, fmt.Errorf("Unable to abort the restore upload of %s/%s: %w", bucket, object, err)) - } -} - -// restoreUploadActive reports whether the restore that owns a stale-aged -// upload still runs: its attempt owns the version's ongoing marker. A long -// download keeps its upload; one whose attempt ended loses it, and an -// ongoing marker alone keeps nothing. -func (er erasureObjects) restoreUploadActive(ctx context.Context, v restoreUploadView) bool { - gen, err := er.readRestoreGeneration(ctx, v.bucket, v.object, v.marker.VersionID) - if err != nil { - // An unreadable version may still be restoring; retry next round. - return !errors.Is(err, errFileVersionNotFound) && !errors.Is(err, errFileNotFound) - } - if gen.split { - // Drives disagree on the restore status: that the attempt ended - // is not established. - return gen.ongoing[v.marker.ID] - } - return v.marker.owns(gen.fi.ToObjectInfo(v.bucket, v.object, false)) -} - // restoreLock is the object lock that restores and their status updates // share: the lock every write of the object takes. func (z *erasureServerPools) restoreLock(bucket, object string) RWLocker { @@ -659,36 +445,50 @@ type restoreRequestStatus struct { // attempt is the restore that must rehydrate the version; nil when a // restored copy was still there and its expiry was renewed. attempt *restoreAttempt + // owner is the restore lock of the version, held for attempt. The + // caller hands it to the worker that runs attempt, or releases it. + owner *restoreOwner + // date is the request date recorded with attempt. + date string } // restoreRequest records a POST restore on target, the version that POST // resolved, on a fresh read under the object lock. The version must still // be target: a replacement under the same version ID is refused. A copy that // still holds its restored data is renewed. Otherwise the version is marked -// ongoing for a new attempt; that also takes over a marker no running -// restore holds, as a running restore holds the object lock for its whole -// download. Only the copy that is to be restored is written, so no copy +// ongoing for a new attempt, which also takes over a marker that no running +// restore holds. Only the copy that is to be restored is written, so no copy // claims a restored copy it does not hold. -func (z *erasureServerPools) restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (restoreRequestStatus, error) { +// +// It takes the restore lock R of the version before the object lock O. A +// running restore holds R for its whole run, so a POST that cannot take R +// within the takeover wait answers from the drives: in progress when they +// show the version ongoing, and a retryable timeout otherwise, such as while +// another POST of the version registers. ctx must outlive the request: a new +// attempt keeps R in the returned status. +func (z *erasureServerPools) restoreRequest(ctx context.Context, bucket, object string, opts ObjectOptions, target ObjectInfo) (status restoreRequestStatus, err error) { object = encodeDirObject(object) versionID := restoreVersionID(target.VersionID) - lk := z.restoreLock(bucket, object) - // Wait only briefly first, whatever the lockless read before saw: a - // restore of this version holds the lock for its whole download. Then - // wait as usual only if the drives show no restore of it running. - lkctx, err := lk.GetLock(ctx, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait)) - if errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil { - running, rerr := z.restoreRunning(ctx, bucket, object, versionID) - switch { - case rerr != nil: - // Whether it is restoring is unknown: answer now, not after - // the lock frees. - return restoreRequestStatus{}, rerr - case running: - return restoreRequestStatus{}, errRestoreInProgress + owner, err := z.takeRestoreOwner(ctx, bucket, object, versionID, newDynamicTimeout(restoreTakeoverWait, restoreTakeoverWait)) + if err != nil { + if errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil { + running, rerr := z.restoreRunning(ctx, bucket, object, versionID) + switch { + case rerr != nil: + return restoreRequestStatus{}, rerr + case running: + return restoreRequestStatus{}, errRestoreInProgress + } } - lkctx, err = lk.GetLock(ctx, globalOperationTimeout) + return restoreRequestStatus{}, err } + defer func() { + if status.attempt == nil { + owner.release() + } + }() + lk := z.restoreLock(bucket, object) + lkctx, err := lk.GetLock(owner.ctx, globalOperationTimeout) if err != nil { return restoreRequestStatus{}, err } @@ -750,7 +550,10 @@ func (z *erasureServerPools) restoreRequest(ctx context.Context, bucket, object } return restoreRequestStatus{}, err } - return restoreRequestStatus{objInfo: oi, attempt: attempt}, nil + if attempt == nil { + return restoreRequestStatus{objInfo: oi}, nil + } + return restoreRequestStatus{objInfo: oi, attempt: attempt, owner: owner, date: date}, nil } // restoreRunning reports, without the object lock and within the takeover @@ -801,7 +604,7 @@ func (z *erasureServerPools) restorePoolIdx(ctx context.Context, bucket, object return -1, restoreResetError{toObjectErr(err, bucket, object, a.VersionID)} } found = true - if !z.IsSuspended(idx) && a.owns(gen.fi.ToObjectInfo(bucket, object, false)) { + if !z.IsSuspended(idx) && a.ownsGeneration(gen, bucket, object) { return idx, nil } } diff --git a/cmd/restore-capability.go b/cmd/restore-capability.go index 3594a6dc9..311c1c6b3 100644 --- a/cmd/restore-capability.go +++ b/cmd/restore-capability.go @@ -36,7 +36,7 @@ import ( // or does not know the question, as an older server does not, blocks it. // nativeRestoreCapability is the native restore protocol this server speaks. -const nativeRestoreCapability = "1" +const nativeRestoreCapability = "2" const ( peerRESTRestoreCapability = "restore" diff --git a/cmd/restore-native-generation_test.go b/cmd/restore-native-generation_test.go index 1cea3faf7..b7383ef8f 100644 --- a/cmd/restore-native-generation_test.go +++ b/cmd/restore-native-generation_test.go @@ -21,17 +21,14 @@ import ( "bytes" "context" "crypto/rand" - "encoding/base64" "errors" "fmt" "maps" "net/http" "net/http/httptest" - "os" "reflect" "runtime" "strings" - "sync" "sync/atomic" "testing" "time" @@ -40,7 +37,6 @@ import ( "github.com/minio/madmin-go/v3" "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/versioning" - "github.com/minio/minio/internal/config/drive" xhttp "github.com/minio/minio/internal/http" ) @@ -56,7 +52,7 @@ func (d restoreGenerationDisk) WriteMetadata(ctx context.Context, orig, vol, pat if d.write != nil && d.write.Load() && vol == d.bucket && path == d.object { return errFaultyDisk } - return d.WriteMetadata(ctx, orig, vol, path, fi) + return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi) } func (d restoreGenerationDisk) UpdateMetadata(ctx context.Context, vol, path string, fi FileInfo, opts UpdateMetadataOpts) error { @@ -99,7 +95,7 @@ func (d restoreGenerationOffline) WriteMetadata(ctx context.Context, orig, bucke if d.fail.Load() && bucket == d.bucket && object == d.object { return errFaultyDisk } - return d.WriteMetadata(ctx, orig, bucket, object, fi) + return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi) } // restoreGenerationMetadata returns the version's metadata on each drive. @@ -259,168 +255,6 @@ func TestRestoreAttemptGenerationFencing(t *testing.T) { } } -// restoreGenerationUploadPath is the stored path of an upload. -func restoreGenerationUploadPath(t *testing.T, r *restoreTest, id string) string { - t.Helper() - b, err := base64.RawURLEncoding.DecodeString(id) - if err != nil { - t.Fatal(err) - } - _, uploadUUID, ok := strings.Cut(string(b), ".") - if !ok { - t.Fatalf("upload ID without UUID: %q", id) - } - return pathJoin(r.z.serverPools[0].getHashedSet(r.object).getMultipartSHADir(r.bucket, r.object), uploadUUID) -} - -// The restore marker is established on a majority of the drives. One drive -// whose copy of the upload lacks it does not open the upload to clients, -// and the restore still completes. -func TestRestoreUploadMarkerQuorum(t *testing.T) { - r := newRestoreSafetyTest(t, 2, versioning.Enabled) - initConfigSubsystem(t.Context(), r.z) - oi, body := restoreIsolationSeed(t, r, 0, false, true) - tier := newRestoreIsolationGateTier(5<<20, nil) - globalTierConfigMgr.drivercache["WARM"] = tier - defer tier.unblock() - layer := newRestoreIsolationLayer(r) - if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { - t.Fatal(rec.Body.String()) - } - restoreIsolationEntered(t, tier) - id, _ := restoreIsolationUpload(t, r, 0) - disk := r.z.serverPools[0].getHashedSet(r.object).getDisks()[0] - path := restoreGenerationUploadPath(t, r, id) - fi, err := disk.ReadVersion(t.Context(), "", minioMetaMultipartBucket, path, "", ReadOptions{}) - if err != nil { - t.Fatal(err) - } - delete(fi.Metadata, restoreUploadKey) - if err := disk.WriteMetadata(t.Context(), "", minioMetaMultipartBucket, path, fi); err != nil { - t.Fatal(err) - } - poison := bytes.Repeat([]byte("p"), 5<<20) - if _, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, id, 1, - mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", ""), ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) { - t.Errorf("public part with one unmarked copy of the upload: %v, want NoSuchUpload", err) - } - for _, op := range []func() error{ - func() error { - _, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{}) - return err - }, - func() error { - _, err := r.z.GetMultipartInfo(t.Context(), r.bucket, r.object, id, ObjectOptions{}) - return err - }, - } { - if err := op(); !errors.As(err, &InvalidUploadID{}) { - t.Errorf("public read with one unmarked copy of the upload: %v, want NoSuchUpload", err) - } - } - if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10); err != nil || len(uploads.Uploads) != 0 { - t.Errorf("listing with one unmarked copy of the upload: %+v, %v", uploads.Uploads, err) - } - tier.unblock() - if err := layer.wait(t); err != nil { - t.Fatal(err) - } - if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { - t.Fatalf("restore contents: %v", err) - } -} - -// When the drives cannot tell an upload's marker, no client lists it, no -// abort removes it and stale cleanup keeps it; the restore completes once -// the drives answer again. -func TestRestoreUploadUnreadableFailsClosed(t *testing.T) { - for _, mode := range []string{"list", "abort", "stale"} { - t.Run(mode, func(t *testing.T) { - r := newRestoreSafetyTest(t, 2, versioning.Enabled) - initConfigSubsystem(t.Context(), r.z) - oi, body := restoreIsolationSeed(t, r, 0, false, true) - er := r.z.serverPools[0].getHashedSet(r.object) - disks := er.getDisks() - var fail atomic.Bool - // Stale cleanup with a short age also removes in-flight temporary - // parts: hold that restore in completion, after its last part and - // before its commit, and the others in their download, outside - // the upload lock. - gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} - defer gate.unblock() - tier := newRestoreIsolationGateTier(5<<20, nil) - defer tier.unblock() - wrapped := make([]StorageAPI, len(disks)) - for i, d := range disks { - disk := d - if mode == "stale" { - disk = restoreIsolationValidationDisk{d, gate} - } - wrapped[i] = restoreGenerationDisk{StorageAPI: disk, uploadRead: &fail} - } - er.getDisks = func() []StorageAPI { return wrapped } - if mode != "stale" { - globalTierConfigMgr.drivercache["WARM"] = tier - } - layer := newRestoreIsolationLayer(r) - if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { - t.Fatal(rec.Body.String()) - } - if mode == "stale" { - select { - case <-gate.entered: - case <-time.After(restoreTestWait): - t.Fatal("restore never reached completion") - } - } else { - restoreIsolationEntered(t, tier) - } - stored := restoreIsolationUploads(t, r, 0) - if len(stored) != 1 { - t.Fatalf("restore upload on disk: %v", stored) - } - var id string - for uploadUUID := range stored { - id = base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)) - } - fail.Store(true) - switch mode { - case "list": - if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 100); len(uploads.Uploads) != 0 { - t.Errorf("an upload with an unknown marker is listed: %+v, %v", uploads.Uploads, err) - } - case "abort": - if err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); err == nil { - t.Error("abort of an upload with an unknown marker succeeded") - } - case "stale": - globalAPIConfig.mu.Lock() - prev := globalAPIConfig.staleUploadsExpiry - globalAPIConfig.staleUploadsExpiry = time.Nanosecond - globalAPIConfig.mu.Unlock() - for _, disk := range wrapped { - er.cleanupStaleUploadsOnDisk(t.Context(), disk) - } - globalAPIConfig.mu.Lock() - globalAPIConfig.staleUploadsExpiry = prev - globalAPIConfig.mu.Unlock() - } - fail.Store(false) - if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 1 { - t.Errorf("%s removed an upload with an unknown marker: %v", mode, uploads) - } - gate.unblock() - tier.unblock() - if err := layer.wait(t); err != nil { - t.Fatalf("restore after %s: %v", mode, err) - } - if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { - t.Fatalf("restore contents: %v", err) - } - }) - } -} - // POST restores the version it resolved with that version's identity. A // replacement under the same version ID between its read and its status // write is refused, and nothing restores it. @@ -633,190 +467,6 @@ func TestRestoreGenerationSplitNeedsQuorum(t *testing.T) { } } -// restoreGenerationStageDisk holds a completion that is staging: as -// xlStorage.RenameData does, a drive that stages writes the committed -// object metadata into the upload before it renames the upload into the -// object. -type restoreGenerationStageDisk struct { - StorageAPI - bucket, object string - stage bool - arrived *atomic.Int32 - entered chan struct{} - release chan struct{} - disks int32 -} - -func (d restoreGenerationStageDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { - if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { - if d.stage { - if err := d.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil { - return RenameDataResp{}, err - } - } - if d.arrived.Add(1) == d.disks { - close(d.entered) - } - select { - case <-d.release: - case <-ctx.Done(): - return RenameDataResp{}, ctx.Err() - } - } - return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) -} - -// While a completion stages its commit, the upload holds object metadata -// without the marker on the drives that staged. Nobody lists, aborts or -// removes it then, also when the copies not staged yet are old. -func TestRestoreUploadStagingFailsClosed(t *testing.T) { - for _, mode := range []string{"staged", "partly-staged", "stalled-commit"} { - t.Run(mode, func(t *testing.T) { testRestoreUploadStaging(t, mode) }) - } -} - -func testRestoreUploadStaging(t *testing.T, mode string) { - r := newRestoreSafetyTest(t, 1, versioning.Enabled) - initConfigSubsystem(t.Context(), r.z) - oi, body := restoreIsolationSeed(t, r, 0, false, true) - er := r.z.serverPools[0].getHashedSet(r.object) - disks := er.getDisks() - var arrived atomic.Int32 - entered, release := make(chan struct{}), make(chan struct{}) - released := false - defer func() { - if !released { - close(release) - } - }() - wrapped := make([]StorageAPI, len(disks)) - for i, d := range disks { - wrapped[i] = restoreGenerationStageDisk{ - StorageAPI: d, bucket: r.bucket, object: r.object, stage: mode != "partly-staged" || i == 0, - arrived: &arrived, entered: entered, release: release, disks: int32(len(disks)), - } - } - er.getDisks = func() []StorageAPI { return wrapped } - layer := newRestoreIsolationLayer(r) - if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { - t.Fatal(rec.Body.String()) - } - select { - case <-entered: - case <-time.After(restoreTestWait): - t.Fatal("completion never staged its commit") - } - stored := restoreIsolationUploads(t, r, 0) - var id string - for uploadUUID := range stored { - id = base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)) - if mode != "staged" { - // partly-staged: a long restore, the copies not staged yet are - // old. stalled-commit: drives that stall the running commit - // past the grace make its staged copies look abandoned. - old := UTCNow().Add(-48 * time.Hour) - for _, d := range disks[1:] { - if err := os.Chtimes(pathJoin(d.Endpoint().Path, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), uploadUUID, xlStorageFormatFile), old, old); err != nil { - t.Fatal(err) - } - } - } - } - if uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, r.object, "", "", "", 10); err != nil || len(uploads.Uploads) != 0 { - t.Errorf("a staged restore upload is listed: %+v, %v", uploads.Uploads, err) - } - // The set's own abort: the pools layer would wait for the upload lock - // the completion holds. - if err := er.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); err == nil { - t.Error("abort of a staged restore upload succeeded") - } - globalAPIConfig.mu.Lock() - prev := globalAPIConfig.staleUploadsExpiry - globalAPIConfig.staleUploadsExpiry = time.Nanosecond - globalAPIConfig.mu.Unlock() - for _, disk := range wrapped { - er.cleanupStaleUploadsOnDisk(t.Context(), disk) - } - globalAPIConfig.mu.Lock() - globalAPIConfig.staleUploadsExpiry = prev - globalAPIConfig.mu.Unlock() - if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) { - t.Errorf("a staged restore upload was removed: %v -> %v", stored, after) - } - released = true - close(release) - if err := layer.wait(t); err != nil { - t.Fatalf("restore after its staging was attacked: %v", err) - } - if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { - t.Fatalf("restore contents: %v", err) - } -} - -// When a drive of an acknowledged attempt becomes unreadable, the drives that -// answer disagree on the restore status. That the attempt ended is then not -// established, and stale cleanup keeps its upload. -func TestRestoreSplitKeepsRunningUpload(t *testing.T) { - r := newRestoreSafetyTest(t, 1, versioning.Enabled) - initConfigSubsystem(t.Context(), r.z) - oi, body := restoreIsolationSeed(t, r, 0, false, true) - er := r.z.serverPools[0].getHashedSet(r.object) - disks := er.getDisks() - var missed, offline atomic.Bool - gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} - defer gate.unblock() - wrapped := make([]StorageAPI, len(disks)) - for i, d := range disks { - var disk StorageAPI = restoreIsolationValidationDisk{d, gate} - switch i { - case 0: - disk = restoreGenerationDisk{StorageAPI: disk, bucket: r.bucket, object: r.object, write: &missed} - case 1: - disk = restoreGenerationOffline{StorageAPI: disk, bucket: r.bucket, object: r.object, fail: &offline} - } - wrapped[i] = disk - } - er.getDisks = func() []StorageAPI { return wrapped } - layer := newRestoreIsolationLayer(r) - missed.Store(true) - rec := restoreIsolationPOST(t, r, layer, oi.VersionID) - missed.Store(false) - if rec.Code != http.StatusOK { - t.Fatal(rec.Body.String()) - } - select { - case <-gate.entered: - case <-time.After(restoreTestWait): - t.Fatal("restore never reached completion") - } - stored := restoreIsolationUploads(t, r, 0) - offline.Store(true) - if gen, err := er.readRestoreGeneration(t.Context(), r.bucket, r.object, oi.VersionID); err != nil || !gen.split { - t.Fatalf("fixture: no split generation: %v", err) - } - globalAPIConfig.mu.Lock() - prev := globalAPIConfig.staleUploadsExpiry - globalAPIConfig.staleUploadsExpiry = time.Nanosecond - globalAPIConfig.mu.Unlock() - for _, disk := range wrapped { - er.cleanupStaleUploadsOnDisk(t.Context(), disk) - } - globalAPIConfig.mu.Lock() - globalAPIConfig.staleUploadsExpiry = prev - globalAPIConfig.mu.Unlock() - offline.Store(false) - if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) { - t.Errorf("stale cleanup removed a running restore's upload: %v -> %v", stored, after) - } - gate.unblock() - if err := layer.wait(t); err != nil { - t.Fatalf("restore after the split: %v", err) - } - if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { - t.Fatalf("restore contents: %v", err) - } -} - // restoreGenerationCleanupDisk misses restored-copy cleanups and metadata // writes of the object while its toggles are set. type restoreGenerationCleanupDisk struct { @@ -836,7 +486,7 @@ func (d restoreGenerationCleanupDisk) WriteMetadata(ctx context.Context, orig, b if bucket == d.bucket && object == d.object && d.metaWrites.Load() { return errFaultyDisk } - return d.WriteMetadata(ctx, orig, bucket, object, fi) + return d.StorageAPI.WriteMetadata(ctx, orig, bucket, object, fi) } // Restored-copy expiry decides on the version's authoritative generation. A @@ -1033,268 +683,6 @@ func TestRestoreHealNeedsRestoreQuorum(t *testing.T) { } } -// restoreGenerationStageFails fails the commit before it stages. -type restoreGenerationStageFails struct { - StorageAPI - bucket, object string -} - -func (d restoreGenerationStageFails) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { - if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { - return RenameDataResp{}, errFaultyDisk - } - return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) -} - -// restoreGenerationFailedCommit stages the committed metadata into the -// upload, as RenameData does, and then fails the commit. -type restoreGenerationFailedCommit struct { - StorageAPI - bucket, object string -} - -func (d restoreGenerationFailedCommit) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { - if srcVolume == minioMetaMultipartBucket && dstVolume == d.bucket && dstPath == d.object { - if err := d.WriteMetadata(ctx, "", srcVolume, srcPath, fi); err != nil { - return RenameDataResp{}, err - } - return RenameDataResp{}, errFaultyDisk - } - return d.StorageAPI.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) -} - -// restoreGenerationLeftover runs a restore whose commit fails after it -// staged on every drive, or on every drive but drive 0, which fails before -// it stages. It returns the upload left behind and its on-disk UUID. -func restoreGenerationLeftover(t *testing.T, r *restoreTest, unstaged bool) (map[string]int, string) { - t.Helper() - initConfigSubsystem(t.Context(), r.z) - oi, _ := restoreIsolationSeed(t, r, 0, false, true) - er := r.z.serverPools[0].getHashedSet(r.object) - disks := er.getDisks() - wrapped := make([]StorageAPI, len(disks)) - for i, d := range disks { - if unstaged && i == 0 { - wrapped[i] = restoreGenerationStageFails{StorageAPI: d, bucket: r.bucket, object: r.object} - continue - } - wrapped[i] = restoreGenerationFailedCommit{StorageAPI: d, bucket: r.bucket, object: r.object} - } - er.getDisks = func() []StorageAPI { return wrapped } - layer := newRestoreIsolationLayer(r) - if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { - t.Fatal(rec.Body.String()) - } - if err := layer.wait(t); err == nil { - t.Fatal("fixture: the failed commit succeeded") - } - er.getDisks = func() []StorageAPI { return disks } - stored := restoreIsolationUploads(t, r, 0) - if len(stored) != 1 { - t.Fatalf("fixture: no leftover of the failed commit: %v", stored) - } - for uploadUUID := range stored { - return stored, uploadUUID - } - return nil, "" -} - -// restoreGenerationAgeUpload makes every drive's copy of an upload's -// metadata look written d ago. -func restoreGenerationAgeUpload(t *testing.T, r *restoreTest, uploadUUID string, d time.Duration) { - t.Helper() - er := r.z.serverPools[0].getHashedSet(r.object) - old := UTCNow().Add(-d) - for _, disk := range er.getDisks() { - if err := os.Chtimes(pathJoin(disk.Endpoint().Path, minioMetaMultipartBucket, er.getMultipartSHADir(r.bucket, r.object), uploadUUID, xlStorageFormatFile), old, old); err != nil { - t.Fatal(err) - } - } -} - -// What a failed commit leaves in an upload can always be removed once every -// copy is staged and older than the commit grace: abort removes it, also -// when stale cleanup could not. A recent staging, a copy that cannot be read -// and a copy not staged yet are refused, and nothing is removed then. -func TestRestoreAbandonedCommitAbort(t *testing.T) { - for _, age := range []string{"recent", "abandoned", "unreadable-copy", "unstaged-copy"} { - t.Run(age, func(t *testing.T) { - r := newRestoreSafetyTest(t, 1, versioning.Enabled) - stored, uploadUUID := restoreGenerationLeftover(t, r, age == "unstaged-copy") - if age != "recent" { - restoreGenerationAgeUpload(t, r, uploadUUID, 2*restoreCommitGrace) - } - id := base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)) - er := r.z.serverPools[0].getHashedSet(r.object) - disks := er.getDisks() - var unreadable atomic.Bool - if age == "unreadable-copy" { - wrapped := append([]StorageAPI(nil), disks...) - wrapped[0] = restoreGenerationDisk{StorageAPI: disks[0], uploadRead: &unreadable} - er.getDisks = func() []StorageAPI { return wrapped } - unreadable.Store(true) - } - err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}) - unreadable.Store(false) - left := restoreIsolationUploads(t, r, 0) - if age != "abandoned" { - if err == nil || !reflect.DeepEqual(left, stored) { - t.Errorf("abort of an upload that may still commit (%s): %v, left %v", age, err, left) - } - return - } - if err != nil || len(left) != 0 { - t.Errorf("abort of an abandoned commit: %v, left %v", err, left) - } - }) - } -} - -// With no commit running, stale cleanup removes what a failed commit left -// drive by drive: a staged copy older than the commit grace. A recent -// staging and a copy that never staged stay. -func TestRestoreFailedCommitStaleCleanup(t *testing.T) { - for _, mode := range []string{"recent", "abandoned", "unstaged-copy"} { - t.Run(mode, func(t *testing.T) { - r := newRestoreSafetyTest(t, 1, versioning.Enabled) - stored, uploadUUID := restoreGenerationLeftover(t, r, mode == "unstaged-copy") - if mode != "recent" { - // The copy that never staged is as old as the others. - restoreGenerationAgeUpload(t, r, uploadUUID, 2*restoreCommitGrace) - } - er := r.z.serverPools[0].getHashedSet(r.object) - globalAPIConfig.mu.Lock() - prev := globalAPIConfig.staleUploadsExpiry - globalAPIConfig.staleUploadsExpiry = time.Nanosecond - globalAPIConfig.mu.Unlock() - for _, disk := range er.getDisks() { - er.cleanupStaleUploadsOnDisk(t.Context(), disk) - } - globalAPIConfig.mu.Lock() - globalAPIConfig.staleUploadsExpiry = prev - globalAPIConfig.mu.Unlock() - want := map[string]int{} - switch mode { - case "recent": - want = stored - case "unstaged-copy": - want = map[string]int{uploadUUID: 1} - } - if left := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(left, want) { - t.Errorf("stale cleanup of a failed commit's leftover: %v -> %v, want %v", stored, left, want) - } - }) - } -} - -// restoreGenerationSnapshotKey marks the context of a stale cleanup whose -// reads of upload metadata restoreGenerationSnapshotDisk holds. -type restoreGenerationSnapshotKey struct{} - -// restoreGenerationSnapshotDisk reads upload metadata at once, and hands the -// reply to a marked stale cleanup only on release. -type restoreGenerationSnapshotDisk struct { - StorageAPI - reads *atomic.Int32 - release chan struct{} -} - -func (d restoreGenerationSnapshotDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) { - fi, err := d.StorageAPI.ReadVersion(ctx, origvolume, volume, path, versionID, opts) - if volume == minioMetaMultipartBucket && ctx.Value(restoreGenerationSnapshotKey{}) != nil { - d.reads.Add(1) - select { - case <-d.release: - case <-ctx.Done(): - return FileInfo{}, ctx.Err() - } - } - return fi, err -} - -// Stale cleanup decides on a removal under the commit lock. A completion -// that starts while cleanup reads an upload past the stale age waits for -// it, and a cleanup that comes later skips the running commit: the upload, -// an ordinary one, still commits with its contents. -func TestCleanupStaleUploadsSkipsRunningCommit(t *testing.T) { - r := newRestoreSafetyTest(t, 1, versioning.Enabled) - initConfigSubsystem(t.Context(), r.z) - er := r.z.serverPools[0].getHashedSet(r.object) - mp, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true, MTime: UTCNow().Add(-48 * time.Hour)}) - if err != nil { - t.Fatal(err) - } - body := []byte("an ordinary upload past the stale age") - part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}) - if err != nil { - t.Fatal(err) - } - disks := er.getDisks() - var reads, staged atomic.Int32 - readRelease := make(chan struct{}) - stageEntered, stageRelease := make(chan struct{}), make(chan struct{}) - var readOnce, stageOnce sync.Once - defer readOnce.Do(func() { close(readRelease) }) - defer stageOnce.Do(func() { close(stageRelease) }) - wrapped := make([]StorageAPI, len(disks)) - for i, d := range disks { - stage := restoreGenerationStageDisk{ - StorageAPI: d, bucket: r.bucket, object: r.object, stage: true, - arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks)), - } - wrapped[i] = restoreGenerationSnapshotDisk{StorageAPI: stage, reads: &reads, release: readRelease} - } - er.getDisks = func() []StorageAPI { return wrapped } - globalAPIConfig.mu.Lock() - prev := globalAPIConfig.staleUploadsExpiry - globalAPIConfig.staleUploadsExpiry = time.Nanosecond - globalAPIConfig.mu.Unlock() - defer func() { - globalAPIConfig.mu.Lock() - globalAPIConfig.staleUploadsExpiry = prev - globalAPIConfig.mu.Unlock() - }() - // Every drive's cleanup at once, each reading the upload on every drive. - ctx := context.WithValue(t.Context(), restoreGenerationSnapshotKey{}, true) - var cleanups sync.WaitGroup - for _, disk := range wrapped { - cleanups.Go(func() { er.cleanupStaleUploadsOnDisk(ctx, disk) }) - } - cleanupDone := make(chan struct{}) - go func() { cleanups.Wait(); close(cleanupDone) }() - // The completion starts once every cleanup that may decide holds its - // read of the upload from before the commit. - for deadline := time.Now().Add(2 * time.Second); reads.Load() < int32(len(disks)*len(disks)) && time.Now().Before(deadline); { - time.Sleep(10 * time.Millisecond) - } - completion := make(chan ObjectInfo, 1) - completionErr := make(chan error, 1) - go func() { - oi, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true}) - completion <- oi - completionErr <- err - }() - select { - case <-stageEntered: - case <-time.After(3 * time.Second): - } - readOnce.Do(func() { close(readRelease) }) - select { - case <-cleanupDone: - case <-time.After(restoreTestWait): - t.Fatal("stale cleanup never returned") - } - stageOnce.Do(func() { close(stageRelease) }) - oi := <-completion - if err := <-completionErr; err != nil { - t.Fatalf("completion of an upload stale cleanup read: %v", err) - } - er.getDisks = func() []StorageAPI { return disks } - if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { - t.Fatalf("completed contents: %q, %v", got, err) - } -} - // restoreGenerationRand draws UUIDs from crypto/rand for the whole test // binary, installed before any goroutine runs. While a pause is set, it holds // the UUID that stale cleanup draws in its deadline-bound worker, for the @@ -1334,91 +722,3 @@ func restoreGenerationInCleanupRename() bool { } } } - -// Stale cleanup holds the commit lock until its rename ends, also when the -// rename outlives the drive deadline and cleanup has moved on: a completion -// does not stage while that rename can still remove its upload. -func TestCleanupStaleUploadsHoldsLockThroughRename(t *testing.T) { - r := newRestoreSafetyTest(t, 1, versioning.Enabled) - initConfigSubsystem(t.Context(), r.z) - er := r.z.serverPools[0].getHashedSet(r.object) - mp, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{Versioned: true, MTime: UTCNow().Add(-48 * time.Hour)}) - if err != nil { - t.Fatal(err) - } - body := []byte("an upload whose stale cleanup outlives the drive deadline") - part, err := r.z.PutObjectPart(t.Context(), r.bucket, r.object, mp.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}) - if err != nil { - t.Fatal(err) - } - disks := er.getDisks() - stored := restoreIsolationUploads(t, r, 0) - prev := globalDriveConfig.GetMaxTimeout() - // The shortest drive deadline the configuration takes. - globalDriveConfig.Update(drive.Config{MaxTimeout: 1100 * time.Millisecond}) - defer globalDriveConfig.Update(drive.Config{MaxTimeout: prev}) - pause := &restoreGenerationPause{entered: make(chan struct{}), release: make(chan struct{})} - var paused sync.Once - defer paused.Do(func() { close(pause.release) }) - restoreGenerationRand.pause.Store(pause) - defer restoreGenerationRand.pause.CompareAndSwap(pause, nil) - - cleanupDone := make(chan struct{}) - go func() { - er.cleanupStaleUploadsOnDisk(t.Context(), disks[0]) - close(cleanupDone) - }() - select { - case <-pause.entered: - case <-time.After(5 * time.Second): - t.Fatal("stale cleanup never reached its rename") - } - select { - case <-cleanupDone: - case <-time.After(5 * time.Second): - t.Fatal("stale cleanup did not move on after the drive deadline") - } - if left := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(left, stored) { - t.Fatalf("fixture: the held rename already ran: %v -> %v", stored, left) - } - - var staged atomic.Int32 - stageEntered, stageRelease := make(chan struct{}), make(chan struct{}) - var stageOnce sync.Once - defer stageOnce.Do(func() { close(stageRelease) }) - wrapped := make([]StorageAPI, len(disks)) - for i, d := range disks { - wrapped[i] = restoreGenerationStageDisk{ - StorageAPI: d, bucket: r.bucket, object: r.object, stage: true, - arrived: &staged, entered: stageEntered, release: stageRelease, disks: int32(len(disks)), - } - } - er.getDisks = func() []StorageAPI { return wrapped } - completion := make(chan ObjectInfo, 1) - completionErr := make(chan error, 1) - go func() { - oi, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, r.object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true}) - completion <- oi - completionErr <- err - }() - select { - case <-stageEntered: - t.Error("a completion staged while stale cleanup's rename could still run") - case <-time.After(2 * time.Second): - } - paused.Do(func() { close(pause.release) }) - select { - case <-stageEntered: - case <-time.After(restoreTestWait): - t.Fatal("completion never staged after the rename ended") - } - stageOnce.Do(func() { close(stageRelease) }) - oi := <-completion - if err := <-completionErr; err != nil { - t.Fatalf("completion after the rename ended: %v", err) - } - er.getDisks = func() []StorageAPI { return disks } - if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { - t.Fatalf("completed contents: %q, %v", got, err) - } -} diff --git a/cmd/restore-native-isolation_test.go b/cmd/restore-native-isolation_test.go index 6e5e412da..2690aefea 100644 --- a/cmd/restore-native-isolation_test.go +++ b/cmd/restore-native-isolation_test.go @@ -20,7 +20,6 @@ package cmd import ( "bytes" "context" - "encoding/base64" "errors" "fmt" "io" @@ -41,12 +40,9 @@ import ( "github.com/minio/mux" ) -// The private metadata of a native restore, as stored. Literal names keep -// these regressions meaningful on a tree without the keys. -const ( - restoreIsolationAttemptKey = "x-minio-internal-restore-attempt" - restoreIsolationUploadKey = "x-minio-internal-restore-upload" -) +// The private attempt metadata of a native restore, as stored. A literal name +// keeps these regressions meaningful on a tree without the key. +const restoreIsolationAttemptKey = "x-minio-internal-restore-attempt" // restoreIsolationLayer hooks the object layer a POST restore handler uses. // afterRead runs once POST has read the version, before POST records its @@ -177,30 +173,6 @@ func restoreIsolationUploads(t *testing.T, r *restoreTest, pool int) map[string] return found } -// restoreIsolationUpload finds the single upload of the object in a pool on -// disk, with its first part, and returns its public upload ID. -func restoreIsolationUpload(t *testing.T, r *restoreTest, pool int) (string, ObjectPartInfo) { - t.Helper() - uploads := restoreIsolationUploads(t, r, pool) - if len(uploads) != 1 { - t.Fatalf("want one upload on disk, found %v", uploads) - } - var uploadUUID string - for uploadUUID = range uploads { - } - er := r.z.serverPools[pool].getHashedSet(r.object) - uploadPath := pathJoin(er.getMultipartSHADir(r.bucket, r.object), uploadUUID) - fi, err := er.getDisks()[0].ReadVersion(t.Context(), "", minioMetaMultipartBucket, uploadPath, "", ReadOptions{}) - if err != nil { - t.Fatal(err) - } - parts, err := readParts(t.Context(), er.getDisks(), minioMetaMultipartBucket, []string{pathJoin(uploadPath, fi.DataDir, "part.1.meta")}, []int{1}, 1) - if err != nil || len(parts) != 1 || parts[0].Error != "" { - t.Fatalf("restore must have published its first part: %+v, %v", parts, err) - } - return base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)), parts[0] -} - // restoreIsolationGateTier holds a download after its first bytes and can // fail it once released. type restoreIsolationGateTier struct { @@ -319,207 +291,6 @@ func installRestoreIsolationFaults(r *restoreTest, pool int, failing int) (reads return reads, writes } -// A public client must not replace a part of a native restore's upload, -// before or after the restore validated it: the bytes would become the -// COMPLIANCE-protected version under its original ETag. -func TestRestoreUploadIsolatedFromPublicParts(t *testing.T) { - r := newRestoreSafetyTest(t, 2, versioning.Enabled) - initConfigSubsystem(t.Context(), r.z) - enableBucketObjectLock(t, r.bucket) - previous := newObjectLayerFn() - t.Cleanup(func() { setObjectLayer(previous) }) - router := initTestAPIEndPoints(r.z, []string{"PutObjectPart", "GetObject"}) - oi, body := nativeRestoreSeed(t, r, 0, false, true) - if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, "retention", retentionBody("COMPLIANCE", UTCNow().Add(48*time.Hour))); rec.Code != http.StatusOK { - t.Fatalf("lock: %d %s", rec.Code, rec.Body.String()) - } - // Hold completion after it validated the parts, before it commits. - er := r.z.serverPools[0].getHashedSet(r.object) - original := er.getDisks - gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} - defer gate.unblock() - disks := original() - wrapped := make([]StorageAPI, len(disks)) - for i, d := range disks { - wrapped[i] = restoreIsolationCompletionDisk{d, gate} - } - er.getDisks = func() []StorageAPI { return wrapped } - defer func() { er.getDisks = original }() - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - restored := make(chan error, 1) - go func() { - restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) - }() - select { - case <-gate.entered: - case err := <-restored: - t.Fatalf("restore stopped before validation: %v", err) - case <-ctx.Done(): - t.Fatal("no completion gate") - } - id, _ := restoreIsolationUpload(t, r, 0) - poison := bytes.Repeat([]byte("z"), 5<<20) - // The request may wait for the upload lock that completion holds. - uploaded := make(chan *httptest.ResponseRecorder, 1) - go func() { - uploaded <- attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodPut, "/"+r.bucket+"/"+r.object+"?uploadId="+url.QueryEscape(id)+"&partNumber=1", poison, nil) - }() - var rec *httptest.ResponseRecorder - select { - case rec = <-uploaded: - case <-time.After(500 * time.Millisecond): - } - gate.unblock() - if err := <-restored; err != nil { - t.Fatalf("restore: %v", err) - } - if rec == nil { - rec = <-uploaded - } - if rec.Code != http.StatusNotFound || !strings.Contains(rec.Body.String(), "NoSuchUpload") { - t.Errorf("public UploadPart on a native restore upload: %d %s, want NoSuchUpload", rec.Code, rec.Body.String()) - } - got := attributesPartsSignedRequest(t, router, globalActiveCred, http.MethodGet, "/"+r.bucket+"/"+r.object+"?versionId="+oi.VersionID, nil, nil) - if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), body) { - t.Errorf("COMPLIANCE version holds other bytes after restore: GET=%d poisoned=%v", got.Code, bytes.HasPrefix(got.Body.Bytes(), poison)) - } -} - -type restoreIsolationCompletionGate struct { - entered, release chan struct{} - enter, open sync.Once -} - -func (g *restoreIsolationCompletionGate) unblock() { g.open.Do(func() { close(g.release) }) } - -type restoreIsolationCompletionDisk struct { - StorageAPI - gate *restoreIsolationCompletionGate -} - -// Completion removes the part metadata it validated just before it renames -// the upload into place. -func (d restoreIsolationCompletionDisk) DeleteBulk(ctx context.Context, bucket string, paths ...string) error { - if bucket == minioMetaMultipartBucket && len(paths) == 2 && strings.HasSuffix(paths[0], ".meta") && strings.HasSuffix(paths[1], ".meta") { - d.gate.enter.Do(func() { close(d.gate.entered) }) - select { - case <-d.gate.release: - case <-ctx.Done(): - return ctx.Err() - } - } - return d.StorageAPI.DeleteBulk(ctx, bucket, paths...) -} - -// restoreIsolationValidationDisk holds a completion where it reads the parts -// it validates: after its object and upload locks, before its commit lock. -// Stale cleanup then decides on the upload itself, not on a running commit. -type restoreIsolationValidationDisk struct { - StorageAPI - gate *restoreIsolationCompletionGate -} - -func (d restoreIsolationValidationDisk) ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error) { - if bucket == minioMetaMultipartBucket { - d.gate.enter.Do(func() { close(d.gate.entered) }) - select { - case <-d.gate.release: - case <-ctx.Done(): - return nil, ctx.Err() - } - } - return d.StorageAPI.ReadParts(ctx, bucket, partMetaPaths...) -} - -// Every public multipart entry point finds no native restore upload, and -// none of them can change it: the restore still completes with its own data. -func TestRestoreUploadHiddenFromPublicEntryPoints(t *testing.T) { - for _, pools := range []int{1, 2} { - t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) { - r := newRestoreSafetyTest(t, pools, versioning.Enabled) - oi, body := nativeRestoreSeed(t, r, 0, false, true) - tier := newRestoreIsolationGateTier(5<<20, nil) - globalTierConfigMgr.drivercache["WARM"] = tier - defer tier.unblock() - ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) - defer cancel() - restored := make(chan error, 1) - go func() { - restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) - }() - restoreIsolationEntered(t, tier) - id, part := restoreIsolationUpload(t, r, 0) - stored := restoreIsolationUploads(t, r, 0) - - for _, prefix := range []string{r.object, ""} { - uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, prefix, "", "", "", 100) - if err != nil { - t.Fatal(err) - } - for _, u := range uploads.Uploads { - if u.UploadID == id { - t.Errorf("ListMultipartUploads(prefix=%q) shows the native restore upload", prefix) - } - } - } - poison := bytes.Repeat([]byte("z"), 5<<20) - noUpload := func(op string, err error) { - t.Helper() - if !errors.As(err, &InvalidUploadID{}) { - t.Errorf("%s on a native restore upload: %v, want NoSuchUpload", op, err) - } - } - // With one pool, part writes first wait for the object lock the - // restore holds; with more they reach the upload at once. - written := make(chan error, 2) - go func() { - _, err := r.z.PutObjectPart(ctx, r.bucket, r.object, id, 1, mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", ""), ObjectOptions{}) - written <- err - }() - go func() { - src := ObjectInfo{PutObjReader: mustGetPutObjReader(t, bytes.NewReader(poison), int64(len(poison)), "", "")} - _, err := r.z.CopyObjectPart(ctx, r.bucket, r.object, r.bucket, r.object, id, 1, 0, int64(len(poison)), src, ObjectOptions{}, ObjectOptions{}) - written <- err - }() - var writes []error - if pools > 1 { - writes = append(writes, <-written, <-written) - } - _, err := r.z.ListObjectParts(t.Context(), r.bucket, r.object, id, 0, 10, ObjectOptions{}) - noUpload("ListObjectParts", err) - _, err = r.z.GetMultipartInfo(t.Context(), r.bucket, r.object, id, ObjectOptions{}) - noUpload("GetMultipartInfo", err) - err = r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}) - noUpload("AbortMultipartUpload", err) - if after := restoreIsolationUploads(t, r, 0); !reflect.DeepEqual(after, stored) { - t.Errorf("public calls changed the restore upload on disk: %v -> %v", stored, after) - } - // Completion waits for the object lock the restore holds. - completed := make(chan error, 1) - go func() { - _, err := r.z.CompleteMultipartUpload(ctx, r.bucket, r.object, id, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, r.opts(oi.VersionID)) - completed <- err - }() - tier.unblock() - if err := <-restored; err != nil { - t.Fatalf("restore after public calls: %v", err) - } - for len(writes) < 2 { - writes = append(writes, <-written) - } - for _, err := range writes { - noUpload("PutObjectPart/CopyObjectPart", err) - } - noUpload("CompleteMultipartUpload", <-completed) - _, got, err := r.read(oi.VersionID) - if err != nil || !bytes.Equal(got, body) { - t.Fatalf("restore contents after public calls: %v", err) - } - }) - } -} - // POST records its status on a fresh read under the object lock, so an // Object Lock update it did not see survives. func TestRestorePOSTKeepsConcurrentProtection(t *testing.T) { @@ -721,6 +492,10 @@ func TestRestoreSupersededAttempt(t *testing.T) { t.Cleanup(unblock) older := newRestoreIsolationLayer(r) older.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + // The queued worker loses the restore lock POST handed it, + // as when its refresh found it gone: the marker then belongs + // to no running restore. + opts.Transition.owner.release() <-release return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) } @@ -984,204 +759,6 @@ func TestRestoreDuplicatePoolCacheStates(t *testing.T) { } } -// Stale-upload cleanup ages an upload by when it started, not by the -// version's ModTime, keeps the upload of a running restore beyond the stale -// age, and still removes ordinary stale uploads and those of ended attempts. -func TestRestoreUploadOutlivesStaleAge(t *testing.T) { - cleanup := func(t *testing.T, r *restoreTest) { - er := r.z.serverPools[0].getHashedSet(r.object) - for _, disk := range er.getDisks() { - er.cleanupStaleUploadsOnDisk(t.Context(), disk) - } - } - shortExpiry := func(t *testing.T) { - globalAPIConfig.mu.Lock() - prev := globalAPIConfig.staleUploadsExpiry - globalAPIConfig.staleUploadsExpiry = time.Nanosecond - globalAPIConfig.mu.Unlock() - t.Cleanup(func() { - globalAPIConfig.mu.Lock() - globalAPIConfig.staleUploadsExpiry = prev - globalAPIConfig.mu.Unlock() - }) - } - for _, mode := range []string{"old-version", "past-stale-age", "public-abort", "ordinary-stale", "unowned-marker"} { - t.Run(mode, func(t *testing.T) { - r := newRestoreSafetyTest(t, 2, versioning.Enabled) - initConfigSubsystem(t.Context(), r.z) - if mode == "ordinary-stale" { - // Control: an abandoned client upload, old by its upload ID. - if _, err := r.z.NewMultipartUpload(t.Context(), r.bucket, r.object, ObjectOptions{MTime: UTCNow().Add(-72 * time.Hour)}); err != nil { - t.Fatal(err) - } - cleanup(t, r) - if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { - t.Errorf("ordinary stale upload survived cleanup: %v", uploads) - } - return - } - var oi ObjectInfo - var body []byte - if mode == "past-stale-age" { - oi, body = restoreIsolationSeed(t, r, 0, false, true) - } else { - oi, body = nativeRestoreSeed(t, r, 0, false, true) - } - old := r.opts(oi.VersionID) - old.MTime = UTCNow().Add(-72 * time.Hour) - old.EvalMetadataFn = func(*ObjectInfo, error) (ReplicateDecision, error) { return ReplicateDecision{}, nil } - if _, err := r.z.PutObjectMetadata(t.Context(), r.bucket, r.object, old); err != nil { - t.Fatal(err) - } - // The cleanup with a short stale age also removes in-flight - // temporary parts; hold such restores in completion instead, - // before their commit. - gated := mode == "past-stale-age" || mode == "unowned-marker" - tier := newRestoreIsolationGateTier(5<<20, nil) - if !gated { - globalTierConfigMgr.drivercache["WARM"] = tier - } - defer tier.unblock() - gate := &restoreIsolationCompletionGate{entered: make(chan struct{}), release: make(chan struct{})} - defer gate.unblock() - if gated { - er := r.z.serverPools[0].getHashedSet(r.object) - original := er.getDisks - disks := original() - wrapped := make([]StorageAPI, len(disks)) - for i, d := range disks { - wrapped[i] = restoreIsolationValidationDisk{d, gate} - } - er.getDisks = func() []StorageAPI { return wrapped } - defer func() { er.getDisks = original }() - } - result := make(chan error, 1) - layer := newRestoreIsolationLayer(r) - if mode == "past-stale-age" { - // A POST publishes the attempt that a running restore owns. - if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { - t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) - } - } else { - go func() { - result <- r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) - }() - } - if gated { - select { - case <-gate.entered: - case <-time.After(restoreTestWait): - t.Fatal("restore never reached completion") - } - } else { - restoreIsolationEntered(t, tier) - } - switch mode { - case "past-stale-age", "unowned-marker": - shortExpiry(t) - cleanup(t, r) - kept := len(restoreIsolationUploads(t, r, 0)) == 1 - if mode == "unowned-marker" && kept { - // The version is ongoing, but not for this direct restore's - // upload: an ongoing marker alone does not keep an upload. - t.Error("stale cleanup kept an upload no attempt owns") - } - if mode == "past-stale-age" && !kept { - t.Error("stale cleanup removed the upload of a running restore") - } - case "old-version": - cleanup(t, r) - case "public-abort": - id, _ := restoreIsolationUpload(t, r, 0) - if err := r.z.AbortMultipartUpload(t.Context(), r.bucket, r.object, id, ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) { - t.Errorf("public abort of a native restore upload: %v, want NoSuchUpload", err) - } - } - tier.unblock() - gate.unblock() - var err error - if mode == "past-stale-age" { - err = layer.wait(t) - } else { - err = <-result - } - if mode == "unowned-marker" { - if err == nil { - t.Error("restore completed without its upload") - } - return - } - if err != nil { - t.Fatalf("restore lost its upload: %v", err) - } - _, got, err := r.read(oi.VersionID) - if err != nil || !bytes.Equal(got, body) { - t.Fatalf("restore contents: %v", err) - } - }) - } -} - -// A failed multipart restore removes its own upload from the drives, also -// when a newer attempt owns the version by then, and leaves that newer -// attempt's marker alone. -func TestRestoreFailureAbortsUpload(t *testing.T) { - for _, mode := range []string{"download-failure", "superseded-mid-download"} { - t.Run(mode, func(t *testing.T) { - r := newRestoreSafetyTest(t, 1, versioning.Enabled) - initConfigSubsystem(t.Context(), r.z) - failure := errors.New("injected remote failure") - if mode == "download-failure" { - oi, _ := nativeRestoreSeed(t, r, 0, false, true) - tier := globalTierConfigMgr.drivercache["WARM"].(*restoreTestTier) - globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, failure} - if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); !errors.Is(err, failure) { - t.Fatalf("restore error=%v, want %v", err, failure) - } - if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { - t.Errorf("failed native restore left its upload on disk: %v", uploads) - } - return - } - oi, _ := restoreIsolationSeed(t, r, 0, false, true) - tier := newRestoreIsolationGateTier(5<<20, failure) - globalTierConfigMgr.drivercache["WARM"] = tier - defer tier.unblock() - layer := newRestoreIsolationLayer(r) - if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { - t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) - } - restoreIsolationEntered(t, tier) - if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 1 { - t.Fatalf("restore upload on disk: %v", uploads) - } - // Model a newer generation that owns the version by now, such as - // one that took over after this restore lost its lock. - newer := r.opts(oi.VersionID) - newer.MTime = oi.ModTime - newer.NoLock = true - newer.EvalMetadataFn = func(o *ObjectInfo, _ error) (ReplicateDecision, error) { - o.UserDefined[restoreIsolationAttemptKey] = "newer-generation" - return ReplicateDecision{}, nil - } - if _, err := r.z.serverPools[0].getHashedSet(r.object).PutObjectMetadata(t.Context(), r.bucket, r.object, newer); err != nil { - t.Fatal(err) - } - tier.unblock() - if err := layer.wait(t); !errors.Is(err, failure) { - t.Fatalf("restore error=%v, want %v", err, failure) - } - if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { - t.Errorf("superseded restore left its upload on disk: %v", uploads) - } - after := r.fileInfo(oi.VersionID) - if !strings.Contains(after.Metadata[xhttp.AmzRestore], "true") || after.Metadata[restoreIsolationAttemptKey] != "newer-generation" { - t.Errorf("failed restore cleared a newer attempt's marker: %q %q", after.Metadata[xhttp.AmzRestore], after.Metadata[restoreIsolationAttemptKey]) - } - }) - } -} - // A restored copy expires only by its own reason, on a fresh read: its // restore deadline passed and nothing else. A renewed deadline survives // another rule falling due and an expiry checked across all pools. @@ -1246,58 +823,6 @@ func TestExpireRestoredNeedsOwnReason(t *testing.T) { } } -// Caller metadata cannot make an upload a native restore upload: the marker -// is dropped, the upload stays an ordinary one and nothing commits it. -func TestRestoreUploadMarkerSpoofing(t *testing.T) { - r := newRestoreSafetyTest(t, 1, versioning.Enabled) - oi, _ := nativeRestoreSeed(t, r, 0, false, true) - forged := fmt.Sprintf(`{"id":%q,"v":%q,"e":%q,"s":%d,"m":%q}`, mustGetUUID(), oi.VersionID, oi.ETag, oi.Size, oi.ModTime.UTC().Format(time.RFC3339Nano)) - for _, target := range []string{"existing-version", "new-object"} { - t.Run(target, func(t *testing.T) { - opts := ObjectOptions{Versioned: true, UserDefined: map[string]string{restoreIsolationUploadKey: forged}} - object := r.object - if target == "existing-version" { - opts.VersionID = oi.VersionID - opts.MTime = oi.ModTime - opts.PreserveETag = oi.ETag - } else { - object = "spoofed-" + r.object - } - upload, err := r.z.NewMultipartUpload(t.Context(), r.bucket, object, opts) - if err != nil { - t.Fatal(err) - } - uploads, err := r.z.ListMultipartUploads(t.Context(), r.bucket, object, "", "", "", 100) - if err != nil || len(uploads.Uploads) != 1 || uploads.Uploads[0].UploadID != upload.UploadID { - t.Fatalf("a caller's upload must stay listed: %+v, %v", uploads.Uploads, err) - } - data := []byte("spoofed part") - part, err := r.z.PutObjectPart(t.Context(), r.bucket, object, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) - if err != nil { - t.Fatal(err) - } - before := nativeRestoreSnapshot(t, r) - done, err := r.z.CompleteMultipartUpload(t.Context(), r.bucket, object, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{Versioned: true, MTime: opts.MTime}) - if target == "existing-version" { - if !errors.Is(err, errAuthentication) { - t.Errorf("spoofed upload overwrote the transitioned version: %v", err) - } - if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) { - t.Error("rejected spoofed completion changed the version") - } - return - } - if err != nil { - t.Fatal(err) - } - info, err := r.z.GetObjectInfo(t.Context(), r.bucket, object, ObjectOptions{VersionID: done.VersionID}) - if err != nil || info.UserDefined[restoreIsolationUploadKey] != "" { - t.Errorf("caller metadata committed a restore upload marker: %q, %v", info.UserDefined[restoreIsolationUploadKey], err) - } - }) - } -} - // Controls: an ordinary POST restore, a renewal, and a client's multipart // upload of the same key while a restore runs. func TestRestorePOSTControls(t *testing.T) { diff --git a/cmd/restore-native_test.go b/cmd/restore-native_test.go index 44676ac7c..61733e094 100644 --- a/cmd/restore-native_test.go +++ b/cmd/restore-native_test.go @@ -373,10 +373,9 @@ func TestNativeRestorePOSTVersionIdentity(t *testing.T) { } } -// A restore request is internal context, not an exception for normal S3 -// multipart overwrites or for a version that has never completed transition. -// Only the upload of a native restore attempt, whose marker names the very -// version being completed, may write that version back. +// A restore request is internal context, not an exception for multipart +// overwrites: a native restore never completes an upload, so no completion, +// whatever restore context it carries, writes an existing version back. func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { for _, mode := range []string{"matching-restore", "ordinary-complete", "restore-without-attempt", "other-attempt", "not-tiered", "changed-time", "changed-size", "changed-etag"} { t.Run(mode, func(t *testing.T) { @@ -446,24 +445,7 @@ func TestNativeRestoreMultipartOverwriteBoundary(t *testing.T) { } before := nativeRestoreSnapshot(t, r) _, err = pool.CompleteMultipartUpload(t.Context(), r.bucket, r.object, upload.UploadID, complete, opts) - if mode == "matching-restore" { - if err != nil { - t.Fatalf("matching restore rejected: %v", err) - } - _, got, err := r.read(oi.VersionID) - if err != nil || !bytes.Equal(got, body) { - t.Fatalf("matching restore contents: %v", err) - } - if after := r.fileInfo(oi.VersionID); after.Metadata[restoreUploadKey] != "" || after.Metadata[restoreAttemptKey] != "" { - t.Error("restore committed its private upload or attempt marker") - } - return - } - if mode == "other-attempt" { - if !errors.As(err, &InvalidUploadID{}) { - t.Fatalf("another attempt completed a private upload: %v", err) - } - } else if !errors.Is(err, errAuthentication) { + if !errors.Is(err, errAuthentication) { t.Fatalf("existing version overwrite: %v", err) } if !reflect.DeepEqual(before, nativeRestoreSnapshot(t, r)) { @@ -525,7 +507,10 @@ func nativeRestoreLockRequest(ctx context.Context, t *testing.T, r *restoreTest, return rec } -func TestNativeRestoreSerializesObjectLock(t *testing.T) { +// A native restore downloads without the object lock. An Object Lock +// update made meanwhile commits at once, and the restore, whether it +// completes or fails, keeps it. +func TestNativeRestoreKeepsObjectLockUpdate(t *testing.T) { for _, kind := range []string{"legal-hold", "retention"} { for _, fail := range []bool{false, true} { t.Run(fmt.Sprintf("%s/failure=%v", kind, fail), func(t *testing.T) { @@ -567,22 +552,19 @@ func TestNativeRestoreSerializesObjectLock(t *testing.T) { case <-ctx.Done(): t.Fatal("restore never started download") } - updateCtx, updateCancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + // The download holds no object lock: an Object Lock update + // commits at once, and the restore's commit keeps it. + updateCtx, updateCancel := context.WithTimeout(t.Context(), 5*time.Second) rec := nativeRestoreLockRequest(updateCtx, t, r, oi.VersionID, kind, updated) updateCancel() - if rec.Code == http.StatusOK { - t.Error("Object Lock update committed during restore's metadata snapshot") - // On the unfixed path this update commits, and the restore - // must not subsequently erase the acknowledged protection. - readOpts := r.opts(oi.VersionID) - readOpts.NoLock = true - before, err = r.z.GetObjectInfo(t.Context(), r.bucket, r.object, readOpts) - if err != nil { - t.Fatal(err) - } + if rec.Code != http.StatusOK { + t.Fatalf("Object Lock update during the restore download: %d %s", rec.Code, rec.Body.String()) } - if rec.Code != http.StatusOK && (rec.Code != http.StatusServiceUnavailable || !strings.Contains(rec.Body.String(), "RequestTimeout")) { - t.Errorf("update failed for an unrelated reason: %d %s", rec.Code, rec.Body.String()) + readOpts := r.opts(oi.VersionID) + readOpts.NoLock = true + during, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, readOpts) + if err != nil { + t.Fatal(err) } unblock() select { @@ -598,26 +580,19 @@ func TestNativeRestoreSerializesObjectLock(t *testing.T) { t.Fatal(err) } for _, key := range keys { - if before.UserDefined[key] == "" || after.UserDefined[key] != before.UserDefined[key] { - t.Errorf("restore changed protection %s: %q -> %q", key, before.UserDefined[key], after.UserDefined[key]) + if during.UserDefined[key] == "" || after.UserDefined[key] != during.UserDefined[key] { + t.Errorf("restore changed protection %s: %q -> %q", key, during.UserDefined[key], after.UserDefined[key]) } } - if rec := nativeRestoreLockRequest(t.Context(), t, r, oi.VersionID, kind, updated); rec.Code != http.StatusOK { - t.Fatalf("protection update after restore: %d %s", rec.Code, rec.Body.String()) - } - final, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) - if err != nil { - t.Fatal(err) - } - if !final.ModTime.Equal(before.ModTime) || final.ETag != before.ETag || final.Size != before.Size { - t.Error("Object Lock update changed content identity") - } changed := keys[0] if kind == "retention" { changed = keys[1] } - if final.UserDefined[changed] == before.UserDefined[changed] || final.UserDefined[keys[len(keys)-1]] == before.UserDefined[keys[len(keys)-1]] { - t.Error("new protection or its timestamp was not stored") + if after.UserDefined[changed] == before.UserDefined[changed] { + t.Error("the update made during the download was not kept") + } + if !after.ModTime.Equal(before.ModTime) || after.ETag != before.ETag || after.Size != before.Size { + t.Error("Object Lock update changed content identity") } }) } @@ -704,80 +679,6 @@ func TestNativeRestoreQueuedBehindDownload(t *testing.T) { } } -func TestNativeRestoreConcurrentCompletion(t *testing.T) { - for _, separateSets := range []bool{false, true} { - t.Run(fmt.Sprintf("separate-set-locks=%v", separateSets), func(t *testing.T) { - r := newRestoreSafetyTest(t, 1, versioning.Enabled) - if separateSets { - // Local production sets share one map. Independent maps model the - // distinct locker groups of distributed sets while reusing test disks. - pool := r.z.serverPools[0] - sets, count := pool.sets, pool.setCount - t.Cleanup(func() { pool.sets, pool.setCount = sets, count }) - first, second := *sets[0], *sets[0] - first.nsMutex, second.nsMutex = newNSLock(false), newNSLock(false) - second.setIndex = 1 - pool.sets, pool.setCount = []*erasureObjects{&first, &second}, 2 - for i := 0; ; i++ { - r.object = fmt.Sprintf("second-set-%d", i) - if pool.getHashedSetIndex(r.object) == 1 { - break - } - } - } - oi, body := nativeRestoreSeed(t, r, 0, false, true) - blocked := &nativeRestoreBlockedTier{WarmBackend: globalTierConfigMgr.drivercache["WARM"], entered: make(chan struct{}), release: make(chan struct{}), afterFirstPart: true} - globalTierConfigMgr.drivercache["WARM"] = blocked - var once sync.Once - unblock := func() { once.Do(func() { close(blocked.release) }) } - defer unblock() - ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) - defer cancel() - restored := make(chan error, 1) - go func() { - restored <- r.z.RestoreTransitionedObject(ctx, r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) - }() - select { - case <-blocked.entered: - case <-ctx.Done(): - t.Fatal("restore never started") - } - // A client that learns the internal upload ID, which listings no - // longer show, can try to complete the first part while native - // restore owns the object. - id, part := restoreIsolationUpload(t, r, 0) - completed := make(chan struct{}) - var completeErr error - go func() { - _, completeErr = r.z.CompleteMultipartUpload(ctx, r.bucket, r.object, id, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, r.opts(oi.VersionID)) - close(completed) - }() - waitForListingReadLock(t, r.z.serverPools[0].getHashedSet(r.object).nsMutex, r.bucket, r.object, completed) - unblock() - select { - case err := <-restored: - if err != nil { - t.Errorf("restore blocked by concurrent completion: %v", err) - } - case <-time.After(5 * time.Second): - t.Fatal("restore did not finish") - } - select { - case <-completed: - case <-time.After(5 * time.Second): - t.Fatal("completion did not finish") - } - if !errors.As(completeErr, &InvalidUploadID{}) { - t.Errorf("completion after native restore: %v, want InvalidUploadID", completeErr) - } - _, got, err := r.read(oi.VersionID) - if err != nil || !bytes.Equal(got, body) { - t.Fatalf("restore contents after concurrent completion: %v", err) - } - }) - } -} - type nativeRestorePartGateReader struct { io.ReadCloser gate *nativeRestoreBlockedTier diff --git a/cmd/restore-stage.go b/cmd/restore-stage.go new file mode 100644 index 000000000..1bc2a443e --- /dev/null +++ b/cmd/restore-stage.go @@ -0,0 +1,900 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "maps" + "net/http" + "slices" + "strconv" + "sync" + "time" + + "github.com/google/uuid" + "github.com/minio/minio/internal/bucket/lifecycle" + xhttp "github.com/minio/minio/internal/http" + "github.com/pgsty/silo-pkg/v3/sync/errgroup" +) + +// A native restore runs in two phases. It downloads the tier copy of a +// version and erasure-codes it into the tmp volume of the version's set +// without the object lock O, then takes O only to check the version again +// and publish the staged copy drive by drive, as heal does. Hot versions of +// the key stay readable and writable during the download. +// +// The restore lock R of the version tells whether a restore of it runs: a +// POST takes R before O, and the worker that rehydrates the version holds R +// for its whole run. Nothing else takes R, so nothing holds O while it waits +// for R. R is a liveness hint, not an exclusion: a lost refresh can let two +// workers overlap. Only the checks under O decide what is written. + +// restoreOwnerPrefix names the restore locks under minioMetaBucket. +const restoreOwnerPrefix = "restore" + +// restorePublicationDelays are the in-memory retries of a convergence after +// the one under the commit's own lock hold, see recoverRestorePublication. +var restorePublicationDelays = []time.Duration{100 * time.Millisecond, time.Second, 5 * time.Second, 30 * time.Second, 2 * time.Minute} + +var ( + // errRestoreMoved: the version moved to another pool after the restore + // staged its copy in this one. + errRestoreMoved = errors.New("restored version moved to another pool") + // errRestoreNotConverged: a publication that reached some drives but not + // a write quorum could not be brought back to the unrestored version yet. + errRestoreNotConverged = errors.New("restore publication did not converge") +) + +// restoreOwner is the restore lock R of one version, held by a POST that +// registers an attempt and then by the worker that runs it. ctx ends when the +// lock is released or when its holder learns that it lost it. +type restoreOwner struct { + lk RWLocker + lkctx LockContext + ctx context.Context + cancel context.CancelFunc + once sync.Once +} + +// release unlocks R once. It is a no-op on a nil owner. +func (o *restoreOwner) release() { + if o == nil { + return + } + o.once.Do(func() { + o.cancel() + o.lk.Unlock(o.lkctx) + }) +} + +// restoreOwnerResource is the name of the restore lock of a version. Bucket +// names hold no slash, object names no ".." element, and the version ID has +// a fixed form, so the name is unambiguous. +func restoreOwnerResource(bucket, object, versionID string) string { + return pathJoin(restoreOwnerPrefix, bucket, object, restoreVersionID(versionID)) +} + +// takeRestoreOwner takes the restore lock of a version. With a nil timeout it +// waits for as long as ctx lives, as a direct caller does; otherwise it gives +// up after timeout. ctx must outlive the restore that the lock is held for. +func (z *erasureServerPools) takeRestoreOwner(ctx context.Context, bucket, object, versionID string, timeout *dynamicTimeout) (*restoreOwner, error) { + for { + // A new instance for each try: a distributed unlock retries in the + // background and could release a later try of the same instance. + lk := z.NewNSLock(minioMetaBucket, restoreOwnerResource(bucket, object, versionID)) + wait := timeout + if wait == nil { + wait = globalRestoreLockTimeout + } + lkctx, err := lk.GetLock(ctx, wait) + if err != nil { + if timeout == nil && errors.As(err, &OperationTimedOut{}) && ctx.Err() == nil { + continue + } + return nil, err + } + octx, cancel := context.WithCancel(lkctx.Context()) + return &restoreOwner{lk: lk, lkctx: lkctx, ctx: octx, cancel: cancel}, nil + } +} + +// restoreStage is the copy of one version that a native restore encoded into +// the tmp volume of the version's set, under tmpID/dataDir, before it takes +// the object lock. +type restoreStage struct { + versionID string + // fi is the version as the download read it: the commit only publishes + // the copy for a version that still has this content, see + // restoreSameContent. + fi FileInfo + tmpID string + dataDir string + // staged tells, in the order of the version's erasure distribution, + // which drives hold every part of the copy. + staged []bool + expiry time.Time + // request is set for an attempt that POST registered: days and date + // are what it recorded. A direct caller keeps the version's own. + request bool + days int + date string +} + +// restoreSameContent reports whether two reads of a version name the same +// encoded bytes, laid out the same way: the version, its tier copy, its parts +// and its erasure layout. The tier copy is never rewritten under the same +// name and remote version, so these decide the bytes. Every other field, such +// as tags, Object Lock, replication state or a rewrapped object key, is taken +// from the version as the commit reads it. +func restoreSameContent(a, b FileInfo) bool { + if a.VersionID != b.VersionID || a.Deleted || b.Deleted || !a.ModTime.Equal(b.ModTime) || + a.Metadata["etag"] != b.Metadata["etag"] || a.Size != b.Size || + a.TransitionStatus != lifecycle.TransitionComplete || b.TransitionStatus != lifecycle.TransitionComplete || + a.TransitionTier != b.TransitionTier || a.TransitionedObjName != b.TransitionedObjName || + a.TransitionVersionID != b.TransitionVersionID { + return false + } + if len(a.Parts) != len(b.Parts) { + return false + } + for i := range a.Parts { + if a.Parts[i].Number != b.Parts[i].Number || a.Parts[i].Size != b.Parts[i].Size { + return false + } + } + ea, eb := a.Erasure, b.Erasure + return ea.Algorithm == eb.Algorithm && ea.DataBlocks == eb.DataBlocks && ea.ParityBlocks == eb.ParityBlocks && + ea.BlockSize == eb.BlockSize && slices.Equal(ea.Distribution, eb.Distribution) +} + +// restoreInlineKey marks a version whose data lives in its xl.meta. +const restoreInlineKey = ReservedMetadataPrefixLower + "inline-data" + +// stageRestore downloads the tier copy of the version opts address and +// erasure-codes it, part by part, into the tmp volume of the drives of this +// set, as heal writes the parts it rebuilds. It holds no object lock. A drive +// that fails to write any part, or that misses a part afterwards, for +// example because its tmp volume was moved away when its server restarted, +// takes no further part and no part in the commit. +func (er erasureObjects) stageRestore(ctx context.Context, bucket, object string, opts ObjectOptions) (*restoreStage, error) { + rreq := opts.Transition.RestoreRequest + if rreq == nil { + return nil, errInvalidArgument + } + gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID) + if err != nil { + return nil, toObjectErr(err, bucket, object, opts.VersionID) + } + fi := gen.fi + if fi.Deleted { + return nil, toObjectErr(errMethodNotAllowed, bucket, object) + } + if a := opts.Transition.attempt; a != nil && !a.ownsGeneration(gen, bucket, object) { + // A later POST took the version over, or it was replaced. + return nil, errRestoreSuperseded + } + disks := er.getDisks() + var total int64 + for _, p := range fi.Parts { + if p.Size < 0 { + return nil, toObjectErr(errFileCorrupt, bucket, object) + } + total += p.Size + } + if fi.TransitionStatus != lifecycle.TransitionComplete || len(fi.Parts) == 0 || total != fi.Size || + len(fi.Erasure.Distribution) != len(disks) { + return nil, InvalidObjectState{Bucket: bucket, Object: object} + } + erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize) + if err != nil { + return nil, toObjectErr(err, bucket, object) + } + writeQuorum := fi.WriteQuorum(er.defaultWQuorum()) + st := &restoreStage{ + versionID: restoreVersionID(fi.VersionID), + fi: fi, + tmpID: mustGetUUID(), + dataDir: mustGetUUID(), + expiry: lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days), + } + if opts.Transition.restoreDate != "" { + st.request, st.days, st.date = true, rreq.Days, opts.Transition.restoreDate + } + staged := false + defer func() { + if !staged { + er.deleteAll(context.Background(), minioMetaTmpBucket, st.tmpID) + } + }() + + // The tier copy is the encoded stream of the parts, encrypted or + // compressed as stored: the parts get exactly these bytes back. + oi := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) + gr, err := getTransitionedObjectReader(ctx, bucket, object, nil, http.Header{}, oi, opts) + if err != nil { + return nil, toObjectErr(err, bucket, object) + } + defer gr.Close() + + var buffer []byte + pool := globalBytePoolCap.Load() + if int64(pool.Width()) < fi.Erasure.BlockSize { + buffer = make([]byte, fi.Erasure.BlockSize) + } else { + buffer = pool.Get() + defer pool.Put(buffer) + } + buffer = buffer[:fi.Erasure.BlockSize] + + // Slots follow the erasure distribution: the drive in slot s gets shard + // s+1 of every part, and the commit gives it erasure index s+1. + slots := shuffleDisks(disks, fi.Erasure.Distribution) + for _, part := range fi.Parts { + partPath := pathJoin(st.tmpID, st.dataDir, fmt.Sprintf("part.%d", part.Number)) + writers := make([]io.Writer, len(slots)) + for s, disk := range slots { + if disk != nil { + writers[s] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, partPath, + erasure.ShardFileSize(part.Size), DefaultBitrotAlgorithm, erasure.ShardSize()) + } + } + n, err := erasure.Encode(ctx, io.LimitReader(gr, part.Size), writers, buffer, writeQuorum) + closeErrs := closeBitrotWriters(writers) + if err != nil { + return nil, toObjectErr(err, bucket, object) + } + if n != part.Size { + return nil, IncompleteBody{Bucket: bucket, Object: object} + } + for s := range slots { + if writers[s] == nil || closeErrs[s] != nil { + slots[s] = nil + } + } + if countOnlineDisks(slots) < writeQuorum { + return nil, toObjectErr(errErasureWriteQuorum, bucket, object) + } + } + // The tier copy must end with the last part. + var extra [1]byte + if n, _ := io.ReadFull(gr, extra[:]); n != 0 { + return nil, InvalidObjectState{Bucket: bucket, Object: object} + } + + // A tmp volume that was moved away and recreated by a later part lacks the + // earlier parts: find them before anything is published. CheckParts sees a + // part that is present and not shorter than its shards; a staged drive + // that wrote every part without an error has exactly that. + check := fi + check.DataDir = st.dataDir + g := errgroup.WithNErrs(len(slots)) + for s, disk := range slots { + if disk == nil { + continue + } + g.Go(func() error { + resp, err := disk.CheckParts(ctx, minioMetaTmpBucket, st.tmpID, check) + if err != nil { + return err + } + for _, result := range resp.Results { + if result != checkPartSuccess { + return errFileCorrupt + } + } + return nil + }, s) + } + st.staged = make([]bool, len(slots)) + count := 0 + for s, err := range g.Wait() { + if slots[s] != nil && err == nil { + st.staged[s] = true + count++ + } + } + if count < writeQuorum { + return nil, toObjectErr(errErasureWriteQuorum, bucket, object) + } + staged = true + return st, nil +} + +// fileInfoRevisions are the revisions the entry that writing fi creates can +// have, as readRestoreGeneration and the convergence identify the entries +// they read. The header flag that tells whether a transitioned version uses +// its data directory follows whether its restored copy has expired at the +// moment a drive writes the entry, so a restore whose copy expires while it +// publishes can find either value: both are this entry. Every other field of +// the header and the full metadata digest still have to match. +func fileInfoRevisions(fi FileInfo) ([]revisionKey, error) { + var x xlMetaV2 + if err := x.AddVersion(fi); err != nil { + return nil, err + } + buf, err := x.AppendTo(nil) + if err != nil { + return nil, err + } + var y xlMetaV2 + if err := y.Load(buf); err != nil { + return nil, err + } + if len(y.versions) != 1 { + return nil, errFileCorrupt + } + ver := y.versions[0] + other := ver + other.header.Flags ^= xlFlagUsesDataDir + return []revisionKey{versionRevision(ver, y.metaV), versionRevision(other, y.metaV)}, nil +} + +// restorePublication remembers, for a commit that published the staged copy +// on some drives but not on a write quorum, what each drive held before the +// commit (Sᵢ), what the commit wrote there (Pᵢ) and what a convergence writes +// there (Vᵢ), in the order of the version's erasure distribution. Only the +// worker that published knows them; a convergence touches a version only +// while every drive holds one of them. +type restorePublication struct { + bucket, object string + versionID string + uv [16]byte + dataDir string + attempt *restoreAttempt + quorum int + dist []int + before []*revisionKey + anyBefore map[revisionKey]bool + published map[revisionKey]bool + conv []FileInfo + convKeys map[revisionKey]bool +} + +// restorePublished is the entry a commit publishes on the drive in slot s: +// the version as the commit read it on that drive, with the staged data +// directory and a completed restore. Restore fields are written alike on +// every drive: the request POST recorded, or for a direct caller the one +// the authoritative generation auth holds. +func restorePublished(base, auth FileInfo, st *restoreStage, slot int) FileInfo { + fi := base + fi.Metadata = maps.Clone(base.Metadata) + if fi.Metadata == nil { + fi.Metadata = make(map[string]string) + } + for _, k := range restoreFieldKeys { + delete(fi.Metadata, k) + } + // An earlier restore may have inlined the data, and expiring that copy + // leaves the inline marker behind; this copy lives in its data directory. + delete(fi.Metadata, restoreInlineKey) + fi.Metadata[xhttp.AmzRestore] = completedRestoreObj(st.expiry).String() + if st.request { + fi.Metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(st.days) + fi.Metadata[xhttp.AmzRestoreRequestDate] = st.date + } else { + for _, k := range []string{xhttp.AmzRestoreExpiryDays, xhttp.AmzRestoreRequestDate} { + if v, ok := auth.Metadata[k]; ok { + fi.Metadata[k] = v + } + } + } + fi.DataDir = st.dataDir + fi.Data = nil + fi.Parts = slices.Clone(base.Parts) + fi.Erasure.Index = slot + 1 + // The staged parts use the default streaming bitrot algorithm. + fi.Erasure.Checksums = nil + return fi +} + +// restoreUnrestored is the entry a convergence writes on the drive in slot s: +// the version as the commit read it, with no restore fields, so that its tier +// copy serves reads again and a later POST can restore it. +func restoreUnrestored(base FileInfo, slot int) FileInfo { + fi := base + fi.Metadata = maps.Clone(base.Metadata) + for _, k := range restoreFieldKeys { + delete(fi.Metadata, k) + } + delete(fi.Metadata, restoreInlineKey) + fi.Data = nil + fi.Erasure.Index = slot + 1 + return fi +} + +// commitRestore publishes a staged copy while the caller holds the object +// lock. It reads the version again, checks that the attempt still owns it and +// that it still has the content the copy was staged for, then publishes the +// copy on every staged drive that holds the version, with RenameData on each +// drive, as heal does: never through erasure renameData, whose undo of a +// replaced version could delete the new data and keep metadata that points +// at it. +// +// It returns nil when the copy reached a write quorum. Otherwise it returns +// an error, and a publication when some drive may already have the copy; the +// caller must then converge, see convergeRestore. +func (er erasureObjects) commitRestore(ctx context.Context, bucket, object string, st *restoreStage, opts ObjectOptions, placed func(context.Context) error) (*restorePublication, error) { + if placed != nil { + if err := placed(ctx); err != nil { + return nil, err + } + } + gen, err := er.readRestoreGeneration(ctx, bucket, object, st.versionID) + if err != nil { + return nil, toObjectErr(err, bucket, object, st.versionID) + } + if gen.fi.Deleted { + return nil, toObjectErr(errMethodNotAllowed, bucket, object) + } + a := opts.Transition.attempt + if a != nil && !a.ownsGeneration(gen, bucket, object) { + return nil, errRestoreSuperseded + } + if !restoreSameContent(st.fi, gen.fi) { + // Replaced, or its tier copy or layout changed since the download. + return nil, InvalidObjectState{Bucket: bucket, Object: object} + } + disks := er.getDisks() + dist := gen.fi.Erasure.Distribution + if len(dist) != len(disks) || len(gen.revs) != len(disks) { + return nil, InvalidObjectState{Bucket: bucket, Object: object} + } + n := len(disks) + writeQuorum := gen.fi.WriteQuorum(er.defaultWQuorum()) + + pub := &restorePublication{ + bucket: bucket, object: object, versionID: st.versionID, dataDir: st.dataDir, + attempt: a, quorum: writeQuorum, dist: slices.Clone(dist), + before: make([]*revisionKey, n), + anyBefore: make(map[revisionKey]bool), + published: make(map[revisionKey]bool), + conv: make([]FileInfo, n), + convKeys: make(map[revisionKey]bool), + } + if st.versionID != nullVersionID { + id, err := uuid.Parse(st.versionID) + if err != nil { + return nil, toObjectErr(errFileVersionNotFound, bucket, object, st.versionID) + } + pub.uv = id + } + targets := make([]StorageAPI, n) + files := make([]FileInfo, n) + for i := range disks { + s := dist[i] - 1 + if s < 0 || s >= n { + return nil, InvalidObjectState{Bucket: bucket, Object: object} + } + if gen.revs[i] != nil { + pub.before[s] = gen.revs[i] + pub.anyBefore[*gen.revs[i]] = true + } + v := restoreUnrestored(gen.fi, s) + keys, err := fileInfoRevisions(v) + if err != nil { + return nil, toObjectErr(err, bucket, object) + } + pub.conv[s] = v + // The unrestored entry has no restore header: its flag never varies. + pub.convKeys[keys[0]] = true + if gen.disks[i] == nil || !st.staged[s] { + continue + } + fi := restorePublished(gen.drives[i], gen.fi, st, s) + if keys, err = fileInfoRevisions(fi); err != nil { + return nil, toObjectErr(err, bucket, object) + } + for _, key := range keys { + pub.published[key] = true + } + targets[s], files[s] = gen.disks[i], fi + } + if countOnlineDisks(targets) < writeQuorum { + // Nothing is published. + return nil, toObjectErr(errErasureWriteQuorum, bucket, object) + } + + errs := make([]error, n) + olds := make([]string, n) + var wg sync.WaitGroup + for s, disk := range targets { + if disk == nil { + errs[s] = errDiskNotFound + continue + } + wg.Add(1) + go func() { + defer wg.Done() + resp, err := disk.RenameData(ctx, minioMetaTmpBucket, st.tmpID, files[s], bucket, object, RenameOptions{}) + olds[s], errs[s] = resp.OldDataDir, err + }() + } + wg.Wait() + + if countErrs(errs, nil) < writeQuorum { + return pub, toObjectErr(errErasureWriteQuorum, bucket, object) + } + // The data directory of the replaced entry, which held a former restored + // copy, goes only when every drive that published reports the same one: + // RenameData reports none where another version shares it, and then the + // directory stays everywhere. It goes only from the drives that reported + // it. + old := "" + cleanup := make([]StorageAPI, n) + for s := range targets { + if errs[s] != nil { + continue + } + if olds[s] == "" || (old != "" && olds[s] != old) { + old = "" + break + } + old = olds[s] + cleanup[s] = targets[s] + } + if old != "" && old != st.dataDir { + er.commitRenameDataDir(ctx, bucket, object, old, cleanup, 1) + } + if countErrs(errs, nil) < n { + er.addPartial(bucket, object, gen.fi.VersionID) + } + return nil, nil +} + +// restoreConvergence is the outcome of one convergence pass. +type restoreConvergence int + +const ( + // restoreIncomplete: the drives are not settled yet; try again. + restoreIncomplete restoreConvergence = iota + // restoreConverged: every drive that holds the version holds it + // unrestored again. + restoreConverged + // restoreCommitted: the copy reached a write quorum after all, such as + // when a drive applied a publication whose answer was lost. + restoreCommitted + // restoreStopped: some drive holds an entry that neither this attempt + // nor the version before it wrote: someone else changed the version. + restoreStopped +) + +// classify reads the entry of the version on every drive and sorts it into +// the slot of its drive: held tells which slots hold an entry this +// publication knows, published and converged count the drives that hold +// what the commit and the convergence write. A drive that cannot be read is +// left out. An entry the publication does not know stops it. +func (pub *restorePublication) classify(ctx context.Context, disks []StorageAPI) (held []bool, published, converged int, err error) { + raws, errs := readAllRawFileInfo(ctx, disks, pub.bucket, pub.object, false) + held = make([]bool, len(disks)) + for i := range disks { + if errs[i] != nil || i >= len(pub.dist) { + continue + } + var xl xlMetaV2 + if xl.LoadOrConvert(raws[i].Buf) != nil { + continue + } + s := pub.dist[i] - 1 + for _, ver := range xl.versions { + if ver.header.VersionID != pub.uv || ver.header.FreeVersion() { + continue + } + key := versionRevision(ver, xl.metaV) + switch { + case pub.published[key]: + published++ + case pub.convKeys[key]: + converged++ + case pub.before[s] != nil && *pub.before[s] == key: + case pub.before[s] == nil && pub.anyBefore[key]: + default: + return nil, 0, 0, errRestoreSuperseded + } + held[s] = true + break + } + } + return held, published, converged, nil +} + +// convergeRestore settles a publication while the caller holds the object +// lock. When every drive still holds what the version held before the commit, +// what the commit wrote, or what an earlier pass wrote, it either finds the +// copy on a write quorum, or writes the unrestored version to every drive +// that holds the version. It deletes no directory: an unreferenced staged +// directory is left to abandoned-data cleanup. Any other entry stops it, so +// that it never overwrites what someone else wrote between two passes. +func (er erasureObjects) convergeRestore(ctx context.Context, pub *restorePublication) (restoreConvergence, error) { + disks := er.getDisks() + held, published, _, err := pub.classify(ctx, disks) + if err != nil { + return restoreStopped, err + } + if published >= pub.quorum { + er.addPartial(pub.bucket, pub.object, pub.versionID) + return restoreCommitted, nil + } + slots := shuffleDisks(disks, pub.dist) + files := make([]FileInfo, len(slots)) + for s := range slots { + if !held[s] { + slots[s] = nil + continue + } + files[s] = pub.conv[s] + } + _, werr := writeUniqueFileInfo(ctx, slots, "", pub.bucket, pub.object, files, pub.quorum) + + held, published, converged, err := pub.classify(ctx, disks) + if err != nil { + return restoreStopped, err + } + holding := 0 + for _, h := range held { + if h { + holding++ + } + } + if published == 0 && converged == holding && converged >= pub.quorum { + return restoreConverged, nil + } + if werr == nil { + werr = errErasureWriteQuorum + } + return restoreIncomplete, werr +} + +// recoverRestorePublication retries a convergence in memory, for a bounded +// time, each pass under the object lock taken afresh. A restart loses the +// retries; the version then stays as the drives hold it, see the design's +// excluded residuals. +func (er erasureObjects) recoverRestorePublication(pub *restorePublication, lock func(context.Context) (LockContext, func(), error)) { + go func() { + err := errRestoreNotConverged + for _, delay := range restorePublicationDelays { + select { + case <-GlobalContext.Done(): + return + case <-time.After(delay): + } + ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout) + lkctx, unlock, lerr := lock(ctx) + if lerr != nil { + cancel() + err = lerr + continue + } + var res restoreConvergence + res, err = er.convergeRestore(lkctx.Context(), pub) + unlock() + cancel() + if res == restoreConverged || res == restoreCommitted { + return + } + if res == restoreStopped { + break + } + } + er.logRestoreNotConverged(pub, err) + }() +} + +func (er erasureObjects) logRestoreNotConverged(pub *restorePublication, err error) { + attempt := "" + if pub.attempt != nil { + attempt = pub.attempt.ID + } + storageLogIf(GlobalContext, fmt.Errorf("%w: %s/%s(%s), attempt %q, data dir %s: %v", + errRestoreNotConverged, pub.bucket, pub.object, pub.versionID, attempt, pub.dataDir, err)) +} + +// restoreHooks tie a restore of one set to its caller. +type restoreHooks struct { + // lock takes the object lock O of the version, in the lock domain of the + // object's writes, and returns its release. + lock func(ctx context.Context) (LockContext, func(), error) + // placed tells, under O, whether the version still lives in this set. + placed func(ctx context.Context) error + // clear clears the attempt's ongoing marker after a failure that + // published nothing, taking O itself, and retries in the background. + clear func() +} + +// restoreVersion rehydrates the transitioned version opts address in this +// set: it stages the copy without the object lock, then commits it under the +// lock. A failure that published nothing clears the attempt's marker. A +// publication that reached some drives but not a write quorum converges +// under the same lock hold, and in bounded background retries after it. +func (er erasureObjects) restoreVersion(ctx context.Context, bucket, object string, opts ObjectOptions, h restoreHooks) error { + st, err := er.stageRestore(ctx, bucket, object, opts) + if err != nil { + return restoreAbandon(err, h) + } + defer er.deleteAll(context.Background(), minioMetaTmpBucket, st.tmpID) + + lkctx, unlock, err := h.lock(ctx) + if err != nil { + return restoreAbandon(err, h) + } + pub, err := er.commitRestore(lkctx.Context(), bucket, object, st, opts, h.placed) + if err == nil { + unlock() + return nil + } + if pub == nil { + unlock() + return restoreAbandon(err, h) + } + res, cerr := er.convergeRestore(lkctx.Context(), pub) + unlock() + switch res { + case restoreCommitted: + return nil + case restoreConverged: + return err + case restoreStopped: + er.logRestoreNotConverged(pub, cerr) + return err + default: + er.recoverRestorePublication(pub, h.lock) + return err + } +} + +// restoreAbandon clears the marker of an attempt that published nothing, +// unless another attempt owns the version. A not-found error may come from the +// tier rather than from the version, and the clear checks the marker's owner +// and finds nothing to do for a version that is gone, so it runs then too. +func restoreAbandon(err error, h restoreHooks) error { + if errors.Is(err, errRestoreSuperseded) { + return err + } + if h.clear != nil { + h.clear() + } + return err +} + +// RestoreTransitionedObject - restore transitioned object content locally on +// this cluster. This is similar to PostObjectRestore from AWS GLACIER storage +// class. When PostObjectRestore API is called, a temporary copy of the object +// is restored locally to the bucket on source cluster until the restore +// expiry date. The copy that was transitioned continues to reside in the +// transitioned tier. +// +// The set entry serves direct callers of one set. It takes the object lock in +// the set's own domain and no restore lock; native restores of a cluster go +// through erasureServerPools. +func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + lock := func(ctx context.Context) (LockContext, func(), error) { + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return LockContext{}, nil, err + } + return lkctx, func() { lk.Unlock(lkctx) }, nil + } + clearMarker := func() { + ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout) + defer cancel() + lkctx, unlock, err := lock(ctx) + if err != nil { + storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, err)) + return + } + defer unlock() + versionID := opts.VersionID + if a := opts.Transition.attempt; a != nil { + versionID = a.VersionID + } + if err := er.clearRestoreStatus(lkctx.Context(), bucket, object, versionID, opts.Transition.attempt); err != nil && + !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !isErrMethodNotAllowed(err) { + storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, versionID, err)) + } + } + return er.restoreVersion(ctx, bucket, object, opts, restoreHooks{lock: lock, clear: clearMarker}) +} + +// RestoreTransitionedObject - restore transitioned object content locally on +// this cluster, see erasureObjects.RestoreTransitionedObject. +// +// A POST hands over the restore lock R of the version it registered in +// opts.Transition.owner; a direct caller takes R here, waiting while ctx +// lives. The restore runs while R is held, and releases it when it returns. +func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + object = encodeDirObject(object) + owner := opts.Transition.owner + opts.Transition.owner = nil + if owner != nil && owner.ctx.Err() != nil { + // The lock POST handed over ended before the restore ran, such as + // when its holder lost it while the restore was queued: take it again + // as a direct caller does. The attempt still has to own the version. + owner = nil + } + defer func() { owner.release() }() + + // Avoid restoring object from a pool being decommissioned. + opts.SkipDecommissioned = true + opts.NoLock = true + clearMarker := func() { + if err := z.resetRestoreStatus(bucket, object, opts); err != nil { + z.recoverRestoreStatus(bucket, object, opts) + } + } + idx, err := z.restorePoolIdx(ctx, bucket, object, opts) + if err != nil { + var reset restoreResetError + if errors.As(err, &reset) { + clearMarker() + return reset.error + } + if isErrObjectNotFound(err) && opts.VersionID != "" { + err = VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID} + } + return err + } + // A server may have been replaced by an older binary while the restore + // was queued: such a server knows neither the restore lock nor the + // staged commit. + if err := nativeRestoreAvailable(ctx); err != nil { + clearMarker() + return err + } + er := z.serverPools[idx].getHashedSet(object) + if owner == nil { + if opts.VersionID == "" { + // Restore the version that is latest now, by its own ID. + gen, err := er.readRestoreGeneration(ctx, bucket, object, "") + if err != nil { + return toObjectErr(err, bucket, object) + } + opts.VersionID = restoreVersionID(gen.fi.VersionID) + } + if owner, err = z.takeRestoreOwner(ctx, bucket, object, opts.VersionID, nil); err != nil { + return err + } + } + return er.restoreVersion(owner.ctx, bucket, object, opts, restoreHooks{ + lock: func(ctx context.Context) (LockContext, func(), error) { + lk := z.restoreLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return LockContext{}, nil, err + } + return lkctx, func() { lk.Unlock(lkctx) }, nil + }, + placed: func(ctx context.Context) error { + got, err := z.restorePoolIdx(ctx, bucket, object, opts) + var reset restoreResetError + if errors.As(err, &reset) { + return reset.error + } + if err != nil { + return err + } + if got != idx { + return errRestoreMoved + } + return nil + }, + clear: clearMarker, + }) +} diff --git a/cmd/restore-two-phase_test.go b/cmd/restore-two-phase_test.go new file mode 100644 index 000000000..ba3e981f1 --- /dev/null +++ b/cmd/restore-two-phase_test.go @@ -0,0 +1,858 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/config/storageclass" + xhttp "github.com/minio/minio/internal/http" +) + +// A native restore downloads its tier copy without the object lock: while +// the download of an old version is paused, the hot versions of the same key +// stay readable and writable. Only the short commit takes the object lock. +func TestRestoreDownloadKeepsKeyAvailable(t *testing.T) { + for _, pools := range []int{1, 2} { + for _, multipart := range []bool{false, true} { + t.Run(fmt.Sprintf("pools=%d/multipart=%v", pools, multipart), func(t *testing.T) { + r := newRestoreSafetyTest(t, pools, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := nativeRestoreSeed(t, r, 0, false, multipart) + hot := putConsistencyObject(t, r.z, r.bucket, r.object, 0, "hot latest", ObjectOptions{Versioned: true, MTime: oi.ModTime.Add(time.Hour)}) + putConsistencyObject(t, r.z, r.bucket, "unrelated", 0, "unrelated hot object", ObjectOptions{Versioned: true}) + gate := newRestoreIsolationGateTier(1, nil) + globalTierConfigMgr.drivercache["WARM"] = gate + defer gate.unblock() + done := make(chan error, 1) + go func() { + done <- r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + restoreIsolationEntered(t, gate) + + within := func(name string, fn func(ctx context.Context) error) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + start := time.Now() + if err := fn(ctx); err != nil { + t.Fatalf("%s during a paused restore download: %v after %s", name, err, time.Since(start)) + } + } + for _, key := range []string{"unrelated", r.object} { + within("GET "+key, func(ctx context.Context) error { + gr, err := r.z.GetObjectNInfo(ctx, r.bucket, key, nil, http.Header{}, ObjectOptions{}) + if err != nil { + return err + } + defer gr.Close() + _, err = io.ReadAll(gr) + return err + }) + } + within("HEAD hot latest", func(ctx context.Context) error { + got, err := r.z.GetObjectInfo(ctx, r.bucket, r.object, ObjectOptions{}) + if err == nil && got.VersionID != hot.VersionID { + err = fmt.Errorf("latest is %s, want %s", got.VersionID, hot.VersionID) + } + return err + }) + within("PutObjectTags hot latest", func(ctx context.Context) error { + _, err := r.z.PutObjectTags(ctx, r.bucket, r.object, "during=restore", ObjectOptions{VersionID: hot.VersionID, Versioned: true}) + return err + }) + // The download publishes nothing: no upload, public or not. + if uploads := restoreIsolationUploads(t, r, 0); len(uploads) != 0 { + t.Errorf("restore created uploads: %v", uploads) + } + within("PUT new version", func(ctx context.Context) error { + data := []byte("written during restore") + _, err := r.z.PutObject(ctx, r.bucket, r.object, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{Versioned: true}) + return err + }) + + gate.unblock() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Minute): + t.Fatal("restore did not complete after releasing the tier") + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored version mismatch: %v", err) + } + }) + } + } +} + +// restoreTwoPhaseStagedCommitFailure makes a multipart completion stage its +// object metadata in the upload and then fail before moving any data. +type restoreTwoPhaseStagedCommitFailure struct { + StorageAPI + bucket, object string +} + +func (d restoreTwoPhaseStagedCommitFailure) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) { + if sv == minioMetaMultipartBucket && dv == d.bucket && dp == d.object { + if err := d.WriteMetadata(ctx, "", sv, sp, fi); err != nil { + return RenameDataResp{}, err + } + return RenameDataResp{}, errFaultyDisk + } + return d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts) +} + +// An ordinary upload whose completion failed after staging stays usable as +// on main: the client re-uploads its parts and completes, or aborts. +func TestMultipartFailedCommitRetriesLikeMain(t *testing.T) { + for _, abort := range []bool{false, true} { + t.Run(fmt.Sprintf("abort=%v", abort), func(t *testing.T) { + z, bucket := consistencyPools(t) + initConfigSubsystem(t.Context(), z) + const object = "public-upload" + er := z.serverPools[0].getHashedSet(object) + ctx := t.Context() + mp, err := er.NewMultipartUpload(ctx, bucket, object, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + body := []byte("ordinary multipart data") + putPart := func() (PartInfo, error) { + return er.PutObjectPart(ctx, bucket, object, mp.UploadID, 1, + mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}) + } + p, err := putPart() + if err != nil { + t.Fatal(err) + } + disks := er.getDisks() + wrapped := make([]StorageAPI, len(disks)) + for i, d := range disks { + wrapped[i] = restoreTwoPhaseStagedCommitFailure{d, bucket, object} + } + previous := er.getDisks + er.getDisks = func() []StorageAPI { return wrapped } + _, err = er.CompleteMultipartUpload(ctx, bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: p.ETag}}, ObjectOptions{}) + er.getDisks = previous + if err == nil { + t.Fatal("fault injection failed") + } + if _, err := er.GetMultipartInfo(ctx, bucket, object, mp.UploadID, ObjectOptions{}); err != nil { + t.Fatalf("upload after a failed commit: %v", err) + } + if abort { + if err := er.AbortMultipartUpload(ctx, bucket, object, mp.UploadID, ObjectOptions{}); err != nil { + t.Fatalf("abort after a failed commit: %v", err) + } + if _, err := er.GetMultipartInfo(ctx, bucket, object, mp.UploadID, ObjectOptions{}); !errors.As(err, &InvalidUploadID{}) { + t.Fatalf("aborted upload still answers: %v", err) + } + return + } + if p, err = putPart(); err != nil { + t.Fatalf("re-upload a part after a failed commit: %v", err) + } + if _, err := er.CompleteMultipartUpload(ctx, bucket, object, mp.UploadID, []CompletePart{{PartNumber: 1, ETag: p.ETag}}, ObjectOptions{}); err != nil { + t.Fatalf("complete after re-uploading: %v", err) + } + gr, err := er.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + defer gr.Close() + if got, err := io.ReadAll(gr); err != nil || !bytes.Equal(got, body) { + t.Fatalf("completed object = %q, %v", got, err) + } + }) + } +} + +// restoreTwoPhaseFaults are the faults restoreTwoPhaseDisk injects, shared by +// a test and its drives so that the test can turn them off. +type restoreTwoPhaseFaults struct { + // rename: "" passes the publication through, "fail" fails it without + // applying it, "apply-fail" applies it and then reports a failure. + rename atomic.Value + // oldDataDir, when set, replaces the old data directory a publication + // reports. + oldDataDir atomic.Pointer[string] + // writeFails fails this many metadata writes of the object. + writeFails atomic.Int32 + // writes counts the metadata writes of the object. + writes atomic.Int32 + // recreateTmp drops the staging directory before the second part is + // written, as a server restart moves the tmp volume away. + recreateTmp atomic.Bool +} + +func (f *restoreTwoPhaseFaults) renameMode() string { + m, _ := f.rename.Load().(string) + return m +} + +type restoreTwoPhaseDisk struct { + StorageAPI + bucket, object string + f *restoreTwoPhaseFaults +} + +func (d restoreTwoPhaseDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) { + if sv != minioMetaTmpBucket || dv != d.bucket || dp != d.object { + return d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts) + } + switch d.f.renameMode() { + case "fail": + return RenameDataResp{}, errFaultyDisk + case "apply-fail": + if _, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts); err != nil { + return RenameDataResp{}, err + } + return RenameDataResp{}, errFaultyDisk + } + resp, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts) + if old := d.f.oldDataDir.Load(); old != nil { + resp.OldDataDir = *old + } + return resp, err +} + +func (d restoreTwoPhaseDisk) WriteMetadata(ctx context.Context, orig, vol, path string, fi FileInfo) error { + if vol == d.bucket && path == d.object { + d.f.writes.Add(1) + if d.f.writeFails.Load() > 0 { + d.f.writeFails.Add(-1) + return errFaultyDisk + } + } + return d.StorageAPI.WriteMetadata(ctx, orig, vol, path, fi) +} + +func (d restoreTwoPhaseDisk) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) error { + if volume == minioMetaTmpBucket && strings.HasSuffix(path, "/part.2") && d.f.recreateTmp.Load() { + tmpID, _, _ := strings.Cut(path, "/") + if err := os.RemoveAll(filepath.Join(d.Endpoint().Path, minioMetaTmpBucket, tmpID)); err != nil { + return err + } + } + return d.StorageAPI.CreateFile(ctx, origvolume, volume, path, size, reader) +} + +// restoreTwoPhaseFaulty wraps the drives of the object's set in the first +// pool; faults[i] is nil for a drive left as it is. It returns the plain +// drives, for reads that see what is stored. +func restoreTwoPhaseFaulty(t *testing.T, r *restoreTest, faults ...*restoreTwoPhaseFaults) []StorageAPI { + t.Helper() + er := r.z.serverPools[0].getHashedSet(r.object) + disks := er.getDisks() + previous := er.getDisks + t.Cleanup(func() { er.getDisks = previous }) + wrapped := append([]StorageAPI(nil), disks...) + for i, f := range faults { + if f != nil { + wrapped[i] = restoreTwoPhaseDisk{StorageAPI: disks[i], bucket: r.bucket, object: r.object, f: f} + } + } + er.getDisks = func() []StorageAPI { return wrapped } + return disks +} + +// restoreTwoPhaseEntries reads the version from each drive. +func restoreTwoPhaseEntries(t *testing.T, r *restoreTest, disks []StorageAPI, versionID string) []FileInfo { + t.Helper() + infos, errs := readAllFileInfo(t.Context(), disks, "", r.bucket, r.object, restoreTestVersionKey(versionID), false, false) + for i, err := range errs { + if err != nil { + t.Fatalf("drive %d: %v", i, err) + } + } + return infos +} + +func restoreTwoPhaseUnrestored(fi FileInfo) bool { + return fi.Metadata[xhttp.AmzRestore] == "" && fi.Metadata[restoreAttemptKey] == "" && fi.IsRemote() +} + +func restoreTwoPhaseRestored(fi FileInfo) bool { + st, err := parseRestoreObjStatus(fi.Metadata[xhttp.AmzRestore]) + return err == nil && !st.Ongoing() && !fi.IsRemote() && fi.Metadata[restoreAttemptKey] == "" +} + +// restoreTwoPhaseDelays shortens the convergence retries for a test. +func restoreTwoPhaseDelays(t *testing.T, delays ...time.Duration) { + previous := restorePublicationDelays + restorePublicationDelays = delays + t.Cleanup(func() { restorePublicationDelays = previous }) +} + +// A publication that reaches fewer drives than the write quorum is brought +// back to the unrestored version on every drive, under the same lock hold. +// No directory is deleted: the staged one is left to abandoned-data cleanup. +// The version restores again once the drives work. +func TestRestorePublicationBelowQuorumReverts(t *testing.T) { + for _, multipart := range []bool{false, true} { + t.Run(fmt.Sprintf("multipart=%v", multipart), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, 0, false, multipart) + faults := &restoreTwoPhaseFaults{} + faults.rename.Store("fail") + disks := restoreTwoPhaseFaulty(t, r, faults, faults) + before := restoreTwoPhaseEntries(t, r, disks, oi.VersionID) + err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + if err == nil { + t.Fatal("a publication on two of four drives succeeded") + } + for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) { + if !restoreTwoPhaseUnrestored(fi) || fi.DataDir != before[i].DataDir { + t.Errorf("drive %d after a failed publication: restore %q, remote %v, data dir %s (was %s)", + i, fi.Metadata[xhttp.AmzRestore], fi.IsRemote(), fi.DataDir, before[i].DataDir) + } + } + // The drives that published keep the staged directory. + for i := 2; i < len(disks); i++ { + entries, err := os.ReadDir(filepath.Join(disks[i].Endpoint().Path, r.bucket, r.object)) + if err != nil { + t.Fatal(err) + } + dirs := 0 + for _, e := range entries { + if e.IsDir() && e.Name() != before[i].DataDir { + dirs++ + } + } + if dirs == 0 { + t.Errorf("drive %d: the published data directory was deleted", i) + } + } + faults.rename.Store("") + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatalf("restore after the drives recovered: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } + }) + } +} + +// A publication whose drives applied it but did not answer is found on a +// write quorum when the worker reads the drives again: the restore stands. +func TestRestorePublicationUncertainCommits(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, 0, false, true) + faults := &restoreTwoPhaseFaults{} + faults.rename.Store("apply-fail") + disks := restoreTwoPhaseFaulty(t, r, faults, faults) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatalf("restore whose answers were lost: %v", err) + } + for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) { + if !restoreTwoPhaseRestored(fi) { + t.Errorf("drive %d: not restored: %q", i, fi.Metadata[xhttp.AmzRestore]) + } + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } +} + +// A convergence that cannot write every drive under the commit's lock hold +// finishes in a background retry. +func TestRestoreConvergenceRetries(t *testing.T) { + restoreTwoPhaseDelays(t, 50*time.Millisecond, 50*time.Millisecond, 50*time.Millisecond) + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + failing, flaky := &restoreTwoPhaseFaults{}, &restoreTwoPhaseFaults{} + failing.rename.Store("fail") + flaky.writeFails.Store(1) + disks := restoreTwoPhaseFaulty(t, r, failing, failing, flaky) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err == nil { + t.Fatal("a publication on two of four drives succeeded") + } + deadline := time.Now().Add(restoreTestWait) + for { + settled := true + for _, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) { + settled = settled && restoreTwoPhaseUnrestored(fi) + } + if settled { + return + } + if time.Now().After(deadline) { + t.Fatal("background convergence did not finish") + } + time.Sleep(20 * time.Millisecond) + } +} + +// A convergence retry never overwrites what someone else wrote between two +// lock holds: a metadata update or a newer attempt's marker stops it. +func TestRestoreConvergenceStopsOnOtherWrites(t *testing.T) { + for _, other := range []string{"tags", "successor"} { + t.Run(other, func(t *testing.T) { + restoreTwoPhaseDelays(t, 300*time.Millisecond, 300*time.Millisecond, 300*time.Millisecond) + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, false) + failing, stuck := &restoreTwoPhaseFaults{}, &restoreTwoPhaseFaults{} + failing.rename.Store("fail") + stuck.writeFails.Store(1 << 20) + disks := restoreTwoPhaseFaulty(t, r, failing, failing, stuck) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err == nil { + t.Fatal("a publication on two of four drives succeeded") + } + var successor string + release := make(chan struct{}) + layer := newRestoreIsolationLayer(r) + switch other { + case "tags": + if _, err := r.z.PutObjectTags(t.Context(), r.bucket, r.object, "kept=yes", r.opts(oi.VersionID)); err != nil { + t.Fatal(err) + } + case "successor": + layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + <-release + return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) + } + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + close(release) + t.Fatalf("successor POST: %d %s", rec.Code, rec.Body.String()) + } + successor = restoreTwoPhaseEntries(t, r, disks, oi.VersionID)[0].Metadata[restoreAttemptKey] + if successor == "" { + close(release) + t.Fatal("successor POST recorded no attempt") + } + } + writes := stuck.writes.Load() + time.Sleep(time.Second) + if got := stuck.writes.Load(); got != writes { + t.Errorf("a stopped convergence kept writing: %d -> %d writes", writes, got) + } + for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) { + if i == 2 { + continue // the drive the convergence could not write + } + switch other { + case "tags": + if fi.Metadata[xhttp.AmzObjectTagging] != "kept=yes" { + t.Errorf("drive %d: the tags written between retries were overwritten: %q", i, fi.Metadata[xhttp.AmzObjectTagging]) + } + case "successor": + if fi.Metadata[restoreAttemptKey] != successor { + t.Errorf("drive %d: the successor's marker was overwritten: %q", i, fi.Metadata[restoreAttemptKey]) + } + } + } + if other == "successor" { + failing.rename.Store("") + stuck.writeFails.Store(0) + close(release) + if err := layer.wait(t); err != nil { + t.Fatalf("successor restore: %v", err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } + } + }) + } +} + +// The data directory of the replaced entry goes only where every drive that +// reports it agrees on it; a drive that does not report it, as when another +// version shares the directory, keeps the directory everywhere. +func TestRestoreRemovesOldDataDirOnlyWhenUnanimous(t *testing.T) { + for _, mixed := range []bool{false, true} { + t.Run(fmt.Sprintf("mixed=%v", mixed), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, 0, false, true) + old := r.fileInfo(oi.VersionID).DataDir + if old == "" { + t.Fatal("fixture: the transitioned version has no data directory") + } + faults := &restoreTwoPhaseFaults{} + if mixed { + none := "" + faults.oldDataDir.Store(&none) + } + restoreTwoPhaseFaulty(t, r, faults) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } + n := r.dataDirs(old) + if mixed && n != 4 { + t.Errorf("drives disagreed on the old data directory, but %d of 4 kept it", n) + } + if !mixed && n != 0 { + t.Errorf("the unanimous old data directory remained on %d drives", n) + } + }) + } +} + +// An earlier restore of a small object may have inlined its data, and the +// expiry of that copy leaves the inline marker behind. A new restore writes +// a data directory and must not publish the marker with it. +func TestRestoreDropsStaleInlineMarker(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, 0, false, false) + disks := r.z.serverPools[0].getHashedSet(r.object).getDisks() + for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) { + fi.SetInlineData() + if err := disks[i].WriteMetadata(t.Context(), "", r.bucket, r.object, fi); err != nil { + t.Fatal(err) + } + } + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatal(err) + } + for i, fi := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) { + if fi.InlineData() { + t.Errorf("drive %d: the restored copy is marked inline", i) + } + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } +} + +// A drive whose staging directory was moved away and recreated by a later +// part misses the earlier parts: it is left out of the commit, and when too +// few drives are left nothing is published at all. +func TestRestoreStagingDropsRecreatedTmp(t *testing.T) { + for _, recreated := range []int{1, 2} { + t.Run(fmt.Sprintf("drives=%d", recreated), func(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, body := nativeRestoreSeed(t, r, 0, false, true) + faults := &restoreTwoPhaseFaults{} + faults.recreateTmp.Store(true) + wrap := make([]*restoreTwoPhaseFaults, recreated) + for i := range wrap { + wrap[i] = faults + } + disks := restoreTwoPhaseFaulty(t, r, wrap...) + err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + entries := restoreTwoPhaseEntries(t, r, disks, oi.VersionID) + if recreated == 2 { + if err == nil { + t.Fatal("a copy staged completely on two of four drives was published") + } + for i, fi := range entries { + if !restoreTwoPhaseUnrestored(fi) { + t.Errorf("drive %d after a failed staging: %q", i, fi.Metadata[xhttp.AmzRestore]) + } + } + return + } + if err != nil { + t.Fatal(err) + } + if restoreTwoPhaseRestored(entries[0]) { + t.Error("the drive with missing parts got the restored copy") + } + for i := 1; i < len(entries); i++ { + if !restoreTwoPhaseRestored(entries[i]) { + t.Errorf("drive %d: not restored", i) + } + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } + }) + } +} + +// A POST while the worker of an earlier POST holds the restore lock answers +// that a restore is in progress; it starts no second worker. +func TestRestoreSecondPOSTWhileRunning(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, body := restoreIsolationSeed(t, r, 0, false, true) + release := make(chan struct{}) + layer := newRestoreIsolationLayer(r) + layer.restore = func(ctx context.Context, bucket, object string, opts ObjectOptions) error { + <-release + return r.z.RestoreTransitionedObject(ctx, bucket, object, opts) + } + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("first POST: %d %s", rec.Code, rec.Body.String()) + } + rec := restoreIsolationPOST(t, r, layer, oi.VersionID) + close(release) + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "RestoreAlreadyInProgress") { + t.Errorf("second POST: %d %s", rec.Code, rec.Body.String()) + } + if err := layer.wait(t); err != nil { + t.Fatal(err) + } + select { + case err := <-layer.done: + t.Fatalf("a second restore ran: %v", err) + default: + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("restored contents: %v", err) + } +} + +// A version deleted while its tier copy downloads is not brought back. +func TestRestoreTargetDeletedDuringDownload(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + gate := newRestoreIsolationGateTier(1, nil) + globalTierConfigMgr.drivercache["WARM"] = gate + defer gate.unblock() + done := make(chan error, 1) + go func() { + done <- r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)) + }() + restoreIsolationEntered(t, gate) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + _, err := r.z.DeleteObject(ctx, r.bucket, r.object, r.opts(oi.VersionID)) + cancel() + if err != nil { + t.Fatalf("delete during the download: %v", err) + } + gate.unblock() + select { + case err := <-done: + if err == nil { + t.Fatal("the restore of a deleted version succeeded") + } + case <-time.After(restoreTestWait): + t.Fatal("restore did not finish") + } + if _, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + t.Fatalf("deleted version after the restore: %v", err) + } +} + +// With several pools, abandoned-data cleanup takes the pools lock that every +// write of the object takes, also where a pool's sets have lockers of their +// own: while a writer holds it, cleanup removes nothing, such as a data +// directory that a commit moved in place and has not yet referenced. +func TestCheckAbandonedPartsTakesPoolsLock(t *testing.T) { + r := newRestoreSafetyTest(t, 2, versioning.Enabled) + nativeRestoreSeed(t, r, 1, false, false) + for _, set := range r.z.serverPools[1].sets { + previous := set.nsMutex + set.nsMutex = newNSLock(false) + t.Cleanup(func() { set.nsMutex = previous }) + } + disks := r.z.serverPools[1].getHashedSet(r.object).getDisks() + moved := mustGetUUID() + for _, disk := range disks { + dir := filepath.Join(disk.Endpoint().Path, r.bucket, r.object, moved) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "part.1"), []byte("staged"), 0o644); err != nil { + t.Fatal(err) + } + } + present := func() int { + n := 0 + for _, disk := range disks { + if _, err := os.Stat(filepath.Join(disk.Endpoint().Path, r.bucket, r.object, moved)); err == nil { + n++ + } + } + return n + } + lk := r.z.NewObjectNSLock(r.bucket, r.object) + lkctx, err := lk.GetLock(t.Context(), globalOperationTimeout) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + r.z.CheckAbandonedParts(ctx, r.bucket, r.object, madmin.HealOpts{Remove: true}) + cancel() + held := present() + lk.Unlock(lkctx) + if held != len(disks) { + t.Fatalf("abandoned-data cleanup removed a moved directory from %d drives while a writer held the object lock", len(disks)-held) + } + if err := r.z.CheckAbandonedParts(t.Context(), r.bucket, r.object, madmin.HealOpts{Remove: true}); err != nil { + t.Fatalf("abandoned-data cleanup without a writer: %v", err) + } + if n := present(); n != 0 { + t.Fatalf("abandoned-data cleanup left the unreferenced directory on %d drives", n) + } +} + +// The header flag that tells whether a transitioned version uses its data +// directory follows whether its restored copy has expired when a drive writes +// the entry. A publication's own entry is recognized with either value. +func TestRestorePublishedRevisionAcrossExpiry(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + oi, _ := nativeRestoreSeed(t, r, 0, false, false) + fi := r.fileInfo(oi.VersionID) + expiry := time.Now().Add(2 * time.Second).Truncate(time.Second).Add(time.Second) + st := &restoreStage{dataDir: mustGetUUID(), expiry: expiry} + published := restorePublished(fi, fi, st, 0) + keys, err := fileInfoRevisions(published) + if err != nil { + t.Fatal(err) + } + stored := func() revisionKey { + var x xlMetaV2 + if err := x.AddVersion(published); err != nil { + t.Fatal(err) + } + buf, err := x.AppendTo(nil) + if err != nil { + t.Fatal(err) + } + var y xlMetaV2 + if err := y.Load(buf); err != nil { + t.Fatal(err) + } + return versionRevision(y.versions[0], y.metaV) + } + before := stored() + time.Sleep(time.Until(expiry) + 100*time.Millisecond) + after := stored() + if before == after { + t.Fatal("fixture: the stored flag did not change at the expiry") + } + for _, got := range []revisionKey{before, after} { + if got != keys[0] && got != keys[1] { + t.Errorf("a publication's own entry is not recognized: %v not in %v", got, keys) + } + } +} + +// A tier that lost the copy answers not found. That is no reason to keep the +// version's ongoing marker: the attempt clears it, and a later POST can +// restore the version again. +func TestRestoreTierNotFoundClearsMarker(t *testing.T) { + r := newRestoreSafetyTest(t, 1, versioning.Enabled) + initConfigSubsystem(t.Context(), r.z) + oi, _ := restoreIsolationSeed(t, r, 0, false, false) + tier := globalTierConfigMgr.drivercache["WARM"] + globalTierConfigMgr.drivercache["WARM"] = nativeRestoreFailTier{tier, ObjectNotFound{Bucket: r.bucket, Object: r.object}} + layer := newRestoreIsolationLayer(r) + if rec := restoreIsolationPOST(t, r, layer, oi.VersionID); rec.Code != http.StatusOK { + t.Fatalf("POST: %d %s", rec.Code, rec.Body.String()) + } + if err := layer.wait(t); !isErrObjectNotFound(err) { + t.Fatalf("restore from a tier without the copy: %v", err) + } + info, err := r.z.GetObjectInfo(t.Context(), r.bucket, r.object, r.opts(oi.VersionID)) + if err != nil || info.RestoreOngoing || info.UserDefined[xhttp.AmzRestore] != "" { + t.Fatalf("version after the failed restore: %q, %v", info.UserDefined[xhttp.AmzRestore], err) + } +} + +// newRestoreWideTest is newRestoreSafetyTest with one pool of twelve drives, +// whose default layout is eight data and four parity shards: a version's +// metadata reaches its read quorum on seven drives, one short of the write +// quorum of eight that its shards need. +func newRestoreWideTest(t *testing.T) *restoreTest { + t.Helper() + prevTier := globalTierConfigMgr + t.Cleanup(func() { globalTierConfigMgr = prevTier }) + dirs, err := getRandomDisks(12) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { removeRoots(dirs) }) + obj, _, err := initObjectLayer(t.Context(), mustGetPoolEndpoints(0, dirs...)) + if err != nil { + t.Fatal(err) + } + z := obj.(*erasureServerPools) + t.Cleanup(func() { z.Shutdown(context.Background()) }) + // Other tests leave a storage class behind; fix the layout to 8+4. + saved := globalStorageClass + globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}}) + t.Cleanup(func() { globalStorageClass.Update(saved) }) + globalTierConfigMgr = NewTierConfigMgr() + globalTierConfigMgr.drivercache["WARM"] = &restoreTestTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}} + r := &restoreTest{t: t, z: z, bucket: getRandomBucketName(), object: "obj", labels: map[string]string{}, bodies: map[string][]byte{}} + if err := z.MakeBucket(t.Context(), r.bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil { + t.Fatal(err) + } + meta, err := globalBucketMetadataSys.Get(r.bucket) + if err != nil { + t.Fatal(err) + } + meta.LifecycleConfigXML = []byte(restoreSafetyRules) + if err := meta.Save(t.Context(), z); err != nil { + t.Fatal(err) + } + globalBucketMetadataSys.Set(r.bucket, meta) + return r +} + +// On twelve drives with eight data shards, a publication that reached seven +// drives has the metadata read quorum but not the eight shards a read needs. +// It is not a restore: the worker brings every drive back to the unrestored +// version, and the version reads from its tier copy. +func TestRestorePublicationBelowWriteQuorumOnWideSet(t *testing.T) { + r := newRestoreWideTest(t) + oi, body := nativeRestoreSeed(t, r, 0, false, true) + fi := r.fileInfo(oi.VersionID) + if fi.Erasure.DataBlocks != 8 || fi.Erasure.ParityBlocks != 4 { + t.Fatalf("fixture layout %d+%d, want 8+4", fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks) + } + faults := &restoreTwoPhaseFaults{} + faults.rename.Store("fail") + disks := restoreTwoPhaseFaulty(t, r, faults, faults, faults, faults, faults) + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err == nil { + t.Fatal("a publication on seven of twelve drives succeeded") + } + for i, e := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) { + if !restoreTwoPhaseUnrestored(e) || e.DataDir != fi.DataDir { + t.Errorf("drive %d after a publication on seven drives: restore %q, remote %v", i, e.Metadata[xhttp.AmzRestore], e.IsRemote()) + } + } + if _, got, err := r.read(oi.VersionID); err != nil || !bytes.Equal(got, body) { + t.Fatalf("the version no longer reads from its tier copy: %v", err) + } + faults.rename.Store("") + if err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID)); err != nil { + t.Fatalf("restore after the drives recovered: %v", err) + } + for i, e := range restoreTwoPhaseEntries(t, r, disks, oi.VersionID) { + if !restoreTwoPhaseRestored(e) { + t.Errorf("drive %d: not restored", i) + } + } +}