mirror of
https://github.com/pgsty/minio.git
synced 2026-11-11 10:05:34 +03:00
Fix pool migration cleanup and physical progress
Count removal only after every source drive confirms version absence, including free records and concurrently deleted parents. Retire successful candidates independently with the existing source, destination and tier ownership checks. Reject ambiguous raw reads, verify remaining metadata directly, and propagate cancellation. Signed-off-by: Feng Ruohang <rh@vonng.com> (cherry picked from commit b1a97c7e6cc7ab5f44110b5b8258afb60a50053e)
This commit is contained in:
Feng Ruohang
@@ -0,0 +1,528 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
)
|
||||
|
||||
// The walk has already captured the version when its data read reaches this
|
||||
// hook. Metadata-only checks made by a concurrent DELETE do not reenter it.
|
||||
type migrationProgressReadDisk struct {
|
||||
StorageAPI
|
||||
bucket, object, version string
|
||||
before func() error
|
||||
}
|
||||
|
||||
func (d migrationProgressReadDisk) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
|
||||
if opts.ReadData && bucket == d.bucket && object == d.object && version == d.version {
|
||||
if err := d.before(); err != nil {
|
||||
return FileInfo{}, err
|
||||
}
|
||||
}
|
||||
return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
|
||||
}
|
||||
|
||||
// Buffer every drive's first walk before publishing any entry. Thus deleting
|
||||
// the parent cannot accidentally let this same walk discover the child.
|
||||
type migrationSnapshotWalk struct {
|
||||
bucket string
|
||||
drives int32
|
||||
calls atomic.Int32
|
||||
ready chan struct{}
|
||||
}
|
||||
|
||||
type migrationSnapshotWalkDisk struct {
|
||||
StorageAPI
|
||||
snapshot *migrationSnapshotWalk
|
||||
}
|
||||
|
||||
func (d migrationSnapshotWalkDisk) WalkDir(ctx context.Context, opts WalkDirOptions, out io.Writer) error {
|
||||
if opts.Bucket != d.snapshot.bucket {
|
||||
return d.StorageAPI.WalkDir(ctx, opts, out)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := d.StorageAPI.WalkDir(ctx, opts, &buf); err != nil {
|
||||
return err
|
||||
}
|
||||
if d.snapshot.calls.Add(1) == d.snapshot.drives {
|
||||
close(d.snapshot.ready)
|
||||
}
|
||||
select {
|
||||
case <-d.snapshot.ready:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
_, err := io.Copy(out, &buf)
|
||||
return err
|
||||
}
|
||||
|
||||
func migrationProgressSnapshot(t *testing.T, z *erasureServerPools, bucket, key string) *migrationSnapshotWalk {
|
||||
t.Helper()
|
||||
s := &migrationSnapshotWalk{bucket: bucket, drives: int32(len(z.serverPools[0].getHashedSet(key).getDisks())), ready: make(chan struct{})}
|
||||
t.Cleanup(conditionalPutSwapDisks(z.serverPools[0], key, func(d StorageAPI) StorageAPI {
|
||||
return migrationSnapshotWalkDisk{StorageAPI: d, snapshot: s}
|
||||
}))
|
||||
return s
|
||||
}
|
||||
|
||||
func migrationProgressPrepare(t *testing.T, z *erasureServerPools, kind string) {
|
||||
t.Helper()
|
||||
migrationTestMover(t, z, 0, kind)
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[0].LastUpdate = UTCNow()
|
||||
z.poolMetaMutex.Unlock()
|
||||
}
|
||||
|
||||
func TestDecommissionSplitMetadataDoesNotAdvance(t *testing.T) {
|
||||
m := newMigrationILM(t, "", false, migrationILMExpireCurrent())
|
||||
const key = "obj-01"
|
||||
m.put(key, "old", ObjectOptions{MTime: UTCNow().Add(-96 * time.Hour)})
|
||||
disks := m.z.serverPools[0].getHashedSet(key).getDisks()
|
||||
old := make([][]byte, len(disks)/2)
|
||||
for i := range old {
|
||||
var err error
|
||||
old[i], err = disks[i].ReadAll(t.Context(), m.bucket, pathJoin(key, xlStorageFormatFile))
|
||||
mustIAM(t, err)
|
||||
}
|
||||
m.put(key, "new-and-longer", ObjectOptions{MTime: UTCNow().Add(-72 * time.Hour)})
|
||||
for i := range old {
|
||||
mustIAM(t, disks[i].WriteAll(t.Context(), m.bucket, pathJoin(key, xlStorageFormatFile), old[i]))
|
||||
}
|
||||
if _, err := m.z.serverPools[0].GetObjectInfo(t.Context(), m.bucket, key, ObjectOptions{VersionID: nullVersionID}); !isErrVersionNotFound(err) {
|
||||
t.Fatalf("fixture must expose the ambiguous logical NotFound, got %v", err)
|
||||
}
|
||||
migrationProgressPrepare(t, m.z, "decommission")
|
||||
walk := migrationProgressSnapshot(t, m.z, m.bucket, key)
|
||||
bi := decomBucketInfo{Name: m.bucket}
|
||||
// Check a single pass first, so the unfixed implementation fails rather
|
||||
// than looping until the test timeout.
|
||||
removed, err := m.z.decommissionPoolWalk(t.Context(), 0, m.z.serverPools[0], bi)
|
||||
if err != nil || removed != 0 {
|
||||
t.Fatalf("split metadata reported progress: removed=%d err=%v", removed, err)
|
||||
}
|
||||
mustIAM(t, m.z.decommissionPool(t.Context(), 0, m.z.serverPools[0], bi))
|
||||
if got := walk.calls.Load(); got != 2*walk.drives {
|
||||
t.Fatalf("unchanged metadata repeated the walk: calls=%d", got)
|
||||
}
|
||||
if err := m.z.checkAfterDecomBucket(t.Context(), 0, bi); err == nil {
|
||||
t.Fatal("final verification accepted split metadata still on disk")
|
||||
}
|
||||
for _, disk := range disks {
|
||||
if _, err := disk.ReadVersion(t.Context(), "", m.bucket, key, nullVersionID, ReadOptions{}); err != nil {
|
||||
t.Fatalf("migration removed an ambiguous source revision: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationConcurrentDeleteCleanup(t *testing.T) {
|
||||
for _, kind := range []string{"decommission", "rebalance"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
v1 := putConsistencyObject(t, z, bucket, "k", 0, "older", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||
v2 := putConsistencyObject(t, z, bucket, "k", 0, "newer", ObjectOptions{Versioned: true})
|
||||
var once sync.Once
|
||||
var deleteErr error
|
||||
before := func() error {
|
||||
once.Do(func() {
|
||||
_, deleteErr = z.DeleteObject(t.Context(), bucket, "k", ObjectOptions{Versioned: true, VersionID: v2.VersionID})
|
||||
})
|
||||
return deleteErr
|
||||
}
|
||||
restore := conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||
return migrationProgressReadDisk{StorageAPI: d, bucket: bucket, object: "k", version: v2.VersionID, before: before}
|
||||
})
|
||||
defer restore()
|
||||
if failed := migrationTestWalk(t, z, bucket, 0, kind); failed != 0 {
|
||||
t.Fatalf("failed moves: %d", failed)
|
||||
}
|
||||
restore()
|
||||
for _, v := range []ObjectInfo{v1, v2} {
|
||||
for _, disk := range z.serverPools[0].getHashedSet("k").getDisks() {
|
||||
_, err := disk.ReadVersion(t.Context(), "", bucket, "k", v.VersionID, ReadOptions{})
|
||||
if !errors.Is(err, errFileVersionNotFound) && !errors.Is(err, errFileNotFound) {
|
||||
t.Fatalf("source version %s remained: %v", v.VersionID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k", v1.VersionID); body != "older" {
|
||||
t.Fatalf("moved version reads %q", body)
|
||||
}
|
||||
if _, err := z.GetObjectInfo(t.Context(), bucket, "k", ObjectOptions{VersionID: v2.VersionID}); !isErrVersionNotFound(err) {
|
||||
t.Fatalf("concurrently deleted version resurrected: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationSiblingFailureCleanup(t *testing.T) {
|
||||
for _, kind := range []string{"decommission", "rebalance"} {
|
||||
for _, loseDestination := range []bool{false, true} {
|
||||
name := kind + "/destination-retained"
|
||||
if loseDestination {
|
||||
name = kind + "/destination-removed"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
v1 := putConsistencyObject(t, z, bucket, "k", 0, "older", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
|
||||
v2 := putConsistencyObject(t, z, bucket, "k", 0, "newer", ObjectOptions{Versioned: true})
|
||||
var once sync.Once
|
||||
var deleteErr error
|
||||
before := func() error {
|
||||
once.Do(func() {
|
||||
if loseDestination {
|
||||
_, deleteErr = z.serverPools[1].DeleteObject(t.Context(), bucket, "k", ObjectOptions{VersionID: v1.VersionID})
|
||||
}
|
||||
})
|
||||
return errDiskNotFound // only the sibling's data read fails
|
||||
}
|
||||
restore := conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||
return migrationProgressReadDisk{StorageAPI: d, bucket: bucket, object: "k", version: v2.VersionID, before: before}
|
||||
})
|
||||
defer restore()
|
||||
migrationProgressPrepare(t, z, kind)
|
||||
if kind == "rebalance" {
|
||||
mustIAM(t, z.rebalanceBucket(t.Context(), bucket, 0))
|
||||
} else {
|
||||
_, err := z.decommissionPoolWalk(t.Context(), 0, z.serverPools[0], decomBucketInfo{Name: bucket})
|
||||
mustIAM(t, err)
|
||||
if err := z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: bucket}); err == nil {
|
||||
t.Fatal("failed sibling must block final decommission verification")
|
||||
}
|
||||
}
|
||||
restore()
|
||||
mustIAM(t, deleteErr)
|
||||
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k", v2.VersionID); body != "newer" {
|
||||
t.Fatalf("failed sibling reads %q", body)
|
||||
}
|
||||
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k", v1.VersionID); body != "older" {
|
||||
t.Fatalf("successful candidate lost: %q", body)
|
||||
}
|
||||
_, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, "k", ObjectOptions{VersionID: v1.VersionID})
|
||||
if loseDestination && err != nil || !loseDestination && !isErrVersionNotFound(err) {
|
||||
t.Fatalf("source cleanup ignored destination state: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecommissionConcurrentParentDeleteRescansChildren(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
putConsistencyObject(t, z, bucket, "k", 0, "parent", ObjectOptions{})
|
||||
putConsistencyObject(t, z, bucket, "k/b", 0, "child", ObjectOptions{})
|
||||
migrationProgressSnapshot(t, z, bucket, "k")
|
||||
var once sync.Once
|
||||
var deleteErr error
|
||||
t.Cleanup(conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||
return migrationProgressReadDisk{StorageAPI: d, bucket: bucket, object: "k", version: nullVersionID, before: func() error {
|
||||
once.Do(func() {
|
||||
_, deleteErr = z.DeleteObject(t.Context(), bucket, "k", ObjectOptions{VersionID: nullVersionID})
|
||||
})
|
||||
return deleteErr
|
||||
}}
|
||||
}))
|
||||
if failed := migrationTestWalk(t, z, bucket, 0, "decommission"); failed != 0 {
|
||||
t.Fatalf("failed moves: %d", failed)
|
||||
}
|
||||
mustIAM(t, z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: bucket}))
|
||||
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k/b", ""); body != "child" {
|
||||
t.Fatalf("child reads %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreeVersionDecommissionRescansChildren(t *testing.T) {
|
||||
f := newFreeVersionTest(t, "")
|
||||
f.bucket = "free-parent"
|
||||
mustIAM(t, f.z.MakeBucket(t.Context(), f.bucket, MakeBucketOptions{VersioningEnabled: true}))
|
||||
f.deleteVersion(f.tiered("k", "parent", UTCNow().Add(-time.Hour)))
|
||||
f.put("k/b", "child", UTCNow())
|
||||
migrationProgressSnapshot(t, f.z, f.bucket, "k")
|
||||
migrationProgressPrepare(t, f.z, "decommission")
|
||||
bi := decomBucketInfo{Name: f.bucket}
|
||||
removed, err := f.z.decommissionPoolWalk(t.Context(), 0, f.z.serverPools[0], bi)
|
||||
if err != nil || removed != 1 {
|
||||
t.Fatalf("free-only parent removal: removed=%d err=%v", removed, err)
|
||||
}
|
||||
if err := f.z.checkAfterDecomBucket(t.Context(), 0, bi); err == nil {
|
||||
t.Fatal("the newly exposed child must still block completion")
|
||||
}
|
||||
mustIAM(t, f.z.decommissionPool(t.Context(), 0, f.z.serverPools[0], bi))
|
||||
mustIAM(t, f.z.checkAfterDecomBucket(t.Context(), 0, bi))
|
||||
if body := migrationTestBody(t, f.z.GetObjectNInfo, f.bucket, "k/b", ""); body != "k/b child" {
|
||||
t.Fatalf("child reads %q", body)
|
||||
}
|
||||
if !slices.Contains(f.tier.removed, f.remote("k", "parent")) {
|
||||
t.Fatal("free version cleanup did not delete its remote object")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsMigrationCancellation(t *testing.T) {
|
||||
for _, kind := range []string{"decommission", "rebalance"} {
|
||||
for _, phase := range []string{"before-walk", "during-read", "deadline"} {
|
||||
name := kind + "/" + phase
|
||||
t.Run(name, func(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
oi := putConsistencyObject(t, z, bucket, "k", 0, "source", ObjectOptions{Versioned: true})
|
||||
migrationProgressPrepare(t, z, kind)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
wantErr := context.Canceled
|
||||
if phase == "deadline" {
|
||||
cancel()
|
||||
ctx, cancel = context.WithDeadline(t.Context(), time.Time{})
|
||||
wantErr = context.DeadlineExceeded
|
||||
}
|
||||
defer cancel()
|
||||
if phase == "during-read" {
|
||||
t.Cleanup(conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||
return migrationProgressReadDisk{StorageAPI: d, bucket: bucket, object: "k", version: oi.VersionID, before: func() error { cancel(); return ctx.Err() }}
|
||||
}))
|
||||
} else {
|
||||
cancel()
|
||||
}
|
||||
var err error
|
||||
if kind == "decommission" {
|
||||
err = z.decommissionPool(ctx, 0, z.serverPools[0], decomBucketInfo{Name: bucket})
|
||||
} else {
|
||||
err = z.rebalanceBucket(ctx, bucket, 0)
|
||||
}
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("cancellation returned %v", err)
|
||||
}
|
||||
if err := z.decommissionInBackground(ctx, 0); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("canceled background migration returned %v", err)
|
||||
}
|
||||
if err := z.rebalanceBuckets(ctx, 0); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("canceled background rebalance returned %v", err)
|
||||
}
|
||||
if n, err := z.countAbsentMigrationVersions(ctx, bucket, "k", 0, []FileInfo{{VersionID: oi.VersionID}}); n != 0 || !errors.Is(err, wantErr) {
|
||||
t.Fatalf("canceled progress verification: removed=%d err=%v", n, err)
|
||||
}
|
||||
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, "k", ObjectOptions{VersionID: oi.VersionID}); err != nil {
|
||||
t.Fatalf("canceled migration lost source: %v", err)
|
||||
}
|
||||
if err := z.checkAfterDecomBucket(ctx, 0, decomBucketInfo{Name: bucket}); !errors.Is(err, wantErr) {
|
||||
t.Fatalf("canceled verification returned %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type migrationProgressRawDisk struct {
|
||||
StorageAPI
|
||||
buf []byte
|
||||
err error
|
||||
reads *atomic.Int32
|
||||
}
|
||||
|
||||
func (d migrationProgressRawDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
|
||||
d.reads.Add(1)
|
||||
if data {
|
||||
return RawFileInfo{}, errors.New("progress check requested object data")
|
||||
}
|
||||
return RawFileInfo{Buf: d.buf}, d.err
|
||||
}
|
||||
|
||||
func (d migrationProgressRawDisk) ListDir(context.Context, string, string, string, int) ([]string, error) {
|
||||
return nil, errFileNotFound
|
||||
}
|
||||
|
||||
func TestPoolsMigrationPhysicalProgress(t *testing.T) {
|
||||
f := newFreeVersionTest(t, "")
|
||||
buf := f.xlMeta(0, "obj-01")
|
||||
versions, err := getFileInfoVersions(buf, f.bucket, "obj-01", true)
|
||||
mustIAM(t, err)
|
||||
if len(versions.Versions) != 3 || !versions.Versions[0].TierFreeVersion() {
|
||||
t.Fatal("fixture must contain two live versions and one free version")
|
||||
}
|
||||
missing := FileInfo{VersionID: mustGetUUID()}
|
||||
snapshot := append(slices.Clone(versions.Versions), missing)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
buf []byte
|
||||
err error
|
||||
want int
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "all-absent", err: errFileNotFound, want: 4},
|
||||
{name: "missing-volume", err: errVolumeNotFound, want: 4},
|
||||
{name: "minority-live-and-free", buf: buf, want: 1},
|
||||
{name: "unavailable", err: errDiskNotFound, wantErr: true},
|
||||
{name: "read-quorum", err: errErasureReadQuorum, wantErr: true},
|
||||
{name: "corrupt-read", err: errFileCorrupt, wantErr: true},
|
||||
{name: "corrupt-metadata", buf: []byte("broken xl.meta"), wantErr: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var reads atomic.Int32
|
||||
disks := make([]StorageAPI, 4)
|
||||
for i := range disks {
|
||||
disks[i] = migrationProgressRawDisk{err: errFileNotFound, reads: &reads}
|
||||
}
|
||||
disks[3] = migrationProgressRawDisk{buf: tc.buf, err: tc.err, reads: &reads}
|
||||
er := erasureObjects{getDisks: func() []StorageAPI { return disks }}
|
||||
n, err := er.countAbsentMigrationVersions(t.Context(), f.bucket, "obj-01", snapshot)
|
||||
if n != tc.want || (err != nil) != tc.wantErr {
|
||||
t.Fatalf("removed=%d err=%v, want %d / error=%t", n, err, tc.want, tc.wantErr)
|
||||
}
|
||||
if reads.Load() != int32(len(disks)) {
|
||||
t.Fatalf("read count %d must not depend on version count", reads.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
putConsistencyObject(t, f.z, f.bucket, "null", 0, "null-data", ObjectOptions{})
|
||||
nullBuf := f.xlMeta(0, "null")
|
||||
var reads atomic.Int32
|
||||
er := erasureObjects{getDisks: func() []StorageAPI {
|
||||
return []StorageAPI{migrationProgressRawDisk{buf: nullBuf, reads: &reads}}
|
||||
}}
|
||||
if n, err := er.countAbsentMigrationVersions(t.Context(), f.bucket, "null", []FileInfo{{VersionID: nullVersionID}, {VersionID: ""}}); n != 0 || err != nil {
|
||||
t.Fatalf("null version normalization: removed=%d err=%v", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A successful quorum deletion can leave one source drive behind. It must not
|
||||
// count as physical removal or pass the final check, nor cause a retry loop.
|
||||
func TestDecommissionQuorumDeletionIsNotPhysicalProgress(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
oi := putConsistencyObject(t, z, bucket, "k", 0, "payload", ObjectOptions{Versioned: true})
|
||||
move := migrationTestMover(t, z, 0, "decommission")
|
||||
mustIAM(t, move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, oi)))
|
||||
first := true
|
||||
restore := conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||
if first {
|
||||
first = false
|
||||
return consistencyDeleteFaultDisk{StorageAPI: d, bucket: bucket, object: "k", version: oi.VersionID}
|
||||
}
|
||||
return d
|
||||
})
|
||||
defer restore()
|
||||
snapshot := []FileInfo{{VersionID: oi.VersionID, ModTime: oi.ModTime}}
|
||||
for range 2 {
|
||||
n, err := z.removeMovedVersions(t.Context(), bucket, "k", 0, snapshot, nil)
|
||||
if err != nil || n != 0 {
|
||||
t.Fatalf("partial deletion reported progress: removed=%d err=%v", n, err)
|
||||
}
|
||||
}
|
||||
if err := z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: bucket}); err == nil {
|
||||
t.Fatal("final verification accepted a minority-drive source copy")
|
||||
}
|
||||
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k", oi.VersionID); body != "payload" {
|
||||
t.Fatalf("destination copy lost: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecommissionProgressReadFailureStopsWalk(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
putConsistencyObject(t, z, bucket, "k", 0, "payload", ObjectOptions{Versioned: true})
|
||||
migrationProgressPrepare(t, z, "decommission")
|
||||
walk := migrationProgressSnapshot(t, z, bucket, "k")
|
||||
restore := conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||
return tierReadFaultDisk{StorageAPI: d, bucket: bucket, object: "k", err: errDiskNotFound}
|
||||
})
|
||||
defer restore()
|
||||
if err := z.decommissionPool(t.Context(), 0, z.serverPools[0], decomBucketInfo{Name: bucket}); err == nil {
|
||||
t.Fatal("unreadable source progress check reported success")
|
||||
}
|
||||
if got := walk.calls.Load(); got != walk.drives {
|
||||
t.Fatalf("verification failure repeated the walk: calls=%d", got)
|
||||
}
|
||||
restore()
|
||||
if body := migrationTestBody(t, z.GetObjectNInfo, bucket, "k", ""); body != "payload" {
|
||||
t.Fatalf("quorum move lost destination: %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
type migrationProgressListFaultDisk struct {
|
||||
StorageAPI
|
||||
}
|
||||
|
||||
func (d migrationProgressListFaultDisk) ListDir(context.Context, string, string, string, int) ([]string, error) {
|
||||
return nil, errDiskAccessDenied
|
||||
}
|
||||
|
||||
func TestDecommissionFinalVerificationFailsClosed(t *testing.T) {
|
||||
for _, state := range []string{"offline-drive", "directory-error", "corrupt-metadata", "truncated-metadata", "legacy-metadata"} {
|
||||
t.Run(state, func(t *testing.T) {
|
||||
z, bucket := migrationTestPools(t)
|
||||
switch state {
|
||||
case "offline-drive", "directory-error":
|
||||
t.Cleanup(conditionalPutSwapDisks(z.serverPools[0], "k", func(d StorageAPI) StorageAPI {
|
||||
if state == "directory-error" {
|
||||
return migrationProgressListFaultDisk{StorageAPI: d}
|
||||
}
|
||||
return nil
|
||||
}))
|
||||
default:
|
||||
// Real metadata on only one drive, without a readable quorum.
|
||||
// This also covers errors that WalkDir deliberately skips.
|
||||
disk := z.serverPools[0].getHashedSet("k").getDisks()[0]
|
||||
file, data := xlStorageFormatFile, []byte("broken xl.meta")
|
||||
if state == "truncated-metadata" {
|
||||
data = nil
|
||||
}
|
||||
if state == "legacy-metadata" {
|
||||
file = xlStorageFormatFileV1
|
||||
}
|
||||
mustIAM(t, disk.WriteAll(t.Context(), bucket, pathJoin("k", file), data))
|
||||
}
|
||||
if err := z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: bucket}); err == nil {
|
||||
t.Fatal("unreadable storage was accepted as physically empty")
|
||||
}
|
||||
if n, err := z.countAbsentMigrationVersions(t.Context(), bucket, "k", 0, []FileInfo{{VersionID: mustGetUUID()}}); n != 0 || err == nil {
|
||||
t.Fatalf("unreadable storage was counted as progress: removed=%d err=%v", n, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreeVersionDecommissionPreservesLiveRemoteOwner(t *testing.T) {
|
||||
f := newFreeVersionTest(t, "")
|
||||
const key = "obj-02"
|
||||
fv, tuple := f.freeRecord(0, key)
|
||||
owner := putConsistencyObject(t, f.z, f.bucket, key, 1, "live-owner", ObjectOptions{Versioned: true})
|
||||
infos, errs := readAllFileInfo(t.Context(), f.z.serverPools[1].getHashedSet(key).getDisks(), "", f.bucket, key, owner.VersionID, false, false)
|
||||
mustIAM(t, errs[0])
|
||||
fi := infos[0]
|
||||
fi.TransitionStatus, fi.TransitionTier = lifecycle.TransitionComplete, tuple.Tier
|
||||
fi.TransitionedObjName, fi.TransitionVersionID = tuple.Object, tuple.VersionID
|
||||
mustIAM(t, f.z.serverPools[1].DecomTieredObject(t.Context(), f.bucket, key, fi, ObjectOptions{}))
|
||||
if err := f.z.removeFreeVersions(t.Context(), f.bucket, key, 0, []FileInfo{fv}); !errors.Is(err, errTierStillReferenced) {
|
||||
t.Fatalf("cleanup with a live remote owner: %v", err)
|
||||
}
|
||||
if n, err := f.z.countAbsentMigrationVersions(t.Context(), f.bucket, key, 0, []FileInfo{fv}); err != nil || n != 0 {
|
||||
t.Fatalf("retained free record reported progress: removed=%d err=%v", n, err)
|
||||
}
|
||||
if slices.Contains(f.tier.removed, tuple.Object) {
|
||||
t.Fatal("cleanup deleted a live remote object")
|
||||
}
|
||||
if err := f.z.checkAfterDecomBucket(t.Context(), 0, decomBucketInfo{Name: f.bucket}); err == nil {
|
||||
t.Fatal("retained free record must block final decommission verification")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user