mirror of
https://github.com/pgsty/minio.git
synced 2026-10-30 04:20:36 +03:00
Preserve maintenance timeout behavior and prepare SILO 20261026
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
@@ -1,3 +1,5 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
|
||||
+10
-20
@@ -1094,19 +1094,11 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
|
||||
|
||||
// Similar to rename but renames data from srcEntry to dstEntry at dataDir
|
||||
func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, []byte, string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, "", err
|
||||
}
|
||||
// Admission is cancelable. After admission use actual disk results for
|
||||
// commit/rollback, even if the client or namespace lease is canceled.
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
g := errgroup.WithNErrs(len(disks))
|
||||
|
||||
fvID := mustGetUUID()
|
||||
for index := range disks {
|
||||
metadata[index].Metadata = maps.Clone(metadata[index].Metadata)
|
||||
metadata[index].SetTierFreeVersionID(fvID)
|
||||
metadata[index].Metadata[rollbackIDKey] = fvID
|
||||
}
|
||||
|
||||
diskVersions := make([][]byte, len(disks))
|
||||
@@ -1564,18 +1556,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
compIndex = opts.IndexCB()
|
||||
}
|
||||
|
||||
if !opts.NoLock {
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
// Mint the generation while holding the commit lock. Staged uploads may
|
||||
// acquire the lock in a different order from their encoding completion.
|
||||
modTime := opts.MTime
|
||||
if opts.MTime.IsZero() {
|
||||
modTime = UTCNow()
|
||||
@@ -1672,6 +1652,16 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.NoLock {
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
if !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement &&
|
||||
(fi.VersionID == "" || fi.VersionID == nullVersionID) {
|
||||
getObjectInfo := er.getObjectInfo
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
|
||||
@@ -26,8 +26,7 @@ func (d compatibilityFailedOverwrite) RenameData(ctx context.Context, sv, sp str
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
|
||||
// Maintenance compatibility requirement: after a completely joined failed
|
||||
// overwrite with one residue, keep the previously acknowledged quorum readable.
|
||||
// Maintenance compatibility requirement: after a failed overwrite with one residue, keep the previously acknowledged quorum readable.
|
||||
// This tests a known history; it does not claim the resulting snapshot proves it.
|
||||
func TestMaintenancePartialFailedOverwriteAvailability(t *testing.T) {
|
||||
// Lock RPC tests configure a single-drive backend (EC:0). Pin this
|
||||
|
||||
@@ -399,7 +399,10 @@ func getCompleteMultipartMD5(parts []CompletePart) string {
|
||||
return s3MD5
|
||||
}
|
||||
|
||||
// Clean unwanted fields from metadata
|
||||
// Keep metadata written by pre-release builds out of public object metadata.
|
||||
const rollbackIDKey = ReservedMetadataPrefixLower + "rollback-id"
|
||||
|
||||
// Clean unwanted fields from metadata.
|
||||
func cleanMetadata(metadata map[string]string) map[string]string {
|
||||
// Remove STANDARD StorageClass
|
||||
metadata = removeStandardStorageClass(metadata)
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
@@ -472,7 +474,7 @@ func TestMutationDeleteJoinsNestedWorker(t *testing.T) {
|
||||
defer globalDriveConfig.Update(drive.Config{MaxTimeout: oldTimeout})
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, e := xioutil.WithDeadlineAndWait(context.Background(), globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
|
||||
_, e := xioutil.WithDeadline(context.Background(), globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, disk.storage.DeleteVersion(ctx, bucket, object, fi, false, DeleteOptions{UndoWrite: true})
|
||||
})
|
||||
done <- e
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"maps"
|
||||
)
|
||||
|
||||
// Compare logical version state, excluding per-shard checksums/index, physical
|
||||
// DataDir placement and properties of other versions in the same xl.meta.
|
||||
// Different DataDir values remain readable for the existing rebalance recovery
|
||||
// contract; rollback separately requires an exact DataDir match.
|
||||
func sameFileInfoVersion(a, b FileInfo) bool {
|
||||
if a.VersionID != b.VersionID || a.Deleted != b.Deleted || a.XLV1 != b.XLV1 || a.Size != b.Size ||
|
||||
a.TransitionStatus != b.TransitionStatus || a.TransitionTier != b.TransitionTier ||
|
||||
a.TransitionedObjName != b.TransitionedObjName || a.TransitionVersionID != b.TransitionVersionID ||
|
||||
!sameVersionMetadata(a.Metadata, b.Metadata) || !bytes.Equal(a.Checksum, b.Checksum) || len(a.Parts) != len(b.Parts) {
|
||||
return false
|
||||
}
|
||||
if !a.Deleted && a.Size != 0 && !a.Erasure.Equal(b.Erasure) {
|
||||
return false
|
||||
}
|
||||
for i, p := range a.Parts {
|
||||
q := b.Parts[i]
|
||||
if p.Number != q.Number || p.Size != q.Size || p.ActualSize != q.ActualSize || p.ETag != q.ETag || !bytes.Equal(p.Index, q.Index) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// The undo capability controls rollback ownership, not the logical contents.
|
||||
// A partial no-op update can revoke it without changing a single public field.
|
||||
func sameVersionMetadata(a, b map[string]string) bool {
|
||||
if a[rollbackIDKey] == b[rollbackIDKey] {
|
||||
return maps.Equal(a, b)
|
||||
}
|
||||
for k, v := range a {
|
||||
if k != rollbackIDKey {
|
||||
if other, ok := b[k]; !ok || other != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
for k := range b {
|
||||
if k != rollbackIDKey {
|
||||
if _, ok := a[k]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -1,439 +0,0 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Keep the old slot calculation here to reproduce the false sharing between
|
||||
// distinct resources, including resources on separate storage drives.
|
||||
func mutationGuardLegacySlot(drive, path string) uint64 {
|
||||
h := uint64(14695981039346656037)
|
||||
for _, part := range []string{drive, path} {
|
||||
for i := range part {
|
||||
h = (h ^ uint64(part[i])) * 1099511628211
|
||||
}
|
||||
h *= 1099511628211
|
||||
}
|
||||
return h % 65536
|
||||
}
|
||||
|
||||
func mutationGuardCollision(t *testing.T, drive string, target uint64) string {
|
||||
t.Helper()
|
||||
for i := 0; i < 2000000; i++ {
|
||||
key := fmt.Sprintf("independent-%d", i)
|
||||
if mutationGuardLegacySlot(drive, pathJoin("bucket", key)) == target {
|
||||
return key
|
||||
}
|
||||
}
|
||||
t.Fatal("no collision found in bounded search")
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestMutationGuardIsolatesStorageResources(t *testing.T) {
|
||||
blocked, rootA, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer blocked.Close()
|
||||
healthy, rootB, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer healthy.Close()
|
||||
defer mutationGuardCheckEmpty(t, rootA, rootB)
|
||||
for _, disk := range []*xlStorageDiskIDCheck{blocked, healthy} {
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const object = "blocked"
|
||||
target := mutationGuardLegacySlot(rootA, pathJoin("bucket", object))
|
||||
type mutation struct {
|
||||
name, key string
|
||||
disk *xlStorageDiskIDCheck
|
||||
done chan error
|
||||
}
|
||||
mutations := []mutation{
|
||||
{name: "same-key", key: object, disk: blocked},
|
||||
{name: "different-key-same-drive-collision", key: mutationGuardCollision(t, rootA, target), disk: blocked},
|
||||
{name: "different-key-healthy-drive-collision", key: mutationGuardCollision(t, rootB, target), disk: healthy},
|
||||
{name: "same-path-healthy-drive", key: object, disk: healthy},
|
||||
}
|
||||
fixture := func(name string) FileInfo {
|
||||
fi := newFileInfo(name, 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Fresh = true
|
||||
fi.Metadata = map[string]string{"x-amz-meta-probe": "before"}
|
||||
return fi
|
||||
}
|
||||
for _, m := range mutations {
|
||||
if err := m.disk.WriteMetadata(t.Context(), "", "bucket", m.key, fixture(m.key)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
var lastSuccess int64
|
||||
gate := &stagedRenameContext{
|
||||
Context: ctx, meta: filepath.Join(rootA, minioMetaTmpBucket, "staged", xlStorageFormatFile),
|
||||
entered: make(chan struct{}), release: make(chan struct{}),
|
||||
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
|
||||
}
|
||||
var once sync.Once
|
||||
release := func() { once.Do(func() { close(gate.release) }) }
|
||||
var workers sync.WaitGroup
|
||||
defer func() {
|
||||
release()
|
||||
workers.Wait()
|
||||
}()
|
||||
holderDone := make(chan error, 1)
|
||||
workers.Go(func() {
|
||||
_, err := blocked.RenameData(gate, minioMetaTmpBucket, "staged", fixture(object), "bucket", object, RenameOptions{})
|
||||
holderDone <- err
|
||||
})
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("RenameData did not reach the staged metadata barrier")
|
||||
}
|
||||
// Cancellation cannot release the guard while the admitted worker can
|
||||
// still commit. Unrelated mutations must nevertheless be able to finish.
|
||||
cancel()
|
||||
for i := range mutations {
|
||||
m := &mutations[i]
|
||||
m.done = make(chan error, 1)
|
||||
workers.Go(func() {
|
||||
m.done <- m.disk.UpdateMetadata(t.Context(), "bucket", m.key, FileInfo{
|
||||
Metadata: map[string]string{"x-amz-meta-probe": "after"},
|
||||
}, UpdateMetadataOpts{})
|
||||
})
|
||||
}
|
||||
mutationGuardWaitRefs(t, rootA, pathJoin("bucket", object), 2)
|
||||
for _, m := range mutations[1:] {
|
||||
select {
|
||||
case err := <-m.done:
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", m.name, err)
|
||||
}
|
||||
t.Logf("%s completed while RenameData remained paused", m.name)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Errorf("%s blocked by an unrelated paused mutation", m.name)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case err := <-holderDone:
|
||||
t.Fatalf("canceled RenameData returned before its worker finished: %v", err)
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case err := <-mutations[0].done:
|
||||
t.Fatalf("same-key mutation escaped the held guard: %v", err)
|
||||
default:
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case err := <-holderDone:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("canceled RenameData result: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("RenameData did not finish after releasing the barrier")
|
||||
}
|
||||
select {
|
||||
case err := <-mutations[0].done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("same-key mutation did not finish after releasing the holder")
|
||||
}
|
||||
workers.Wait()
|
||||
for _, m := range mutations {
|
||||
fi, err := m.disk.ReadVersion(t.Context(), "", "bucket", m.key, "", ReadOptions{})
|
||||
if err != nil || fi.Metadata["x-amz-meta-probe"] != "after" {
|
||||
t.Errorf("%s persisted metadata: %q, %v", m.name, fi.Metadata["x-amz-meta-probe"], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mutationGuardCheckEmpty(t *testing.T, drives ...string) {
|
||||
t.Helper()
|
||||
storageMutationLocks.Lock()
|
||||
defer storageMutationLocks.Unlock()
|
||||
for key, lock := range storageMutationLocks.locks {
|
||||
if slices.Contains(drives, key.drive) {
|
||||
t.Errorf("unused guard retained for %+v with %d references", key, lock.refs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mutationGuardWaitRefs(t *testing.T, drive, path string, refs int) *storageMutationLock {
|
||||
t.Helper()
|
||||
deadline := time.NewTimer(5 * time.Second)
|
||||
defer deadline.Stop()
|
||||
tick := time.NewTicker(time.Millisecond)
|
||||
defer tick.Stop()
|
||||
key := storageMutationKey{drive, path}
|
||||
for {
|
||||
storageMutationLocks.Lock()
|
||||
lock := storageMutationLocks.locks[key]
|
||||
ready := lock != nil && lock.refs == refs
|
||||
storageMutationLocks.Unlock()
|
||||
if ready {
|
||||
return lock
|
||||
}
|
||||
select {
|
||||
case <-deadline.C:
|
||||
t.Fatalf("guard %+v did not reach %d references", key, refs)
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationGuardRetainsWaitersAndRemovesIdleEntries(t *testing.T) {
|
||||
s := &xlStorage{drivePath: t.TempDir()}
|
||||
defer mutationGuardCheckEmpty(t, s.drivePath)
|
||||
unlock := sync.OnceFunc(s.lockMutations("object"))
|
||||
defer unlock()
|
||||
original := mutationGuardWaitRefs(t, s.drivePath, "object", 1)
|
||||
acquired, release, finished := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
go func() {
|
||||
defer close(finished)
|
||||
defer s.lockMutations("object", "object")()
|
||||
close(acquired)
|
||||
<-release
|
||||
}()
|
||||
if lock := mutationGuardWaitRefs(t, s.drivePath, "object", 2); lock != original {
|
||||
t.Error("waiting operation registered a second mutex for the same resource")
|
||||
}
|
||||
select {
|
||||
case <-acquired:
|
||||
t.Error("waiter acquired before the holder released")
|
||||
default:
|
||||
}
|
||||
unlock()
|
||||
select {
|
||||
case <-acquired:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("waiter did not acquire after release")
|
||||
}
|
||||
if lock := mutationGuardWaitRefs(t, s.drivePath, "object", 1); lock != original {
|
||||
t.Error("holder release discarded the queued operation's mutex")
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case <-finished:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("waiter did not release")
|
||||
}
|
||||
mutationGuardCheckEmpty(t, s.drivePath)
|
||||
// The registry must not grow with the historical object count.
|
||||
for i := 0; i < 1000; i++ {
|
||||
s.lockMutations(fmt.Sprintf("object-%d", i))()
|
||||
}
|
||||
s.lockMutations()()
|
||||
}
|
||||
|
||||
func TestMutationGuardOrdersAndDeduplicatesPaths(t *testing.T) {
|
||||
s := &xlStorage{drivePath: t.TempDir()}
|
||||
defer mutationGuardCheckEmpty(t, s.drivePath)
|
||||
paths := []string{"b", "a", "b", "a"}
|
||||
before := slices.Clone(paths)
|
||||
unlock := s.lockMutations(paths...)
|
||||
mutationGuardWaitRefs(t, s.drivePath, "a", 1)
|
||||
mutationGuardWaitRefs(t, s.drivePath, "b", 1)
|
||||
unlock()
|
||||
if !slices.Equal(paths, before) {
|
||||
t.Fatalf("changed caller's path slice: %v", paths)
|
||||
}
|
||||
const workers, iterations = 8, 500
|
||||
start, done := make(chan struct{}), make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
var protected atomic.Int64
|
||||
var active atomic.Int32
|
||||
for i := 0; i < workers; i++ {
|
||||
worker := i
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
order := []string{"a", "b", "a"}
|
||||
if worker%2 != 0 {
|
||||
order = []string{"b", "a", "b"}
|
||||
}
|
||||
for j := 0; j < iterations; j++ {
|
||||
unlock := s.lockMutations(order...)
|
||||
if active.Add(1) != 1 {
|
||||
t.Error("overlapping guards entered concurrently")
|
||||
}
|
||||
protected.Add(1)
|
||||
active.Add(-1)
|
||||
unlock()
|
||||
}
|
||||
})
|
||||
}
|
||||
go func() { wg.Wait(); close(done) }()
|
||||
close(start)
|
||||
select {
|
||||
case <-done:
|
||||
if got := protected.Load(); got != workers*iterations {
|
||||
t.Fatalf("overlapping guards lost updates: got %d, want %d", got, workers*iterations)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("opposing path orders or duplicate paths deadlocked")
|
||||
}
|
||||
}
|
||||
|
||||
// Measure registry/allocation cost separately from storage I/O. Independent
|
||||
// workers use two resources, as RenameData does, and shared workers use one.
|
||||
func BenchmarkMutationGuard(b *testing.B) {
|
||||
s := &xlStorage{drivePath: "/benchmark/drive"}
|
||||
b.Run("single", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
s.lockMutations("bucket/object")()
|
||||
}
|
||||
})
|
||||
b.Run("parallel-independent", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
var worker atomic.Uint64
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
id := worker.Add(1)
|
||||
paths := []string{fmt.Sprintf("staging/%d", id), fmt.Sprintf("bucket/object-%d", id)}
|
||||
for pb.Next() {
|
||||
s.lockMutations(paths...)()
|
||||
}
|
||||
})
|
||||
})
|
||||
b.Run("parallel-shared", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
s.lockMutations("bucket/object")()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// A call must retain paths it has not reached yet, even across xlStorage
|
||||
// instances for the same drive. No resource wait may hold the registry mutex.
|
||||
func TestMutationGuardRetainsUnacquiredPaths(t *testing.T) {
|
||||
s := &xlStorage{drivePath: t.TempDir()}
|
||||
sameDrive := &xlStorage{drivePath: s.drivePath}
|
||||
defer mutationGuardCheckEmpty(t, s.drivePath)
|
||||
unlockA := sync.OnceFunc(s.lockMutations("a"))
|
||||
defer unlockA()
|
||||
acquired, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
go func() {
|
||||
defer close(done)
|
||||
defer sameDrive.lockMutations("b", "a")()
|
||||
close(acquired)
|
||||
<-release
|
||||
}()
|
||||
mutationGuardWaitRefs(t, s.drivePath, "a", 2)
|
||||
original := mutationGuardWaitRefs(t, s.drivePath, "b", 1)
|
||||
// The queued call has not locked b yet: sorting must put a first.
|
||||
unlockB := sync.OnceFunc(s.lockMutations("b"))
|
||||
defer unlockB()
|
||||
if got := mutationGuardWaitRefs(t, s.drivePath, "b", 2); got != original {
|
||||
t.Fatal("unacquired path got a different mutex")
|
||||
}
|
||||
unlockA()
|
||||
select {
|
||||
case <-acquired:
|
||||
t.Fatal("multi-path call escaped the independently held second path")
|
||||
default:
|
||||
}
|
||||
unlockB()
|
||||
select {
|
||||
case <-acquired:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("multi-path call did not acquire after both paths were released")
|
||||
}
|
||||
if got := mutationGuardWaitRefs(t, s.drivePath, "b", 1); got != original {
|
||||
t.Fatal("release replaced an entry retained by a pending multi-path call")
|
||||
}
|
||||
unblock()
|
||||
<-done
|
||||
// The variadic helper has no hidden four-path limit.
|
||||
s.lockMutations("e", "d", "c", "b", "a", "a")()
|
||||
}
|
||||
|
||||
func TestMutationGuardCanceledWaiterRetainsSerialization(t *testing.T) {
|
||||
disk, root, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
defer mutationGuardCheckEmpty(t, root)
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi := newFileInfo("object", 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Fresh = true
|
||||
fi.Metadata = map[string]string{"x-amz-meta-probe": "before"}
|
||||
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Use a separate instance with the same drive identity as the API wrapper.
|
||||
s := &xlStorage{drivePath: root}
|
||||
unlock := sync.OnceFunc(s.lockMutations("bucket/object"))
|
||||
defer unlock()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- disk.UpdateMetadata(ctx, "bucket", "object", FileInfo{
|
||||
Metadata: map[string]string{"x-amz-meta-probe": "canceled"},
|
||||
}, UpdateMetadataOpts{})
|
||||
}()
|
||||
mutationGuardWaitRefs(t, root, "bucket/object", 2)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("canceled API returned with worker still queued: %v", err)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
mutationGuardWaitRefs(t, root, "bucket/object", 2)
|
||||
unlock()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled waiter: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("canceled waiter did not finish after holder release")
|
||||
}
|
||||
got, err := disk.ReadVersion(t.Context(), "", "bucket", "object", "", ReadOptions{})
|
||||
if err != nil || got.Metadata["x-amz-meta-probe"] != "before" {
|
||||
t.Fatalf("canceled metadata update persisted: %v, %v", got.Metadata, err)
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
|
||||
@@ -1,218 +0,0 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// This is an undo capability, not a distributed fencing token. Any subsequent
|
||||
// mutation of the same version revokes it, including a no-op metadata update.
|
||||
const rollbackIDKey = ReservedMetadataPrefixLower + "rollback-id"
|
||||
|
||||
type storageMutationKey struct {
|
||||
drive, path string
|
||||
}
|
||||
|
||||
type storageMutationLock struct {
|
||||
mu sync.Mutex
|
||||
refs int // Holders and waiters; protected by the registry mutex.
|
||||
}
|
||||
|
||||
// Keep exact resource locks only while they have holders or waiters. The
|
||||
// registry mutex protects references, never storage I/O or resource waits.
|
||||
var storageMutationLocks = struct {
|
||||
sync.Mutex
|
||||
locks map[storageMutationKey]*storageMutationLock
|
||||
}{locks: make(map[storageMutationKey]*storageMutationLock)}
|
||||
|
||||
func (s *xlStorage) lockMutations(paths ...string) func() {
|
||||
// Each call uses one drive. Sorting its unique paths gives all overlapping
|
||||
// calls the same acquisition order without changing the caller's slice.
|
||||
paths = slices.Clone(paths)
|
||||
slices.Sort(paths)
|
||||
paths = slices.Compact(paths)
|
||||
drive := s.drivePath
|
||||
locks := make([]*storageMutationLock, len(paths))
|
||||
// Retain every entry before waiting so unlock cannot remove a mutex that
|
||||
// a queued operation still needs.
|
||||
storageMutationLocks.Lock()
|
||||
for i, path := range paths {
|
||||
key := storageMutationKey{drive, path}
|
||||
lock := storageMutationLocks.locks[key]
|
||||
if lock == nil {
|
||||
lock = &storageMutationLock{}
|
||||
storageMutationLocks.locks[key] = lock
|
||||
}
|
||||
lock.refs++
|
||||
locks[i] = lock
|
||||
}
|
||||
storageMutationLocks.Unlock()
|
||||
for _, lock := range locks {
|
||||
lock.mu.Lock()
|
||||
}
|
||||
return func() {
|
||||
for i := len(locks) - 1; i >= 0; i-- {
|
||||
locks[i].mu.Unlock()
|
||||
}
|
||||
storageMutationLocks.Lock()
|
||||
for i, path := range paths {
|
||||
locks[i].refs--
|
||||
if locks[i].refs == 0 {
|
||||
delete(storageMutationLocks.locks, storageMutationKey{drive, path})
|
||||
}
|
||||
}
|
||||
storageMutationLocks.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Compare logical version state, excluding per-shard checksums/index, physical
|
||||
// DataDir placement and properties of other versions in the same xl.meta.
|
||||
// Different DataDir values remain readable for the existing rebalance recovery
|
||||
// contract; rollback separately requires an exact DataDir match.
|
||||
func sameFileInfoVersion(a, b FileInfo) bool {
|
||||
if a.VersionID != b.VersionID || a.Deleted != b.Deleted || a.XLV1 != b.XLV1 || a.Size != b.Size ||
|
||||
a.TransitionStatus != b.TransitionStatus || a.TransitionTier != b.TransitionTier ||
|
||||
a.TransitionedObjName != b.TransitionedObjName || a.TransitionVersionID != b.TransitionVersionID ||
|
||||
!sameVersionMetadata(a.Metadata, b.Metadata) || !bytes.Equal(a.Checksum, b.Checksum) || len(a.Parts) != len(b.Parts) {
|
||||
return false
|
||||
}
|
||||
if !a.Deleted && a.Size != 0 && !a.Erasure.Equal(b.Erasure) {
|
||||
return false
|
||||
}
|
||||
for i, p := range a.Parts {
|
||||
q := b.Parts[i]
|
||||
if p.Number != q.Number || p.Size != q.Size || p.ActualSize != q.ActualSize || p.ETag != q.ETag || !bytes.Equal(p.Index, q.Index) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// The undo capability controls rollback ownership, not the logical contents.
|
||||
// A partial no-op update can revoke it without changing a single public field.
|
||||
func sameVersionMetadata(a, b map[string]string) bool {
|
||||
if a[rollbackIDKey] == b[rollbackIDKey] {
|
||||
return maps.Equal(a, b)
|
||||
}
|
||||
for k, v := range a {
|
||||
if k != rollbackIDKey {
|
||||
if other, ok := b[k]; !ok || other != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
for k := range b {
|
||||
if k != rollbackIDKey {
|
||||
if _, ok := a[k]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Legacy callers without an undo capability must match the persisted state,
|
||||
// rather than transient coordinator-only fields in their original FileInfo.
|
||||
func samePersistedVersion(current, expected FileInfo) bool {
|
||||
if current.Metadata[rollbackIDKey] != expected.Metadata[rollbackIDKey] {
|
||||
return false
|
||||
}
|
||||
var meta xlMetaV2
|
||||
if err := meta.AddVersion(expected); err != nil {
|
||||
return false
|
||||
}
|
||||
fi, err := meta.ToFileInfo(current.Volume, current.Name, expected.VersionID, true, true)
|
||||
return err == nil && sameFileInfoVersion(current, fi)
|
||||
}
|
||||
|
||||
// Caller holds the object's mutation guard. Restore just the overwritten
|
||||
// version, and persist that decision before removing any external shards.
|
||||
func (s *xlStorage) undoWrite(ctx context.Context, volume, path string, fi FileInfo, opts DeleteOptions, meta *xlMetaV2) error {
|
||||
versionID := fi.VersionID
|
||||
if versionID == "" {
|
||||
versionID = nullVersionID
|
||||
}
|
||||
current, err := meta.ToFileInfo(volume, path, versionID, true, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current.DataDir != fi.DataDir || !current.ModTime.Equal(fi.ModTime) || !samePersistedVersion(current, fi) {
|
||||
return errFileVersionNotFound
|
||||
}
|
||||
var previous xlMetaV2
|
||||
var oldVersion *xlMetaV2Version
|
||||
var oldData []byte
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.OldDataDir != "" {
|
||||
buf, err := s.readAllData(ctx, volume, volumeDir, pathJoin(volumeDir, path, opts.OldDataDir, xlStorageFormatFileBackup))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer metaDataPoolPut(buf)
|
||||
if err := previous.LoadOrConvert(buf); err != nil {
|
||||
return err
|
||||
}
|
||||
_, oldVersion, err = previous.findVersionStr(versionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldData = previous.data.find(versionID)
|
||||
if oldData == nil {
|
||||
oldData = previous.data.find(oldVersion.getDataDir())
|
||||
}
|
||||
}
|
||||
dataDir, err := meta.DeleteVersion(fi)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dataDir != "" {
|
||||
meta.data.remove(versionID, dataDir)
|
||||
}
|
||||
if oldVersion != nil {
|
||||
if err := meta.addVersion(*oldVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
if oldData != nil {
|
||||
meta.data.replace(versionID, oldData)
|
||||
}
|
||||
if dataDir == oldVersion.getDataDir() {
|
||||
dataDir = ""
|
||||
}
|
||||
}
|
||||
if len(meta.versions) > 0 {
|
||||
buf, err := meta.AppendTo(metaDataPoolGet())
|
||||
defer metaDataPoolPut(buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.writeAllMeta(ctx, volume, pathJoin(path, xlStorageFormatFile), buf, true); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := s.deleteFile(volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile), true, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if dataDir != "" {
|
||||
if err := s.moveToTrash(pathJoin(volumeDir, path, dataDir), true, false); err != nil && !errors.Is(err, errFileNotFound) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if opts.OldDataDir != "" {
|
||||
backupDir := pathJoin(volumeDir, path, opts.OldDataDir)
|
||||
Remove(pathJoin(backupDir, xlStorageFormatFileBackup))
|
||||
if strings.HasSuffix(opts.OldDataDir, "-rollback") {
|
||||
Remove(backupDir)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
|
||||
@@ -408,11 +408,8 @@ func (client *storageRESTClient) CreateFile(ctx context.Context, origvolume, vol
|
||||
}
|
||||
|
||||
func (client *storageRESTClient) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
// As with RenameData, admitted metadata writes must finish before unlock.
|
||||
ctx = grid.WithoutDefaultTimeout(context.WithoutCancel(ctx))
|
||||
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
|
||||
defer cancel()
|
||||
|
||||
_, err := storageWriteMetadataRPC.Call(ctx, client.gridConn, &MetadataHandlerParams{
|
||||
DiskID: *client.diskID.Load(),
|
||||
@@ -425,10 +422,8 @@ func (client *storageRESTClient) WriteMetadata(ctx context.Context, origvolume,
|
||||
}
|
||||
|
||||
func (client *storageRESTClient) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = grid.WithoutDefaultTimeout(context.WithoutCancel(ctx))
|
||||
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
|
||||
defer cancel()
|
||||
|
||||
_, err := storageUpdateMetadataRPC.Call(ctx, client.gridConn, &MetadataHandlerParams{
|
||||
DiskID: *client.diskID.Load(),
|
||||
@@ -441,14 +436,8 @@ func (client *storageRESTClient) UpdateMetadata(ctx context.Context, volume, pat
|
||||
}
|
||||
|
||||
func (client *storageRESTClient) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) (err error) {
|
||||
if opts.UndoWrite {
|
||||
// Rollback is part of the commit and must finish before unlocking.
|
||||
ctx = grid.WithoutDefaultTimeout(context.WithoutCancel(ctx))
|
||||
} else {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
|
||||
defer cancel()
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
|
||||
defer cancel()
|
||||
|
||||
_, err = storageDeleteVersionRPC.Call(ctx, client.gridConn, &DeleteVersionHandlerParams{
|
||||
DiskID: *client.diskID.Load(),
|
||||
@@ -498,13 +487,6 @@ func (client *storageRESTClient) CheckParts(ctx context.Context, volume string,
|
||||
func (client *storageRESTClient) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo,
|
||||
dstVolume, dstPath string, opts RenameOptions,
|
||||
) (res RenameDataResp, err error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, err
|
||||
}
|
||||
// The remote handler cannot cancel an admitted RenameData. Keep waiting
|
||||
// for its result so renameData can roll back every completed mutation
|
||||
// before the caller cleans staging files and releases the namespace lock.
|
||||
ctx = grid.WithoutDefaultTimeout(context.WithoutCancel(ctx))
|
||||
params := RenameDataHandlerParams{
|
||||
DiskID: *client.diskID.Load(),
|
||||
SrcVolume: srcVolume,
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
type cancelableRenameDisk struct {
|
||||
StorageAPI
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d cancelableRenameDisk) RenameData(ctx context.Context, _, _ string, _ FileInfo, _, _ string, _ RenameOptions) (RenameDataResp, error) {
|
||||
d.entered <- struct{}{}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return RenameDataResp{}, ctx.Err()
|
||||
case <-d.release:
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageTimeoutCoordinatorCancellation(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
entered, release := make(chan struct{}, 4), make(chan struct{})
|
||||
defer close(release)
|
||||
disks := make([]StorageAPI, 4)
|
||||
metadata := make([]FileInfo, 4)
|
||||
for i := range disks {
|
||||
disks[i] = cancelableRenameDisk{entered: entered, release: release}
|
||||
metadata[i] = FileInfo{Deleted: true}
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, _, err := renameData(ctx, disks, minioMetaTmpBucket, "staged", metadata, "bucket", "object", 3)
|
||||
done <- err
|
||||
}()
|
||||
for range disks {
|
||||
<-entered
|
||||
}
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("canceled commit unexpectedly succeeded")
|
||||
}
|
||||
default:
|
||||
t.Fatal("coordinator did not propagate cancellation to disk calls")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type rollbackOptionsDisk struct {
|
||||
StorageAPI
|
||||
mu sync.Mutex
|
||||
opts DeleteOptions
|
||||
}
|
||||
|
||||
func (d *rollbackOptionsDisk) DeleteVersion(_ context.Context, _, _ string, _ FileInfo, _ bool, opts DeleteOptions) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.opts = opts
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestStorageRollbackRPCOptions(t *testing.T) {
|
||||
endpoint, err := NewEndpoint(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disk := &rollbackOptionsDisk{}
|
||||
globalLocalDrivesMu.Lock()
|
||||
oldDrives, oldSets := globalLocalDrivesMap, globalLocalSetDrives
|
||||
globalLocalDrivesMap = map[string]StorageAPI{endpoint.String(): disk}
|
||||
globalLocalSetDrives = nil
|
||||
globalLocalDrivesMu.Unlock()
|
||||
defer func() {
|
||||
globalLocalDrivesMu.Lock()
|
||||
globalLocalDrivesMap, globalLocalSetDrives = oldDrives, oldSets
|
||||
globalLocalDrivesMu.Unlock()
|
||||
}()
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tg.Cleanup()
|
||||
server := storageRESTServer{endpoint: endpoint}
|
||||
const route = "/rollback-options"
|
||||
if err := storageDeleteVersionRPC.Register(tg.Managers[1], server.DeleteVersionHandler, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
|
||||
client.diskID.Store(new(string))
|
||||
want := DeleteOptions{UndoWrite: true, OldDataDir: mustGetUUID()}
|
||||
if err := client.DeleteVersion(t.Context(), "bucket", "object", FileInfo{}, false, want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disk.mu.Lock()
|
||||
defer disk.mu.Unlock()
|
||||
if disk.opts != want {
|
||||
t.Fatalf("remote handler lost rollback options: got %+v, want %+v", disk.opts, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoragePrereleaseMetadataHidden(t *testing.T) {
|
||||
disk, _, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi := newFileInfo("object", 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Metadata = map[string]string{rollbackIDKey: mustGetUUID(), "user-key": "value"}
|
||||
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := disk.ReadVersion(t.Context(), "", "bucket", "object", "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
public := cleanMetadata(got.Metadata)
|
||||
if _, ok := public[rollbackIDKey]; ok || public["user-key"] != "value" {
|
||||
t.Fatalf("pre-release metadata exposed or user metadata lost: %+v", public)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
// TrackDiskHealth makes the first lookup. Pause the real disk worker at its
|
||||
// next health callback, without requiring a broken drive or a wall-clock sleep.
|
||||
type timeoutDiskContext struct {
|
||||
context.Context
|
||||
calls atomic.Int32
|
||||
entered, release chan struct{}
|
||||
tracker healthDiskCtxValue
|
||||
pauseDeadline bool
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (c *timeoutDiskContext) Deadline() (time.Time, bool) {
|
||||
if c.pauseDeadline {
|
||||
// Metadata operations create a nested deadline in their disk worker.
|
||||
c.once.Do(func() { close(c.entered); <-c.release })
|
||||
}
|
||||
return c.Context.Deadline()
|
||||
}
|
||||
|
||||
func (c *timeoutDiskContext) Value(key any) any {
|
||||
if _, ok := key.(healthDiskCtxKey); ok {
|
||||
if !c.pauseDeadline && c.calls.Add(1) == 2 {
|
||||
close(c.entered)
|
||||
<-c.release
|
||||
}
|
||||
return &c.tracker
|
||||
}
|
||||
return c.Context.Value(key)
|
||||
}
|
||||
|
||||
func TestStorageTimeoutLocal(t *testing.T) {
|
||||
for _, operation := range []string{"rename", "rename-cancel", "write-metadata", "update-metadata", "undo"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
disk, _, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi := newFileInfo("object", 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Metadata = map[string]string{"test": "original"}
|
||||
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
var lastSuccess int64
|
||||
gate := &timeoutDiskContext{
|
||||
Context: ctx, entered: make(chan struct{}), release: make(chan struct{}),
|
||||
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
|
||||
pauseDeadline: operation != "rename" && operation != "rename-cancel",
|
||||
}
|
||||
unblock := sync.OnceFunc(func() { close(gate.release) })
|
||||
defer unblock()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
var err error
|
||||
switch operation {
|
||||
case "rename", "rename-cancel":
|
||||
_, err = disk.RenameData(gate, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
|
||||
case "write-metadata":
|
||||
err = disk.WriteMetadata(gate, "", "bucket", "object", fi)
|
||||
case "update-metadata":
|
||||
err = disk.UpdateMetadata(gate, "bucket", "object", fi, UpdateMetadataOpts{})
|
||||
case "undo":
|
||||
err = disk.DeleteVersion(gate, "bucket", "object", fi, false, DeleteOptions{UndoWrite: true})
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("operation did not reach the disk worker: %v", err)
|
||||
}
|
||||
want := context.DeadlineExceeded
|
||||
if operation == "rename-cancel" {
|
||||
cancel()
|
||||
want = context.Canceled
|
||||
} else {
|
||||
time.Sleep(globalDriveConfig.GetMaxTimeout() + time.Second)
|
||||
}
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("expected %v, got %v", want, err)
|
||||
}
|
||||
default:
|
||||
t.Fatal("operation still waits for the blocked worker after its deadline")
|
||||
}
|
||||
// A timed-out worker must not leave a new physical mutation lock
|
||||
// preventing subsequent work on this object.
|
||||
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
|
||||
t.Fatalf("subsequent metadata write: %v", err)
|
||||
}
|
||||
unblock()
|
||||
// synctest waits for the released worker before disk teardown.
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageTimeoutRPC(t *testing.T) {
|
||||
for _, operation := range []string{"rename", "inline", "write-metadata", "update-metadata", "undo"} {
|
||||
for _, deadline := range []bool{false, true} {
|
||||
mode := "cancel"
|
||||
if deadline {
|
||||
mode = "deadline"
|
||||
}
|
||||
t.Run(operation+"/"+mode, func(t *testing.T) {
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tg.Cleanup()
|
||||
entered, release, finished := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
handle := func() {
|
||||
close(entered)
|
||||
<-release
|
||||
close(finished)
|
||||
}
|
||||
const route = "/storage-timeout"
|
||||
if err := storageRenameDataRPC.Register(tg.Managers[1], func(*RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
handle()
|
||||
return &RenameDataResp{}, nil
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
defer p.Recycle()
|
||||
handle()
|
||||
return &RenameDataResp{}, nil
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta := func(*MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
|
||||
handle()
|
||||
return grid.NewNoPayload(), nil
|
||||
}
|
||||
if err := storageWriteMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageUpdateMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageDeleteVersionRPC.Register(tg.Managers[1], func(p *DeleteVersionHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
|
||||
if !p.Opts.UndoWrite || p.Opts.OldDataDir != "previous" {
|
||||
t.Error("rollback options were lost on the wire")
|
||||
}
|
||||
return meta(nil)
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
|
||||
client.diskID.Store(new(string))
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
if deadline {
|
||||
var stop context.CancelFunc
|
||||
ctx, stop = context.WithTimeout(ctx, 500*time.Millisecond)
|
||||
defer stop()
|
||||
}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
var err error
|
||||
fi := FileInfo{}
|
||||
switch operation {
|
||||
case "write-metadata":
|
||||
err = client.WriteMetadata(ctx, "", "bucket", "object", fi)
|
||||
case "update-metadata":
|
||||
err = client.UpdateMetadata(ctx, "bucket", "object", fi, UpdateMetadataOpts{})
|
||||
case "undo":
|
||||
err = client.DeleteVersion(ctx, "bucket", "object", fi, false, DeleteOptions{UndoWrite: true, OldDataDir: "previous"})
|
||||
default:
|
||||
if operation == "inline" {
|
||||
fi.Data = []byte("payload")
|
||||
}
|
||||
_, err = client.RenameData(ctx, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("remote worker did not start")
|
||||
}
|
||||
want := context.DeadlineExceeded
|
||||
if !deadline {
|
||||
cancel()
|
||||
want = context.Canceled
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, want) {
|
||||
t.Fatalf("expected %v, got %v", want, err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("RPC still waits for the remote worker")
|
||||
}
|
||||
unblock()
|
||||
<-finished
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -495,7 +495,7 @@ func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPat
|
||||
if len(fi.Data) > 0 {
|
||||
fi.Data = append(grid.GetByteBufferCap(len(fi.Data))[:0], fi.Data...)
|
||||
}
|
||||
return xioutil.WithDeadlineAndWait[RenameDataResp](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (res RenameDataResp, err error) {
|
||||
return xioutil.WithDeadline[RenameDataResp](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (res RenameDataResp, err error) {
|
||||
if len(fi.Data) > 0 {
|
||||
defer grid.PutByteBuffer(fi.Data)
|
||||
}
|
||||
@@ -634,12 +634,6 @@ func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path s
|
||||
}
|
||||
}()
|
||||
|
||||
if opts.UndoWrite {
|
||||
_, err = xioutil.WithDeadlineAndWait(ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
|
||||
})
|
||||
return err
|
||||
}
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
return w.Run(func() error { return p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts) })
|
||||
}
|
||||
@@ -651,10 +645,8 @@ func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path
|
||||
}
|
||||
defer done(0, &err)
|
||||
|
||||
_, err = xioutil.WithDeadlineAndWait(ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, p.storage.UpdateMetadata(ctx, volume, path, fi, opts)
|
||||
})
|
||||
return err
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
return w.Run(func() error { return p.storage.UpdateMetadata(ctx, volume, path, fi, opts) })
|
||||
}
|
||||
|
||||
func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) (err error) {
|
||||
@@ -664,10 +656,8 @@ func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, origvolume, vo
|
||||
}
|
||||
defer done(0, &err)
|
||||
|
||||
_, err = xioutil.WithDeadlineAndWait(ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, p.storage.WriteMetadata(ctx, origvolume, volume, path, fi)
|
||||
})
|
||||
return err
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
return w.Run(func() error { return p.storage.WriteMetadata(ctx, origvolume, volume, path, fi) })
|
||||
}
|
||||
|
||||
func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (fi FileInfo, err error) {
|
||||
|
||||
@@ -1570,11 +1570,7 @@ func (x *xlMetaV2) UpdateObjectVersion(fi FileInfo) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(ver.ObjectV2.MetaSys, rollbackIDKey)
|
||||
for k, v := range fi.Metadata {
|
||||
if k == rollbackIDKey {
|
||||
continue
|
||||
}
|
||||
if len(k) > len(ReservedMetadataPrefixLower) && strings.EqualFold(k[:len(ReservedMetadataPrefixLower)], ReservedMetadataPrefixLower) {
|
||||
ver.ObjectV2.MetaSys[k] = []byte(v)
|
||||
} else {
|
||||
|
||||
+11
-45
@@ -25,7 +25,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
pathutil "path"
|
||||
"path/filepath"
|
||||
@@ -1101,7 +1100,6 @@ func (s *xlStorage) ListDir(ctx context.Context, origvolume, volume, dirPath str
|
||||
}
|
||||
|
||||
func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis ...FileInfo) error {
|
||||
defer s.lockMutations(pathJoin(volume, path))()
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1310,10 +1308,11 @@ func (s *xlStorage) readAllData(ctx context.Context, volume, volumeDir string, f
|
||||
})
|
||||
}
|
||||
|
||||
func (s *xlStorage) moveToTrash(filePath string, recursive, immediatePurge bool) error {
|
||||
// The caller must retain its mutation guard until this rename completes.
|
||||
// Only purging the already detached unique trash directory is asynchronous.
|
||||
return s.moveToTrashNoDeadline(filePath, recursive, immediatePurge)
|
||||
func (s *xlStorage) moveToTrash(filePath string, recursive, immediatePurge bool) (err error) {
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
return w.Run(func() (err error) {
|
||||
return s.moveToTrashNoDeadline(filePath, recursive, immediatePurge)
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteVersion - deletes FileInfo metadata for path at `xl.meta`. forceDelMarker
|
||||
@@ -1326,8 +1325,6 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
})
|
||||
}
|
||||
|
||||
defer s.lockMutations(pathJoin(volume, path))()
|
||||
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1348,7 +1345,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
metaDataPoolPut(buf) // Never used, return it
|
||||
if fi.Deleted && forceDelMarker {
|
||||
// Create a new xl.meta with a delete marker in it
|
||||
return s.writeMetadataNoLock(ctx, "", volume, path, fi)
|
||||
return s.WriteMetadata(ctx, "", volume, path, fi)
|
||||
}
|
||||
|
||||
s.RLock()
|
||||
@@ -1384,10 +1381,6 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
return err
|
||||
}
|
||||
|
||||
if opts.UndoWrite {
|
||||
return s.undoWrite(ctx, volume, path, fi, opts, &xlMeta)
|
||||
}
|
||||
|
||||
dataDir, err := xlMeta.DeleteVersion(fi)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1428,12 +1421,15 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
return s.writeAllMeta(ctx, volume, pathJoin(path, xlStorageFormatFile), buf, true)
|
||||
}
|
||||
|
||||
if opts.UndoWrite && opts.OldDataDir != "" {
|
||||
return renameAll(pathJoin(filePath, opts.OldDataDir, xlStorageFormatFileBackup), pathJoin(filePath, xlStorageFormatFile), filePath)
|
||||
}
|
||||
|
||||
return s.deleteFile(volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile), true, false)
|
||||
}
|
||||
|
||||
// Updates only metadata for a given version.
|
||||
func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) (err error) {
|
||||
defer s.lockMutations(pathJoin(volume, path))()
|
||||
if len(fi.Metadata) == 0 {
|
||||
return errInvalidArgument
|
||||
}
|
||||
@@ -1482,16 +1478,6 @@ func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi
|
||||
|
||||
// WriteMetadata - writes FileInfo metadata for path at `xl.meta`
|
||||
func (s *xlStorage) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) (err error) {
|
||||
defer s.lockMutations(pathJoin(volume, path))()
|
||||
return s.writeMetadataNoLock(ctx, origvolume, volume, path, fi)
|
||||
}
|
||||
|
||||
func (s *xlStorage) writeMetadataNoLock(ctx context.Context, origvolume, volume, path string, fi FileInfo) (err error) {
|
||||
// Any later write of this version revokes the original writer's undo.
|
||||
if _, ok := fi.Metadata[rollbackIDKey]; ok {
|
||||
fi.Metadata = maps.Clone(fi.Metadata)
|
||||
delete(fi.Metadata, rollbackIDKey)
|
||||
}
|
||||
if fi.Fresh {
|
||||
if origvolume != "" {
|
||||
origvolumeDir, err := s.getVolDir(origvolume)
|
||||
@@ -2554,10 +2540,7 @@ func (s *xlStorage) DeleteBulk(ctx context.Context, volume string, paths ...stri
|
||||
return err
|
||||
}
|
||||
|
||||
unlock := s.lockMutations(pathJoin(volume, fp), pathJoin(volume, pathutil.Dir(fp)))
|
||||
err = s.moveToTrash(filePath, false, false)
|
||||
unlock()
|
||||
if err != nil {
|
||||
if err = s.moveToTrash(filePath, false, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -2567,7 +2550,6 @@ func (s *xlStorage) DeleteBulk(ctx context.Context, volume string, paths ...stri
|
||||
|
||||
// Delete - delete a file at path.
|
||||
func (s *xlStorage) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
|
||||
defer s.lockMutations(pathJoin(volume, path), pathJoin(volume, pathutil.Dir(path)))()
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -2597,7 +2579,6 @@ func skipAccessChecks(volume string) (ok bool) {
|
||||
|
||||
// RenameData - rename source path to destination path atomically, metadata and data directory.
|
||||
func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (res RenameDataResp, err error) {
|
||||
defer s.lockMutations(pathJoin(srcVolume, srcPath), pathJoin(dstVolume, dstPath))()
|
||||
defer func() {
|
||||
ignoredErrs := []error{
|
||||
errFileNotFound,
|
||||
@@ -2841,12 +2822,6 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
|
||||
xlMeta.data.remove(reqVID, dataDir)
|
||||
}
|
||||
}
|
||||
if !healing && res.OldDataDir == "" {
|
||||
// Shared data must survive both commit and rollback. Keep the
|
||||
// previous metadata in a separate transaction directory instead
|
||||
// of using a data directory still owned by another version.
|
||||
res.OldDataDir = fi.DataDir + "-rollback"
|
||||
}
|
||||
}
|
||||
|
||||
if err = xlMeta.AddVersion(fi); err != nil {
|
||||
@@ -3299,7 +3274,6 @@ func (s *xlStorage) CleanAbandonedData(ctx context.Context, volume string, path
|
||||
if volume == "" || path == "" {
|
||||
return nil // Ignore
|
||||
}
|
||||
defer s.lockMutations(pathJoin(volume, path))()
|
||||
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
@@ -3351,14 +3325,6 @@ func (s *xlStorage) CleanAbandonedData(ctx context.Context, volume string, path
|
||||
// Do not abort on context errors.
|
||||
for dir := range foundDirs {
|
||||
toRemove := pathJoin(volumeDir, path, dir+SlashSeparator)
|
||||
// A completed disk mutation may still be awaiting its quorum result.
|
||||
// Its previous generation is needed if that transaction must undo.
|
||||
// Only commit/undo cleanup owns a directory containing this backup.
|
||||
if _, err := Lstat(pathJoin(toRemove, xlStorageFormatFileBackup)); err == nil {
|
||||
continue
|
||||
} else if !osIsNotExist(err) {
|
||||
return osErrToFileErr(err)
|
||||
}
|
||||
err = s.deleteFile(volumeDir, toRemove, true, true)
|
||||
diskHealthCheckOK(ctx, err)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user