mirror of
https://github.com/pgsty/minio.git
synced 2026-10-31 21:02:06 +03:00
fix: retain maintenance read compatibility with write safety fixes
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
@@ -53,6 +53,16 @@ Before opening a pull request:
|
|||||||
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
|
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
|
||||||
`.minio.sys`, ARN, module/import-path, or serialized compatibility name.
|
`.minio.sys`, ARN, module/import-path, or serialized compatibility name.
|
||||||
|
|
||||||
|
The maintenance branch retains the existing read-quorum and healing policy.
|
||||||
|
Deferred strict-read and cross-pool recovery tests are preserved behind the
|
||||||
|
`consistency_strict` build tag. Run them explicitly with
|
||||||
|
`go test -tags kqueue,dev,consistency_strict ./cmd`; known failures must be
|
||||||
|
reported separately, not described as fixed or included in a passing result.
|
||||||
|
They cover late commits after lease loss, stale metadata reads, and stricter
|
||||||
|
read refusal. The normal suite covers the retained write/undo/cleanup fixes
|
||||||
|
and continued readability after a failed partial overwrite. This scope does
|
||||||
|
not provide distributed fencing or repair previously inconsistent objects.
|
||||||
|
|
||||||
### Commit changes
|
### Commit changes
|
||||||
|
|
||||||
After verification, commit your changes with a concise message and a DCO
|
After verification, commit your changes with a concise message and a DCO
|
||||||
|
|||||||
@@ -185,7 +185,7 @@ func listObjectModtimes(partsMetadata []FileInfo, errs []error) (modTimes []time
|
|||||||
|
|
||||||
func filterOnlineDisksInplace(fi FileInfo, partsMetadata []FileInfo, onlineDisks []StorageAPI) {
|
func filterOnlineDisksInplace(fi FileInfo, partsMetadata []FileInfo, onlineDisks []StorageAPI) {
|
||||||
for i, meta := range partsMetadata {
|
for i, meta := range partsMetadata {
|
||||||
if sameFileInfoVersion(fi, meta) {
|
if fi.XLV1 == meta.XLV1 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
onlineDisks[i] = nil
|
onlineDisks[i] = nil
|
||||||
|
|||||||
@@ -373,9 +373,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return result, err
|
return result, err
|
||||||
}
|
}
|
||||||
if err := checkObjectGenerationQuorum(latestMeta, partsMetadata, errs); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// No modtime quorum
|
// No modtime quorum
|
||||||
filterDisksByETag := quorumETag != ""
|
filterDisksByETag := quorumETag != ""
|
||||||
|
|||||||
+57
-34
@@ -19,6 +19,7 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/hex"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -27,6 +28,8 @@ import (
|
|||||||
"github.com/minio/minio/internal/amztime"
|
"github.com/minio/minio/internal/amztime"
|
||||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||||
"github.com/minio/minio/internal/bucket/replication"
|
"github.com/minio/minio/internal/bucket/replication"
|
||||||
|
"github.com/minio/minio/internal/crypto"
|
||||||
|
"github.com/minio/minio/internal/hash/sha256"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||||
)
|
)
|
||||||
@@ -316,48 +319,68 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
|
|||||||
if quorum < 1 {
|
if quorum < 1 {
|
||||||
return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInsufficientOnlineDrives}
|
return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInsufficientOnlineDrives}
|
||||||
}
|
}
|
||||||
// Group by exact logical state rather than a layout-only hash. There are
|
metaHashes := make([]string, len(metaArr))
|
||||||
// at most one erasure set's worth of candidates; healthy objects compare
|
h := sha256.New()
|
||||||
// against a single representative without hashing or a second XL decode.
|
|
||||||
groups := make([]int, len(metaArr))
|
|
||||||
representatives := make([]int, 0, len(metaArr))
|
|
||||||
counts := make([]int, len(metaArr)+1)
|
|
||||||
for i, meta := range metaArr {
|
for i, meta := range metaArr {
|
||||||
if !meta.IsValid() {
|
if !meta.IsValid() {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
etagOnly := modTime.Equal(timeSentinel) && etag != "" && etag == meta.Metadata["etag"]
|
etagOnly := modTime.Equal(timeSentinel) && (etag != "" && etag == meta.Metadata["etag"])
|
||||||
if !meta.ModTime.Equal(modTime) && !etagOnly {
|
mtimeValid := meta.ModTime.Equal(modTime)
|
||||||
continue
|
if mtimeValid || etagOnly {
|
||||||
}
|
fmt.Fprintf(h, "%v", meta.XLV1)
|
||||||
group := 0
|
for _, part := range meta.Parts {
|
||||||
for j, representative := range representatives {
|
fmt.Fprintf(h, "part.%d", part.Number)
|
||||||
if sameFileInfoVersion(metaArr[representative], meta) {
|
fmt.Fprintf(h, "part.%d", part.Size)
|
||||||
group = j + 1
|
|
||||||
break
|
|
||||||
}
|
}
|
||||||
|
// Previously we checked if we had quorum on DataDir value.
|
||||||
|
// We have removed this check to allow reading objects with different DataDir
|
||||||
|
// values in a few drives (due to a rebalance-stop race bug)
|
||||||
|
// provided their their etags or ModTimes match.
|
||||||
|
|
||||||
|
if !meta.Deleted && meta.Size != 0 {
|
||||||
|
fmt.Fprintf(h, "%v+%v", meta.Erasure.DataBlocks, meta.Erasure.ParityBlocks)
|
||||||
|
fmt.Fprintf(h, "%v", meta.Erasure.Distribution)
|
||||||
|
}
|
||||||
|
|
||||||
|
if meta.IsRemote() {
|
||||||
|
// ILM transition fields
|
||||||
|
fmt.Fprint(h, meta.TransitionStatus)
|
||||||
|
fmt.Fprint(h, meta.TransitionTier)
|
||||||
|
fmt.Fprint(h, meta.TransitionedObjName)
|
||||||
|
fmt.Fprint(h, meta.TransitionVersionID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If metadata says encrypted, ask for it in quorum.
|
||||||
|
if etyp, ok := crypto.IsEncrypted(meta.Metadata); ok {
|
||||||
|
fmt.Fprint(h, etyp)
|
||||||
|
}
|
||||||
|
|
||||||
|
// If compressed, look for compressed FileInfo only
|
||||||
|
if meta.IsCompressed() {
|
||||||
|
fmt.Fprint(h, meta.Metadata[ReservedMetadataPrefix+"compression"])
|
||||||
|
}
|
||||||
|
|
||||||
|
metaHashes[i] = hex.EncodeToString(h.Sum(nil))
|
||||||
|
h.Reset()
|
||||||
}
|
}
|
||||||
if group == 0 {
|
|
||||||
representatives = append(representatives, i)
|
|
||||||
group = len(representatives)
|
|
||||||
}
|
|
||||||
groups[i] = group
|
|
||||||
counts[group]++
|
|
||||||
}
|
}
|
||||||
maxGroup, maxCount := 0, 0
|
|
||||||
tied := false
|
metaHashCountMap := make(map[string]int)
|
||||||
for group, count := range counts {
|
for _, hash := range metaHashes {
|
||||||
if group == 0 {
|
if hash == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if count > maxCount {
|
metaHashCountMap[hash]++
|
||||||
maxGroup, maxCount, tied = group, count, false
|
|
||||||
} else if count == maxCount {
|
|
||||||
tied = true
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if tied {
|
|
||||||
return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInconsistentMeta}
|
maxHash := ""
|
||||||
|
maxCount := 0
|
||||||
|
for hash, count := range metaHashCountMap {
|
||||||
|
if count > maxCount {
|
||||||
|
maxCount = count
|
||||||
|
maxHash = hash
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if maxCount < quorum {
|
if maxCount < quorum {
|
||||||
@@ -374,8 +397,8 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
|
|||||||
otherPropsMap := make(counterMap[objProps])
|
otherPropsMap := make(counterMap[objProps])
|
||||||
var candidate FileInfo
|
var candidate FileInfo
|
||||||
var found bool
|
var found bool
|
||||||
for i, group := range groups {
|
for i, hash := range metaHashes {
|
||||||
if group == maxGroup {
|
if hash == maxHash {
|
||||||
if metaArr[i].IsValid() {
|
if metaArr[i].IsValid() {
|
||||||
if !found {
|
if !found {
|
||||||
candidate = metaArr[i]
|
candidate = metaArr[i]
|
||||||
|
|||||||
+25
-34
@@ -126,9 +126,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return oi, toObjectErr(err, srcBucket, srcObject)
|
return oi, toObjectErr(err, srcBucket, srcObject)
|
||||||
}
|
}
|
||||||
if err := checkObjectGenerationQuorum(fi, metaArr, errs); err != nil {
|
|
||||||
return oi, toObjectErr(err, srcBucket, srcObject)
|
|
||||||
}
|
|
||||||
if fi.Deleted {
|
if fi.Deleted {
|
||||||
if srcOpts.VersionID == "" {
|
if srcOpts.VersionID == "" {
|
||||||
return oi, toObjectErr(errFileNotFound, srcBucket, srcObject)
|
return oi, toObjectErr(errFileNotFound, srcBucket, srcObject)
|
||||||
@@ -620,18 +617,6 @@ func readAllRawFileInfo(ctx context.Context, disks []StorageAPI, bucket, object
|
|||||||
return rawFileInfos, g.Wait()
|
return rawFileInfos, g.Wait()
|
||||||
}
|
}
|
||||||
|
|
||||||
// An older quorum cannot resolve a later overwrite of the same version.
|
|
||||||
// Reads and healing must preserve that uncertainty rather than roll back
|
|
||||||
// the visible object or erase the newer fragments.
|
|
||||||
func checkObjectGenerationQuorum(fi FileInfo, metadata []FileInfo, errs []error) error {
|
|
||||||
for i, meta := range metadata {
|
|
||||||
if errs[i] == nil && meta.IsValid() && meta.VersionID == fi.VersionID && meta.ModTime.After(fi.ModTime) {
|
|
||||||
return InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInconsistentMeta}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo, errs []error, bucket, object string, readData, inclFreeVers bool) ([]FileInfo, []error) {
|
func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo, errs []error, bucket, object string, readData, inclFreeVers bool) ([]FileInfo, []error) {
|
||||||
metadataArray := make([]*xlMetaV2, len(rawFileInfos))
|
metadataArray := make([]*xlMetaV2, len(rawFileInfos))
|
||||||
metaFileInfos := make([]FileInfo, len(rawFileInfos))
|
metaFileInfos := make([]FileInfo, len(rawFileInfos))
|
||||||
@@ -674,17 +659,6 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
|
|||||||
for i := range errs {
|
for i := range errs {
|
||||||
if errs[i] == nil {
|
if errs[i] == nil {
|
||||||
errs[i] = err
|
errs[i] = err
|
||||||
// An empty quorum merge does not prove absence when this
|
|
||||||
// disk contains a readable object generation. Preserve that
|
|
||||||
// uncertainty instead of hiding it behind FileNotFound.
|
|
||||||
if errors.Is(err, errFileNotFound) && metadataArray[i] != nil {
|
|
||||||
for _, ver := range metadataArray[i].versions {
|
|
||||||
if inclFreeVers || !ver.header.FreeVersion() {
|
|
||||||
errs[i] = errErasureReadQuorum
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return metaFileInfos, errs
|
return metaFileInfos, errs
|
||||||
@@ -917,6 +891,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
validResp := 0
|
||||||
totalResp := 0
|
totalResp := 0
|
||||||
|
|
||||||
// minDisks value is only to reduce the number of calls
|
// minDisks value is only to reduce the number of calls
|
||||||
@@ -949,9 +924,6 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return FileInfo{}, nil, nil, time.Time{}, "", err
|
return FileInfo{}, nil, nil, time.Time{}, "", err
|
||||||
}
|
}
|
||||||
if err := checkObjectGenerationQuorum(fi, metaArr, errs); err != nil {
|
|
||||||
return FileInfo{}, nil, nil, time.Time{}, "", err
|
|
||||||
}
|
|
||||||
|
|
||||||
onlineMeta := make([]FileInfo, len(metaArr))
|
onlineMeta := make([]FileInfo, len(metaArr))
|
||||||
for i, disk := range onlineDisks {
|
for i, disk := range onlineDisks {
|
||||||
@@ -972,8 +944,11 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
|
|||||||
err error
|
err error
|
||||||
)
|
)
|
||||||
|
|
||||||
for range done {
|
for success := range done {
|
||||||
totalResp++
|
totalResp++
|
||||||
|
if success {
|
||||||
|
validResp++
|
||||||
|
}
|
||||||
|
|
||||||
if totalResp >= minDisks && opts.FastGetObjInfo {
|
if totalResp >= minDisks && opts.FastGetObjInfo {
|
||||||
rw.Lock()
|
rw.Lock()
|
||||||
@@ -989,21 +964,27 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
|
|||||||
}
|
}
|
||||||
|
|
||||||
if totalResp < er.setDriveCount {
|
if totalResp < er.setDriveCount {
|
||||||
// A fast quorum may contain only an older generation. Include
|
if !opts.FastGetObjInfo {
|
||||||
// slower metadata responses before selecting a readable value.
|
continue
|
||||||
continue
|
}
|
||||||
|
if validResp < minDisks {
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rw.Lock()
|
rw.Lock()
|
||||||
// when its a versioned bucket and empty versionID - at totalResp == setDriveCount
|
// when its a versioned bucket and empty versionID - at totalResp == setDriveCount
|
||||||
// we must use rawFileInfo to resolve versions to figure out the latest version.
|
// we must use rawFileInfo to resolve versions to figure out the latest version.
|
||||||
if opts.VersionID == "" && totalResp == er.setDriveCount && !uniformLatestVersion(metaArr, errs) {
|
if opts.VersionID == "" && totalResp == er.setDriveCount {
|
||||||
fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(pickLatestQuorumFilesInfo(ctx,
|
fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(pickLatestQuorumFilesInfo(ctx,
|
||||||
rawArr, errs, bucket, object, readData, opts.InclFreeVersions))
|
rawArr, errs, bucket, object, readData, opts.InclFreeVersions))
|
||||||
} else {
|
} else {
|
||||||
fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(metaArr, errs)
|
fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(metaArr, errs)
|
||||||
}
|
}
|
||||||
rw.Unlock()
|
rw.Unlock()
|
||||||
|
if err == nil && (fi.InlineData() || len(fi.Data) > 0) {
|
||||||
|
break
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1019,6 +1000,16 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// when we have insufficient read quorum and inconsistent metadata return
|
||||||
|
// file not found, since we can't possibly have a way to recover this object
|
||||||
|
// anyway.
|
||||||
|
if v, ok := err.(InsufficientReadQuorum); ok && v.Type == RQInconsistentMeta {
|
||||||
|
if opts.VersionID != "" {
|
||||||
|
err = errFileVersionNotFound
|
||||||
|
} else {
|
||||||
|
err = errFileNotFound
|
||||||
|
}
|
||||||
|
}
|
||||||
return fi, nil, nil, toObjectErr(err, bucket, object)
|
return fi, nil, nil, toObjectErr(err, bucket, object)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,8 @@
|
|||||||
|
//go:build consistency_strict
|
||||||
|
|
||||||
|
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||||
|
// maintenance release gate; failures record intentionally unresolved risks.
|
||||||
|
|
||||||
// Copyright (c) 2026 Feng Ruohang
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
//
|
//
|
||||||
// This file is part of Silo Object Storage.
|
// This file is part of Silo Object Storage.
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ func TestPoolsConditionalPutHTTP(t *testing.T) {
|
|||||||
opts.MTime = UTCNow().Add(-time.Minute)
|
opts.MTime = UTCNow().Add(-time.Minute)
|
||||||
current := putConsistencyObject(t, z, bucket, object, currentPool, "current", opts)
|
current := putConsistencyObject(t, z, bucket, object, currentPool, "current", opts)
|
||||||
defer conditionalPutPool(t, z, object, target)()
|
defer conditionalPutPool(t, z, object, target)()
|
||||||
idx, _, err := z.getWritePoolIdx(t.Context(), bucket, object, 11, false, true)
|
idx, err := z.getWritePoolIdx(t.Context(), bucket, object, 11, false)
|
||||||
if err != nil || idx != target {
|
if err != nil || idx != target {
|
||||||
t.Fatalf("allocation target=%d: idx=%d err=%v", target, idx, err)
|
t.Fatalf("allocation target=%d: idx=%d err=%v", target, idx, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,8 @@
|
|||||||
|
//go:build consistency_strict
|
||||||
|
|
||||||
|
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||||
|
// maintenance release gate; failures record intentionally unresolved risks.
|
||||||
|
|
||||||
// Copyright (c) 2026 Feng Ruohang
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
//
|
//
|
||||||
// This file is part of Silo Object Storage stack
|
// This file is part of Silo Object Storage stack
|
||||||
|
|||||||
@@ -493,10 +493,6 @@ type poolErrs struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (PoolObjInfo, []poolErrs, error) {
|
func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (PoolObjInfo, []poolErrs, error) {
|
||||||
return z.getPoolInfoExisting(ctx, bucket, object, opts, false)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (z *erasureServerPools) getPoolInfoExisting(ctx context.Context, bucket, object string, opts ObjectOptions, forWrite bool) (PoolObjInfo, []poolErrs, error) {
|
|
||||||
var noReadQuorumPools []poolErrs
|
var noReadQuorumPools []poolErrs
|
||||||
poolObjInfos := make([]PoolObjInfo, len(z.serverPools))
|
poolObjInfos := make([]PoolObjInfo, len(z.serverPools))
|
||||||
poolOpts := make([]ObjectOptions, len(z.serverPools))
|
poolOpts := make([]ObjectOptions, len(z.serverPools))
|
||||||
@@ -524,27 +520,6 @@ func (z *erasureServerPools) getPoolInfoExisting(ctx context.Context, bucket, ob
|
|||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
if forWrite && !z.SinglePool() {
|
|
||||||
// Reuse the metadata already read for allocation. A healthy duplicate
|
|
||||||
// must not hide an unreadable pool that will veto the next GET.
|
|
||||||
recovery := -1
|
|
||||||
for i, info := range poolObjInfos {
|
|
||||||
if info.Err == nil || isErrObjectNotFound(info.Err) || isErrVersionNotFound(info.Err) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
rq, inconsistent := info.Err.(InsufficientReadQuorum)
|
|
||||||
if !inconsistent || rq.Type != RQInconsistentMeta || recovery >= 0 || z.IsSuspended(i) || z.IsPoolRebalancing(i) {
|
|
||||||
return info, nil, info.Err
|
|
||||||
}
|
|
||||||
recovery = i
|
|
||||||
}
|
|
||||||
if recovery >= 0 {
|
|
||||||
// Repair the sole unreadable copy in place. One write cannot repair
|
|
||||||
// several pools, and draining pools cannot accept recovery writes.
|
|
||||||
return poolObjInfos[recovery], nil, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort the objInfos such that we always serve latest
|
// Sort the objInfos such that we always serve latest
|
||||||
// this is a defensive change to handle any duplicate
|
// this is a defensive change to handle any duplicate
|
||||||
// content that may have been created, we always serve
|
// content that may have been created, we always serve
|
||||||
@@ -661,21 +636,20 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec
|
|||||||
// if none are found falls back to most available space pool, this function is
|
// if none are found falls back to most available space pool, this function is
|
||||||
// designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
|
// designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
|
||||||
func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
|
func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
|
||||||
idx, _, err = z.getWritePoolIdx(ctx, bucket, object, size, false, false)
|
return z.getWritePoolIdx(ctx, bucket, object, size, false)
|
||||||
return idx, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// getWritePoolIdx keeps the write-allocation policy when the caller already
|
// getWritePoolIdx keeps the write-allocation policy when the caller already
|
||||||
// holds the pools-layer object lock and must not reacquire a set read lock.
|
// holds the pools-layer object lock and must not reacquire a set read lock.
|
||||||
func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object string, size int64, noLock, recoverWrite bool) (idx int, recovery bool, err error) {
|
func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object string, size int64, noLock bool) (idx int, err error) {
|
||||||
pinfo, _, err := z.getPoolInfoExisting(ctx, bucket, object, ObjectOptions{
|
pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, ObjectOptions{
|
||||||
NoLock: noLock,
|
NoLock: noLock,
|
||||||
SkipDecommissioned: true,
|
SkipDecommissioned: true,
|
||||||
SkipRebalancing: true,
|
SkipRebalancing: true,
|
||||||
}, recoverWrite)
|
})
|
||||||
|
|
||||||
if err != nil && !isErrObjectNotFound(err) {
|
if err != nil && !isErrObjectNotFound(err) {
|
||||||
return -1, false, err
|
return -1, err
|
||||||
}
|
}
|
||||||
|
|
||||||
idx = pinfo.Index
|
idx = pinfo.Index
|
||||||
@@ -683,11 +657,11 @@ func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object
|
|||||||
// will generate a temp object
|
// will generate a temp object
|
||||||
idx = z.getAvailablePoolIdx(ctx, bucket, object, size)
|
idx = z.getAvailablePoolIdx(ctx, bucket, object, size)
|
||||||
if idx < 0 {
|
if idx < 0 {
|
||||||
return -1, false, toObjectErr(errDiskFull)
|
return -1, toObjectErr(errDiskFull)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return idx, recoverWrite && isErrReadQuorum(pinfo.Err), nil
|
return idx, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (z *erasureServerPools) Shutdown(ctx context.Context) error {
|
func (z *erasureServerPools) Shutdown(ctx context.Context) error {
|
||||||
@@ -1074,17 +1048,6 @@ func (z *erasureServerPools) getLatestObjectInfoWithIdx(ctx context.Context, buc
|
|||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
|
||||||
// An unreadable pool may contain a newer generation. Check every pool
|
|
||||||
// before choosing a readable copy; sorting first would hide errors behind
|
|
||||||
// an older object's nonzero ModTime. Delete markers are resolved metadata
|
|
||||||
// and still participate in the ordering below.
|
|
||||||
for _, res := range results {
|
|
||||||
if res.err != nil && !isErrObjectNotFound(res.err) && !isErrVersionNotFound(res.err) &&
|
|
||||||
(!res.oi.DeleteMarker || !isErrMethodNotAllowed(res.err)) {
|
|
||||||
return res.oi, res.zIdx, res.err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort the objInfos such that we always serve latest
|
// Sort the objInfos such that we always serve latest
|
||||||
// this is a defensive change to handle any duplicate
|
// this is a defensive change to handle any duplicate
|
||||||
// content that may have been created, we always serve
|
// content that may have been created, we always serve
|
||||||
@@ -1229,8 +1192,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
|||||||
opts.replicaObjectInfo = resolve
|
opts.replicaObjectInfo = resolve
|
||||||
}
|
}
|
||||||
|
|
||||||
recoverWrite := !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement
|
idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true)
|
||||||
idx, recovery, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true, recoverWrite)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ObjectInfo{}, err
|
return ObjectInfo{}, err
|
||||||
}
|
}
|
||||||
@@ -1253,22 +1215,6 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
|||||||
opts.replicaObjectInfo = z.replicaObjectInfo
|
opts.replicaObjectInfo = z.replicaObjectInfo
|
||||||
}
|
}
|
||||||
oi, err := z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
|
oi, err := z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
|
||||||
if err == nil && recovery {
|
|
||||||
// Only recovery pays for a second read. A partial repair or a new
|
|
||||||
// unreadable pool must not be acknowledged as recovered. Keep any
|
|
||||||
// committed write on failure; a read failure is not permission to undo.
|
|
||||||
var latest ObjectInfo
|
|
||||||
var latestIdx int
|
|
||||||
latest, latestIdx, err = z.getLatestObjectInfoWithIdx(ctx, bucket, object, ObjectOptions{
|
|
||||||
NoLock: true,
|
|
||||||
NoAuditLog: true,
|
|
||||||
Versioned: opts.Versioned,
|
|
||||||
VersionSuspended: opts.VersionSuspended,
|
|
||||||
})
|
|
||||||
if err == nil && (latestIdx != idx || latest.VersionID != oi.VersionID || latest.ETag != oi.ETag || !latest.ModTime.Equal(oi.ModTime)) {
|
|
||||||
err = InsufficientReadQuorum{Bucket: bucket, Object: decodeDirObject(object), Err: errErasureReadQuorum, Type: RQInconsistentMeta}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err == nil && opts.ReplicaLockReconcile && !opts.DataMovement {
|
if err == nil && opts.ReplicaLockReconcile && !opts.DataMovement {
|
||||||
err = z.retireReplicaCopies(ctx, bucket, object, idx, oi)
|
err = z.retireReplicaCopies(ctx, bucket, object, idx, oi)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type compatibilityFailedOverwrite struct {
|
||||||
|
StorageAPI
|
||||||
|
commit bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d compatibilityFailedOverwrite) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||||
|
if d.commit {
|
||||||
|
if _, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts); err != nil {
|
||||||
|
return RenameDataResp{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return RenameDataResp{}, errDiskNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
// Maintenance compatibility requirement: after a completely joined failed
|
||||||
|
// overwrite with one residue, keep the previously acknowledged quorum readable.
|
||||||
|
// This tests a known history; it does not claim the resulting snapshot proves it.
|
||||||
|
func TestMaintenancePartialFailedOverwriteAvailability(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
dirs, err := getRandomDisks(16)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, dirs...))
|
||||||
|
if err != nil {
|
||||||
|
removeRoots(dirs)
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
z := obj.(*erasureServerPools)
|
||||||
|
previous := newObjectLayerFn()
|
||||||
|
setObjectLayer(z)
|
||||||
|
defer func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) }()
|
||||||
|
bucket := "maintenance-compatibility"
|
||||||
|
key := "partial-overwrite"
|
||||||
|
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
oldBody := bytes.Repeat([]byte("old-A"), 65536)
|
||||||
|
newBody := bytes.Repeat([]byte("new-B"), 65536)
|
||||||
|
put := func(body []byte) error {
|
||||||
|
_, err := z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := put(oldBody); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
set := z.serverPools[0].getHashedSet(key)
|
||||||
|
getDisks := set.getDisks
|
||||||
|
disks := getDisks()
|
||||||
|
wrapped := append([]StorageAPI(nil), disks...)
|
||||||
|
for i, d := range disks {
|
||||||
|
wrapped[i] = compatibilityFailedOverwrite{d, i == 0}
|
||||||
|
}
|
||||||
|
set.getDisks = func() []StorageAPI { return wrapped }
|
||||||
|
err = put(newBody)
|
||||||
|
set.getDisks = getDisks
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("partial overwrite must fail")
|
||||||
|
}
|
||||||
|
t.Logf("Overwrite completed with failure; one new residue, fifteen original shards: %v", err)
|
||||||
|
r, err := z.GetObjectNInfo(ctx, bucket, key, nil, nil, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("MAINTENANCE COMPATIBILITY: previously readable acknowledged value now refused: %v", err)
|
||||||
|
}
|
||||||
|
data, err := io.ReadAll(r)
|
||||||
|
r.Close()
|
||||||
|
if err != nil || !bytes.Equal(data, oldBody) {
|
||||||
|
t.Fatalf("old acknowledged payload mismatch: bytes=%d err=%v", len(data), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
//go:build consistency_strict
|
||||||
|
|
||||||
|
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||||
|
// maintenance release gate; failures record intentionally unresolved risks.
|
||||||
|
|
||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStrictListFallbackRejectsSplitGenerations(t *testing.T) {
|
||||||
|
z, bucket, _ := nullQuorumBackend(t)
|
||||||
|
set := z.serverPools[0].sets[0]
|
||||||
|
disks := set.getDisks()
|
||||||
|
const object = "object"
|
||||||
|
mtime := time.Now().UTC().Add(-time.Hour)
|
||||||
|
var generations [3][][]byte
|
||||||
|
for generation := range generations {
|
||||||
|
body := bytes.Repeat([]byte{byte('a' + generation)}, 8192)
|
||||||
|
nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime.Add(time.Duration(generation) * time.Minute)})
|
||||||
|
generations[generation] = make([][]byte, len(disks))
|
||||||
|
for i, disk := range disks {
|
||||||
|
generations[generation][i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// All sixteen drives are online, but the 7/5/4 split leaves no generation
|
||||||
|
// with the eight matching snapshots required by the listing resolver.
|
||||||
|
for i, disk := range disks {
|
||||||
|
generation := 0
|
||||||
|
if i >= 7 {
|
||||||
|
generation = 1
|
||||||
|
}
|
||||||
|
if i >= 12 {
|
||||||
|
generation = 2
|
||||||
|
}
|
||||||
|
if err := disk.WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, generations[generation][i]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrReadQuorum(err) {
|
||||||
|
t.Fatalf("split metadata must remain a read-quorum failure: %v", err)
|
||||||
|
}
|
||||||
|
entries := make(metaCacheEntries, len(disks))
|
||||||
|
for i, disk := range disks {
|
||||||
|
entries[i] = metaCacheEntry{name: object, metadata: mustReadNullQuorumMeta(t, disk, bucket, object)}
|
||||||
|
if _, err := entries[i].xlmeta(); err != nil {
|
||||||
|
t.Fatalf("drive %d no longer has valid metadata: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolver := metadataResolutionParams{dirQuorum: 8, objQuorum: 8, bucket: bucket, requestedVersions: 1}
|
||||||
|
entry, err := z.resolveListingGroup(t.Context(), bucket, []*metaCacheEntry{set.resolveListEntry(entries, nil, &resolver)})
|
||||||
|
if entry != nil || !isErrReadQuorum(err) {
|
||||||
|
t.Fatalf("split live metadata became absence: entry=%v err=%v", entry, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -115,8 +115,8 @@ func TestListFallbackRejectsSplitGenerations(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrReadQuorum(err) {
|
if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrObjectNotFound(err) {
|
||||||
t.Fatalf("split metadata must remain a read-quorum failure: %v", err)
|
t.Fatalf("split metadata did not exercise ambiguous not-found: %v", err)
|
||||||
}
|
}
|
||||||
entries := make(metaCacheEntries, len(disks))
|
entries := make(metaCacheEntries, len(disks))
|
||||||
for i, disk := range disks {
|
for i, disk := range disks {
|
||||||
|
|||||||
@@ -0,0 +1,167 @@
|
|||||||
|
//go:build consistency_strict
|
||||||
|
|
||||||
|
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||||
|
// maintenance release gate; failures record intentionally unresolved risks.
|
||||||
|
|
||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
madmin "github.com/minio/madmin-go/v3"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStrictSingleNullQuorumReadAndConsumers(t *testing.T) {
|
||||||
|
z, bucket, router := nullQuorumBackend(t)
|
||||||
|
set := z.serverPools[0].sets[0]
|
||||||
|
disks := set.getDisks()
|
||||||
|
const object = "object"
|
||||||
|
oldBody, newBody := bytes.Repeat([]byte{'a'}, 8192), bytes.Repeat([]byte{'b'}, 8192)
|
||||||
|
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||||
|
put := func(body []byte, modTime time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
_, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: modTime})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
put(oldBody, oldTime)
|
||||||
|
oldMeta := make([][]byte, len(disks))
|
||||||
|
for i, disk := range disks {
|
||||||
|
var err error
|
||||||
|
oldMeta[i], err = disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
put(newBody, oldTime.Add(time.Minute))
|
||||||
|
newMeta := make([][]byte, len(disks))
|
||||||
|
for i, disk := range disks {
|
||||||
|
newMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||||
|
}
|
||||||
|
// Model an incomplete overwrite using real inline shards: fifteen old
|
||||||
|
// copies retain a read quorum; the final disk contains the newer minority.
|
||||||
|
// The old quorum does not prove that the newer overwrite was uncommitted.
|
||||||
|
for i := range 15 {
|
||||||
|
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
checkRead := func(body []byte, modTime time.Time) {
|
||||||
|
t.Helper()
|
||||||
|
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object))
|
||||||
|
head := nullQuorumRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, object))
|
||||||
|
if get.Code != 200 || !bytes.Equal(get.Body.Bytes(), body) || head.Code != 200 || head.Header().Get("Content-Length") != "8192" {
|
||||||
|
t.Fatalf("GET/HEAD lost readable quorum: GET=%d HEAD=%d body=%q", get.Code, head.Code, get.Body.String())
|
||||||
|
}
|
||||||
|
oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||||
|
if err != nil || !oi.ModTime.Equal(modTime) {
|
||||||
|
t.Fatalf("wrong readable generation: %v, %v", oi.ModTime, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
||||||
|
rec := nullQuorumRequest(t, router, method, getGetObjectURL("", bucket, object))
|
||||||
|
if rec.Code != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("%s exposed an unresolved old quorum: %d", method, rec.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := z.HealObject(t.Context(), bucket, object, "", madmin.HealOpts{Remove: true}); !isErrReadQuorum(err) {
|
||||||
|
t.Fatalf("heal accepted an unresolved old quorum: %v", err)
|
||||||
|
}
|
||||||
|
// Reading must not reinterpret the minority as absent or delete it.
|
||||||
|
minority, err := disks[15].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||||
|
if err != nil || !minority.ModTime.Equal(oldTime.Add(time.Minute)) {
|
||||||
|
t.Fatalf("read mutated the minority: %+v, %v", minority, err)
|
||||||
|
}
|
||||||
|
// Now model the safe direction: the latest generation has the quorum.
|
||||||
|
// Keep the migration, Walk and scanner assertions on that readable state.
|
||||||
|
for i := range 15 {
|
||||||
|
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMeta[i]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := disks[15].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[15]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
newTime := oldTime.Add(time.Minute)
|
||||||
|
checkRead(newBody, newTime)
|
||||||
|
for _, kind := range []string{"rebalance", "decommission"} {
|
||||||
|
t.Run(kind, func(t *testing.T) {
|
||||||
|
var names []string
|
||||||
|
consume := func(entry metaCacheEntry) {
|
||||||
|
versions, err := entry.fileInfoVersions(bucket)
|
||||||
|
if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(newTime) {
|
||||||
|
t.Errorf("invalid migration metadata: %+v, %v", versions, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Migration reopens the selected name/version on all drives;
|
||||||
|
// it must not treat the selected listing metadata as disk agreement.
|
||||||
|
reader, err := set.GetObjectNInfo(t.Context(), bucket, entry.name, nil, nil, ObjectOptions{VersionID: nullVersionID, NoLock: true, NoDecryption: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(reader)
|
||||||
|
reader.Close()
|
||||||
|
if err != nil || !bytes.Equal(body, newBody) {
|
||||||
|
t.Errorf("migration could not read selected object data: %v", err)
|
||||||
|
}
|
||||||
|
names = append(names, entry.name)
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
if kind == "rebalance" {
|
||||||
|
err = set.listObjectsToRebalance(t.Context(), bucket, consume)
|
||||||
|
} else {
|
||||||
|
err = set.listObjectsToDecommission(t.Context(), decomBucketInfo{Name: bucket}, consume)
|
||||||
|
}
|
||||||
|
if err != nil || !slices.Equal(names, []string{object}) {
|
||||||
|
t.Fatalf("migration listing dropped the quorum: %v, %v", names, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, latestOnly := range []bool{false, true} {
|
||||||
|
results := make(chan itemOrErr[ObjectInfo], 16)
|
||||||
|
if err := z.Walk(t.Context(), bucket, "", results, WalkOptions{AskDisks: "strict", LatestOnly: latestOnly}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var names []string
|
||||||
|
for result := range results {
|
||||||
|
if result.Err != nil || !result.Item.ModTime.Equal(newTime) {
|
||||||
|
t.Fatalf("Walk returned invalid metadata: %+v", result)
|
||||||
|
}
|
||||||
|
names = append(names, result.Item.Name)
|
||||||
|
}
|
||||||
|
if !slices.Equal(names, []string{object}) {
|
||||||
|
t.Fatalf("Walk dropped the quorum: %v", names)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exercise the scanner's actual abandoned-child path. Make the scan drive
|
||||||
|
// miss the object, while keeping the latest quorum and an older minority on
|
||||||
|
// the remaining disks. Its queued heal must read all disks and repair both.
|
||||||
|
if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[14]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(filepath.Join(disks[15].Endpoint().Path, bucket, object)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
scanNullQuorumAbandoned(t, z, bucket, object, newTime)
|
||||||
|
for i, disk := range disks {
|
||||||
|
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{Healing: true})
|
||||||
|
if err != nil || !fi.ModTime.Equal(newTime) {
|
||||||
|
t.Fatalf("scanner heal did not reconcile disk %d: %v, %v", i, fi.ModTime, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
checkRead(newBody, newTime)
|
||||||
|
put(newBody, oldTime.Add(2*time.Minute))
|
||||||
|
checkRead(newBody, oldTime.Add(2*time.Minute))
|
||||||
|
}
|
||||||
@@ -701,13 +701,9 @@ func TestSingleNullQuorumReadAndConsumers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
put(newBody, oldTime.Add(time.Minute))
|
put(newBody, oldTime.Add(time.Minute))
|
||||||
newMeta := make([][]byte, len(disks))
|
newMinorityMeta := mustReadNullQuorumMeta(t, disks[14], bucket, object)
|
||||||
for i, disk := range disks {
|
|
||||||
newMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
|
||||||
}
|
|
||||||
// Model an incomplete overwrite using real inline shards: fifteen old
|
// Model an incomplete overwrite using real inline shards: fifteen old
|
||||||
// copies retain a read quorum; the final disk contains the newer minority.
|
// copies retain a read quorum; the final disk contains the newer minority.
|
||||||
// The old quorum does not prove that the newer overwrite was uncommitted.
|
|
||||||
for i := range 15 {
|
for i := range 15 {
|
||||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -725,38 +721,18 @@ func TestSingleNullQuorumReadAndConsumers(t *testing.T) {
|
|||||||
t.Fatalf("wrong readable generation: %v, %v", oi.ModTime, err)
|
t.Fatalf("wrong readable generation: %v, %v", oi.ModTime, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
checkRead(oldBody, oldTime)
|
||||||
rec := nullQuorumRequest(t, router, method, getGetObjectURL("", bucket, object))
|
|
||||||
if rec.Code != http.StatusServiceUnavailable {
|
|
||||||
t.Fatalf("%s exposed an unresolved old quorum: %d", method, rec.Code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if _, err := z.HealObject(t.Context(), bucket, object, "", madmin.HealOpts{Remove: true}); !isErrReadQuorum(err) {
|
|
||||||
t.Fatalf("heal accepted an unresolved old quorum: %v", err)
|
|
||||||
}
|
|
||||||
// Reading must not reinterpret the minority as absent or delete it.
|
// Reading must not reinterpret the minority as absent or delete it.
|
||||||
minority, err := disks[15].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
minority, err := disks[15].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||||
if err != nil || !minority.ModTime.Equal(oldTime.Add(time.Minute)) {
|
if err != nil || !minority.ModTime.Equal(oldTime.Add(time.Minute)) {
|
||||||
t.Fatalf("read mutated the minority: %+v, %v", minority, err)
|
t.Fatalf("read mutated the minority: %+v, %v", minority, err)
|
||||||
}
|
}
|
||||||
// Now model the safe direction: the latest generation has the quorum.
|
|
||||||
// Keep the migration, Walk and scanner assertions on that readable state.
|
|
||||||
for i := range 15 {
|
|
||||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMeta[i]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := disks[15].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[15]); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
newTime := oldTime.Add(time.Minute)
|
|
||||||
checkRead(newBody, newTime)
|
|
||||||
for _, kind := range []string{"rebalance", "decommission"} {
|
for _, kind := range []string{"rebalance", "decommission"} {
|
||||||
t.Run(kind, func(t *testing.T) {
|
t.Run(kind, func(t *testing.T) {
|
||||||
var names []string
|
var names []string
|
||||||
consume := func(entry metaCacheEntry) {
|
consume := func(entry metaCacheEntry) {
|
||||||
versions, err := entry.fileInfoVersions(bucket)
|
versions, err := entry.fileInfoVersions(bucket)
|
||||||
if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(newTime) {
|
if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(oldTime) {
|
||||||
t.Errorf("invalid migration metadata: %+v, %v", versions, err)
|
t.Errorf("invalid migration metadata: %+v, %v", versions, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -769,7 +745,7 @@ func TestSingleNullQuorumReadAndConsumers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
body, err := io.ReadAll(reader)
|
body, err := io.ReadAll(reader)
|
||||||
reader.Close()
|
reader.Close()
|
||||||
if err != nil || !bytes.Equal(body, newBody) {
|
if err != nil || !bytes.Equal(body, oldBody) {
|
||||||
t.Errorf("migration could not read selected object data: %v", err)
|
t.Errorf("migration could not read selected object data: %v", err)
|
||||||
}
|
}
|
||||||
names = append(names, entry.name)
|
names = append(names, entry.name)
|
||||||
@@ -792,7 +768,7 @@ func TestSingleNullQuorumReadAndConsumers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
var names []string
|
var names []string
|
||||||
for result := range results {
|
for result := range results {
|
||||||
if result.Err != nil || !result.Item.ModTime.Equal(newTime) {
|
if result.Err != nil || !result.Item.ModTime.Equal(oldTime) {
|
||||||
t.Fatalf("Walk returned invalid metadata: %+v", result)
|
t.Fatalf("Walk returned invalid metadata: %+v", result)
|
||||||
}
|
}
|
||||||
names = append(names, result.Item.Name)
|
names = append(names, result.Item.Name)
|
||||||
@@ -803,22 +779,22 @@ func TestSingleNullQuorumReadAndConsumers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Exercise the scanner's actual abandoned-child path. Make the scan drive
|
// Exercise the scanner's actual abandoned-child path. Make the scan drive
|
||||||
// miss the object, while keeping the latest quorum and an older minority on
|
// miss the object, while keeping an older quorum and a newer minority on
|
||||||
// the remaining disks. Its queued heal must read all disks and repair both.
|
// the remaining disks. Its queued heal must read all disks and repair both.
|
||||||
if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[14]); err != nil {
|
if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMinorityMeta); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.RemoveAll(filepath.Join(disks[15].Endpoint().Path, bucket, object)); err != nil {
|
if err := os.RemoveAll(filepath.Join(disks[15].Endpoint().Path, bucket, object)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
scanNullQuorumAbandoned(t, z, bucket, object, newTime)
|
scanNullQuorumAbandoned(t, z, bucket, object, oldTime)
|
||||||
for i, disk := range disks {
|
for i, disk := range disks {
|
||||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{Healing: true})
|
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{Healing: true})
|
||||||
if err != nil || !fi.ModTime.Equal(newTime) {
|
if err != nil || !fi.ModTime.Equal(oldTime) {
|
||||||
t.Fatalf("scanner heal did not reconcile disk %d: %v, %v", i, fi.ModTime, err)
|
t.Fatalf("scanner heal did not reconcile disk %d: %v, %v", i, fi.ModTime, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
checkRead(newBody, newTime)
|
checkRead(oldBody, oldTime)
|
||||||
put(newBody, oldTime.Add(2*time.Minute))
|
put(newBody, oldTime.Add(2*time.Minute))
|
||||||
checkRead(newBody, oldTime.Add(2*time.Minute))
|
checkRead(newBody, oldTime.Add(2*time.Minute))
|
||||||
}
|
}
|
||||||
@@ -914,5 +890,5 @@ func scanNullQuorumAbandoned(t *testing.T, z *erasureServerPools, bucket, object
|
|||||||
if flat == nil || flat.Objects != 1 || flat.Size != 8192 {
|
if flat == nil || flat.Objects != 1 || flat.Size != 8192 {
|
||||||
t.Fatalf("scanner lost the healed object from usage: %+v", flat)
|
t.Fatalf("scanner lost the healed object from usage: %+v", flat)
|
||||||
}
|
}
|
||||||
t.Logf("scanner queued %v; healed latest quorum, repaired minority/missing copies, and retained one 8192-byte object", healed)
|
t.Logf("scanner queued %v; healed old quorum, repaired minority/missing copies, and retained one 8192-byte object", healed)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,112 +69,6 @@ func mutationPut(t testing.TB, z *erasureServerPools, bucket, obj string, pool i
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ordinary successful tag write with one temporarily unavailable drive. No
|
|
||||||
// metadata is fabricated. The stale drive returns first on a subsequent GET.
|
|
||||||
func TestMutationReadPreservesAcknowledgedTags(t *testing.T) {
|
|
||||||
z, bucket := mutationPools(t)
|
|
||||||
const object = "tag-read"
|
|
||||||
mutationPut(t, z, bucket, object, 0, "payload", time.Now())
|
|
||||||
set := z.serverPools[0].getHashedSet(object)
|
|
||||||
disks, swap := installListingDiskSwitch(set)
|
|
||||||
defer swap(disks)
|
|
||||||
partial := append([]StorageAPI(nil), disks...)
|
|
||||||
partial[0] = nil
|
|
||||||
swap(partial)
|
|
||||||
const tags = "owner=acknowledged-B"
|
|
||||||
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
swap(disks)
|
|
||||||
before, err := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
after, err := disks[1].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Logf("old/new tags=%q/%q; equal tuple=%t", before.Metadata[xhttp.AmzObjectTagging], after.Metadata[xhttp.AmzObjectTagging], before.ModTime.Equal(after.ModTime) && before.DataDir == after.DataDir && before.Metadata["etag"] == after.Metadata["etag"])
|
|
||||||
for _, fast := range []bool{false, true} {
|
|
||||||
for _, version := range []string{"", nullVersionID} {
|
|
||||||
oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if oi.UserTags != tags {
|
|
||||||
t.Fatalf("acknowledged tags replaced by stale metadata (fast=%t version=%q): %q", fast, version, oi.UserTags)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A healthy version history must reuse the already decoded latest version.
|
|
||||||
// A stale same-version tag copy must still take the full consistency path.
|
|
||||||
func TestMutationVersionHistoryRead(t *testing.T) {
|
|
||||||
z, bucket := mutationPools(t)
|
|
||||||
const object = "history-read"
|
|
||||||
set := z.serverPools[0].getHashedSet(object)
|
|
||||||
var versions []string
|
|
||||||
for i := range 12 {
|
|
||||||
id := mustGetUUID()
|
|
||||||
body := "version-" + strconv.Itoa(i)
|
|
||||||
_, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, VersionID: id})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
versions = append(versions, id)
|
|
||||||
}
|
|
||||||
disks, swap := installListingDiskSwitch(set)
|
|
||||||
defer swap(disks)
|
|
||||||
uniform := func() bool {
|
|
||||||
raw := make([]RawFileInfo, len(disks))
|
|
||||||
meta := make([]FileInfo, len(disks))
|
|
||||||
errs := make([]error, len(disks))
|
|
||||||
for i, disk := range disks {
|
|
||||||
var err error
|
|
||||||
raw[i], err = disk.ReadXL(t.Context(), bucket, object, false)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
meta[i], errs[i] = fileInfoFromRaw(raw[i], bucket, object, false, false)
|
|
||||||
}
|
|
||||||
return uniformLatestVersion(meta, errs)
|
|
||||||
}
|
|
||||||
if !uniform() {
|
|
||||||
t.Fatal("healthy version history requires redundant full metadata decoding")
|
|
||||||
}
|
|
||||||
partial := append([]StorageAPI(nil), disks...)
|
|
||||||
partial[0] = nil
|
|
||||||
swap(partial)
|
|
||||||
const tags = "owner=latest-acknowledged"
|
|
||||||
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
swap(disks)
|
|
||||||
if uniform() {
|
|
||||||
t.Fatal("stale same-version metadata incorrectly accepted as uniform")
|
|
||||||
}
|
|
||||||
for _, fast := range []bool{false, true} {
|
|
||||||
for _, version := range []string{"", versions[len(versions)-1]} {
|
|
||||||
oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version})
|
|
||||||
if err != nil || oi.VersionID != versions[len(versions)-1] || oi.UserTags != tags {
|
|
||||||
t.Fatalf("latest version/tags changed: version=%q tags=%q err=%v", oi.VersionID, oi.UserTags, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i, id := range versions {
|
|
||||||
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: id})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(r)
|
|
||||||
r.Close()
|
|
||||||
if err != nil || string(body) != "version-"+strconv.Itoa(i) {
|
|
||||||
t.Fatalf("history version %d changed: %q err=%v", i, body, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type mutationAmbiguousRename struct {
|
type mutationAmbiguousRename struct {
|
||||||
StorageAPI
|
StorageAPI
|
||||||
index int
|
index int
|
||||||
@@ -730,68 +624,6 @@ func (d orderedRenameDisk) RenameData(ctx context.Context, sv, sp string, fi Fil
|
|||||||
return r, err
|
return r, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMutationCrossDiskOrderHasStableRead(t *testing.T) {
|
|
||||||
z, bucket := mutationPools(t)
|
|
||||||
const object = "cross-disk-order"
|
|
||||||
base := time.Now().Add(-time.Hour)
|
|
||||||
mutationPut(t, z, bucket, object, 0, "initial", base)
|
|
||||||
set := z.serverPools[0].getHashedSet(object)
|
|
||||||
disks, swap := installListingDiskSwitch(set)
|
|
||||||
defer swap(disks)
|
|
||||||
earlyDone, release := make(chan struct{}, 4), make(chan struct{})
|
|
||||||
var once sync.Once
|
|
||||||
unblock := func() { once.Do(func() { close(release) }) }
|
|
||||||
defer unblock()
|
|
||||||
aDisks := append([]StorageAPI(nil), disks...)
|
|
||||||
for i, d := range disks {
|
|
||||||
aDisks[i] = orderedRenameDisk{d, i < 4, earlyDone, release}
|
|
||||||
}
|
|
||||||
swap(aDisks)
|
|
||||||
put := func(body string, mtime time.Time) error {
|
|
||||||
_, e := z.serverPools[0].PutObject(t.Context(), bucket, object,
|
|
||||||
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
|
|
||||||
ObjectOptions{NoLock: true, MTime: mtime})
|
|
||||||
return e
|
|
||||||
}
|
|
||||||
aDone := make(chan error, 1)
|
|
||||||
go func() { aDone <- put("writer-A", base.Add(time.Minute)) }()
|
|
||||||
for range 4 {
|
|
||||||
select {
|
|
||||||
case <-earlyDone:
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("A did not reach first-half barrier")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
swap(disks)
|
|
||||||
if err := put("writer-B", base.Add(2*time.Minute)); err != nil {
|
|
||||||
t.Fatalf("B failed: %v", err)
|
|
||||||
}
|
|
||||||
unblock()
|
|
||||||
select {
|
|
||||||
case err := <-aDone:
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("A failed: %v", err)
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("A did not finish")
|
|
||||||
}
|
|
||||||
t.Log("both PUTs succeeded; final per-disk order is A→B on four disks and B→A on four disks; no writer remains")
|
|
||||||
swap(disks)
|
|
||||||
for range 8 {
|
|
||||||
for _, version := range []string{"", nullVersionID} {
|
|
||||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{FastGetObjInfo: true, VersionID: version})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("quiescent successful writes became unreadable: %v", err)
|
|
||||||
}
|
|
||||||
body, err := io.ReadAll(r)
|
|
||||||
r.Close()
|
|
||||||
if err != nil || string(body) != "writer-B" {
|
|
||||||
t.Fatalf("completed reads changed value: %q %v", body, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Partial no-op updates may revoke undo on only some disks. That internal
|
// Partial no-op updates may revoke undo on only some disks. That internal
|
||||||
// capability must not split the quorum of otherwise identical object contents.
|
// capability must not split the quorum of otherwise identical object contents.
|
||||||
func TestMutationPartialNoopPreservesReadability(t *testing.T) {
|
func TestMutationPartialNoopPreservesReadability(t *testing.T) {
|
||||||
|
|||||||
@@ -1,3 +1,8 @@
|
|||||||
|
//go:build consistency_strict
|
||||||
|
|
||||||
|
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||||
|
// maintenance release gate; failures record intentionally unresolved risks.
|
||||||
|
|
||||||
// Copyright (c) 2026 Feng Ruohang
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
//
|
//
|
||||||
// This file is part of Silo Object Storage.
|
// This file is part of Silo Object Storage.
|
||||||
|
|||||||
@@ -0,0 +1,198 @@
|
|||||||
|
//go:build consistency_strict
|
||||||
|
|
||||||
|
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||||
|
// maintenance release gate; failures record intentionally unresolved risks.
|
||||||
|
|
||||||
|
// Copyright (c) 2026 Feng Ruohang
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"io"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMutationReadPreservesAcknowledgedTags(t *testing.T) {
|
||||||
|
z, bucket := mutationPools(t)
|
||||||
|
const object = "tag-read"
|
||||||
|
mutationPut(t, z, bucket, object, 0, "payload", time.Now())
|
||||||
|
set := z.serverPools[0].getHashedSet(object)
|
||||||
|
disks, swap := installListingDiskSwitch(set)
|
||||||
|
defer swap(disks)
|
||||||
|
partial := append([]StorageAPI(nil), disks...)
|
||||||
|
partial[0] = nil
|
||||||
|
swap(partial)
|
||||||
|
const tags = "owner=acknowledged-B"
|
||||||
|
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
swap(disks)
|
||||||
|
before, err := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
after, err := disks[1].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Logf("old/new tags=%q/%q; equal tuple=%t", before.Metadata[xhttp.AmzObjectTagging], after.Metadata[xhttp.AmzObjectTagging], before.ModTime.Equal(after.ModTime) && before.DataDir == after.DataDir && before.Metadata["etag"] == after.Metadata["etag"])
|
||||||
|
for _, fast := range []bool{false, true} {
|
||||||
|
for _, version := range []string{"", nullVersionID} {
|
||||||
|
oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if oi.UserTags != tags {
|
||||||
|
t.Fatalf("acknowledged tags replaced by stale metadata (fast=%t version=%q): %q", fast, version, oi.UserTags)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMutationVersionHistoryRead(t *testing.T) {
|
||||||
|
z, bucket := mutationPools(t)
|
||||||
|
const object = "history-read"
|
||||||
|
set := z.serverPools[0].getHashedSet(object)
|
||||||
|
var versions []string
|
||||||
|
for i := range 12 {
|
||||||
|
id := mustGetUUID()
|
||||||
|
body := "version-" + strconv.Itoa(i)
|
||||||
|
_, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, VersionID: id})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
versions = append(versions, id)
|
||||||
|
}
|
||||||
|
disks, swap := installListingDiskSwitch(set)
|
||||||
|
defer swap(disks)
|
||||||
|
uniform := func() bool {
|
||||||
|
raw := make([]RawFileInfo, len(disks))
|
||||||
|
meta := make([]FileInfo, len(disks))
|
||||||
|
errs := make([]error, len(disks))
|
||||||
|
for i, disk := range disks {
|
||||||
|
var err error
|
||||||
|
raw[i], err = disk.ReadXL(t.Context(), bucket, object, false)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
meta[i], errs[i] = fileInfoFromRaw(raw[i], bucket, object, false, false)
|
||||||
|
}
|
||||||
|
return uniformLatestVersion(meta, errs)
|
||||||
|
}
|
||||||
|
if !uniform() {
|
||||||
|
t.Fatal("healthy version history requires redundant full metadata decoding")
|
||||||
|
}
|
||||||
|
partial := append([]StorageAPI(nil), disks...)
|
||||||
|
partial[0] = nil
|
||||||
|
swap(partial)
|
||||||
|
const tags = "owner=latest-acknowledged"
|
||||||
|
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
swap(disks)
|
||||||
|
if uniform() {
|
||||||
|
t.Fatal("stale same-version metadata incorrectly accepted as uniform")
|
||||||
|
}
|
||||||
|
for _, fast := range []bool{false, true} {
|
||||||
|
for _, version := range []string{"", versions[len(versions)-1]} {
|
||||||
|
oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version})
|
||||||
|
if err != nil || oi.VersionID != versions[len(versions)-1] || oi.UserTags != tags {
|
||||||
|
t.Fatalf("latest version/tags changed: version=%q tags=%q err=%v", oi.VersionID, oi.UserTags, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for i, id := range versions {
|
||||||
|
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: id})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(r)
|
||||||
|
r.Close()
|
||||||
|
if err != nil || string(body) != "version-"+strconv.Itoa(i) {
|
||||||
|
t.Fatalf("history version %d changed: %q err=%v", i, body, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMutationCrossDiskOrderHasStableRead(t *testing.T) {
|
||||||
|
z, bucket := mutationPools(t)
|
||||||
|
const object = "cross-disk-order"
|
||||||
|
base := time.Now().Add(-time.Hour)
|
||||||
|
mutationPut(t, z, bucket, object, 0, "initial", base)
|
||||||
|
set := z.serverPools[0].getHashedSet(object)
|
||||||
|
disks, swap := installListingDiskSwitch(set)
|
||||||
|
defer swap(disks)
|
||||||
|
earlyDone, release := make(chan struct{}, 4), make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
unblock := func() { once.Do(func() { close(release) }) }
|
||||||
|
defer unblock()
|
||||||
|
aDisks := append([]StorageAPI(nil), disks...)
|
||||||
|
for i, d := range disks {
|
||||||
|
aDisks[i] = orderedRenameDisk{d, i < 4, earlyDone, release}
|
||||||
|
}
|
||||||
|
swap(aDisks)
|
||||||
|
put := func(body string, mtime time.Time) error {
|
||||||
|
_, e := z.serverPools[0].PutObject(t.Context(), bucket, object,
|
||||||
|
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
|
||||||
|
ObjectOptions{NoLock: true, MTime: mtime})
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
aDone := make(chan error, 1)
|
||||||
|
go func() { aDone <- put("writer-A", base.Add(time.Minute)) }()
|
||||||
|
for range 4 {
|
||||||
|
select {
|
||||||
|
case <-earlyDone:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("A did not reach first-half barrier")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
swap(disks)
|
||||||
|
if err := put("writer-B", base.Add(2*time.Minute)); err != nil {
|
||||||
|
t.Fatalf("B failed: %v", err)
|
||||||
|
}
|
||||||
|
unblock()
|
||||||
|
select {
|
||||||
|
case err := <-aDone:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("A failed: %v", err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("A did not finish")
|
||||||
|
}
|
||||||
|
t.Log("both PUTs succeeded; final per-disk order is A→B on four disks and B→A on four disks; no writer remains")
|
||||||
|
swap(disks)
|
||||||
|
for range 8 {
|
||||||
|
for _, version := range []string{"", nullVersionID} {
|
||||||
|
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{FastGetObjInfo: true, VersionID: version})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("quiescent successful writes became unreadable: %v", err)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(r)
|
||||||
|
r.Close()
|
||||||
|
if err != nil || string(body) != "writer-B" {
|
||||||
|
t.Fatalf("completed reads changed value: %q %v", body, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func uniformLatestVersion(meta []FileInfo, errs []error) bool {
|
||||||
|
if len(meta) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
first := meta[0]
|
||||||
|
for i, fi := range meta {
|
||||||
|
if errs[i] != nil || !fi.IsValid() || fi.XLV1 || fi.Deleted || !fi.IsLatest || fi.NumVersions < 1 ||
|
||||||
|
!fi.ModTime.Equal(first.ModTime) || !sameFileInfoVersion(first, fi) ||
|
||||||
|
fi.NumVersions != first.NumVersions || !fi.SuccessorModTime.Equal(first.SuccessorModTime) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -193,23 +193,3 @@ func (s *xlStorage) undoWrite(ctx context.Context, volume, path string, fi FileI
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// All drives have replied, and fileInfoFromRaw has already validated each raw
|
|
||||||
// buffer and selected its latest visible version. Reuse that decoded state:
|
|
||||||
// revalidating the raw history here adds work proportional to version count.
|
|
||||||
// Any missing response, logical disagreement, or history-property disagreement
|
|
||||||
// retains the full merge. Legacy objects and delete markers also use that path.
|
|
||||||
func uniformLatestVersion(meta []FileInfo, errs []error) bool {
|
|
||||||
if len(meta) == 0 {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
first := meta[0]
|
|
||||||
for i, fi := range meta {
|
|
||||||
if errs[i] != nil || !fi.IsValid() || fi.XLV1 || fi.Deleted || !fi.IsLatest || fi.NumVersions < 1 ||
|
|
||||||
!fi.ModTime.Equal(first.ModTime) || !sameFileInfoVersion(first, fi) ||
|
|
||||||
fi.NumVersions != first.NumVersions || !fi.SuccessorModTime.Equal(first.SuccessorModTime) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user