diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 52612257e..e9ef251a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,6 +53,16 @@ Before opening a pull request: - Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`, `.minio.sys`, ARN, module/import-path, or serialized compatibility name. +The maintenance branch retains the existing read-quorum and healing policy. +Deferred strict-read and cross-pool recovery tests are preserved behind the +`consistency_strict` build tag. Run them explicitly with +`go test -tags kqueue,dev,consistency_strict ./cmd`; known failures must be +reported separately, not described as fixed or included in a passing result. +They cover late commits after lease loss, stale metadata reads, and stricter +read refusal. The normal suite covers the retained write/undo/cleanup fixes +and continued readability after a failed partial overwrite. This scope does +not provide distributed fencing or repair previously inconsistent objects. + ### Commit changes After verification, commit your changes with a concise message and a DCO diff --git a/cmd/erasure-healing-common.go b/cmd/erasure-healing-common.go index fd204383d..0bde3107d 100644 --- a/cmd/erasure-healing-common.go +++ b/cmd/erasure-healing-common.go @@ -185,7 +185,7 @@ func listObjectModtimes(partsMetadata []FileInfo, errs []error) (modTimes []time func filterOnlineDisksInplace(fi FileInfo, partsMetadata []FileInfo, onlineDisks []StorageAPI) { for i, meta := range partsMetadata { - if sameFileInfoVersion(fi, meta) { + if fi.XLV1 == meta.XLV1 { continue } onlineDisks[i] = nil diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go index f6792aee7..8fb371270 100644 --- a/cmd/erasure-healing.go +++ b/cmd/erasure-healing.go @@ -373,9 +373,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object if err != nil { return result, err } - if err := checkObjectGenerationQuorum(latestMeta, partsMetadata, errs); err != nil { - return result, err - } // No modtime quorum filterDisksByETag := quorumETag != "" diff --git a/cmd/erasure-metadata.go b/cmd/erasure-metadata.go index 5f1c92ac2..1c3209f24 100644 --- a/cmd/erasure-metadata.go +++ b/cmd/erasure-metadata.go @@ -19,6 +19,7 @@ package cmd import ( "context" + "encoding/hex" "fmt" "sort" "strings" @@ -27,6 +28,8 @@ import ( "github.com/minio/minio/internal/amztime" "github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash/sha256" xhttp "github.com/minio/minio/internal/http" "github.com/pgsty/silo-pkg/v3/sync/errgroup" ) @@ -316,48 +319,68 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time. if quorum < 1 { return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInsufficientOnlineDrives} } - // Group by exact logical state rather than a layout-only hash. There are - // at most one erasure set's worth of candidates; healthy objects compare - // against a single representative without hashing or a second XL decode. - groups := make([]int, len(metaArr)) - representatives := make([]int, 0, len(metaArr)) - counts := make([]int, len(metaArr)+1) + metaHashes := make([]string, len(metaArr)) + h := sha256.New() for i, meta := range metaArr { if !meta.IsValid() { continue } - etagOnly := modTime.Equal(timeSentinel) && etag != "" && etag == meta.Metadata["etag"] - if !meta.ModTime.Equal(modTime) && !etagOnly { - continue - } - group := 0 - for j, representative := range representatives { - if sameFileInfoVersion(metaArr[representative], meta) { - group = j + 1 - break + etagOnly := modTime.Equal(timeSentinel) && (etag != "" && etag == meta.Metadata["etag"]) + mtimeValid := meta.ModTime.Equal(modTime) + if mtimeValid || etagOnly { + fmt.Fprintf(h, "%v", meta.XLV1) + for _, part := range meta.Parts { + fmt.Fprintf(h, "part.%d", part.Number) + fmt.Fprintf(h, "part.%d", part.Size) } + // Previously we checked if we had quorum on DataDir value. + // We have removed this check to allow reading objects with different DataDir + // values in a few drives (due to a rebalance-stop race bug) + // provided their their etags or ModTimes match. + + if !meta.Deleted && meta.Size != 0 { + fmt.Fprintf(h, "%v+%v", meta.Erasure.DataBlocks, meta.Erasure.ParityBlocks) + fmt.Fprintf(h, "%v", meta.Erasure.Distribution) + } + + if meta.IsRemote() { + // ILM transition fields + fmt.Fprint(h, meta.TransitionStatus) + fmt.Fprint(h, meta.TransitionTier) + fmt.Fprint(h, meta.TransitionedObjName) + fmt.Fprint(h, meta.TransitionVersionID) + } + + // If metadata says encrypted, ask for it in quorum. + if etyp, ok := crypto.IsEncrypted(meta.Metadata); ok { + fmt.Fprint(h, etyp) + } + + // If compressed, look for compressed FileInfo only + if meta.IsCompressed() { + fmt.Fprint(h, meta.Metadata[ReservedMetadataPrefix+"compression"]) + } + + metaHashes[i] = hex.EncodeToString(h.Sum(nil)) + h.Reset() } - if group == 0 { - representatives = append(representatives, i) - group = len(representatives) - } - groups[i] = group - counts[group]++ } - maxGroup, maxCount := 0, 0 - tied := false - for group, count := range counts { - if group == 0 { + + metaHashCountMap := make(map[string]int) + for _, hash := range metaHashes { + if hash == "" { continue } - if count > maxCount { - maxGroup, maxCount, tied = group, count, false - } else if count == maxCount { - tied = true - } + metaHashCountMap[hash]++ } - if tied { - return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInconsistentMeta} + + maxHash := "" + maxCount := 0 + for hash, count := range metaHashCountMap { + if count > maxCount { + maxCount = count + maxHash = hash + } } if maxCount < quorum { @@ -374,8 +397,8 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time. otherPropsMap := make(counterMap[objProps]) var candidate FileInfo var found bool - for i, group := range groups { - if group == maxGroup { + for i, hash := range metaHashes { + if hash == maxHash { if metaArr[i].IsValid() { if !found { candidate = metaArr[i] diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index c2dbcfdcb..16dc41326 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -126,9 +126,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d if err != nil { return oi, toObjectErr(err, srcBucket, srcObject) } - if err := checkObjectGenerationQuorum(fi, metaArr, errs); err != nil { - return oi, toObjectErr(err, srcBucket, srcObject) - } if fi.Deleted { if srcOpts.VersionID == "" { return oi, toObjectErr(errFileNotFound, srcBucket, srcObject) @@ -620,18 +617,6 @@ func readAllRawFileInfo(ctx context.Context, disks []StorageAPI, bucket, object return rawFileInfos, g.Wait() } -// An older quorum cannot resolve a later overwrite of the same version. -// Reads and healing must preserve that uncertainty rather than roll back -// the visible object or erase the newer fragments. -func checkObjectGenerationQuorum(fi FileInfo, metadata []FileInfo, errs []error) error { - for i, meta := range metadata { - if errs[i] == nil && meta.IsValid() && meta.VersionID == fi.VersionID && meta.ModTime.After(fi.ModTime) { - return InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInconsistentMeta} - } - } - return nil -} - func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo, errs []error, bucket, object string, readData, inclFreeVers bool) ([]FileInfo, []error) { metadataArray := make([]*xlMetaV2, len(rawFileInfos)) metaFileInfos := make([]FileInfo, len(rawFileInfos)) @@ -674,17 +659,6 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo, for i := range errs { if errs[i] == nil { errs[i] = err - // An empty quorum merge does not prove absence when this - // disk contains a readable object generation. Preserve that - // uncertainty instead of hiding it behind FileNotFound. - if errors.Is(err, errFileNotFound) && metadataArray[i] != nil { - for _, ver := range metadataArray[i].versions { - if inclFreeVers || !ver.header.FreeVersion() { - errs[i] = errErasureReadQuorum - break - } - } - } } } return metaFileInfos, errs @@ -917,6 +891,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s } }() + validResp := 0 totalResp := 0 // minDisks value is only to reduce the number of calls @@ -949,9 +924,6 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s if err != nil { return FileInfo{}, nil, nil, time.Time{}, "", err } - if err := checkObjectGenerationQuorum(fi, metaArr, errs); err != nil { - return FileInfo{}, nil, nil, time.Time{}, "", err - } onlineMeta := make([]FileInfo, len(metaArr)) for i, disk := range onlineDisks { @@ -972,8 +944,11 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s err error ) - for range done { + for success := range done { totalResp++ + if success { + validResp++ + } if totalResp >= minDisks && opts.FastGetObjInfo { rw.Lock() @@ -989,21 +964,27 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s } if totalResp < er.setDriveCount { - // A fast quorum may contain only an older generation. Include - // slower metadata responses before selecting a readable value. - continue + if !opts.FastGetObjInfo { + continue + } + if validResp < minDisks { + continue + } } rw.Lock() // when its a versioned bucket and empty versionID - at totalResp == setDriveCount // we must use rawFileInfo to resolve versions to figure out the latest version. - if opts.VersionID == "" && totalResp == er.setDriveCount && !uniformLatestVersion(metaArr, errs) { + if opts.VersionID == "" && totalResp == er.setDriveCount { fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(pickLatestQuorumFilesInfo(ctx, rawArr, errs, bucket, object, readData, opts.InclFreeVersions)) } else { fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(metaArr, errs) } rw.Unlock() + if err == nil && (fi.InlineData() || len(fi.Data) > 0) { + break + } } if err != nil { @@ -1019,6 +1000,16 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s } } } + // when we have insufficient read quorum and inconsistent metadata return + // file not found, since we can't possibly have a way to recover this object + // anyway. + if v, ok := err.(InsufficientReadQuorum); ok && v.Type == RQInconsistentMeta { + if opts.VersionID != "" { + err = errFileVersionNotFound + } else { + err = errFileNotFound + } + } return fi, nil, nil, toObjectErr(err, bucket, object) } diff --git a/cmd/erasure-read-generation_test.go b/cmd/erasure-read-generation_test.go index 33cb2b1bd..f65e9a63b 100644 --- a/cmd/erasure-read-generation_test.go +++ b/cmd/erasure-read-generation_test.go @@ -1,3 +1,8 @@ +//go:build consistency_strict + +// Preserved diagnostic for the deferred strict-read policy. This is not a +// maintenance release gate; failures record intentionally unresolved risks. + // Copyright (c) 2026 Feng Ruohang // // This file is part of Silo Object Storage. diff --git a/cmd/erasure-server-pool-put-conditional_test.go b/cmd/erasure-server-pool-put-conditional_test.go index 3b7fd91fc..cd4897921 100644 --- a/cmd/erasure-server-pool-put-conditional_test.go +++ b/cmd/erasure-server-pool-put-conditional_test.go @@ -136,7 +136,7 @@ func TestPoolsConditionalPutHTTP(t *testing.T) { opts.MTime = UTCNow().Add(-time.Minute) current := putConsistencyObject(t, z, bucket, object, currentPool, "current", opts) defer conditionalPutPool(t, z, object, target)() - idx, _, err := z.getWritePoolIdx(t.Context(), bucket, object, 11, false, true) + idx, err := z.getWritePoolIdx(t.Context(), bucket, object, 11, false) if err != nil || idx != target { t.Fatalf("allocation target=%d: idx=%d err=%v", target, idx, err) } diff --git a/cmd/erasure-server-pool-recovery_test.go b/cmd/erasure-server-pool-recovery_test.go index bf1159e72..2dde55d1a 100644 --- a/cmd/erasure-server-pool-recovery_test.go +++ b/cmd/erasure-server-pool-recovery_test.go @@ -1,3 +1,8 @@ +//go:build consistency_strict + +// Preserved diagnostic for the deferred strict-read policy. This is not a +// maintenance release gate; failures record intentionally unresolved risks. + // Copyright (c) 2026 Feng Ruohang // // This file is part of Silo Object Storage stack diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index edf48848c..a75af7339 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -493,10 +493,6 @@ type poolErrs struct { } func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (PoolObjInfo, []poolErrs, error) { - return z.getPoolInfoExisting(ctx, bucket, object, opts, false) -} - -func (z *erasureServerPools) getPoolInfoExisting(ctx context.Context, bucket, object string, opts ObjectOptions, forWrite bool) (PoolObjInfo, []poolErrs, error) { var noReadQuorumPools []poolErrs poolObjInfos := make([]PoolObjInfo, len(z.serverPools)) poolOpts := make([]ObjectOptions, len(z.serverPools)) @@ -524,27 +520,6 @@ func (z *erasureServerPools) getPoolInfoExisting(ctx context.Context, bucket, ob } wg.Wait() - if forWrite && !z.SinglePool() { - // Reuse the metadata already read for allocation. A healthy duplicate - // must not hide an unreadable pool that will veto the next GET. - recovery := -1 - for i, info := range poolObjInfos { - if info.Err == nil || isErrObjectNotFound(info.Err) || isErrVersionNotFound(info.Err) { - continue - } - rq, inconsistent := info.Err.(InsufficientReadQuorum) - if !inconsistent || rq.Type != RQInconsistentMeta || recovery >= 0 || z.IsSuspended(i) || z.IsPoolRebalancing(i) { - return info, nil, info.Err - } - recovery = i - } - if recovery >= 0 { - // Repair the sole unreadable copy in place. One write cannot repair - // several pools, and draining pools cannot accept recovery writes. - return poolObjInfos[recovery], nil, nil - } - } - // Sort the objInfos such that we always serve latest // this is a defensive change to handle any duplicate // content that may have been created, we always serve @@ -661,21 +636,20 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec // if none are found falls back to most available space pool, this function is // designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload. func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) { - idx, _, err = z.getWritePoolIdx(ctx, bucket, object, size, false, false) - return idx, err + return z.getWritePoolIdx(ctx, bucket, object, size, false) } // getWritePoolIdx keeps the write-allocation policy when the caller already // holds the pools-layer object lock and must not reacquire a set read lock. -func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object string, size int64, noLock, recoverWrite bool) (idx int, recovery bool, err error) { - pinfo, _, err := z.getPoolInfoExisting(ctx, bucket, object, ObjectOptions{ +func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object string, size int64, noLock bool) (idx int, err error) { + pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, ObjectOptions{ NoLock: noLock, SkipDecommissioned: true, SkipRebalancing: true, - }, recoverWrite) + }) if err != nil && !isErrObjectNotFound(err) { - return -1, false, err + return -1, err } idx = pinfo.Index @@ -683,11 +657,11 @@ func (z *erasureServerPools) getWritePoolIdx(ctx context.Context, bucket, object // will generate a temp object idx = z.getAvailablePoolIdx(ctx, bucket, object, size) if idx < 0 { - return -1, false, toObjectErr(errDiskFull) + return -1, toObjectErr(errDiskFull) } } - return idx, recoverWrite && isErrReadQuorum(pinfo.Err), nil + return idx, nil } func (z *erasureServerPools) Shutdown(ctx context.Context) error { @@ -1074,17 +1048,6 @@ func (z *erasureServerPools) getLatestObjectInfoWithIdx(ctx context.Context, buc } wg.Wait() - // An unreadable pool may contain a newer generation. Check every pool - // before choosing a readable copy; sorting first would hide errors behind - // an older object's nonzero ModTime. Delete markers are resolved metadata - // and still participate in the ordering below. - for _, res := range results { - if res.err != nil && !isErrObjectNotFound(res.err) && !isErrVersionNotFound(res.err) && - (!res.oi.DeleteMarker || !isErrMethodNotAllowed(res.err)) { - return res.oi, res.zIdx, res.err - } - } - // Sort the objInfos such that we always serve latest // this is a defensive change to handle any duplicate // content that may have been created, we always serve @@ -1229,8 +1192,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec opts.replicaObjectInfo = resolve } - recoverWrite := !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement - idx, recovery, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true, recoverWrite) + idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true) if err != nil { return ObjectInfo{}, err } @@ -1253,22 +1215,6 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec opts.replicaObjectInfo = z.replicaObjectInfo } oi, err := z.serverPools[idx].PutObject(ctx, bucket, object, data, opts) - if err == nil && recovery { - // Only recovery pays for a second read. A partial repair or a new - // unreadable pool must not be acknowledged as recovered. Keep any - // committed write on failure; a read failure is not permission to undo. - var latest ObjectInfo - var latestIdx int - latest, latestIdx, err = z.getLatestObjectInfoWithIdx(ctx, bucket, object, ObjectOptions{ - NoLock: true, - NoAuditLog: true, - Versioned: opts.Versioned, - VersionSuspended: opts.VersionSuspended, - }) - if err == nil && (latestIdx != idx || latest.VersionID != oi.VersionID || latest.ETag != oi.ETag || !latest.ModTime.Equal(oi.ModTime)) { - err = InsufficientReadQuorum{Bucket: bucket, Object: decodeDirObject(object), Err: errErasureReadQuorum, Type: RQInconsistentMeta} - } - } if err == nil && opts.ReplicaLockReconcile && !opts.DataMovement { err = z.retireReplicaCopies(ctx, bucket, object, idx, oi) } diff --git a/cmd/maintenance-read-compatibility_test.go b/cmd/maintenance-read-compatibility_test.go new file mode 100644 index 000000000..3c0a27e8e --- /dev/null +++ b/cmd/maintenance-read-compatibility_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2026 Feng Ruohang +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "bytes" + "context" + "io" + "testing" +) + +type compatibilityFailedOverwrite struct { + StorageAPI + commit bool +} + +func (d compatibilityFailedOverwrite) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) { + if d.commit { + if _, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts); err != nil { + return RenameDataResp{}, err + } + } + return RenameDataResp{}, errDiskNotFound +} + +// Maintenance compatibility requirement: after a completely joined failed +// overwrite with one residue, keep the previously acknowledged quorum readable. +// This tests a known history; it does not claim the resulting snapshot proves it. +func TestMaintenancePartialFailedOverwriteAvailability(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + dirs, err := getRandomDisks(16) + if err != nil { + t.Fatal(err) + } + obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, dirs...)) + if err != nil { + removeRoots(dirs) + t.Fatal(err) + } + z := obj.(*erasureServerPools) + previous := newObjectLayerFn() + setObjectLayer(z) + defer func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) }() + bucket := "maintenance-compatibility" + key := "partial-overwrite" + if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + oldBody := bytes.Repeat([]byte("old-A"), 65536) + newBody := bytes.Repeat([]byte("new-B"), 65536) + put := func(body []byte) error { + _, err := z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{}) + return err + } + if err := put(oldBody); err != nil { + t.Fatal(err) + } + set := z.serverPools[0].getHashedSet(key) + getDisks := set.getDisks + disks := getDisks() + wrapped := append([]StorageAPI(nil), disks...) + for i, d := range disks { + wrapped[i] = compatibilityFailedOverwrite{d, i == 0} + } + set.getDisks = func() []StorageAPI { return wrapped } + err = put(newBody) + set.getDisks = getDisks + if err == nil { + t.Fatal("partial overwrite must fail") + } + t.Logf("Overwrite completed with failure; one new residue, fifteen original shards: %v", err) + r, err := z.GetObjectNInfo(ctx, bucket, key, nil, nil, ObjectOptions{}) + if err != nil { + t.Fatalf("MAINTENANCE COMPATIBILITY: previously readable acknowledged value now refused: %v", err) + } + data, err := io.ReadAll(r) + r.Close() + if err != nil || !bytes.Equal(data, oldBody) { + t.Fatalf("old acknowledged payload mismatch: bytes=%d err=%v", len(data), err) + } +} diff --git a/cmd/metacache-listing-delete-strict_test.go b/cmd/metacache-listing-delete-strict_test.go new file mode 100644 index 000000000..1f44a0355 --- /dev/null +++ b/cmd/metacache-listing-delete-strict_test.go @@ -0,0 +1,61 @@ +//go:build consistency_strict + +// Preserved diagnostic for the deferred strict-read policy. This is not a +// maintenance release gate; failures record intentionally unresolved risks. + +// Copyright (c) 2026 Feng Ruohang +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "bytes" + "testing" + "time" +) + +func TestStrictListFallbackRejectsSplitGenerations(t *testing.T) { + z, bucket, _ := nullQuorumBackend(t) + set := z.serverPools[0].sets[0] + disks := set.getDisks() + const object = "object" + mtime := time.Now().UTC().Add(-time.Hour) + var generations [3][][]byte + for generation := range generations { + body := bytes.Repeat([]byte{byte('a' + generation)}, 8192) + nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime.Add(time.Duration(generation) * time.Minute)}) + generations[generation] = make([][]byte, len(disks)) + for i, disk := range disks { + generations[generation][i] = mustReadNullQuorumMeta(t, disk, bucket, object) + } + } + // All sixteen drives are online, but the 7/5/4 split leaves no generation + // with the eight matching snapshots required by the listing resolver. + for i, disk := range disks { + generation := 0 + if i >= 7 { + generation = 1 + } + if i >= 12 { + generation = 2 + } + if err := disk.WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, generations[generation][i]); err != nil { + t.Fatal(err) + } + } + if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrReadQuorum(err) { + t.Fatalf("split metadata must remain a read-quorum failure: %v", err) + } + entries := make(metaCacheEntries, len(disks)) + for i, disk := range disks { + entries[i] = metaCacheEntry{name: object, metadata: mustReadNullQuorumMeta(t, disk, bucket, object)} + if _, err := entries[i].xlmeta(); err != nil { + t.Fatalf("drive %d no longer has valid metadata: %v", i, err) + } + } + resolver := metadataResolutionParams{dirQuorum: 8, objQuorum: 8, bucket: bucket, requestedVersions: 1} + entry, err := z.resolveListingGroup(t.Context(), bucket, []*metaCacheEntry{set.resolveListEntry(entries, nil, &resolver)}) + if entry != nil || !isErrReadQuorum(err) { + t.Fatalf("split live metadata became absence: entry=%v err=%v", entry, err) + } +} diff --git a/cmd/metacache-listing-delete_test.go b/cmd/metacache-listing-delete_test.go index 81220c5cc..c59e64844 100644 --- a/cmd/metacache-listing-delete_test.go +++ b/cmd/metacache-listing-delete_test.go @@ -115,8 +115,8 @@ func TestListFallbackRejectsSplitGenerations(t *testing.T) { t.Fatal(err) } } - if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrReadQuorum(err) { - t.Fatalf("split metadata must remain a read-quorum failure: %v", err) + if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrObjectNotFound(err) { + t.Fatalf("split metadata did not exercise ambiguous not-found: %v", err) } entries := make(metaCacheEntries, len(disks)) for i, disk := range disks { diff --git a/cmd/metacache-null-quorum-strict_test.go b/cmd/metacache-null-quorum-strict_test.go new file mode 100644 index 000000000..91d3f7ba1 --- /dev/null +++ b/cmd/metacache-null-quorum-strict_test.go @@ -0,0 +1,167 @@ +//go:build consistency_strict + +// Preserved diagnostic for the deferred strict-read policy. This is not a +// maintenance release gate; failures record intentionally unresolved risks. + +// Copyright (c) 2026 Feng Ruohang +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "bytes" + madmin "github.com/minio/madmin-go/v3" + "io" + "net/http" + "os" + "path/filepath" + "slices" + "testing" + "time" +) + +func TestStrictSingleNullQuorumReadAndConsumers(t *testing.T) { + z, bucket, router := nullQuorumBackend(t) + set := z.serverPools[0].sets[0] + disks := set.getDisks() + const object = "object" + oldBody, newBody := bytes.Repeat([]byte{'a'}, 8192), bytes.Repeat([]byte{'b'}, 8192) + oldTime := time.Now().UTC().Add(-time.Hour) + put := func(body []byte, modTime time.Time) { + t.Helper() + _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: modTime}) + if err != nil { + t.Fatal(err) + } + } + put(oldBody, oldTime) + oldMeta := make([][]byte, len(disks)) + for i, disk := range disks { + var err error + oldMeta[i], err = disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile) + if err != nil { + t.Fatal(err) + } + } + put(newBody, oldTime.Add(time.Minute)) + newMeta := make([][]byte, len(disks)) + for i, disk := range disks { + newMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object) + } + // Model an incomplete overwrite using real inline shards: fifteen old + // copies retain a read quorum; the final disk contains the newer minority. + // The old quorum does not prove that the newer overwrite was uncommitted. + for i := range 15 { + if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil { + t.Fatal(err) + } + } + checkRead := func(body []byte, modTime time.Time) { + t.Helper() + get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object)) + head := nullQuorumRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, object)) + if get.Code != 200 || !bytes.Equal(get.Body.Bytes(), body) || head.Code != 200 || head.Header().Get("Content-Length") != "8192" { + t.Fatalf("GET/HEAD lost readable quorum: GET=%d HEAD=%d body=%q", get.Code, head.Code, get.Body.String()) + } + oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}) + if err != nil || !oi.ModTime.Equal(modTime) { + t.Fatalf("wrong readable generation: %v, %v", oi.ModTime, err) + } + } + for _, method := range []string{http.MethodGet, http.MethodHead} { + rec := nullQuorumRequest(t, router, method, getGetObjectURL("", bucket, object)) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("%s exposed an unresolved old quorum: %d", method, rec.Code) + } + } + if _, err := z.HealObject(t.Context(), bucket, object, "", madmin.HealOpts{Remove: true}); !isErrReadQuorum(err) { + t.Fatalf("heal accepted an unresolved old quorum: %v", err) + } + // Reading must not reinterpret the minority as absent or delete it. + minority, err := disks[15].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{}) + if err != nil || !minority.ModTime.Equal(oldTime.Add(time.Minute)) { + t.Fatalf("read mutated the minority: %+v, %v", minority, err) + } + // Now model the safe direction: the latest generation has the quorum. + // Keep the migration, Walk and scanner assertions on that readable state. + for i := range 15 { + if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMeta[i]); err != nil { + t.Fatal(err) + } + } + if err := disks[15].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[15]); err != nil { + t.Fatal(err) + } + newTime := oldTime.Add(time.Minute) + checkRead(newBody, newTime) + for _, kind := range []string{"rebalance", "decommission"} { + t.Run(kind, func(t *testing.T) { + var names []string + consume := func(entry metaCacheEntry) { + versions, err := entry.fileInfoVersions(bucket) + if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(newTime) { + t.Errorf("invalid migration metadata: %+v, %v", versions, err) + return + } + // Migration reopens the selected name/version on all drives; + // it must not treat the selected listing metadata as disk agreement. + reader, err := set.GetObjectNInfo(t.Context(), bucket, entry.name, nil, nil, ObjectOptions{VersionID: nullVersionID, NoLock: true, NoDecryption: true}) + if err != nil { + t.Error(err) + return + } + body, err := io.ReadAll(reader) + reader.Close() + if err != nil || !bytes.Equal(body, newBody) { + t.Errorf("migration could not read selected object data: %v", err) + } + names = append(names, entry.name) + } + var err error + if kind == "rebalance" { + err = set.listObjectsToRebalance(t.Context(), bucket, consume) + } else { + err = set.listObjectsToDecommission(t.Context(), decomBucketInfo{Name: bucket}, consume) + } + if err != nil || !slices.Equal(names, []string{object}) { + t.Fatalf("migration listing dropped the quorum: %v, %v", names, err) + } + }) + } + for _, latestOnly := range []bool{false, true} { + results := make(chan itemOrErr[ObjectInfo], 16) + if err := z.Walk(t.Context(), bucket, "", results, WalkOptions{AskDisks: "strict", LatestOnly: latestOnly}); err != nil { + t.Fatal(err) + } + var names []string + for result := range results { + if result.Err != nil || !result.Item.ModTime.Equal(newTime) { + t.Fatalf("Walk returned invalid metadata: %+v", result) + } + names = append(names, result.Item.Name) + } + if !slices.Equal(names, []string{object}) { + t.Fatalf("Walk dropped the quorum: %v", names) + } + } + + // Exercise the scanner's actual abandoned-child path. Make the scan drive + // miss the object, while keeping the latest quorum and an older minority on + // the remaining disks. Its queued heal must read all disks and repair both. + if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[14]); err != nil { + t.Fatal(err) + } + if err := os.RemoveAll(filepath.Join(disks[15].Endpoint().Path, bucket, object)); err != nil { + t.Fatal(err) + } + scanNullQuorumAbandoned(t, z, bucket, object, newTime) + for i, disk := range disks { + fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{Healing: true}) + if err != nil || !fi.ModTime.Equal(newTime) { + t.Fatalf("scanner heal did not reconcile disk %d: %v, %v", i, fi.ModTime, err) + } + } + checkRead(newBody, newTime) + put(newBody, oldTime.Add(2*time.Minute)) + checkRead(newBody, oldTime.Add(2*time.Minute)) +} diff --git a/cmd/metacache-null-quorum_test.go b/cmd/metacache-null-quorum_test.go index d6f2d33ef..ee88f0cfa 100644 --- a/cmd/metacache-null-quorum_test.go +++ b/cmd/metacache-null-quorum_test.go @@ -701,13 +701,9 @@ func TestSingleNullQuorumReadAndConsumers(t *testing.T) { } } put(newBody, oldTime.Add(time.Minute)) - newMeta := make([][]byte, len(disks)) - for i, disk := range disks { - newMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object) - } + newMinorityMeta := mustReadNullQuorumMeta(t, disks[14], bucket, object) // Model an incomplete overwrite using real inline shards: fifteen old // copies retain a read quorum; the final disk contains the newer minority. - // The old quorum does not prove that the newer overwrite was uncommitted. for i := range 15 { if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil { t.Fatal(err) @@ -725,38 +721,18 @@ func TestSingleNullQuorumReadAndConsumers(t *testing.T) { t.Fatalf("wrong readable generation: %v, %v", oi.ModTime, err) } } - for _, method := range []string{http.MethodGet, http.MethodHead} { - rec := nullQuorumRequest(t, router, method, getGetObjectURL("", bucket, object)) - if rec.Code != http.StatusServiceUnavailable { - t.Fatalf("%s exposed an unresolved old quorum: %d", method, rec.Code) - } - } - if _, err := z.HealObject(t.Context(), bucket, object, "", madmin.HealOpts{Remove: true}); !isErrReadQuorum(err) { - t.Fatalf("heal accepted an unresolved old quorum: %v", err) - } + checkRead(oldBody, oldTime) // Reading must not reinterpret the minority as absent or delete it. minority, err := disks[15].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{}) if err != nil || !minority.ModTime.Equal(oldTime.Add(time.Minute)) { t.Fatalf("read mutated the minority: %+v, %v", minority, err) } - // Now model the safe direction: the latest generation has the quorum. - // Keep the migration, Walk and scanner assertions on that readable state. - for i := range 15 { - if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMeta[i]); err != nil { - t.Fatal(err) - } - } - if err := disks[15].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[15]); err != nil { - t.Fatal(err) - } - newTime := oldTime.Add(time.Minute) - checkRead(newBody, newTime) for _, kind := range []string{"rebalance", "decommission"} { t.Run(kind, func(t *testing.T) { var names []string consume := func(entry metaCacheEntry) { versions, err := entry.fileInfoVersions(bucket) - if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(newTime) { + if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(oldTime) { t.Errorf("invalid migration metadata: %+v, %v", versions, err) return } @@ -769,7 +745,7 @@ func TestSingleNullQuorumReadAndConsumers(t *testing.T) { } body, err := io.ReadAll(reader) reader.Close() - if err != nil || !bytes.Equal(body, newBody) { + if err != nil || !bytes.Equal(body, oldBody) { t.Errorf("migration could not read selected object data: %v", err) } names = append(names, entry.name) @@ -792,7 +768,7 @@ func TestSingleNullQuorumReadAndConsumers(t *testing.T) { } var names []string for result := range results { - if result.Err != nil || !result.Item.ModTime.Equal(newTime) { + if result.Err != nil || !result.Item.ModTime.Equal(oldTime) { t.Fatalf("Walk returned invalid metadata: %+v", result) } names = append(names, result.Item.Name) @@ -803,22 +779,22 @@ func TestSingleNullQuorumReadAndConsumers(t *testing.T) { } // Exercise the scanner's actual abandoned-child path. Make the scan drive - // miss the object, while keeping the latest quorum and an older minority on + // miss the object, while keeping an older quorum and a newer minority on // the remaining disks. Its queued heal must read all disks and repair both. - if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[14]); err != nil { + if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMinorityMeta); err != nil { t.Fatal(err) } if err := os.RemoveAll(filepath.Join(disks[15].Endpoint().Path, bucket, object)); err != nil { t.Fatal(err) } - scanNullQuorumAbandoned(t, z, bucket, object, newTime) + scanNullQuorumAbandoned(t, z, bucket, object, oldTime) for i, disk := range disks { fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{Healing: true}) - if err != nil || !fi.ModTime.Equal(newTime) { + if err != nil || !fi.ModTime.Equal(oldTime) { t.Fatalf("scanner heal did not reconcile disk %d: %v, %v", i, fi.ModTime, err) } } - checkRead(newBody, newTime) + checkRead(oldBody, oldTime) put(newBody, oldTime.Add(2*time.Minute)) checkRead(newBody, oldTime.Add(2*time.Minute)) } @@ -914,5 +890,5 @@ func scanNullQuorumAbandoned(t *testing.T, z *erasureServerPools, bucket, object if flat == nil || flat.Objects != 1 || flat.Size != 8192 { t.Fatalf("scanner lost the healed object from usage: %+v", flat) } - t.Logf("scanner queued %v; healed latest quorum, repaired minority/missing copies, and retained one 8192-byte object", healed) + t.Logf("scanner queued %v; healed old quorum, repaired minority/missing copies, and retained one 8192-byte object", healed) } diff --git a/cmd/object-mutation-consistency_test.go b/cmd/object-mutation-consistency_test.go index 3c1c121cb..b624965a1 100644 --- a/cmd/object-mutation-consistency_test.go +++ b/cmd/object-mutation-consistency_test.go @@ -69,112 +69,6 @@ func mutationPut(t testing.TB, z *erasureServerPools, bucket, obj string, pool i } } -// Ordinary successful tag write with one temporarily unavailable drive. No -// metadata is fabricated. The stale drive returns first on a subsequent GET. -func TestMutationReadPreservesAcknowledgedTags(t *testing.T) { - z, bucket := mutationPools(t) - const object = "tag-read" - mutationPut(t, z, bucket, object, 0, "payload", time.Now()) - set := z.serverPools[0].getHashedSet(object) - disks, swap := installListingDiskSwitch(set) - defer swap(disks) - partial := append([]StorageAPI(nil), disks...) - partial[0] = nil - swap(partial) - const tags = "owner=acknowledged-B" - if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil { - t.Fatal(err) - } - swap(disks) - before, err := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{}) - if err != nil { - t.Fatal(err) - } - after, err := disks[1].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{}) - if err != nil { - t.Fatal(err) - } - t.Logf("old/new tags=%q/%q; equal tuple=%t", before.Metadata[xhttp.AmzObjectTagging], after.Metadata[xhttp.AmzObjectTagging], before.ModTime.Equal(after.ModTime) && before.DataDir == after.DataDir && before.Metadata["etag"] == after.Metadata["etag"]) - for _, fast := range []bool{false, true} { - for _, version := range []string{"", nullVersionID} { - oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version}) - if err != nil { - t.Fatal(err) - } - if oi.UserTags != tags { - t.Fatalf("acknowledged tags replaced by stale metadata (fast=%t version=%q): %q", fast, version, oi.UserTags) - } - } - } -} - -// A healthy version history must reuse the already decoded latest version. -// A stale same-version tag copy must still take the full consistency path. -func TestMutationVersionHistoryRead(t *testing.T) { - z, bucket := mutationPools(t) - const object = "history-read" - set := z.serverPools[0].getHashedSet(object) - var versions []string - for i := range 12 { - id := mustGetUUID() - body := "version-" + strconv.Itoa(i) - _, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, VersionID: id}) - if err != nil { - t.Fatal(err) - } - versions = append(versions, id) - } - disks, swap := installListingDiskSwitch(set) - defer swap(disks) - uniform := func() bool { - raw := make([]RawFileInfo, len(disks)) - meta := make([]FileInfo, len(disks)) - errs := make([]error, len(disks)) - for i, disk := range disks { - var err error - raw[i], err = disk.ReadXL(t.Context(), bucket, object, false) - if err != nil { - t.Fatal(err) - } - meta[i], errs[i] = fileInfoFromRaw(raw[i], bucket, object, false, false) - } - return uniformLatestVersion(meta, errs) - } - if !uniform() { - t.Fatal("healthy version history requires redundant full metadata decoding") - } - partial := append([]StorageAPI(nil), disks...) - partial[0] = nil - swap(partial) - const tags = "owner=latest-acknowledged" - if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil { - t.Fatal(err) - } - swap(disks) - if uniform() { - t.Fatal("stale same-version metadata incorrectly accepted as uniform") - } - for _, fast := range []bool{false, true} { - for _, version := range []string{"", versions[len(versions)-1]} { - oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version}) - if err != nil || oi.VersionID != versions[len(versions)-1] || oi.UserTags != tags { - t.Fatalf("latest version/tags changed: version=%q tags=%q err=%v", oi.VersionID, oi.UserTags, err) - } - } - } - for i, id := range versions { - r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: id}) - if err != nil { - t.Fatal(err) - } - body, err := io.ReadAll(r) - r.Close() - if err != nil || string(body) != "version-"+strconv.Itoa(i) { - t.Fatalf("history version %d changed: %q err=%v", i, body, err) - } - } -} - type mutationAmbiguousRename struct { StorageAPI index int @@ -730,68 +624,6 @@ func (d orderedRenameDisk) RenameData(ctx context.Context, sv, sp string, fi Fil return r, err } -func TestMutationCrossDiskOrderHasStableRead(t *testing.T) { - z, bucket := mutationPools(t) - const object = "cross-disk-order" - base := time.Now().Add(-time.Hour) - mutationPut(t, z, bucket, object, 0, "initial", base) - set := z.serverPools[0].getHashedSet(object) - disks, swap := installListingDiskSwitch(set) - defer swap(disks) - earlyDone, release := make(chan struct{}, 4), make(chan struct{}) - var once sync.Once - unblock := func() { once.Do(func() { close(release) }) } - defer unblock() - aDisks := append([]StorageAPI(nil), disks...) - for i, d := range disks { - aDisks[i] = orderedRenameDisk{d, i < 4, earlyDone, release} - } - swap(aDisks) - put := func(body string, mtime time.Time) error { - _, e := z.serverPools[0].PutObject(t.Context(), bucket, object, - mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), - ObjectOptions{NoLock: true, MTime: mtime}) - return e - } - aDone := make(chan error, 1) - go func() { aDone <- put("writer-A", base.Add(time.Minute)) }() - for range 4 { - select { - case <-earlyDone: - case <-time.After(5 * time.Second): - t.Fatal("A did not reach first-half barrier") - } - } - swap(disks) - if err := put("writer-B", base.Add(2*time.Minute)); err != nil { - t.Fatalf("B failed: %v", err) - } - unblock() - select { - case err := <-aDone: - if err != nil { - t.Fatalf("A failed: %v", err) - } - case <-time.After(5 * time.Second): - t.Fatal("A did not finish") - } - t.Log("both PUTs succeeded; final per-disk order is A→B on four disks and B→A on four disks; no writer remains") - swap(disks) - for range 8 { - for _, version := range []string{"", nullVersionID} { - r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{FastGetObjInfo: true, VersionID: version}) - if err != nil { - t.Fatalf("quiescent successful writes became unreadable: %v", err) - } - body, err := io.ReadAll(r) - r.Close() - if err != nil || string(body) != "writer-B" { - t.Fatalf("completed reads changed value: %q %v", body, err) - } - } - } -} - // Partial no-op updates may revoke undo on only some disks. That internal // capability must not split the quorum of otherwise identical object contents. func TestMutationPartialNoopPreservesReadability(t *testing.T) { diff --git a/cmd/object-mutation-late-commit_test.go b/cmd/object-mutation-late-commit_test.go index 1ab20f24f..62f2f2a48 100644 --- a/cmd/object-mutation-late-commit_test.go +++ b/cmd/object-mutation-late-commit_test.go @@ -1,3 +1,8 @@ +//go:build consistency_strict + +// Preserved diagnostic for the deferred strict-read policy. This is not a +// maintenance release gate; failures record intentionally unresolved risks. + // Copyright (c) 2026 Feng Ruohang // // This file is part of Silo Object Storage. diff --git a/cmd/object-mutation-read-strict_test.go b/cmd/object-mutation-read-strict_test.go new file mode 100644 index 000000000..df47e4f72 --- /dev/null +++ b/cmd/object-mutation-read-strict_test.go @@ -0,0 +1,198 @@ +//go:build consistency_strict + +// Preserved diagnostic for the deferred strict-read policy. This is not a +// maintenance release gate; failures record intentionally unresolved risks. + +// Copyright (c) 2026 Feng Ruohang +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + xhttp "github.com/minio/minio/internal/http" + "io" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +func TestMutationReadPreservesAcknowledgedTags(t *testing.T) { + z, bucket := mutationPools(t) + const object = "tag-read" + mutationPut(t, z, bucket, object, 0, "payload", time.Now()) + set := z.serverPools[0].getHashedSet(object) + disks, swap := installListingDiskSwitch(set) + defer swap(disks) + partial := append([]StorageAPI(nil), disks...) + partial[0] = nil + swap(partial) + const tags = "owner=acknowledged-B" + if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil { + t.Fatal(err) + } + swap(disks) + before, err := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{}) + if err != nil { + t.Fatal(err) + } + after, err := disks[1].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{}) + if err != nil { + t.Fatal(err) + } + t.Logf("old/new tags=%q/%q; equal tuple=%t", before.Metadata[xhttp.AmzObjectTagging], after.Metadata[xhttp.AmzObjectTagging], before.ModTime.Equal(after.ModTime) && before.DataDir == after.DataDir && before.Metadata["etag"] == after.Metadata["etag"]) + for _, fast := range []bool{false, true} { + for _, version := range []string{"", nullVersionID} { + oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version}) + if err != nil { + t.Fatal(err) + } + if oi.UserTags != tags { + t.Fatalf("acknowledged tags replaced by stale metadata (fast=%t version=%q): %q", fast, version, oi.UserTags) + } + } + } +} + +func TestMutationVersionHistoryRead(t *testing.T) { + z, bucket := mutationPools(t) + const object = "history-read" + set := z.serverPools[0].getHashedSet(object) + var versions []string + for i := range 12 { + id := mustGetUUID() + body := "version-" + strconv.Itoa(i) + _, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, VersionID: id}) + if err != nil { + t.Fatal(err) + } + versions = append(versions, id) + } + disks, swap := installListingDiskSwitch(set) + defer swap(disks) + uniform := func() bool { + raw := make([]RawFileInfo, len(disks)) + meta := make([]FileInfo, len(disks)) + errs := make([]error, len(disks)) + for i, disk := range disks { + var err error + raw[i], err = disk.ReadXL(t.Context(), bucket, object, false) + if err != nil { + t.Fatal(err) + } + meta[i], errs[i] = fileInfoFromRaw(raw[i], bucket, object, false, false) + } + return uniformLatestVersion(meta, errs) + } + if !uniform() { + t.Fatal("healthy version history requires redundant full metadata decoding") + } + partial := append([]StorageAPI(nil), disks...) + partial[0] = nil + swap(partial) + const tags = "owner=latest-acknowledged" + if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil { + t.Fatal(err) + } + swap(disks) + if uniform() { + t.Fatal("stale same-version metadata incorrectly accepted as uniform") + } + for _, fast := range []bool{false, true} { + for _, version := range []string{"", versions[len(versions)-1]} { + oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version}) + if err != nil || oi.VersionID != versions[len(versions)-1] || oi.UserTags != tags { + t.Fatalf("latest version/tags changed: version=%q tags=%q err=%v", oi.VersionID, oi.UserTags, err) + } + } + } + for i, id := range versions { + r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: id}) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(r) + r.Close() + if err != nil || string(body) != "version-"+strconv.Itoa(i) { + t.Fatalf("history version %d changed: %q err=%v", i, body, err) + } + } +} + +func TestMutationCrossDiskOrderHasStableRead(t *testing.T) { + z, bucket := mutationPools(t) + const object = "cross-disk-order" + base := time.Now().Add(-time.Hour) + mutationPut(t, z, bucket, object, 0, "initial", base) + set := z.serverPools[0].getHashedSet(object) + disks, swap := installListingDiskSwitch(set) + defer swap(disks) + earlyDone, release := make(chan struct{}, 4), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + defer unblock() + aDisks := append([]StorageAPI(nil), disks...) + for i, d := range disks { + aDisks[i] = orderedRenameDisk{d, i < 4, earlyDone, release} + } + swap(aDisks) + put := func(body string, mtime time.Time) error { + _, e := z.serverPools[0].PutObject(t.Context(), bucket, object, + mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), + ObjectOptions{NoLock: true, MTime: mtime}) + return e + } + aDone := make(chan error, 1) + go func() { aDone <- put("writer-A", base.Add(time.Minute)) }() + for range 4 { + select { + case <-earlyDone: + case <-time.After(5 * time.Second): + t.Fatal("A did not reach first-half barrier") + } + } + swap(disks) + if err := put("writer-B", base.Add(2*time.Minute)); err != nil { + t.Fatalf("B failed: %v", err) + } + unblock() + select { + case err := <-aDone: + if err != nil { + t.Fatalf("A failed: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("A did not finish") + } + t.Log("both PUTs succeeded; final per-disk order is A→B on four disks and B→A on four disks; no writer remains") + swap(disks) + for range 8 { + for _, version := range []string{"", nullVersionID} { + r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{FastGetObjInfo: true, VersionID: version}) + if err != nil { + t.Fatalf("quiescent successful writes became unreadable: %v", err) + } + body, err := io.ReadAll(r) + r.Close() + if err != nil || string(body) != "writer-B" { + t.Fatalf("completed reads changed value: %q %v", body, err) + } + } + } +} + +func uniformLatestVersion(meta []FileInfo, errs []error) bool { + if len(meta) == 0 { + return false + } + first := meta[0] + for i, fi := range meta { + if errs[i] != nil || !fi.IsValid() || fi.XLV1 || fi.Deleted || !fi.IsLatest || fi.NumVersions < 1 || + !fi.ModTime.Equal(first.ModTime) || !sameFileInfoVersion(first, fi) || + fi.NumVersions != first.NumVersions || !fi.SuccessorModTime.Equal(first.SuccessorModTime) { + return false + } + } + return true +} diff --git a/cmd/object-mutation.go b/cmd/object-mutation.go index be187d8be..788ebcdc6 100644 --- a/cmd/object-mutation.go +++ b/cmd/object-mutation.go @@ -193,23 +193,3 @@ func (s *xlStorage) undoWrite(ctx context.Context, volume, path string, fi FileI } return nil } - -// All drives have replied, and fileInfoFromRaw has already validated each raw -// buffer and selected its latest visible version. Reuse that decoded state: -// revalidating the raw history here adds work proportional to version count. -// Any missing response, logical disagreement, or history-property disagreement -// retains the full merge. Legacy objects and delete markers also use that path. -func uniformLatestVersion(meta []FileInfo, errs []error) bool { - if len(meta) == 0 { - return false - } - first := meta[0] - for i, fi := range meta { - if errs[i] != nil || !fi.IsValid() || fi.XLV1 || fi.Deleted || !fi.IsLatest || fi.NumVersions < 1 || - !fi.ModTime.Equal(first.ModTime) || !sameFileInfoVersion(first, fi) || - fi.NumVersions != first.NumVersions || !fi.SuccessorModTime.Equal(first.SuccessorModTime) { - return false - } - } - return true -}