Compare commits

..

109 Commits

Author SHA1 Message Date
Feng Ruohang a47bacbc0f test: satisfy lint for lifecycle migration follow-up
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-11-02 14:37:04 +08:00
Feng Ruohang 0a546c7a76 fix(pools): preserve ILM cleanup during migration
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-11-02 14:20:17 +08:00
Feng Ruohang 8dac7f7c14 Merge lifecycle migration fix (#266)
Merge the contributor's lifecycle migration fix after forward-porting it onto current main.

The contributor commits are retained in the merge commit. Maintainer follow-up work will address the remaining replication, tier cleanup, lifecycle configuration, and DeleteAll completion-fence cases.
2026-11-02 12:50:39 +08:00
Feng Ruohang 4cc3bc1de6 Merge remote-tracking branch 'origin/main' into HEAD
Signed-off-by: Feng Ruohang <rh@vonng.com>

# Conflicts:
#	cmd/erasure-server-pool-decom.go
2026-11-02 12:29:08 +08:00
Feng Ruohang 0cfca29cb2 Merge pull request #264 from botdeniska/fix/free-versions-not-versions
fix(ilm): stop taking the free versions of deleted tiered versions for versions
2026-11-01 14:44:22 +08:00
Feng Ruohang a7310d59fe Fix cross-parity version listing after pool removal (#267)
Availability-optimized writes can leave a readable null version across separate EC groups when a disk is unavailable. Recount ordinary null versions for all non-strict reads and cover version listing and complete pool removal with end-to-end tests.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-11-01 14:08:09 +08:00
Denis Babchuk a995921e57 fix(pools): move ILM-expired current versions during rebalance and remove moved copies
Signed-off-by: Denis Babchuk <babchukdenis@icloud.com>
2026-10-29 22:43:26 +03:00
Denis Babchuk 19910230aa fix(pools): move ILM-expired current versions during decommission
Signed-off-by: Denis Babchuk <babchukdenis@icloud.com>
2026-10-29 22:42:17 +03:00
Denis Babchuk 7ce7fc8d1e fix(pools): do not list or move free versions as delete markers
Signed-off-by: Denis Babchuk <babchukdenis@icloud.com>
2026-10-29 22:32:44 +03:00
Denis Babchuk fb1a619ee1 fix(listing): skip free versions when looking for a latest delete marker
Signed-off-by: Denis Babchuk <babchukdenis@icloud.com>
2026-10-29 22:08:34 +03:00
Denis Babchuk 3f13c26eed fix(ilm): do not let a free version make the latest version noncurrent
Signed-off-by: Denis Babchuk <babchukdenis@icloud.com>
2026-10-29 22:07:56 +03:00
Feng Ruohang dd905b29f4 docs: align README and changelog with published 20261026 release
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-27 08:03:22 +08:00
Feng Ruohang 2f728c7643 docs: refresh contributor records for 20261026
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-27 07:58:56 +08:00
Feng Ruohang c3ad836aed Accept full RPM fingerprints in release finalization
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-27 00:51:13 +08:00
Feng Ruohang dc6dc3d806 Pin published Console v2.5.1 for SILO 20261026
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-27 00:12:04 +08:00
Feng Ruohang dcc2f20ba8 Preserve maintenance timeout behavior and prepare SILO 20261026
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-26 22:43:51 +08:00
Feng Ruohang a13efd5760 fix: isolate storage mutation guards by exact resource (#258)
Integrate the exact drive/path mutation guard after PR CI.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-26 21:25:30 +08:00
Feng Ruohang 39278e068e fix: isolate storage mutation guards by exact resource
Replace process-wide hash-slot mutexes with reference-counted locks keyed
by drive and path. Retain entries before waiting, acquire unique paths in
order, and reclaim entries only after all holders and waiters finish.

Add real storage API regressions for unrelated colliding objects and drives,
plus coverage for cancellation, multi-path order and waiter lifetimes.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-26 21:11:49 +08:00
Feng Ruohang 263ee0510d docs: complete the 20261025 change log and contributor records
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-26 00:01:28 +08:00
Feng Ruohang 641eb51a40 test: pin the rollback fixture to its required erasure layout
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 23:47:55 +08:00
Feng Ruohang 45d6e5d065 release: prepare SILO 20261025 with the published component stack
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 23:44:14 +08:00
Feng Ruohang dabc894132 Merge delivered Console coordination fixes into local main
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 22:30:08 +08:00
Feng Ruohang fb08ca7b6a deps: embed the Console bucket-search navigation repair
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 22:13:28 +08:00
Feng Ruohang 41822710e4 deps: embed the Console bucket-search navigation repair
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 22:13:27 +08:00
Feng Ruohang 034f7bbf73 deps: select the Console candidate with native keyboard regression coverage
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 21:50:10 +08:00
Feng Ruohang 79eca5e920 deps: select the Console candidate with native keyboard regression coverage
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 21:50:10 +08:00
Feng Ruohang aec7ddac46 deps: integrate Console repairs and the released SILO libraries
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 21:33:40 +08:00
Feng Ruohang 475640e7e1 deps: embed the repaired Console 2.5.0 candidate
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 21:33:34 +08:00
Feng Ruohang 62124c779c docs: include coordinated Console dependency attributions
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 21:06:21 +08:00
Feng Ruohang 030c6d740e deps: integrate pkg 3.15.1 and Console 2.5.0 candidate for acceptance
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 21:01:22 +08:00
Feng Ruohang 5f1f4178ae fix: return current bucket inventory from account info
Signed-off-by: Feng Ruohang <rh@vonng.com>
(cherry picked from commit 6921f8c186)
2026-10-25 21:01:09 +08:00
Feng Ruohang 0828861bd0 iam: restore native policy immediately after resetting overrides
Signed-off-by: Feng Ruohang <rh@vonng.com>
(cherry picked from commit 1e3ab56b06)
2026-10-25 21:01:09 +08:00
Feng Ruohang 6921f8c186 fix: return current bucket inventory from account info
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 21:01:01 +08:00
Feng Ruohang 25d928c9df test: pin failed-overwrite compatibility fixture to 12+4
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 18:48:39 +08:00
Feng Ruohang ee3e7ea716 test: keep deferred read diagnostic helpers with their build tag
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 18:40:40 +08:00
Feng Ruohang 2a04d5f485 fix: retain maintenance read compatibility with write safety fixes
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 18:39:31 +08:00
Feng Ruohang 9449e6a4a8 test: preserve reads across late commits and rollback
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 18:23:14 +08:00
Feng Ruohang 9c4bb39571 fix: preserve versioning context when verifying recovery
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 16:05:28 +08:00
Feng Ruohang 86ce8d6012 test: cover versioned and encrypted multi-pool recovery
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 16:03:13 +08:00
Feng Ruohang b15aa97c94 test: follow recovery regression lint conventions
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 15:59:45 +08:00
Feng Ruohang f54c04846e fix: recover inconsistent multi-pool copies before acknowledging PUT
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 15:56:28 +08:00
Feng Ruohang 6fdea3b8d9 test: reject recovery ACK when another pool still wins reads
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 15:55:10 +08:00
Feng Ruohang 380fcbfe47 test: require safe refusal when pool recovery cannot be confirmed
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 15:49:29 +08:00
Feng Ruohang a02fa9644c test: pin multi-pool overwrite recovery after a minority commit
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 15:48:33 +08:00
Feng Ruohang 652a1c09d3 fix: protect unfinished commits from abandoned data cleanup
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 13:23:20 +08:00
Feng Ruohang c86df14f4d test: pin abandoned cleanup races with commit and undo
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 13:22:27 +08:00
Feng Ruohang 03875d61e3 fix: reuse validated latest state without rescanning version history
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 13:02:23 +08:00
Feng Ruohang 1865985aa1 fix: separate rollback ownership from readable metadata identity
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 12:41:54 +08:00
Feng Ruohang 076a13f9d2 test: preserve reads after partial no-op metadata updates
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 12:40:00 +08:00
Feng Ruohang 9a4e089f56 fix: reuse unanimous latest metadata for healthy version histories
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 12:33:49 +08:00
Feng Ruohang 9255f443a6 test: guard healthy history reads and same-version divergence
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 12:31:10 +08:00
Feng Ruohang ea7c794a94 fix: preserve version state across late rollback and nested mutations
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 12:22:47 +08:00
Feng Ruohang 16c21428d9 test: pin metadata rollback and nested mutation lifetime regressions
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 12:07:50 +08:00
Feng Ruohang 7e83f9950a merge: restore built-in IAM policies after Reset
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 11:02:53 +08:00
Feng Ruohang 1e3ab56b06 iam: restore native policy immediately after resetting overrides
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 03:32:53 +08:00
Feng Ruohang 5ad1f344b7 fix: keep admitted mutation RPCs alive until their result arrives
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 01:12:58 +08:00
Feng Ruohang 8a4097309f fix: preserve shared-version metadata during overwrite rollback
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 01:11:27 +08:00
Feng Ruohang 0501a198e3 fix: reject COPY-to-self from unresolved object generations
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 01:11:27 +08:00
Feng Ruohang d2eb0ade91 fix: assign PUT generations while holding the commit lock
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 01:08:36 +08:00
Feng Ruohang 5c074efd1d test: preserve external shards in generation conflict fixtures
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 01:03:14 +08:00
Feng Ruohang 03fe5e9c21 test: isolate endpoint locality fixtures from public DNS
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 00:50:41 +08:00
Feng Ruohang 0b1c21cb4b fix: retain unresolved generations during fast reads and healing
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 00:48:16 +08:00
Feng Ruohang b55495aef0 fix: preserve remote rollback options and reject stale read quorums
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 00:37:21 +08:00
Feng Ruohang d8b77e5f3b test: require read-quorum errors for split listing generations
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 00:24:28 +08:00
Feng Ruohang cd1f6cec3c fix: drain canceled storage commits before releasing object locks
Wait for admitted data and metadata mutations, preserve overwrite backups during rollback, and reject rollback of a different generation. Report conflicting or unreadable pool metadata as read-quorum failures instead of exposing stale copies. Add barrier-driven local/RPC and multi-pool regressions for cancellation, self-copy, rollback and old-value reads.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-25 00:19:42 +08:00
Feng Ruohang 4f17a23e33 test: isolate release acceptance fixtures across backends
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-24 22:39:29 +08:00
Feng Ruohang be44ff2669 build: update silo-pkg to v3.15.0
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-24 22:17:20 +08:00
Feng Ruohang d16f5ed9c8 fix(ilm): keep tier data when expiry only adds a delete marker (#253)
Preserve transitioned version data when lifecycle expiration only creates a delete marker, and apply suspended-versioning semantics consistently.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-24 17:46:10 +08:00
Feng Ruohang 6d890fc860 Merge pull request #255: preserve FULL_OBJECT checksum type in replication
Forward FULL_OBJECT checksum type for multipart CRC replication so the target validates the whole-object checksum correctly. Restore the existing single-PUT replication path for server-side copies retaining these checksums.

Fixes #254

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-24 17:39:38 +08:00
Feng Ruohang aafe4f8886 fix(pools): preserve authoritative copies across failed deletes (#257)
Preserve the latest logical object when cleanup of another pool fails, report each bulk error with its original replication identity, and support retries after partial physical deletion. Reuse existing tier-aware reconciliation to avoid premature remote-object collection.

Includes the original contribution and signed-off maintainer fixes.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-24 16:04:09 +08:00
Feng Ruohang a2073d2514 fix(pools): preserve shared tier references during failed deletes
Reuse deleteObjectReconciled for tiered bulk entries and unversioned single deletes. Its reference-aware cleanup avoids scheduling remote contents for garbage collection while another copy still needs them, including restored copies. Keep the normal local-data bulk path unchanged.

Extend existing tier GC regressions to bulk and unversioned deletes and include them in the multi-pool race CI step.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-24 15:47:51 +08:00
Feng Ruohang 3c82748e14 fix(pools): retain authoritative copies until bulk cleanup succeeds
Resolve each addressed version under the shared write lock, keep bulk I/O parallel within cleanup and authoritative-deletion stages, and preserve replication response identity on errors. Treat reliable absence as successful physical cleanup on retries without accepting it for marker or replication-state writes.

Cover execution-time faults, partial cleanup recovery, mixed authoritative pools, response identity, null marker ordering, metadata uncertainty, cancellation, and concurrent PUT. Run the delete consistency regressions under race in CI.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-24 15:36:07 +08:00
Feng Ruohang 1e94c5a30d Merge remote-tracking branch 'origin/main' into codex/pr257-delete-safety
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-24 15:13:21 +08:00
Feng Ruohang 000e653038 ci: use runner Python for repository card checks
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-24 12:42:50 +08:00
Feng Ruohang 945dc16060 Update SFTP and Go dependencies with object-operation regression coverage
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-24 11:50:39 +08:00
Denis Babchuk a30270d9ba fix(pools): put a delete marker into a pool that can take it
Signed-off-by: Denis Babchuk <babchukdenis@icloud.com>
2026-10-23 22:29:49 +03:00
Denis Babchuk 1841fce4c4 fix(pools): keep the latest copy when a delete fails in one pool
Signed-off-by: Denis Babchuk <babchukdenis@icloud.com>
2026-10-23 21:28:46 +03:00
Denis Babchuk f2b8d4b9cb fix(replication): forward FULL_OBJECT checksum type for multipart replicas
Signed-off-by: Denis Babchuk <babchukdenis@icloud.com>
2026-10-23 17:52:58 +03:00
Denis Babchuk 8ed5c90dcd fix(ilm): keep tier data when expiry only adds a delete marker
Signed-off-by: Denis Babchuk <babchukdenis@icloud.com>
2026-10-23 17:36:21 +03:00
Feng Ruohang 789a2327ba docs: replace the single-node report with the maintained migration entry point
Keep the original PR #228 report at a fixed commit link with contributor credit. Direct operators to the bilingual migration guide and its recovery requirements, and record the merged contribution.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 19:27:20 +08:00
Feng Ruohang e3e0c6215d Merge remote-tracking branch 'origin/main'
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 19:16:00 +08:00
Feng Ruohang b91e1c6f2a Merge pull request #228 from yupoet/docs/single-node-upgrade-from-archived-minio
Preserve the single-node migration report contributed by @yupoet.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 19:15:50 +08:00
Feng Ruohang 969a6a42d3 build: package Windows SILO archives as ZIP
Use ZIP for Windows amd64 and arm64. Validate the platform archive matrix, executable names, documentation, SBOMs and checksum coverage in the release pipeline.

Validated with local GoReleaser snapshots, six SPDX SBOMs and packaging regression checks.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 16:13:15 +08:00
Feng Ruohang 1b5477f3df fix(iam): isolate policy reloads and propagate KMS cancellation
Release the authentication-cache lock while retaining IAM write ordering. Carry request cancellation through IAM and SSE-S3 list decryption, and publish policy replacements only after a complete successful load.

Preserve caller deadlines for slow policy queries and keep the existing etcd read budget scoped to its RPC. Cover concurrent refresh, updates, revocation, cancellation, and deadline compatibility.

Validation: 36 focused and 23 adversarial scenarios each passed three race runs; real 32-second object/etcd probes passed. Full suite passed with the Go DNS resolver after reproducing native DNS failures on the baseline; existing etcd suites, make verifiers, and make build also passed.
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 15:50:12 +08:00
Feng Ruohang a07d655dc6 Merge PR #250 while preserving local dependency updates
Retain the merged upstream LDAP fix and original contributor history. Keep the existing Go/dependency refresh and published-release documentation; combine the changelog credit without repeating the Go 1.27.3 note.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 11:22:52 +08:00
Feng Ruohang d43adc0f2b Merge pull request #250 from 910patin/fix/ldap-long-dn
Restore long-DN LDAP login while preserving IAM revision read errors.

Retain the original contribution by @910patin, add the bounded ASCII component guard and regression tests, and require the patched Go 1.27.3 toolchain. DCO, Go CI, VulnCheck and release snapshot validation passed on the final PR head.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 11:21:50 +08:00
Feng Ruohang 3df51549bb build: require Go 1.27.3 for patched certificate verification
The PR CI found GO-2026-6809 in the remote main baseline using Go 1.27.2. Require the patched toolchain already used for local acceptance; keep module versions and the separate dependency refresh unchanged.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 11:04:31 +08:00
Feng Ruohang 324bb308dd docs: credit LDAP long-DN fix contributor
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 10:53:41 +08:00
Feng Ruohang 625c11e7e1 fix(iam): preserve revision errors for storable record names
Only classify a valid IAM key as absent when a length error accompanies an ASCII path component longer than 255 bytes. Normalized LDAP DNs satisfy this condition, while total-path errors and Unicode names can refer to existing revocation records.

Cover component boundaries, wrapped and storage errors, and real revocation records moved beneath a deep Darwin drive path.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 10:35:31 +08:00
Feng Ruohang 4059c25e22 docs: reconcile published releases and unreleased dependencies
Replace stale draft status with the published 20261018 source boundary and
summarize the subsequent Go, dependency, and compatibility changes. Keep
historical details in the canonical English and Chinese release records.

Validated new links and anchors and the rebrand compatibility guard.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 02:15:22 +08:00
Feng Ruohang 011b0df5f5 build: update Go to 1.27.3 and refresh dependencies
Update the server and five debugging modules, regenerate dependency credits,
and refresh static curl and the native Red Hat CA mitigation.

Preserve Kafka's default broker protocol and terminal widths. Copy the
returned DARE plaintext when unsealing legacy keys, and cover old encrypted
keys, Kafka producer exchanges, and MinKMS TLS identity with regression tests.
Keep the published PGSTY component references for reproducible builds.

Validated with make test, targeted race tests, module integrity and tidy
checks, and govulncheck v1.9.0 source analysis. Rechecked the frozen source
and artifact hashes behind the earlier whole-stack compatibility evidence.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-23 02:15:16 +08:00
GL 9faeb7a114 fix(iam): treat IAM record names too long for the drive as absent
Since RELEASE.2026-09-16T00-00-00Z, LDAP users whose normalized DN is
longer than 255 bytes cannot log in. NormDN hex-escapes every non-ASCII
byte (go-ldap DN.String()), so one Cyrillic letter takes six characters.
IAM looks up records named after the DN: the parent revision check added
with cmd/iam-revision.go reads config/iam/users/<DN>/identity.json, and
PolicyDBGet reads config/iam/policydb/sts-users/<DN>.json when the
mapping is not cached.

A drive rejects such an element in one of two ways. checkPathLength
refuses more than 255 counted characters, but outside Windows it never
counts a backslash, so an escaped DN usually passes it. The filesystem
then fails with ENAMETOOLONG once the parent directory exists.
toObjectErr reports both as ObjectNameInvalid, and the callers treat
only errConfigNotFound as "no record". As a result,
AssumeRoleWithLDAPIdentity, access-key creation and STS loading fail.

Such a record cannot have been stored. When the object layer's own name
validation (checkBucketAndObjectNames) accepts the path,
ObjectNameInvalid can only mean a length rejection, so report it as
errConfigNotFound. Invalid names and storage errors are returned
unchanged, and saving such a record still fails. The etcd store has no
such limit and is unchanged.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: GL <g@lopatin.info>
2026-10-19 18:49:20 +03:00
Feng Ruohang 856f8f4c55 docs: refresh release and contributor acknowledgments
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-19 07:52:26 +08:00
Feng Ruohang 0cd388c666 fix(iam): persist and propagate scoped STS revocation
Enumerate durable STS identities for parent/type revocation, retain the
operation cutoff, and notify sibling caches for each committed deletion.
Repair missed notifications during periodic object/etcd IAM refresh.

Carry immutable STS tombstones and expiry through the existing IAM revision
protocol and deletion healing. Delayed creation cannot undo revocation,
while unrelated and newly issued keys stay valid. Keep partial failures
visible while propagating the successfully committed deletions.

Cover uncached credentials, warm sibling caches, dropped notifications,
object/etcd backends, reordered replay, concurrent issuance and partial
persistence failure. No new dependencies or authentication-path I/O.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 22:39:12 +08:00
Feng Ruohang 1748f62c7e fix(pools): preserve delete markers and recheck migration cleanup
Move solitary and null delete markers instead of dropping them, including
idempotent UUID marker retries. Check that a reader's source version still
exists before moving it, and preserve the established read winner for
conflicting timestamp ties.

Retire only the versions that were successfully moved while holding the
pooled object lock. Recheck the source and retained copies before cleanup,
so concurrent deletes and uncertain destination reads cannot revive data.
Count remaining markers when verifying decommission completion.

Add two/three-pool walker, history, purge, concurrent-delete, read-fault,
and timestamp-tie regressions.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 22:39:12 +08:00
Denis Babchuk 60c7179ddb fix(pools): do not let data movement overwrite a newer copy
Apply PR #248 from bca40a4ac9.

Signed-off-by: Denis Babchuk <babchukdenis@icloud.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 22:39:07 +08:00
Feng Ruohang 7007c164a5 build: remove ripgrep from rebrand verification
Use the existing Git tool to scan source files, including untracked files and exported source trees. Preserve hidden-file exclusions and worktree ignore rules, and reject scanner diagnostics before filtering matches. Remove the CI installation step.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 09:50:18 +08:00
Feng Ruohang 51b7de6a0d ci: install ripgrep for rebrand verification
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 08:46:30 +08:00
Feng Ruohang 4733e1423c fix(multipart): defer persistent listing from public requests
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 08:36:58 +08:00
Feng Ruohang 4189869e43 Merge multipart parent cleanup fix
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 08:22:31 +08:00
Feng Ruohang 0f9199e0c3 fix(multipart): remove empty upload parents promptly
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 08:21:35 +08:00
Feng Ruohang bb60871f57 build: update silo-pkg to v3.14.3
Adopt the published policy-evaluation optimization and record the source update separately from existing component releases.

Verified with repository checks, builds and relevant race tests. No component release is created.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 07:52:13 +08:00
Feng Ruohang 2c820895d7 fix(buildscripts): fail rebrand verification when ripgrep is missing
Check the existing ripgrep prerequisite before searches whose conditional
status handling would otherwise hide command-not-found failures. Preserve
untracked-file coverage and verification from a source archive without Git.

Addresses the missing-tool finding recorded in commit
4fb3a7fa3b175727db987f3ba2523c84f74d4168 without its scan-scope changes.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 07:23:05 +08:00
Feng Ruohang 2080b3d031 test(listing): cover mixed object and directory recovery groups
Extend the pending-directory collision test with absent recovery and a smaller
key. Add all six normal-object, directory, and pending-object channel orders,
checking complete-group recovery and preservation of trailing entries.

Test changes adapted from commit 557bc274a32330321fa5e5ac7636068c7bd98eb0.
Retain the complete-group directory handling from 3dd53eb81f.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 07:23:05 +08:00
Feng Ruohang 9954392605 fix(iam): normalize bulk self access-key listings
Use the existing self-service authorization helper for policyless regular
users and normalize explicit service-account and STS aliases to their parent.
Keep explicit denials and cross-user authorization intact. Assert decrypted
identity contents and negative boundaries in the existing handler tests.

Adapted from commit 5d0b692d8cd387622a3219e72430dffaaf5a25ce.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-18 07:23:05 +08:00
Feng Ruohang 3dd53eb81f fix(listing): preserve directories after absent object recovery
Signed-off-by: Feng Ruohang <rh@vonng.com>
(cherry picked from commit 1445fc1d85d5166de24c7a378cee3479cfc95a43)
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-17 21:25:01 +08:00
Feng Ruohang 86fb2b833f Prepare SILO 20261018 with the coordinated October dependency stack
Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-17 19:28:33 +08:00
Feng Ruohang fbcb511978 Merge pull request #239: confirm unresolved LIST groups
Confirm eligible unresolved LIST groups without changing S3 parameters, storage formats or ordinary writer routing. Preserve original PR #221 commits and authorship.

All eight PR checks, local full/race tests and the nonpublishing snapshot pipeline pass. Current-source normal and rolling correctness pass. The strict 5% performance qualification remains inconclusive and is documented separately from the source-merge decision; no production release is authorized.

Closes #218.

Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-17 11:57:20 +08:00
Paris Yu fec1e0804a docs: add field-validated single-node upgrade guide from archived MinIO community edition
The compatibility audit does not cover the single-node/single-drive (FS mode)
case explicitly. This documents a production upgrade from the final community
release (RELEASE.2025-10-15T17-29-55Z) to Silo RELEASE.2026-09-16T00-00-00Z:
why standalone deployments are low-risk (schema retention, no multi-node REST
concern, IAM tightening not applicable), the validated procedure with layered
verification, field results (~5.9k objects, zero migration, zero hardening
rejections, full app regression passed), and the post-upgrade behavior changes
to review per client (TLS defaults, If-Match on DeleteObject, multipart listing
cap, TRUSTED_PROXIES default).
2026-09-26 19:41:43 +08:00
131 changed files with 13635 additions and 1788 deletions
+4
View File
@@ -27,6 +27,10 @@ archives:
ids: ids:
- silo - silo
name_template: "silo_{{ .Env.PKG_VERSION }}_{{ .Os }}_{{ .Arch }}" name_template: "silo_{{ .Env.PKG_VERSION }}_{{ .Os }}_{{ .Arch }}"
formats: ["tar.gz"]
format_overrides:
- goos: windows
formats: ["zip"]
# Explicit so the license materials cannot silently drop out of the # Explicit so the license materials cannot silently drop out of the
# binary archives: GoReleaser's default file globs would miss NOTICE. # binary archives: GoReleaser's default file globs would miss NOTICE.
# CREDITS stays out deliberately -- at 1.8MB it would dominate the # CREDITS stays out deliberately -- at 1.8MB it would dominate the
+3 -1
View File
@@ -148,7 +148,9 @@ jobs:
for rpm_file in finalize/packages/*.rpm; do for rpm_file in finalize/packages/*.rpm; do
signature_output="$(sudo rpmkeys --checksig --verbose "${rpm_file}")" signature_output="$(sudo rpmkeys --checksig --verbose "${rpm_file}")"
printf '%s\n' "${signature_output}" printf '%s\n' "${signature_output}"
if ! printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]' | grep -q "key id ${key_id}: ok"; then normalized_signature="$(printf '%s\n' "${signature_output}" | tr '[:upper:]' '[:lower:]')"
full_fingerprint="$(printf '%s' "${expected_fingerprint}" | tr '[:upper:]' '[:lower:]')"
if ! printf '%s\n' "${normalized_signature}" | grep -Eq "key id ${key_id}: ok|key fingerprint: ${full_fingerprint}: ok"; then
echo "Signature verification failed for ${rpm_file}" >&2 echo "Signature verification failed for ${rpm_file}" >&2
exit 1 exit 1
fi fi
+16 -3
View File
@@ -93,11 +93,16 @@ jobs:
- name: Run conditional PUT tests under race detector - name: Run conditional PUT tests under race detector
run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP)' -count=1 -timeout=5m run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP)' -count=1 -timeout=5m
- name: Run multipart listing and cancellation tests under race detector - name: Run multi-pool delete consistency tests under race detector
run: go test -race ./cmd -run '^Test(MultipartListing|MultipartAbort|PaginateMultipartUploads|ListMultipartUploads)' -count=1 -timeout=5m run: go test -race ./cmd -run '^Test(Pools(Delete|BulkDelete|RetiringCopyPreservesSharedTierObject)|BulkDeleteQuorumEvidence)' -count=1 -timeout=5m
- name: Run multipart listing, cancellation and cleanup tests under race detector
env:
MINIO_API_REQUESTS_MAX: "10000"
run: go test -race ./cmd -run '^Test(Multipart|PaginateMultipartUploads|ListMultipartUploads|CleanupStaleUploads|ExtractModTime)' -count=1 -timeout=5m
- name: Run listing metadata recovery, pagination and cancellation tests under race detector - name: Run listing metadata recovery, pagination and cancellation tests under race detector
run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity)' -count=1 -timeout=5m run: go test -race ./cmd -run '^Test(ListFallback|ListingCache|ListingMerge|ListingRecovery|ListingPending|ListingEntry|ListingSecondLock|StreamMetadataParts|LocalLockerListing|DistributedListing|MergeXLV2LatestNullCrossParity|CountXLV2MixedECGroups|ListCrossParity|ListObjectVersionsCrossParityNull|MigrationCrossParity)' -count=1 -timeout=5m
- name: Run lock cancellation and late-grant cleanup tests under race detector - name: Run lock cancellation and late-grant cleanup tests under race detector
run: go test -race ./internal/dsync ./internal/lsync -run '^Test(DRWMutex|LRWMutex)' -count=1 -timeout=5m run: go test -race ./internal/dsync ./internal/lsync -run '^Test(DRWMutex|LRWMutex)' -count=1 -timeout=5m
@@ -113,6 +118,14 @@ jobs:
MINIO_API_REQUESTS_MAX: "10000" MINIO_API_REQUESTS_MAX: "10000"
run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
- name: Run maintenance storage timeout and compatibility tests under race detector
env:
MINIO_API_REQUESTS_MAX: "10000"
run: go test -race -tags kqueue,dev ./cmd ./internal/grid ./internal/ioutil -run '^Test(StorageTimeout|StorageRollback|StoragePrerelease|MaintenancePartialFailedOverwriteAvailability|SingleRequest|WithDeadline)' -count=1 -timeout=5m
- name: Compile deferred consistency diagnostics
run: go test -tags kqueue,dev,consistency_strict ./cmd -run '^$'
crosscompile: crosscompile:
name: Cross Compile name: Cross Compile
runs-on: ubuntu-latest runs-on: ubuntu-latest
+2 -2
View File
@@ -140,7 +140,7 @@ jobs:
- name: Build Draft release with GoReleaser - name: Build Draft release with GoReleaser
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
with: with:
version: v2.18.1 version: v2.18.3
args: release --clean --skip=validate --config .github/goreleaser.yml args: release --clean --skip=validate --config .github/goreleaser.yml
env: env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -156,7 +156,7 @@ jobs:
- name: Install nFPM - name: Install nFPM
run: | run: |
set -euo pipefail set -euo pipefail
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}" echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
- name: Build nFPM packages - name: Build nFPM packages
+10 -8
View File
@@ -34,10 +34,11 @@ jobs:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with: with:
persist-credentials: false persist-credentials: false
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - name: Use the runner's Python in an isolated environment
with: run: |
python-version: "3.13" python3 -m venv "$RUNNER_TEMP/repository-cards-venv"
- run: python -m pip install -r buildscripts/repository-cards/requirements.txt "$RUNNER_TEMP/repository-cards-venv/bin/python" -m pip install -r buildscripts/repository-cards/requirements.txt
echo "$RUNNER_TEMP/repository-cards-venv/bin" >> "$GITHUB_PATH"
- run: python -m unittest discover -s buildscripts/repository-cards -p 'test_*.py' -v - run: python -m unittest discover -s buildscripts/repository-cards -p 'test_*.py' -v
publish: publish:
@@ -54,10 +55,11 @@ jobs:
OUTPUT_BRANCH: codex/repository-cards OUTPUT_BRANCH: codex/repository-cards
steps: steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - name: Use the runner's Python in an isolated environment
with: run: |
python-version: "3.13" python3 -m venv "$RUNNER_TEMP/repository-cards-venv"
- run: python -m pip install -r buildscripts/repository-cards/requirements.txt "$RUNNER_TEMP/repository-cards-venv/bin/python" -m pip install -r buildscripts/repository-cards/requirements.txt
echo "$RUNNER_TEMP/repository-cards-venv/bin" >> "$GITHUB_PATH"
- name: Load the generated-assets branch - name: Load the generated-assets branch
shell: bash shell: bash
+28 -7
View File
@@ -60,7 +60,7 @@ jobs:
docker build --build-arg TARGETARCH=${{ matrix.arch }} \ docker build --build-arg TARGETARCH=${{ matrix.arch }} \
--target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test . --target curl-runtime -f Dockerfile.goreleaser -t silo-curl-test .
docker run --rm silo-curl-test --version | tee curl-version.txt docker run --rm silo-curl-test --version | tee curl-version.txt
grep -F 'curl 8.22.0 ' curl-version.txt grep -F 'curl 8.23.0 ' curl-version.txt
grep -F 'HTTP2' curl-version.txt grep -F 'HTTP2' curl-version.txt
docker run --rm silo-curl-test --fail --silent --show-error \ docker run --rm silo-curl-test --fail --silent --show-error \
--connect-timeout 15 --max-time 60 https://curl.se/robots.txt --connect-timeout 15 --max-time 60 https://curl.se/robots.txt
@@ -97,7 +97,7 @@ jobs:
- name: GoReleaser config check - name: GoReleaser config check
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
with: with:
version: v2.18.1 version: v2.18.3
args: check --config .github/goreleaser.yml args: check --config .github/goreleaser.yml
- name: Validate Helm chart and legacy upgrade identity - name: Validate Helm chart and legacy upgrade identity
@@ -111,7 +111,7 @@ jobs:
- name: Build snapshot artifacts - name: Build snapshot artifacts
uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3
with: with:
version: v2.18.1 version: v2.18.3
# A pull-request snapshot has no trusted release identity. Exercise # A pull-request snapshot has no trusted release identity. Exercise
# the SBOM/checksum pipeline here, and reserve keyless signing for # the SBOM/checksum pipeline here, and reserve keyless signing for
# the tag-triggered release workflow with GitHub OIDC. # the tag-triggered release workflow with GitHub OIDC.
@@ -120,13 +120,34 @@ jobs:
LDFLAGS: ${{ env.LDFLAGS }} LDFLAGS: ${{ env.LDFLAGS }}
PKG_VERSION: ${{ env.PKG_VERSION }} PKG_VERSION: ${{ env.PKG_VERSION }}
- name: Verify archive SBOM and checksum coverage - name: Verify archive formats, documentation, SBOMs, and checksums
run: | run: |
set -euo pipefail set -euo pipefail
mapfile -t archives < <(find dist -maxdepth 1 -type f -name 'silo_*.tar.gz' | sort) mapfile -t archives < <(find dist -maxdepth 1 -type f \( -name 'silo_*.tar.gz' -o -name 'silo_*.zip' \) | sort)
mapfile -t sboms < <(find dist -maxdepth 1 -type f -name 'silo_*.tar.gz.sbom.json' | sort) mapfile -t sboms < <(find dist -maxdepth 1 -type f \( -name 'silo_*.tar.gz.sbom.json' -o -name 'silo_*.zip.sbom.json' \) | sort)
test "${#archives[@]}" -eq 6 test "${#archives[@]}" -eq 6
test "${#sboms[@]}" -eq 6 test "${#sboms[@]}" -eq 6
for os in linux darwin windows; do
format=tar.gz
binary=silo
if [ "${os}" = windows ]; then
format=zip
binary=silo.exe
fi
for arch in amd64 arm64; do
archive="dist/silo_${PKG_VERSION}_${os}_${arch}.${format}"
test -s "${archive}"
jq -e '.spdxVersion | startswith("SPDX-")' "${archive}.sbom.json" >/dev/null
if [ "${format}" = zip ]; then
listing="$(unzip -Z1 "${archive}")"
else
listing="$(tar -tzf "${archive}")"
fi
for member in "${binary}" LICENSE NOTICE README.md; do
grep -Fx "${member}" <<< "${listing}"
done
done
done
manifest="dist/silo_${PKG_VERSION}_checksums.txt" manifest="dist/silo_${PKG_VERSION}_checksums.txt"
test -s "${manifest}" test -s "${manifest}"
( (
@@ -143,7 +164,7 @@ jobs:
- name: Install package validation tools - name: Install package validation tools
run: | run: |
set -euo pipefail set -euo pipefail
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.47.0 go install github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.48.0
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}" echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
sudo apt-get update sudo apt-get update
sudo apt-get install --yes rpm binutils sudo apt-get install --yes rpm binutils
+1 -1
View File
@@ -29,7 +29,7 @@ jobs:
- name: Install govulncheck - name: Install govulncheck
run: | run: |
go install golang.org/x/vuln/cmd/govulncheck@v1.8.0 go install golang.org/x/vuln/cmd/govulncheck@v1.9.0
echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}" echo "$(go env GOPATH)/bin" >> "${GITHUB_PATH}"
- name: Run govulncheck - name: Run govulncheck
+146 -204
View File
@@ -1,222 +1,164 @@
# Changelog # Changelog
## Unreleased ## RELEASE.2026-10-26T00-00-00Z
Preparation target: `RELEASE.2026-09-16T00-00-00Z` (package version Package version: `20261026000000.0.0`; published from source `c3ad836aed0d`.
`20260916000000.0.0`). The entries below describe the candidate changes since Artifacts are listed in the [GitHub release](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z).
the latest published Server. The coordinated [English release notes](https://silo.pgsty.com/blog/release/silo-20261026/)
**The latest published Server remains 20260903.** These changes are not in its and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261026/)
binaries, packages or images. See the [component matrix](https://silo.pgsty.com/compatibility/versions/) cover all components, upgrade guidance, and validation scope.
and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-09-03T13-18-01Z...main).
### Authorization and security - Fix LDAP login for long normalized DNs without hiding unreadable IAM
revocation records. Only length errors for valid keys with an ASCII path
component over 255 bytes are treated as absent; long-DN writes still fail.
Thanks to [@910patin](https://github.com/910patin) for the original fix and
reproduction in [#250](https://github.com/pgsty/silo/pull/250).
- Build the server and standalone debugging tools with Go 1.27.3 and refresh
Go dependencies, including silo-pkg v3.15.2, mcli 20261026
and upstream minio-go `320af050c445` for multipart, read/seek,
batch-delete and bucket-replication removal fixes. Retain output-format and NetBSD compatibility selections.
- Embed Console v2.5.1 source `fff1618d9abb`, including its GUI update,
diagnostic WebSocket closing handshake, complete health-report decoding,
group-membership preservation, OIDC group permissions, TLS-proxy session
cookies, credential exports and accessibility fixes.
- Bundle mcli 20261026 in container builds, the verified client installer and
Helm client jobs. Include client identity isolation, resync status and rate
fixes, and bounded batch-status fallback. Scripts must handle the additive
batch `pending` state and nonzero exits for partial resync mutations.
- Read current bucket inventory in account-info responses so Console refreshes
immediately reflect successful creation and deletion. This management endpoint
now performs a live list instead of returning a cached inventory.
- Restore the native built-in policy immediately after resetting an override.
Peer and etcd notifications reload committed state, so delayed delete events
preserve subsequent edits. Keep deletion revisions, policy-in-use checks and
custom-policy deletion behavior; storage failures retain the previous policy.
- Reload policies without holding the authentication-cache lock, retain IAM
write ordering and publish replacements only after a complete successful load.
Propagate cancellation through IAM/KMS and SSE-S3 decryption without shortening
the caller's policy-query deadline.
- Preserve authoritative copies when a multi-pool delete or bulk cleanup fails,
choose an available pool for a new delete marker, and keep tier data referenced
by surviving copies. This does not introduce strict cross-pool read recovery.
- Preserve remote tier data when lifecycle expiry only adds a delete marker,
including non-null versions in suspended buckets. Forward `FULL_OBJECT`
checksum type for multipart replicas. Thanks to
[@botdeniska](https://github.com/botdeniska) for these lifecycle, replication
and multi-pool deletion contributions.
- Preserve Kafka's default broker protocol 2.1 and terminal-width behavior;
fix legacy DARE v1 key unsealing and verify DARE v1/v2 and MinKMS TLS compatibility.
- Forward storage RPC delete options, including the previous data directory
needed by overwrite rollback. Preserve existing drive/RPC timeouts, request
cancellation, object locking, read-quorum and healing behavior.
- Update bundled curl to 8.23.0 and require patched static libraries. Apply
[Red Hat's compromised-CA mitigation](https://access.redhat.com/security/vulnerabilities/RHSB-2026-012);
the root blocklist also rejects legitimate chains under the affected roots.
- Ship Windows platform archives as ZIP; Linux and macOS retain tar.gz.
- Synchronize CPU metrics reads with resource-metrics updates (#210), preventing Storage timeouts retain their existing meaning: a returned timeout does not
concurrent map access from terminating the server during Prometheus scraping. prove that the underlying disk operation stopped or that a write did not
Metric names, values and authentication requirements are unchanged. commit. Late commits or delayed rollback can affect newer writes; stale reads
- Restrict embedded Console's anonymous sharing proxy to object-content GETs remain possible after partial failures, lease loss or network partitions. This
at the configured S3 origin, and reject every redirect. Internal metrics, release does not provide distributed fencing or repair previously inconsistent
system paths and non-download S3 operations cannot be reached through it. objects.
Normal public, presigned and versioned downloads remain available without a
new setting; a full sharing-disable switch is not introduced. See
[Console #56](https://github.com/pgsty/silo-console/pull/56) and the
[design record](https://github.com/pgsty/silo-console/issues/52).
Thanks to Jiri Pejchal (@jiri-pejchal) for the report.
- Persist IAM deletion revisions and parent revocation boundaries so stale site
events cannot restore deleted identities, policies or their older grants
(#191, #192). Peer deletion notifications reload committed storage; deliberate
recreation requires a newer revision, and credentials issued before the
parent's revocation remain invalid.
**Coordinated upgrade required:** upgrade every participating node and site.
Mixed old/new nodes sharing an IAM backend and rolling downgrade are
unsupported. Back up complete IAM storage and encryption material; an admin
export of live records omits deletion history. Reissue credentials for
recreated parents and explicitly reconcile pre-upgrade revocations whose
history is already lost. Restoring an older backup can lose later revocations;
keep affected sites isolated until reconciliation/rekeying is complete. See
[the operator runbook](https://silo.pgsty.com/operations/replication/iam-upgrade/).
- Enforce an absolute HTTP/1 request-header deadline through the connection
wrapper (#196). Repeated small reads no longer extend that deadline, and
`--read-header-timeout` / `MINIO_READ_HEADER_TIMEOUT` now reaches the HTTP
server. HTTP/1 request bodies retain the rolling idle timeout; this does not
impose a total upload/download duration. A shorter setting also constrains
TLS handshake reads. The wrapper's strict header mode is not applied to HTTP/2.
- Reject unsigned `x-amz-*` request headers that could turn a signed PUT into a
copy of another object accessible to the signer (SN-2026-011). The latest
public Server is affected; the fix is on main. See [the advisory ledger](https://silo.pgsty.com/about/security-advisories/).
- Align signed request fields with policy conditions and enforce header-only
presigned payload checksums. See [the signed-header review](https://silo.pgsty.com/blog/design/signed-header-coverage/).
- **Breaking policy semantics:** separate self-service `admin:ChangeMyPassword`
from `admin:CreateUser`. Built-in read-only policies follow the split. Preserve
both denies if the previous combined restriction must survive upgrades or
rollback. Saved policies are not rewritten. Deploy with the matching Console
and pkg; see [the migration guide](docs/iam/password-permissions.md).
### Object storage and replication These source changes are not included in the published 20261018 artifacts.
- Make durable `ListMultipartUploads` scanning the only listing path (#79): ## RELEASE.2026-10-18T00-00-00Z
apply prefix, delimiter, markers, ordering and the 1,000-entry page limit
globally across pools and sets, independently of node-local caches. Omit
readable historical uploads only when both bucket/key identity fields are
absent; malformed identities, uncertain coverage and resource limits remain
errors. Old uploads still support part upload, completion and cancellation
with a known bucket/key/upload ID. No backfill or new storage layout is added.
`MINIO_API_MULTIPART_LISTING` is retired and ignored with a startup warning;
historical shared `api multipart_listing` keys remain tolerated. The read-only
admin `multipart-preflight` reports `mode=persistent`; `ready` diagnoses the
absence of old records and no longer gates listing. Each page still scans
durable state within the existing 100,000-directory-entry, 30-second and
per-process concurrency budgets. See the
[design and limits](https://silo.pgsty.com/blog/design/list-multipart-uploads/#implementation).
Thanks to mr javad seydi (@mrjavadseydi) for the original metadata-and-scan
implementation (#198).
- Keep the default read-quorum, best-effort multipart cancellation and
first-matching-pool behavior when retiring the listing switch. The optional
strict cancellation branch is removed. Wrong-target protection, upload-ID
path validation and peer notification with the request context remain.
HTTP 204 for an absent upload remains idempotent and does not prove physical
cleanup. **Known boundary:** delayed creation writes can still restore a
readable upload after cancellation; this change adds no creation fence.
- Preserve object tags during multi-pool metadata reconciliation by reading the
resolved tag field together with its revision (#189). Previously, reconciliation
could replace existing tags with an empty value.
- Preserve the tag revision on SSE-KMS metadata replication (#193), and advance
tag revisions monotonically on local PUT/DELETE tagging (#196). Empty tags
participate in reconciliation as an ordered deletion, preventing older
events from restoring removed tags. SSE-C key rotation also retains the tag
revision. Malformed historical revisions can fail and retry; their missing
history is not reconstructed by the upgrade.
- Complete delete-marker version purges and preserve their identity and retry
state through MRF recovery (#196). Recovery accepts a 405 marker response only
when its version, bucket, object name and modification time match the task.
Purge audit status is normalized from `COMPLETE` to `COMPLETED`.
Thanks to Julien Laurenceau (@julienlau) for the investigation and proposed
fix in #184 that helped shape this follow-up.
- Restore only the six replication-specific metadata fields after ordinary
request metadata extraction (#194). This prevents transport-only `aws-chunked`
from being stored as Content-Encoding while preserving the signed-header
protections. Trusted Snowball entries no longer inherit the outer archive's
ordinary metadata. Thanks to Mikhail Khadarenka (@chodorenko) for the fix in #187.
**Existing data:** these repairs prevent new errors; they do not scan or rewrite
historical object metadata, recover lost tags or prove that old purge work has
converged. Follow the [read-only audit procedure](https://silo.pgsty.com/operations/replication/replica-metadata-audit/)
before planning any repair of stored state.
- Make exact-version delete-marker purges converge in replicated buckets Package version: `20261018000000.0.0`; published on 2026-10-18 from source
(`eb4f5e5b3`, `254b19ac0`, `358ab38fb`). A purge no longer creates the `0cd388c666f8211f552604ae73448a066e9c25ce`. This section covers the changes
marker on drives that lacked it; a retried purge of a missing version is after the public 20260916 source, `2a4d51406b7ed87af5fe6fe0f801f3290f96eb3c`.
acknowledged only when a write-quorum majority of drives report it absent;
purge results merge removed and reliably absent replies; healing a marker
preserves its stored replication and purge metadata; and a queued marker
creation is re-checked against the source under the replication lock before
it is sent, so a purge that already reached the targets is not undone by a
stale task from another frontend, a GET/LIST heal, the scanner or MRF.
Purging a data version whose earlier purge is still pending reports it as a
data version. **Known limitations:** creations already in flight or replayed
from another site, and minority marker copies left by a crash after a
majority-acknowledged purge, are tracked in #217. See
[the replication reliability record](https://silo.pgsty.com/blog/design/replication-reliability/).
- Keep a null object version that has listing quorum when a newer minority of
drives sorts first (`8d06424b1`). The resolver recounts per header only when
the original selection lacks quorum, every non-empty drive stream holds
exactly one ordinary null version and all share the same erasure layout;
mixed histories keep their previous behavior. **Known limitation:** a
successful ListObjects can still omit readable keys during rolling restarts
with concurrent overwrites (#218). Do not run destination-deleting sync tools
against a listing taken during a rolling restart; list again once the
cluster is stable.
- Carry object tags through rebalance and decommission for ordinary and
multipart writes (`fced86303`). Both migration entry points restore the tags
and their revision fields when rewriting the object in the destination pool.
Tags dropped by earlier migrations are not recovered; audit tag-dependent
lifecycle and policy rules for pools migrated with an older build.
- Evaluate conditional multipart completion against the logical current object Full [English release notes](https://silo.pgsty.com/blog/release/silo-20261018/)
across all pools while holding the existing object lock. A stale `If-Match` and [Chinese release notes](https://silo.pgsty.com/zh/blog/release/silo-20261018/)
can no longer replace newer data in another pool, and the current ETag is no cover the upgrade contract, evidence and remaining limits.
longer rejected because the upload resides next to an older copy. Conditions
are evaluated once; a current delete marker counts as an absent object.
**Availability change:** if any pool's metadata cannot be read, conditional
completion fails even when another pool can still serve GET/HEAD. This also
applies when the unreadable pool may not hold the object: absence cannot be
verified. Retry after the pool recovers. Unconditional completion and the
single-pool path retain their existing behavior.
- Evaluate ordinary multi-pool conditional PUT against the logical current - **Object Lock (#234, #235):** require `s3:PutObjectRetention` with request-body
object across all pools, including draining pools, under the existing object conditions before object lookup; weakening active GOVERNANCE additionally
lock (#207). A stale destination copy no longer accepts a stale ETag or rejects requires bypass intent and permission. Protect caller-addressed write versions,
the current one; a current delete marker is treated as absence. directory null versions and persisted multipart targets under the write lock;
**Availability change:** if any pool's object metadata cannot be verified, fail closed on unreadable lock state. See SN-2026-015/016 in the
the condition fails even when GET can use another pool; read-quorum failures [advisory ledger](https://silo.pgsty.com/about/security-advisories/).
return 503. Restore readability or heal before retrying. Unconditional PUT, - **IAM Admin authorization (#242, `995439260`):** enforce explicit self denies,
single-pool conditions and internal replication retain their existing behavior. session restrictions and plugin decisions; prevent restricted delegation and
A public condition with a destination `versionId` compares the current object non-root mutation of root-derived credentials under built-in authorization.
while preserving the requested write version. This change does not retire Preserve empty-policy inheritance and damaged-key cleanup. Normalize bulk
stale copies in other pools, undo historical accepted overwrites or provide self-list aliases and allow ordinary policyless self inspection. See
a new global clock-ordering guarantee. The multipart-completion repair in #190 [SN-2026-017](https://silo.pgsty.com/blog/security/20261018-release-hardening/#sn-2026-017).
neither introduced nor repaired this separate PUT defect. - **Scoped STS revocation (`0cd388c66`):** enumerate durable parent/type-selected
tokens, propagate each committed tombstone to sibling nodes and sites, and
recover missed notifications through existing refresh/healing. Tokens issued
only at an isolated site are outside the requesting cluster's snapshot;
propagation is eventual and newly issued keys remain valid.
- **Pool migration (#248, `60c7179dd`, `1748f62c7`):** prevent stale moved
payloads from overwriting newer copies, preserve null/UUID delete markers,
recheck source versions and clean only confirmed moved versions under the
object lock. Added per-pool reads and per-version cleanup have maintenance
costs; earlier data loss is not reconstructed.
- Reconcile ordinary single-object version DELETE across all pools, including - **LIST recovery (#239, `3dd53eb81`, `2080b3d03`):** confirm eligible uncertain
null versions, delete markers and unqualified directory-marker DELETE. This latest-object groups under the appropriate read locks, distinguish absence
applies the deletion to every resolved pool copy under existing quorum from failure, preserve directory/common-prefix entries and terminal pagination
rules. Pending outbound delete replication retains versions until the errors, and make lock backoff cancelable. Resolved entries keep their fast path;
existing replication worker completes their purge; a successful response version history, quorum and disk formats retain their contracts. The strict
does not imply immediate physical removal from every drive. Unreadable sub-5% performance target remains unconfirmed in the recorded comparison.
pools now consistently return 503 instead of depending on pool traversal - **Multipart listing (`4733e1423`):** withdraw #243's default persistent scan.
order; insufficient read quorum returns `SlowDownRead`. This extends the Public listing uses node-local cache or exact-key lookup; restart omissions,
existing failure surface. Retry after recovery. prefix and cache-pagination limits remain. The retired strict switch is ignored
Cleanup failures also return an error. Batch deletion already fans out across with a startup warning. **Former strict users lose durable listing and strict
pools; replication and scanner cleanup keep their existing contracts. See cancellation.** Keep identity metadata, path/wrong-target safety, default
[scope and limitations](docs/bucket/lifecycle/access-tiering-removal.md#version-deletion-scope). best-effort Abort and peer notification. Diagnostic `multipart-preflight`
reports `mode=legacy`; its budgets do not gate public operations. #79 remains
open, including delayed creation after cancellation.
- **Multipart cleanup (#236, `0f9199e0c`):** parse native timestamps before the
Base64 fallback and remove empty hash parents without a second expiry wait.
Keep active/unreadable uploads; default expiry/check intervals stay 24h/6h.
- **Commit errors (#232):** missing staged `xl.meta` at final commit becomes
HTTP 503 `SlowDownWrite`; preserve this classification across storage RPC and
healing. Real incomplete bodies retain their existing errors.
- **Reused metadata (#233):** clear absent legacy `EcM`/`Signature` and locally
reset mutually exclusive scanner response fields without changing disk/RPC formats.
- **Checksum and Console errors (#225, #229, #238):** align Content-MD5 and
algorithm-specific messages, retaining the generic unknown-algorithm message;
embed Console's wrapped-error and `InvalidBucketState` handling.
- **Lambda and erasure documentation (#220, `2fde3cf53`):** fix inverted
`TargetIDSet.IsEmpty`; document parity/quorum per object and erasure set.
- **Coordinated dependencies:** Console v2.4.2 (`ef2dd2c41d21`), mcli 20261018
(`fa3b8bf72458`), silo-pkg v3.14.3, upstream minio-go v7.4.1 and Go 1.27.2.
Standalone Console/mcli still declare pkg v3.14.2. The Server package update
speeds admin-action classification without changing policy behavior. Refresh
Go x/*, go-jose v4.1.6, LDAP v3.4.15, compress v1.20.3, msgp v1.6.5, etcd v3.7.2,
fwd v1.2.1 and cpuid v2.4.1. OIDC RSA JWKs above 4096 bits are rejected.
Keep the NetBSD go-systemd v22.6.0 replacement; parallel SDK GET/Compose remain opt-in.
- **Build and delivery:** gate releases on Object Lock regressions; use
Git for rebrand scanning without a ripgrep dependency (`7007c164a`). Refresh UBI 9 pins,
GoReleaser v2.18.3, nFPM v2.48.0, golangci-lint v2.15.0 and govulncheck v1.9.0.
Update bundled client hashes/helper and the Helm client image to mcli 20261018.
The chart's Server default stays at the public 20260916 image until publication.
- Remove the opt-in GET-frequency pool-tiering feature from PR #60, including Upgrade every writer and replication site; follow the
its tracker, mover, scanner hooks, configuration, XML actions and metrics. [IAM recovery guide](https://silo.pgsty.com/operations/replication/iam-upgrade/).
Accept and ignore retired configuration/XML and preserve ordinary statistics Old strict multipart users must review the capability loss before upgrading.
when reading v9 caches. See [migration notes](docs/bucket/lifecycle/access-tiering-removal.md). #217's multi-site delete-marker late creation and crash residue remain unresolved.
The [decision record](https://silo.pgsty.com/compatibility/access-tiering-removal/) preserves Source tests, draft artifacts, public images and production rollout are separate
the feature's introduction, subsequent fixes, rollback scope and review history. acceptance boundaries; historical 20261015 test counts do not certify this source.
- Preserve the independent multi-pool write, metadata, healing and conditional
deletion fixes from PR #178, including shared remote-tier reference protection.
- Enforce `If-Match` on DELETE, preserve retention and independently ordered
Object Lock/tag updates, and correctly retransmit encrypted replicas.
- Preserve plaintext part sizes and raw SSE-C replicas; prevent SSE-C
compression, honor key-rotation checksums, and complete attributes pagination.
- Repair federated CopyObject checksums, destination timestamps, reserved
metadata, encrypted-object forwarding, legal hold and KMS context.
- Make resync counters, target selection, cancellation and worker lifetimes
reflect actual work, and report bounded MRF drops.
- Converge bucket metadata with deterministic source state, deletion tombstones,
creation time recovery and diagnostics. The mixed-version export gate requires
coordinated upgrades before tombstones are exported. See [the #77 record](https://silo.pgsty.com/blog/design/bucket-metadata-convergence/).
- Include per-bucket CORS in metadata export/import, close metadata publication
and logger races, and report effective bucket quotas in metrics.
### Console, dependencies and delivery ## RELEASE.2026-09-16T00-00-00Z
- Restore embedded Console login over loopback TLS, trusted-proxy handling and Package version: `20260916000000.0.0`; published source:
all four WebSocket connection limits. Preserve Go TLS defaults across transports. `2a4d51406b7ed87af5fe6fe0f801f3290f96eb3c`.
- Directly require `github.com/pgsty/silo-pkg/v3` v3.14.1; select released Console
v2.4.1 (`v0.0.0-20260916075814-1360e26d976d`) and mcli 20260916
(`v0.0.0-20260916070421-e952aa78f10a`) with explicit PGSTY replacements.
The embedded frontend identifies itself as Console v2.4.1.
- Pin upstream minio-go `v7.3.1-0.20260915093545-32e1f32cb176` to handle
CopyObject errors embedded in HTTP 200 responses. Update JWX to v3.3.0 for
JSON field-name escaping, strfmt to v0.27.2 for Go 1.27 hostname validation,
and LZ4 to v4.1.30 for frame-reader, partial-read and concurrency fixes.
Retain the earlier Go x/* and bounded AMQP frame updates, Go 1.27.1, and
go-systemd v22.6.0's NetBSD compatibility replacement.
- Refresh container base digests and build static curl 8.22.0 from verified
source for both Linux architectures. Pin the published mcli 20260916 archives
and hashes in the container and update the client installer default.
- Prepare Helm chart 7.0.3 with Server and client defaults for the September 16
batch. Publish the chart only after the corresponding Server image exists.
- Pin GoReleaser v2.18.1 and its action commit identically in snapshot and release
workflows. Dependency-only PRs now run the Test Release Pipeline too.
Validation of earlier source revisions does not establish acceptance of this This release includes signed-header and payload-policy hardening (SN-2026-011/012),
candidate. Final source, package, image and multi-process checks are tracked durable IAM revocations (SN-2026-013), the bounded Console sharing proxy
separately in [#203](https://github.com/pgsty/silo/issues/203). No Server release (SN-2026-014), password-permission separation, multi-pool conditions/deletion/tag
or production rollout is implied by this preparation target. repairs, encrypted/federated copy repairs, delete-marker retry fixes, opt-in strict
multipart discovery, Go-default TLS key exchange, absolute HTTP/1 header deadlines
and synchronized CPU metrics. It embeds Console v2.4.1, mcli 20260916 and pkg v3.14.1.
The complete release, contributor credits, historical artifact acceptance and
upgrade requirements are preserved in the canonical
[English](https://silo.pgsty.com/blog/release/silo-20260916/) and
[Chinese](https://silo.pgsty.com/zh/blog/release/silo-20260916/) release records.
## RELEASE.2026-09-03T13-18-01Z ## RELEASE.2026-09-03T13-18-01Z
+14
View File
@@ -53,6 +53,20 @@ Before opening a pull request:
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`, - Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
`.minio.sys`, ARN, module/import-path, or serialized compatibility name. `.minio.sys`, ARN, module/import-path, or serialized compatibility name.
The maintenance branch retains existing storage timeouts, cancellation,
object locking, read-quorum and healing behavior. The normal suite checks local
and RPC timeout/cancellation, remote rollback options, and readability after a
failed partial overwrite. A timeout does not prove that a disk worker stopped
or that a write did not commit.
Stronger mutation lifetime, rollback ownership, generation ordering, strict-read
and cross-pool recovery diagnostics are preserved behind `consistency_strict`.
Compile them with `go test -tags kqueue,dev,consistency_strict ./cmd -run '^$'`
and run individual scenarios explicitly. They describe guarantees outside the
maintenance contract and may fail; report their results separately from the
normal suite. They must not be used to claim distributed fencing or repair of
previously inconsistent objects.
### Commit changes ### Commit changes
After verification, commit your changes with a concise message and a DCO After verification, commit your changes with a concise message and a DCO
+72 -38
View File
File diff suppressed because one or more lines are too long
+835 -276
View File
File diff suppressed because it is too large Load Diff
+19 -7
View File
@@ -1,4 +1,4 @@
FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS curl-build FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS curl-build
ARG TARGETARCH ARG TARGETARCH
COPY dockerscripts/build-static-curl.sh /build/build-static-curl COPY dockerscripts/build-static-curl.sh /build/build-static-curl
RUN /bin/sh /build/build-static-curl RUN /bin/sh /build/build-static-curl
@@ -9,7 +9,7 @@ COPY --from=curl-build /go/bin/curl /curl
COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=curl-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
ENTRYPOINT ["/curl"] ENTRYPOINT ["/curl"]
FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS build FROM golang:1.27.3-alpine@sha256:d530cf60723abf1ffae0cd16bb546b379ec0550d2f3815e5b16ed69dbc00fba4 AS build
ARG TARGETARCH ARG TARGETARCH
@@ -17,9 +17,9 @@ ENV GOPATH=/go
ENV CGO_ENABLED=0 ENV CGO_ENABLED=0
ARG MC_REPO=pgsty/mc ARG MC_REPO=pgsty/mc
ARG MC_VERSION=RELEASE.2026-09-16T00-00-00Z ARG MC_VERSION=RELEASE.2026-10-26T00-00-00Z
ARG MC_AMD64_SHA256=4ba2814fd5507fbe6b4d237c359750b9119d28d7217495fa5b48002fcbd397ef ARG MC_AMD64_SHA256=54bd131039dad6677f083a75207b28f82818c067c8e65fae1973f51510d268e1
ARG MC_ARM64_SHA256=b7008ca2a1bc5735b6585981c59a3640a0daa152dc789df3d1a0d0438787de82 ARG MC_ARM64_SHA256=9bb294f1912f8c6b9ac08c9a0b8d2964b86306a7c440b987ee5f0570d18de3c5
RUN apk add -U --no-cache \ RUN apk add -U --no-cache \
ca-certificates \ ca-certificates \
@@ -67,14 +67,26 @@ RUN apk add -U --no-cache \
chmod +x /go/bin/mcli && \ chmod +x /go/bin/mcli && \
ln -sf mcli /go/bin/mc ln -sf mcli /go/bin/mc
FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:206b65b8ee0f04b992818c9a51b29081b14974630d4850bc358097d0c44ea156 AS certs FROM registry.access.redhat.com/ubi9/ubi:latest@sha256:16e9e43d99e181463513cc84a514859b1abb9460f1c4d9e208da4b4e51cd5cc3 AS certs
# Native root blocklist mitigation: https://access.redhat.com/security/vulnerabilities/RHSB-2026-012
RUN dnf -y install ca-certificates && \ RUN dnf -y install ca-certificates && \
if [ -d /etc/pki/ca-trust/source/blocklist ]; then \
blocklist=/etc/pki/ca-trust/source/blocklist; \
else \
blocklist=/etc/pki/ca-trust/source/blacklist; \
fi && \
trust extract --overwrite --format=pem-bundle \
--filter='pkcs11:object=Comodo%20AAA%20Services%20root;type=cert' \
"$blocklist/Comodo_AAA_Services_root.pem" && \
trust extract --overwrite --format=pem-bundle \
--filter='pkcs11:object=emSign%20ECC%20Root%20CA%20-%20G3;type=cert' \
"$blocklist/emSign_ECC_Root_CA_-_G3.pem" && \
update-ca-trust && \ update-ca-trust && \
cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \ cp /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /tmp/ca-certificates.crt && \
dnf clean all && \ dnf clean all && \
rm -rf /var/cache/dnf rm -rf /var/cache/dnf
FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:f332c99eb8f798a8486821c91937f10ad64ee83d7e739303be2df051040918f6 FROM registry.access.redhat.com/ubi9/ubi-micro:latest@sha256:5be877a2a280ae36eb9ab80ca4efb28bbf77c57abbba38137cdc37a03b8465e6
LABEL org.opencontainers.image.title="Silo" \ LABEL org.opencontainers.image.title="Silo" \
org.opencontainers.image.description="S3-Interface Libre Object Storage" \ org.opencontainers.image.description="S3-Interface Libre Object Storage" \
+1 -1
View File
@@ -4,7 +4,7 @@ LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
GOOS ?= $(shell go env GOOS) GOOS ?= $(shell go env GOOS)
GOARCH ?= $(shell go env GOARCH) GOARCH ?= $(shell go env GOARCH)
GOLANGCI_VERSION ?= v2.14.0 GOLANGCI_VERSION ?= v2.15.0
VERSION ?= $(shell git describe --tags) VERSION ?= $(shell git describe --tags)
REPO ?= docker.io/pgsty REPO ?= docker.io/pgsty
+11 -11
View File
@@ -24,7 +24,7 @@
<p align="center"> <p align="center">
<a href="https://silo.pgsty.com/"><img alt="Website" src="https://img.shields.io/badge/Website-silo.pgsty.com-1d588c"></a> <a href="https://silo.pgsty.com/"><img alt="Website" src="https://img.shields.io/badge/Website-silo.pgsty.com-1d588c"></a>
<a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a> <a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a>
<a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a> <a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/silo?logo=docker"></a>
<a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a> <a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a>
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a> <a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
</p> </p>
@@ -35,13 +35,13 @@
> [!NOTE] > [!NOTE]
> Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z). > Renamed from `pgsty/minio` to `pgsty/silo`, default branch `master` → `main`, on 2026-08-06. Artifacts under the original MinIO identity stay published on the archived [`minio`](https://github.com/pgsty/silo/tree/minio) branch and in releases up to [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z).
## Current release and main branch ## SILO 20261026000000
The latest published Server is [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z). [SILO 20261026000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z) is available as platform archives, Linux packages, and standard or distroless Docker images for amd64 and arm64. The release tag is `RELEASE.2026-10-26T00-00-00Z`; the package version is `20261026000000.0.0`, built from source `c3ad836aed0d`.
As of 2026-09-16, the main branch has newer security, storage, Console and
shared-package changes that have not shipped in a Server release. See This release fixes LDAP long-DN login, IAM policy refresh, multi-pool deletion safety, tiered-object expiry, multipart replication checksums, and remote rollback-option propagation. It embeds Console v2.5.1 (`fff1618d9abb`) and uses silo-pkg v3.15.2, Go 1.27.3, and upstream minio-go `v7.4.2-0.20261025165926-320af050c445`. Use mcli 20261026 for administration; the container images bundle it as `mcli`.
[CHANGELOG.md](CHANGELOG.md) and the [component version matrix](https://silo.pgsty.com/compatibility/versions/)
for the exact release/source boundary, including SN-2026-011 and password-policy migration. The Server IAM cache repair enables Console's **Reset to default** action. Existing storage timeouts and cancellation behavior remain: a timeout does not prove that a disk operation stopped or a write did not commit. Read the [coordinated release notes](https://silo.pgsty.com/blog/release/silo-20261026/) and [remaining storage limits](CHANGELOG.md#release2026-10-26t00-00-00z) before upgrading. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people behind the code, reports, and documentation.
## Overview ## Overview
@@ -62,7 +62,7 @@ docker run -d --name silo -p 9000:9000 -p 9001:9001 \
-e MINIO_ROOT_USER=minioadmin \ -e MINIO_ROOT_USER=minioadmin \
-e MINIO_ROOT_PASSWORD=change-me-long-password \ -e MINIO_ROOT_PASSWORD=change-me-long-password \
-v "$PWD/data:/data" \ -v "$PWD/data:/data" \
docker.io/pgsty/silo:latest server /data --console-address ":9001" docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z server /data --console-address ":9001"
``` ```
<p align="center"> <p align="center">
@@ -101,8 +101,8 @@ Every divergence from upstream is listed in the code-verified [compatibility aud
### TLS and Go upgrades ### TLS and Go upgrades
The following TLS repair is on main and is not included in Server 20260903. Server 20260916 restored Go's default key exchange; Server 20261018 retains it with Go 1.27.2.
With that repair, TLS key exchange follows Go's defaults across the S3 listener, node links, TLS key exchange follows Go's defaults across the S3 listener, node links,
replication, identity providers, etcd, and external HTTP services. If an endpoint replication, identity providers, etcd, and external HTTP services. If an endpoint
cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges cannot accept ML-KEM, `GODEBUG=tlsmlkem=0` disables the default hybrid exchanges
for the process; certificate verification remains enabled. This option does not for the process; certificate verification remains enabled. This option does not
@@ -137,7 +137,7 @@ Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); e
<!-- Generated by silo.pgsty.com/bin/contributors.py. --> <!-- Generated by silo.pgsty.com/bin/contributors.py. -->
Every human issue or pull-request author is part of the SILO community, including open and unmerged work. Merged fixes, adopted proposals, and actionable reports receive priority, with first participation guiding the remaining order. Gold rings highlight reviewed significant contributions. Every human issue or pull-request author is part of the SILO community, including open and unmerged work. We also recognize security reports, including reports sent by email, with the reporter's permission. Merged fixes, adopted proposals, and actionable reports receive priority, with first participation guiding the remaining order. Gold rings highlight reviewed significant contributions.
<a href="CONTRIBUTORS.md"> <a href="CONTRIBUTORS.md">
<picture> <picture>
+10 -9
View File
@@ -24,7 +24,7 @@
<p align="center"> <p align="center">
<a href="https://silo.pgsty.com/zh/"><img alt="官网" src="https://img.shields.io/badge/%E5%AE%98%E7%BD%91-silo.pgsty.com%2Fzh-1d588c"></a> <a href="https://silo.pgsty.com/zh/"><img alt="官网" src="https://img.shields.io/badge/%E5%AE%98%E7%BD%91-silo.pgsty.com%2Fzh-1d588c"></a>
<a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a> <a href="https://github.com/pgsty/silo/releases"><img alt="GitHub Release" src="https://img.shields.io/github/v/release/pgsty/silo?include_prereleases&label=release&logo=github"></a>
<a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/minio?logo=docker"></a> <a href="https://hub.docker.com/r/pgsty/silo"><img alt="Docker Pulls" src="https://img.shields.io/docker/pulls/pgsty/silo?logo=docker"></a>
<a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a> <a href="go.mod"><img alt="Go Version" src="https://img.shields.io/github/go-mod/go-version/pgsty/silo?logo=go"></a>
<a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a> <a href="LICENSE"><img alt="License" src="https://img.shields.io/badge/license-AGPLv3-blue"></a>
</p> </p>
@@ -35,12 +35,13 @@
> [!NOTE] > [!NOTE]
> 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。 > 2026-08-06,本仓库由 `pgsty/minio` 更名为 `pgsty/silo`,默认分支由 `master` 更名为 `main`。以原 MinIO 形态维持的归档构件仍位于归档的 [`minio`](https://github.com/pgsty/silo/tree/minio) 分支,以及截止 [`RELEASE.2026-08-04T00-00-00Z`](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-08-04T00-00-00Z) 的历次发布中。
## 当前发行版与主分支 ## SILO 20261026000000
最新已发布的 Server 仍为 [20260903](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-09-03T13-18-01Z)。 [SILO 20261026000000](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-26T00-00-00Z) 已正式发布,提供各平台归档包、Linux 软件包,以及 amd64/arm64 标准版和 distroless Docker 镜像。版本标签为 `RELEASE.2026-10-26T00-00-00Z`,包版本为 `20261026000000.0.0`,构建源码为 `c3ad836aed0d`。
截至 2026-09-16,主分支已合入更新的安全、存储、Console 与共享包改动,但尚未发布新 Server。
准确的已发布/源码边界见 [CHANGELOG.md](CHANGELOG.md) 与[组件版本矩阵](https://silo.pgsty.com/zh/compatibility/versions/), 本版修复 LDAP 长 DN 登录、IAM 策略刷新、多池删除安全、分层对象过期、分片复制校验和及远端回滚参数透传。内置 Console v2.5.1(`fff1618d9abb`),使用 silo-pkg v3.15.2、Go 1.27.3 与上游 minio-go `v7.4.2-0.20261025165926-320af050c445`。配套管理客户端为 mcli 20261026,容器镜像内置该版本的 `mcli`。
其中包括 SN-2026-011 修复状态与密码权限迁移要求。
Server IAM 缓存修复使 Console **Reset to default(恢复默认)** 操作即时生效。存储超时与取消行为保持既有语义:超时不代表底层磁盘操作已停止,也不代表写入未提交。升级前请阅读[联合发布说明](https://silo.pgsty.com/zh/blog/release/silo-20261026/)与[仍存的存储限制](CHANGELOG.md#release2026-10-26t00-00-00z)。代码、问题报告和文档贡献者见 [CONTRIBUTORS.md](CONTRIBUTORS.md)。
## 概述 ## 概述
@@ -61,7 +62,7 @@ docker run -d --name silo -p 9000:9000 -p 9001:9001 \
-e MINIO_ROOT_USER=minioadmin \ -e MINIO_ROOT_USER=minioadmin \
-e MINIO_ROOT_PASSWORD=change-me-long-password \ -e MINIO_ROOT_PASSWORD=change-me-long-password \
-v "$PWD/data:/data" \ -v "$PWD/data:/data" \
docker.io/pgsty/silo:latest server /data --console-address ":9001" docker.io/pgsty/silo:RELEASE.2026-10-26T00-00-00Z server /data --console-address ":9001"
``` ```
<p align="center"> <p align="center">
@@ -100,7 +101,7 @@ Silo 保留 S3 与存储格式兼容性,包括既有 `MINIO_*` 环境变量、
### TLS 与 Go 升级 ### TLS 与 Go 升级
Server 恢复 Go 默认密钥交换策略的修复已在 main,尚未包含在 Server 20260903。 Server 20260916 已恢复 Go 默认密钥交换策略,20261018 在 Go 1.27.2 下保留该行为。
`GODEBUG=tlsmlkem=0`、`tlssecpmlkem=0` 的适用范围、macOS 根证书来源变化, `GODEBUG=tlsmlkem=0`、`tlssecpmlkem=0` 的适用范围、macOS 根证书来源变化,
以及 OIDC discovery 的诊断方法见 [Go 1.27 TLS 与 OIDC 指南](https://silo.pgsty.com/zh/blog/design/go127-tls-oidc-discovery/)。 以及 OIDC discovery 的诊断方法见 [Go 1.27 TLS 与 OIDC 指南](https://silo.pgsty.com/zh/blog/design/go127-tls-oidc-discovery/)。
@@ -119,7 +120,7 @@ Server 恢复 Go 默认密钥交换策略的修复已在 main,尚未包含在
<!-- Generated by silo.pgsty.com/bin/contributors.py. --> <!-- Generated by silo.pgsty.com/bin/contributors.py. -->
每位 issue 或 PR 的作者都是 SILO 社区的一员,包括尚未合并的工作。已合并的修复、被采纳的方案和有效报告优先展示,其余参考首次参与时间;金色圆环突出经过审核的显著贡献。 每位 issue 或 PR 的作者,以及同意公开署名的安全报告者,都是 SILO 社区的一员,包括尚未合并的工作。已合并的修复、被采纳的方案和有效报告优先展示,其余参考首次参与时间;金色圆环突出经过审核的显著贡献。
<a href="CONTRIBUTORS.md"> <a href="CONTRIBUTORS.md">
<picture> <picture>
+1 -1
View File
@@ -7,7 +7,7 @@ _init() {
## Minimum required versions for build dependencies ## Minimum required versions for build dependencies
GIT_VERSION="1.0" GIT_VERSION="1.0"
GO_VERSION="1.27.2" GO_VERSION="1.27.3"
OSX_VERSION="10.8" OSX_VERSION="10.8"
KNAME=$(uname -s) KNAME=$(uname -s)
ARCH=$(uname -m) ARCH=$(uname -m)
+1 -1
View File
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
exit 0 exit 0
fi fi
release=${MCLI_RELEASE:-RELEASE.2026-09-16T00-00-00Z} release=${MCLI_RELEASE:-RELEASE.2026-10-26T00-00-00Z}
version_hyphen=${release#RELEASE.} version_hyphen=${release#RELEASE.}
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/') package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
if [ "${package_version}" = "${version_hyphen}" ]; then if [ "${package_version}" = "${version_hyphen}" ]; then
+45 -14
View File
@@ -11,6 +11,33 @@ fail() {
exit 1 exit 1
} }
# Git is already a build prerequisite. --no-index also scans untracked files
# and exported source trees. As before, honor .gitignore only in a worktree.
search_ignore=--no-exclude-standard
if git rev-parse --show-toplevel >/dev/null 2>&1; then
search_ignore=--exclude-standard
fi
search_errors="$(mktemp)"
trap 'rm -f "${search_errors}"' EXIT
search_text() {
local status=0
git --no-pager grep --no-index "${search_ignore}" --no-color -nIHE "$@" 2>"${search_errors}" || status=$?
# Git can return 1 for both no matches and a file read error. Check stderr
# as well, before callers filter matches through their allowlists.
if [ "${status}" -gt 1 ] || [ -s "${search_errors}" ]; then
cat "${search_errors}" >&2
fail "could not scan source files"
fi
}
# Keep hidden files and build output out of the repository-wide scans.
source_excludes=(
':(exclude,glob)**/.*'
':(exclude,glob)**/.*/**'
':(exclude)dist/**'
)
require_file() { require_file() {
[ -f "$1" ] || fail "missing required file: $1" [ -f "$1" ] || fail "missing required file: $1"
} }
@@ -152,7 +179,11 @@ if grep -Ev '^[[:space:]]*(#|$)' silo.env | grep -q '='; then
fail "silo.env must not contain active assignments that shadow /etc/default/minio" fail "silo.env must not contain active assignments that shadow /etc/default/minio"
fi fi
if rg -n 'pgsty/minio:' .github/workflows Dockerfile.goreleaser helm/silo; then delivery_hits="$(search_text 'pgsty/minio:' -- .github/workflows Dockerfile.goreleaser helm/silo \
':(exclude,glob).github/workflows/**/.*' ':(exclude,glob).github/workflows/**/.*/**' \
':(exclude,glob)helm/silo/**/.*' ':(exclude,glob)helm/silo/**/.*/**')"
if [ -n "${delivery_hits}" ]; then
printf '%s\n' "${delivery_hits}" >&2
fail "an active delivery surface still publishes the frozen pgsty/minio image" fail "an active delivery surface still publishes the frozen pgsty/minio image"
fi fi
@@ -165,10 +196,10 @@ fi
# So two rules. First, no live URL may resolve to the old repository anywhere, # So two rules. First, no live URL may resolve to the old repository anywhere,
# READMEs and CONTRIBUTORS.md included. # READMEs and CONTRIBUTORS.md included.
old_repo_pattern='pgsty/minio(\.git)?([^[:alnum:]_.-]|$)' old_repo_pattern='pgsty/minio(\.git)?([^[:alnum:]_.-]|$)'
stale_repo_url="$(rg -n -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" \ stale_repo_url="$(search_text -e "github\.com/${old_repo_pattern}" -e "hub\.docker\.com/r/${old_repo_pattern}" -- . \
--glob '!.git/**' --glob '!dist/**' \ "${source_excludes[@]}" ':(exclude)SILO_REBRANDING_MIGRATION.md' \
--glob '!SILO_REBRANDING_MIGRATION.md' \ ':(exclude)buildscripts/rebrand-guard/compat-baseline.json')"
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . | stale_repo_url="$(printf '%s\n' "${stale_repo_url}" |
sed 's#^\./##' | grep -v '^buildscripts/verify-rebrand\.sh:' || true)" sed 's#^\./##' | grep -v '^buildscripts/verify-rebrand\.sh:' || true)"
if [ -n "${stale_repo_url}" ]; then if [ -n "${stale_repo_url}" ]; then
printf '%s\n' "${stale_repo_url}" >&2 printf '%s\n' "${stale_repo_url}" >&2
@@ -180,32 +211,32 @@ fi
# legacy image, the two READMEs that document the rename and the archived # legacy image, the two READMEs that document the rename and the archived
# minio branch, and historical issue titles in CONTRIBUTORS.md. # minio branch, and historical issue titles in CONTRIBUTORS.md.
repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md|CONTRIBUTORS\.md):' repo_guard_allowlist='^(buildscripts/minio-upgrade\.sh|buildscripts/verify-rebrand\.sh|buildscripts/helm-migration-guard/main\.go|README\.md|README_ZH\.md|CONTRIBUTORS\.md):'
stale_repo="$(rg -n "${old_repo_pattern}" --glob '!.git/**' --glob '!dist/**' \ stale_repo="$(search_text "${old_repo_pattern}" -- . "${source_excludes[@]}" \
--glob '!SILO_REBRANDING_MIGRATION.md' \ ':(exclude)SILO_REBRANDING_MIGRATION.md' ':(exclude)buildscripts/rebrand-guard/compat-baseline.json')"
--glob '!buildscripts/rebrand-guard/compat-baseline.json' . | stale_repo="$(printf '%s\n' "${stale_repo}" |
sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)" sed 's#^\./##' | grep -Ev "${repo_guard_allowlist}" || true)"
if [ -n "${stale_repo}" ]; then if [ -n "${stale_repo}" ]; then
printf '%s\n' "${stale_repo}" >&2 printf '%s\n' "${stale_repo}" >&2
fail "a source reference still names the pre-rename pgsty/minio repository" fail "a source reference still names the pre-rename pgsty/minio repository"
fi fi
stale_branch="$(rg -n 'pgsty/silo/(blob/|tree/|raw/)?master' \ stale_branch="$(search_text 'pgsty/silo/(blob/|tree/|raw/)?master' -- . "${source_excludes[@]}")"
--glob '!.git/**' --glob '!dist/**' . || true)"
if [ -n "${stale_branch}" ]; then if [ -n "${stale_branch}" ]; then
printf '%s\n' "${stale_branch}" >&2 printf '%s\n' "${stale_branch}" >&2
fail "a link still targets the retired master branch; raw and Actions URLs do not follow a branch rename" fail "a link still targets the retired master branch; raw and Actions URLs do not follow a branch rename"
fi fi
for workflow in .github/workflows/go.yml .github/workflows/vulncheck.yml; do for workflow in .github/workflows/go.yml .github/workflows/vulncheck.yml; do
if rg -q '^\s+- master$' "${workflow}"; then stale_filter="$(search_text '^[[:space:]]+- master$' -- "${workflow}")"
if [ -n "${stale_filter}" ]; then
fail "${workflow} still filters on master and would go silently dormant on main" fail "${workflow} still filters on master and would go silently dormant on main"
fi fi
require_text "${workflow}" " - main" require_text "${workflow}" " - main"
done done
network_hits="$(rg -n --glob '*.go' --glob '!**/*_test.go' \ network_hits="$(search_text \
'https?://[^"`[:space:]]*(dl\.min\.io|subnet\.min\.io|api\.min\.io|slack\.min\.io|play\.min\.io)' \ 'https?://[^"`[:space:]]*(dl\.min\.io|subnet\.min\.io|api\.min\.io|slack\.min\.io|play\.min\.io)' -- \
cmd internal || true)" ':(glob)cmd/**/*.go' ':(glob)internal/**/*.go' ':(exclude,glob)**/*_test.go' "${source_excludes[@]}")"
network_hits="$(printf '%s\n' "${network_hits}" | grep -Ev '^[^:]+:[0-9]+:[[:space:]]*//' || true)" network_hits="$(printf '%s\n' "${network_hits}" | grep -Ev '^[^:]+:[0-9]+:[[:space:]]*//' || true)"
if [ -n "${network_hits}" ]; then if [ -n "${network_hits}" ]; then
printf '%s\n' "${network_hits}" >&2 printf '%s\n' "${network_hits}" >&2
+184
View File
@@ -0,0 +1,184 @@
//go:build consistency_strict
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"io"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
type abandonedCleanupCommitDisk struct {
StorageAPI
entered chan<- struct{}
release <-chan struct{}
}
func (d abandonedCleanupCommitDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
var success int64
gate := &scopeStagedContext{
Context: ctx, meta: filepath.Join(d.Endpoint().Path, dv, dp, fi.DataDir, "part.1"),
entered: d.entered, release: d.release,
tracker: healthDiskCtxValue{lastSuccess: &success},
}
return d.StorageAPI.RenameData(gate, sv, sp, fi, dv, dp, o)
}
// A lost namespace lease can admit cleanup while a disk transaction is between
// moving its external shards and replacing xl.meta. Those shards are live work,
// even though the old metadata does not refer to them yet. Cleanup must wait.
func TestMutationAbandonedCleanupWaitsForCommit(t *testing.T) {
z, bucket := mutationPools(t)
const object = "cleanup-during-commit"
body := strings.Repeat("new payload", 200000)
mutationPut(t, z, bucket, object, 0, strings.Repeat("old payload", 200000), time.Now().Add(-time.Minute))
set := z.serverPools[0].getHashedSet(object)
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
entered, release := make(chan struct{}, len(disks)), make(chan struct{})
unblock := sync.OnceFunc(func() { close(release) })
defer unblock()
wrapped := append([]StorageAPI(nil), disks...)
for i, d := range disks {
wrapped[i] = abandonedCleanupCommitDisk{d, entered, release}
}
swap(wrapped)
committed := make(chan error, 1)
go func() {
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true})
committed <- err
}()
for range disks {
select {
case <-entered:
case err := <-committed:
t.Fatalf("PUT returned before shard barrier: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("PUT did not move all external shards")
}
}
t.Log("All eight new external shard directories exist; old xl.meta is still active")
cleaned := make(chan error, len(disks))
for _, d := range disks {
go func(d StorageAPI) { cleaned <- d.CleanAbandonedData(context.Background(), bucket, object) }(d)
}
early := 0
timer := time.NewTimer(150 * time.Millisecond)
wait:
for {
select {
case err := <-cleaned:
early++
t.Errorf("abandoned-data cleanup overtook unfinished commit: %v", err)
if early == len(disks) {
break wait
}
case <-timer.C:
break wait
}
}
timer.Stop()
unblock()
select {
case err := <-committed:
if err != nil {
t.Fatalf("PUT failed: %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("PUT did not finish")
}
for i := early; i < len(disks); i++ {
select {
case err := <-cleaned:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("cleanup did not finish")
}
}
swap(disks)
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
if err != nil {
t.Fatalf("acknowledged PUT unreadable after abandoned-data cleanup: %v", err)
}
defer r.Close()
data, err := io.ReadAll(r)
if err != nil || string(data) != body {
t.Fatalf("acknowledged external shards lost: bytes=%d err=%v", len(data), err)
}
}
// Cleanup can also arrive after a disk has committed but before its delayed
// reply lets the coordinator decide to undo a failed quorum. The previous
// generation's backup is still owned by that unfinished transaction.
func TestMutationAbandonedCleanupPreservesUndoBackup(t *testing.T) {
z, bucket := mutationPools(t)
const object = "cleanup-before-undo"
body := strings.Repeat("last acknowledged", 150000)
base := time.Now().Add(-time.Minute)
mutationPut(t, z, bucket, object, 0, body, base)
set := z.serverPools[0].getHashedSet(object)
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
entered, release := make(chan struct{}, 4), make(chan struct{})
unblock := sync.OnceFunc(func() { close(release) })
defer unblock()
wrapped := append([]StorageAPI(nil), disks...)
for i, d := range disks {
wrapped[i] = delayedRenameResult{d, i >= 4, entered, release}
}
swap(wrapped)
committed := make(chan error, 1)
go func() {
next := strings.Repeat("failed update", 180000)
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(next), int64(len(next)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Second)})
committed <- err
}()
for range 4 {
select {
case <-entered:
case err := <-committed:
t.Fatalf("PUT returned before reply barrier: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("disk commit did not reach reply barrier")
}
}
for _, d := range disks[:4] {
if err := d.CleanAbandonedData(t.Context(), bucket, object); err != nil {
t.Fatal(err)
}
}
unblock()
select {
case err := <-committed:
if err == nil {
t.Fatal("four-of-eight commit unexpectedly reached write quorum")
}
case <-time.After(5 * time.Second):
t.Fatal("failed PUT did not finish undo")
}
swap(disks)
for i, d := range disks {
fi, err := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
if err != nil || !fi.ModTime.Equal(base) {
t.Errorf("disk%d lost last acknowledged generation after failed PUT: mtime=%v err=%v", i, fi.ModTime, err)
}
}
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
if err != nil {
t.Fatalf("last acknowledged value unreadable after cleanup and undo: %v", err)
}
defer r.Close()
data, err := io.ReadAll(r)
if err != nil || string(data) != body {
t.Fatalf("last acknowledged value lost after cleanup and undo: bytes=%d err=%v", len(data), err)
}
}
+80
View File
@@ -0,0 +1,80 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/cachevalue"
)
func TestAccountInfoReflectsBucketMutations(t *testing.T) {
ctx := t.Context()
bed, err := prepareAdminErasureTestBed(ctx)
if err != nil {
t.Fatal(err)
}
defer bed.TearDown()
previousCache := listBucketsCache
defer func() { listBucketsCache = previousCache }()
// Hold a real pre-mutation snapshot past the assertions so the regression
// cannot disappear merely because a slow test outlasted the cache TTL.
freezeListing := func() {
t.Helper()
buckets, err := bed.objLayer.ListBuckets(ctx, BucketOptions{})
if err != nil {
t.Fatal(err)
}
listBucketsCache = cachevalue.NewFromFunc(time.Hour, cachevalue.Opts{}, func(context.Context) ([]BucketInfo, error) {
return buckets, nil
})
if _, err := listBucketsCache.GetWithCtx(ctx); err != nil {
t.Fatal(err)
}
}
const bucket = "account-info-freshness"
assertPresent := func(want bool) {
t.Helper()
req, err := buildAdminRequest(nil, http.MethodGet, "/accountinfo", 0, nil)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
bed.router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("account info: %d %s", rec.Code, rec.Body.String())
}
var info madmin.AccountInfo
if err := json.Unmarshal(rec.Body.Bytes(), &info); err != nil {
t.Fatal(err)
}
found := false
for _, item := range info.Buckets {
if item.Name == bucket {
found = true
}
}
if found != want {
t.Errorf("account info bucket present = %v, want %v", found, want)
}
}
freezeListing()
if err := bed.objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
assertPresent(true)
freezeListing()
if err := bed.objLayer.DeleteBucket(ctx, bucket, DeleteBucketOptions{}); err != nil {
t.Fatal(err)
}
assertPresent(false)
}
+50 -1
View File
@@ -11,6 +11,7 @@ import (
"io" "io"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"net/url"
"strings" "strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
@@ -248,8 +249,20 @@ func TestIAMAdminPolicylessSelf(t *testing.T) {
ctx, sys := iamAdminFixture(t) ctx, sys := iamAdminFixture(t)
caller := iamAdminUser(ctx, t, sys, "review-policyless", "") caller := iamAdminUser(ctx, t, sys, "review-policyless", "")
svc := iamAdminServiceAccount(ctx, t, sys, caller.AccessKey, "rv-policyless-child", "") svc := iamAdminServiceAccount(ctx, t, sys, caller.AccessKey, "rv-policyless-child", "")
other := iamAdminUser(ctx, t, sys, "bulk-other", "readwrite")
_ = iamAdminServiceAccount(ctx, t, sys, other.AccessKey, "bulk-other-child", "")
api := adminAPIHandlers{} api := adminAPIHandlers{}
assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 200) assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 200)
rec := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?listType=all", nil, nil, api.ListAccessKeysBulk)
assertIAMAdminStatus(t, "list bulk self", rec, 200)
data, err := madmin.DecryptData(caller.SecretKey, rec.Body)
mustIAM(t, err)
var listed map[string]madmin.ListAccessKeysResp
mustIAM(t, json.Unmarshal(data, &listed))
entry, ok := listed[caller.AccessKey]
if len(listed) != 1 || !ok || len(entry.ServiceAccounts) != 1 || entry.ServiceAccounts[0].AccessKey != svc.AccessKey || len(entry.STSKeys) != 0 {
t.Fatalf("policyless bulk self returned unexpected identities: %+v", listed)
}
assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoServiceAccount), 200) assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoServiceAccount), 200)
assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoAccessKey), 200) assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.InfoAccessKey), 200)
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204) assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+svc.AccessKey, nil, map[string]string{"accessKey": svc.AccessKey}, api.DeleteServiceAccount), 204)
@@ -384,6 +397,7 @@ func TestIAMAdminSelfDenials(t *testing.T) {
} }
api := adminAPIHandlers{} api := adminAPIHandlers{}
assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 403) assertIAMAdminStatus(t, "list", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-service-accounts", nil, nil, api.ListServiceAccounts), 403)
assertIAMAdminStatus(t, "list bulk self", iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?listType=all", nil, nil, api.ListAccessKeysBulk), 403)
assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoServiceAccount), 403) assertIAMAdminStatus(t, "info service", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoServiceAccount), 403)
assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoAccessKey), 403) assertIAMAdminStatus(t, "info access key", iamAdminCall(t, caller, "GET", "/minio/admin/v3/info-access-key?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.InfoAccessKey), 403)
assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 403) assertIAMAdminStatus(t, "delete", iamAdminCall(t, caller, "DELETE", "/minio/admin/v3/delete-service-account?accessKey="+target.AccessKey, nil, map[string]string{"accessKey": target.AccessKey}, api.DeleteServiceAccount), 403)
@@ -477,7 +491,10 @@ func TestIAMAdminListSelfAliases(t *testing.T) {
ctx, sys := iamAdminFixture(t) ctx, sys := iamAdminFixture(t)
parent := iamAdminUser(ctx, t, sys, "alias-parent", "readwrite") parent := iamAdminUser(ctx, t, sys, "alias-parent", "readwrite")
target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-target", "") target := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-target", "")
caller := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-caller", "") serviceCaller := iamAdminServiceAccount(ctx, t, sys, parent.AccessKey, "alias-caller", "")
caller := serviceCaller
other := iamAdminUser(ctx, t, sys, "alias-other", "readwrite")
_ = iamAdminServiceAccount(ctx, t, sys, other.AccessKey, "alias-other-child", "")
if kind == "sts" { if kind == "sts" {
var err error var err error
caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix()}, globalActiveCred.SecretKey) caller, err = auth.GetNewCredentialsWithMetadata(map[string]any{parentClaim: parent.AccessKey, "exp": UTCNow().Add(time.Hour).Unix()}, globalActiveCred.SecretKey)
@@ -501,6 +518,38 @@ func TestIAMAdminListSelfAliases(t *testing.T) {
if !found { if !found {
t.Fatalf("self alias %q did not list the parent's service accounts", alias) t.Fatalf("self alias %q did not list the parent's service accounts", alias)
} }
query := url.Values{"listType": {madmin.AccessKeyListAll}}
if alias != "" {
query.Set("users", alias)
}
bulk := iamAdminCall(t, caller, "GET", "/minio/admin/v3/list-access-keys-bulk?"+query.Encode(), nil, nil, api.ListAccessKeysBulk)
assertIAMAdminStatus(t, "list bulk self alias", bulk, 200)
data, err = madmin.DecryptData(caller.SecretKey, bulk.Body)
mustIAM(t, err)
var listed map[string]madmin.ListAccessKeysResp
mustIAM(t, json.Unmarshal(data, &listed))
entry, ok := listed[parent.AccessKey]
if len(listed) != 1 || !ok || len(entry.ServiceAccounts) != 2 {
t.Fatalf("bulk self alias %q returned unexpected identities: %+v", alias, listed)
}
want := map[string]bool{target.AccessKey: true, serviceCaller.AccessKey: true}
for _, account := range entry.ServiceAccounts {
if !want[account.AccessKey] {
t.Fatalf("bulk self alias %q returned unexpected or duplicate service account %s", alias, account.AccessKey)
}
delete(want, account.AccessKey)
}
if kind == "sts" {
if len(entry.STSKeys) != 1 || entry.STSKeys[0].AccessKey != caller.AccessKey {
t.Fatalf("bulk self alias %q returned unexpected STS accounts: %+v", alias, entry.STSKeys)
}
} else if len(entry.STSKeys) != 0 {
t.Fatalf("service caller returned unexpected STS accounts: %+v", entry.STSKeys)
}
if bytes.Contains(data, []byte(serviceCaller.SecretKey)) || bytes.Contains(data, []byte(`"secretKey"`)) || bytes.Contains(data, []byte(`"sessionToken"`)) {
t.Fatal("bulk listing returned credential secrets")
}
} }
}) })
} }
+7 -4
View File
@@ -1237,7 +1237,7 @@ func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Requ
} }
} }
if !globalIAMSys.IsAllowed(policy.Args{ if !isAllowedServiceAccountAction(cred, policy.Args{
AccountName: cred.AccessKey, AccountName: cred.AccessKey,
Groups: cred.Groups, Groups: cred.Groups,
Action: policy.ListServiceAccountsAdminAction, Action: policy.ListServiceAccountsAdminAction,
@@ -1250,12 +1250,12 @@ func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Requ
return return
} }
if selfOnly && len(users) == 0 { if selfOnly {
selfUser := cred.AccessKey selfUser := cred.AccessKey
if cred.ParentUser != "" { if cred.ParentUser != "" {
selfUser = cred.ParentUser selfUser = cred.ParentUser
} }
users = append(users, selfUser) users = []string{selfUser}
} }
var checkedUserList []string var checkedUserList []string
@@ -1447,7 +1447,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
return buckets[i].Name < buckets[j].Name return buckets[i].Name < buckets[j].Name
}) })
} else { } else {
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{Cached: true}) // Console refreshes this inventory immediately after bucket mutations.
// A stale cached list can otherwise hide a creation or retain a deletion
// indefinitely in the client's snapshot.
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
if err != nil { if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return return
+12 -15
View File
@@ -511,14 +511,13 @@ func testSiteReplicationStatusDetectsCorsTimestampMismatch(obj ObjectLayer, _ st
})) }))
defer remote.Close() defer remote.Close()
serviceCred, err := auth.CreateCredentials("cors-status-service", "cors-status-service-secret-key") serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
accessKey: "cors-status-service",
secretKey: "cors-status-service-secret-key",
})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
serviceCred.ParentUser = credentials.AccessKey
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
t.Fatal(err)
}
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false) defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
globalSiteReplicationSys.Lock() globalSiteReplicationSys.Lock()
@@ -585,14 +584,13 @@ func testSiteReplicationStatusCountsCorsPerSite(obj ObjectLayer, _ string, bucke
})) }))
defer remote.Close() defer remote.Close()
serviceCred, err := auth.CreateCredentials("cors-count-service", "cors-count-service-secret-key") serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
accessKey: "cors-count-service",
secretKey: "cors-count-service-secret-key",
})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
serviceCred.ParentUser = credentials.AccessKey
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
t.Fatal(err)
}
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false) defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
globalSiteReplicationSys.Lock() globalSiteReplicationSys.Lock()
@@ -1190,14 +1188,13 @@ func testCorsReplicationDispatchStatusHealReload(obj ObjectLayer, _ string, buck
})) }))
defer remote.Close() defer remote.Close()
serviceCred, err := auth.CreateCredentials("cors-integration-svc", "cors-integration-service-secret") serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
accessKey: "cors-integration-svc",
secretKey: "cors-integration-service-secret",
})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
serviceCred.ParentUser = credentials.AccessKey
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
t.Fatal(err)
}
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false) defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
globalSiteReplicationSys.Lock() globalSiteReplicationSys.Lock()
+55 -10
View File
@@ -37,6 +37,7 @@ import (
"github.com/minio/minio/internal/amztime" "github.com/minio/minio/internal/amztime"
sse "github.com/minio/minio/internal/bucket/encryption" sse "github.com/minio/minio/internal/bucket/encryption"
"github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/bucket/replication"
"github.com/minio/minio/internal/event" "github.com/minio/minio/internal/event"
xhttp "github.com/minio/minio/internal/http" xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger" "github.com/minio/minio/internal/logger"
@@ -631,21 +632,65 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
return err return err
} }
// Delete remote object from warm-tier // Recheck the lifecycle decision and create the hot-tier tombstone before
err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier) // touching the remote tier. If the lifecycle configuration changed after
if err == nil { // the scanner queued this task, DeleteObject rejects the stale action while
// Skip adding free version since we successfully deleted the // the remote object is still protected.
// remote object opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name)
opts.SkipFreeVersion = true keepRemote := opts.VersionID == "" &&
} else { (opts.Versioned || (opts.VersionSuspended && oi.VersionID != "" && oi.VersionID != nullVersionID))
transitionLogIf(ctx, err) if lcEvent.Action == lifecycle.DeleteAction || lcEvent.Action == lifecycle.DeleteVersionAction {
opts.SetEvalMetadataFn(func(goi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
return checkReplicateDelete(ctx, oi.Bucket, ObjectToDelete{
ObjectV: ObjectV{
ObjectName: goi.Name,
VersionID: opts.VersionID,
},
}, *goi, opts, gerr), nil
})
} }
// Now, delete object from hot-tier namespace // Delete the hot-tier namespace first. For a transitioned version this
if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil { // leaves a durable free-version record until the remote delete succeeds;
// that record is the retry point if the process stops between the two
// operations.
delInfo, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
if err != nil {
return err return err
} }
if !keepRemote {
if err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier); err != nil {
transitionLogIf(ctx, err)
}
}
// Queue replication of the ILM-created delete marker or version purge,
// mirroring the DELETE handler's post-delete scheduling.
if delInfo.ReplicationStatus == replication.Pending || delInfo.VersionPurgeStatus == replication.VersionPurgePending {
dmVersionID, versionID := "", ""
if delInfo.DeleteMarker && delInfo.VersionPurgeStatus.Empty() {
dmVersionID = delInfo.VersionID
} else {
versionID = delInfo.VersionID
}
if delInfo.Name == "" {
delInfo.Name = oi.Name
}
scheduleReplicationDelete(ctx, DeletedObjectReplicationInfo{
DeletedObject: DeletedObject{
ObjectName: delInfo.Name,
VersionID: versionID,
DeleteMarkerVersionID: dmVersionID,
DeleteMarkerMTime: DeleteMarkerMTime{delInfo.ModTime},
DeleteMarker: delInfo.DeleteMarker,
ReplicationState: delInfo.ReplicationState(),
},
Bucket: oi.Bucket,
EventType: ReplicateILMDelete,
}, objectAPI)
}
// Send audit for the lifecycle delete operation // Send audit for the lifecycle delete operation
defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn) defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
+183
View File
@@ -19,11 +19,17 @@ package cmd
import ( import (
"bytes" "bytes"
"context"
"io"
"net/http" "net/http"
"slices"
"strings"
"sync"
"testing" "testing"
"time" "time"
"github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/lifecycle"
objectlock "github.com/minio/minio/internal/bucket/object/lock"
xhttp "github.com/minio/minio/internal/http" xhttp "github.com/minio/minio/internal/http"
) )
@@ -245,3 +251,180 @@ func TestValidateTransitionTier(t *testing.T) {
} }
} }
} }
// tierExpiryTestBackend is a remote tier held in memory.
type tierExpiryTestBackend struct {
mu sync.Mutex
objects map[string][]byte
removed []string
}
func (b *tierExpiryTestBackend) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
return b.PutWithMeta(ctx, object, r, length, nil)
}
func (b *tierExpiryTestBackend) PutWithMeta(_ context.Context, object string, r io.Reader, _ int64, _ map[string]string) (remoteVersionID, error) {
data, err := io.ReadAll(r)
if err != nil {
return "", err
}
b.mu.Lock()
defer b.mu.Unlock()
b.objects[object] = data
return "", nil
}
func (b *tierExpiryTestBackend) Get(_ context.Context, object string, _ remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) {
b.mu.Lock()
defer b.mu.Unlock()
data, ok := b.objects[object]
if !ok {
return nil, ObjectNotFound{Bucket: "tier", Object: object}
}
if opts.length > 0 {
data = data[opts.startOffset : opts.startOffset+opts.length]
}
return io.NopCloser(bytes.NewReader(data)), nil
}
func (b *tierExpiryTestBackend) Remove(_ context.Context, object string, _ remoteVersionID) error {
b.mu.Lock()
defer b.mu.Unlock()
delete(b.objects, object)
b.removed = append(b.removed, object)
return nil
}
func (b *tierExpiryTestBackend) InUse(context.Context) (bool, error) { return false, nil }
// The expiry of a transitioned latest version that only puts a delete marker
// on top of it must keep its remote object, so the version stays readable as
// a noncurrent version. The remote object goes when the version is deleted.
func TestExpireTransitionedObjectKeepsNoncurrentData(t *testing.T) {
z, _ := migrationTestPools(t)
ctx := t.Context()
tier := &tierExpiryTestBackend{objects: map[string][]byte{}}
prevTier := globalTierConfigMgr
globalTierConfigMgr = NewTierConfigMgr()
globalTierConfigMgr.drivercache["WARM"] = tier
t.Cleanup(func() { globalTierConfigMgr = prevTier })
const expire = `<LifecycleConfiguration><Rule><ID>exp</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter>` +
`<Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration></Rule></LifecycleConfiguration>`
testCases := []struct {
name string
versioning string // "", Enabled or Suspended
nullLatest bool // the latest version is the null version
keep bool // the latest version survives as noncurrent
want []string // versions after the expiry, newest first
}{
{name: "versioned", versioning: "Enabled", keep: true, want: []string{"DM", "v2", "v1"}},
{name: "suspended", versioning: "Suspended", keep: true, want: []string{"DM", "v2", "v1"}},
{name: "suspended-null", versioning: "Suspended", nullLatest: true, want: []string{"DM", "v1"}},
{name: "unversioned", nullLatest: true, want: nil},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
bucket := "tier-expiry-" + tc.name
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: tc.versioning != ""}); err != nil {
t.Fatal(err)
}
meta, err := globalBucketMetadataSys.Get(bucket)
if err != nil {
t.Fatal(err)
}
if tc.versioning == "Suspended" {
meta.VersioningConfigXML = []byte("<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
}
meta.LifecycleConfigXML = []byte(expire)
if err := meta.Save(ctx, z); err != nil {
t.Fatal(err)
}
globalBucketMetadataSys.Set(bucket, meta)
// tiered writes a version whose data is in the remote tier.
labels := map[string]string{}
tiered := func(label string, opts ObjectOptions) ObjectInfo {
t.Helper()
oi := putConsistencyObject(t, z, bucket, "obj", 0, "obj "+label, opts)
vid := migrationTestVersion(oi)
infos, errs := readAllFileInfo(ctx, z.serverPools[0].getHashedSet("obj").getDisks(), "", bucket, "obj", vid, false, false)
if errs[0] != nil {
t.Fatal(errs[0])
}
fi := infos[0]
fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", bucket+"/"+label
if _, err := tier.Put(ctx, fi.TransitionedObjName, strings.NewReader("obj "+label), -1); err != nil {
t.Fatal(err)
}
if err := z.serverPools[0].DecomTieredObject(ctx, bucket, "obj", fi, ObjectOptions{}); err != nil {
t.Fatal(err)
}
labels[vid] = label
return oi
}
var latest ObjectInfo
if tc.versioning != "" {
tiered("v1", ObjectOptions{Versioned: true})
}
if tc.nullLatest {
latest = tiered("v2", ObjectOptions{})
} else {
latest = tiered("v2", ObjectOptions{Versioned: true})
}
// Expire the latest version as the expiry worker does for the scanner.
getOpts := ObjectOptions{Versioned: tc.versioning == "Enabled", VersionSuspended: tc.versioning == "Suspended"}
oi, err := z.GetObjectInfo(ctx, bucket, "obj", getOpts)
if err != nil {
t.Fatal(err)
}
lc, err := globalLifecycleSys.Get(bucket)
if err != nil {
t.Fatal(err)
}
evt := evalActionFromLifecycle(ctx, *lc, objectlock.Retention{}, nil, oi)
if evt.Action != lifecycle.DeleteAction || oi.TransitionedObject.Status != lifecycle.TransitionComplete {
t.Fatalf("fixture: action %v, transition status %q", evt.Action, oi.TransitionedObject.Status)
}
if err := expireTransitionedObject(ctx, z, &oi, evt, lcEventSrc_Scanner); err != nil {
t.Errorf("expiry: %v", err)
}
var got []string
for _, set := range z.serverPools[0].sets {
err := set.listObjectsToDecommission(ctx, decomBucketInfo{Name: bucket}, func(entry metaCacheEntry) {
fivs, err := entry.fileInfoVersions(bucket)
if err != nil {
t.Error(err)
return
}
for _, fi := range fivs.Versions {
if fi.Deleted {
got = append(got, "DM")
} else {
got = append(got, labels[migrationTestVersion(fi.ToObjectInfo(bucket, "obj", true))])
}
}
})
if err != nil {
t.Fatal(err)
}
}
if !slices.Equal(got, tc.want) {
t.Errorf("versions after expiry: got %v, want %v", got, tc.want)
}
remote := bucket + "/v2"
if removed := slices.Contains(tier.removed, remote); removed == tc.keep {
t.Errorf("remote object of the expired version removed: %v, want %v", removed, !tc.keep)
}
if tc.keep {
body := migrationTestBody(t, z.GetObjectNInfo, bucket, "obj", migrationTestVersion(latest))
if body != "obj v2" {
t.Errorf("v2 reads %q after the expiry, want %q", body, "obj v2")
}
}
})
}
}
+257
View File
@@ -0,0 +1,257 @@
// Copyright (c) 2026 Denis Babchuk
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"maps"
"net/http"
"strings"
"testing"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio/internal/hash"
xhttp "github.com/minio/minio/internal/http"
)
// checksumTestObject builds the xl.meta checksum of an object the way
// PutObject and CompleteMultipartUpload store it.
func checksumTestObject(t *testing.T, alg hash.ChecksumType, full bool, parts ...[]byte) FileInfo {
t.Helper()
fi := FileInfo{Volume: "bucket", Name: "object"}
if len(parts) == 1 {
fi.Size = int64(len(parts[0]))
fi.Checksum = hash.NewChecksumFromData(alg, parts[0]).AppendTo(nil, nil)
return fi
}
var combined, all []byte
for i, p := range parts {
combined = append(combined, hash.NewChecksumFromData(alg, p).Raw...)
all = append(all, p...)
fi.Parts = append(fi.Parts, ObjectPartInfo{Number: i + 1, Size: int64(len(p)), ActualSize: int64(len(p))})
fi.Size += int64(len(p))
}
typ := alg | hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
if full {
typ |= hash.ChecksumFullObject
}
cs := hash.NewChecksumFromData(typ, combined)
if typ.FullObjectRequested() {
// Full object CRC, as merged from the parts on completion.
whole := hash.NewChecksumFromData(alg, all)
cs = &hash.Checksum{Type: typ, Raw: whole.Raw, Encoded: whole.Encoded}
}
fi.Checksum = cs.AppendTo(nil, combined)
return fi
}
// TestGetCRCMetaChecksumType checks the checksum headers that replication
// sends for an object: a FULL_OBJECT checksum type only for multipart full
// object CRCs, where the target needs it to create a matching upload.
func TestGetCRCMetaChecksumType(t *testing.T) {
p1, p2 := []byte("first part"), []byte("second part")
enc := func(alg hash.ChecksumType, b []byte) string { return hash.NewChecksumFromData(alg, b).Encoded }
whole := append(append([]byte{}, p1...), p2...)
composite := func(alg hash.ChecksumType) string {
var combined []byte
combined = append(combined, hash.NewChecksumFromData(alg, p1).Raw...)
combined = append(combined, hash.NewChecksumFromData(alg, p2).Raw...)
return enc(alg, combined) + "-2"
}
full := map[string]string{xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject}
tests := []struct {
name string
fi FileInfo
alg hash.ChecksumType
isMP bool
value string
extra map[string]string
}{
{"single-part SHA256", checksumTestObject(t, hash.ChecksumSHA256, false, p1), hash.ChecksumSHA256, false, enc(hash.ChecksumSHA256, p1), nil},
{"single-part CRC32", checksumTestObject(t, hash.ChecksumCRC32, false, p1), hash.ChecksumCRC32, false, enc(hash.ChecksumCRC32, p1), nil},
{"single-part CRC64NVME", checksumTestObject(t, hash.ChecksumCRC64NVME, false, p1), hash.ChecksumCRC64NVME, false, enc(hash.ChecksumCRC64NVME, p1), nil},
{"multipart COMPOSITE CRC32C", checksumTestObject(t, hash.ChecksumCRC32C, false, p1, p2), hash.ChecksumCRC32C, true, composite(hash.ChecksumCRC32C), nil},
{"multipart COMPOSITE SHA256", checksumTestObject(t, hash.ChecksumSHA256, false, p1, p2), hash.ChecksumSHA256, true, composite(hash.ChecksumSHA256), nil},
{"multipart FULL_OBJECT CRC32", checksumTestObject(t, hash.ChecksumCRC32, true, p1, p2), hash.ChecksumCRC32, true, enc(hash.ChecksumCRC32, whole), full},
{"multipart FULL_OBJECT CRC32C", checksumTestObject(t, hash.ChecksumCRC32C, true, p1, p2), hash.ChecksumCRC32C, true, enc(hash.ChecksumCRC32C, whole), full},
{"multipart CRC64NVME", checksumTestObject(t, hash.ChecksumCRC64NVME, false, p1, p2), hash.ChecksumCRC64NVME, true, enc(hash.ChecksumCRC64NVME, whole), full},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
oi := tt.fi.ToObjectInfo("bucket", "object", true)
want := map[string]string{tt.alg.Key(): tt.value, xhttp.AmzChecksumAlgo: tt.alg.String()}
maps.Copy(want, tt.extra)
got, isMP := getCRCMeta(oi, 0, nil)
if isMP != tt.isMP {
t.Errorf("isMP: got %t, want %t", isMP, tt.isMP)
}
if !maps.Equal(got, want) {
t.Errorf("object headers: got %v, want %v", got, want)
}
if !tt.isMP {
return
}
// Part headers carry the part checksum and never a type.
want = map[string]string{tt.alg.Key(): enc(tt.alg, p2), xhttp.AmzChecksumAlgo: tt.alg.String()}
if got, _ := getCRCMeta(oi, 2, nil); !maps.Equal(got, want) {
t.Errorf("part 2 headers: got %v, want %v", got, want)
}
})
}
}
// TestReplicateObjectChecksumTypes replicates objects with checksums through
// the S3 API of a test server, and checks that the replica gets the same
// checksum as the source.
func TestReplicateObjectChecksumTypes(t *testing.T) {
ctx := context.Background()
// Leave no object layer behind: unit tests that run later in the package
// rely on newObjectLayerFn() returning nil.
defer resetTestGlobals()
ts := StartTestServer(t, ErasureSDStr)
defer ts.Stop()
clnt, err := minio.New(strings.TrimPrefix(ts.Server.URL, "http://"), &minio.Options{
Creds: credentials.NewStaticV4(ts.AccessKey, ts.SecretKey, ""),
})
if err != nil {
t.Fatal(err)
}
core := &minio.Core{Client: clnt}
for _, bucket := range []string{"src", "dst"} {
if err := clnt.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
if err := clnt.EnableVersioning(ctx, bucket); err != nil {
t.Fatal(err)
}
}
small := []byte("single part object")
p1 := bytes.Repeat([]byte("a"), 5<<20)
p2 := []byte("last part")
tests := []struct {
name string
alg hash.ChecksumType
multipart bool
full bool
copyOf string // server-side copy of an object created above
}{
{"put-SHA256", hash.ChecksumSHA256, false, false, ""},
{"put-CRC32", hash.ChecksumCRC32, false, false, ""},
{"mp-COMPOSITE-CRC32C", hash.ChecksumCRC32C, true, false, ""},
{"mp-COMPOSITE-SHA256", hash.ChecksumSHA256, true, false, ""},
{"mp-FULL_OBJECT-CRC32", hash.ChecksumCRC32, true, true, ""},
{"mp-FULL_OBJECT-CRC32C", hash.ChecksumCRC32C, true, true, ""},
{"mp-CRC64NVME", hash.ChecksumCRC64NVME, true, false, ""},
// The copy keeps the multipart FULL_OBJECT checksum of its source
// but is not multipart, so it replicates with a single PUT.
{"copy-FULL_OBJECT-CRC32", hash.ChecksumCRC32, false, true, "mp-FULL_OBJECT-CRC32"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
object := tt.name
// Upload the source object like an S3 client does.
switch {
case tt.copyOf != "":
_, err := core.CopyObject(ctx, "src", tt.copyOf, "src", object, nil, minio.CopySrcOptions{}, minio.PutObjectOptions{})
if err != nil {
t.Fatal(err)
}
case !tt.multipart:
_, err := core.PutObject(ctx, "src", object, bytes.NewReader(small), int64(len(small)), "", "", minio.PutObjectOptions{
UserMetadata: map[string]string{tt.alg.Key(): hash.NewChecksumFromData(tt.alg, small).Encoded},
})
if err != nil {
t.Fatal(err)
}
default:
meta := map[string]string{xhttp.AmzChecksumAlgo: tt.alg.String()}
if tt.full {
meta[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject
}
uploadID, err := core.NewMultipartUpload(ctx, "src", object, minio.PutObjectOptions{UserMetadata: meta})
if err != nil {
t.Fatal(err)
}
var parts []minio.CompletePart
for i, p := range [][]byte{p1, p2} {
h := http.Header{}
h.Set(tt.alg.Key(), hash.NewChecksumFromData(tt.alg, p).Encoded)
pi, err := core.PutObjectPart(ctx, "src", object, uploadID, i+1, bytes.NewReader(p), int64(len(p)), minio.PutObjectPartOptions{CustomHeader: h})
if err != nil {
t.Fatal(err)
}
parts = append(parts, minio.CompletePart{
PartNumber: pi.PartNumber, ETag: pi.ETag,
ChecksumCRC32: pi.ChecksumCRC32, ChecksumCRC32C: pi.ChecksumCRC32C,
ChecksumSHA1: pi.ChecksumSHA1, ChecksumSHA256: pi.ChecksumSHA256,
ChecksumCRC64NVME: pi.ChecksumCRC64NVME,
})
}
if _, err := core.CompleteMultipartUpload(ctx, "src", object, uploadID, parts, minio.PutObjectOptions{}); err != nil {
t.Fatal(err)
}
}
// Replicate it the way replicateObject does.
gr, err := ts.Obj.GetObjectNInfo(ctx, "src", object, nil, http.Header{}, ObjectOptions{Versioned: true, ReplicationRequest: true})
if err != nil {
t.Fatal(err)
}
defer gr.Close()
objInfo := gr.ObjInfo
putOpts, isMP, err := putReplicationOpts(ctx, "", objInfo)
if err != nil {
t.Fatal(err)
}
if isMP != tt.multipart {
t.Fatalf("multipart replication: got %t, want %t", isMP, tt.multipart)
}
if isMP {
err = replicateObjectWithMultipart(ctx, core, "dst", object, gr, objInfo, putOpts)
} else {
size, serr := objInfo.GetActualSize()
if serr != nil {
t.Fatal(serr)
}
_, err = core.PutObject(ctx, "dst", object, gr, size, "", "", putOpts)
}
if err != nil {
t.Fatalf("replication failed: %v", err)
}
src, err := clnt.StatObject(ctx, "src", object, minio.StatObjectOptions{Checksum: true})
if err != nil {
t.Fatal(err)
}
dst, err := clnt.StatObject(ctx, "dst", object, minio.StatObjectOptions{Checksum: true})
if err != nil {
t.Fatal(err)
}
checksums := func(oi minio.ObjectInfo) string {
return strings.Join([]string{oi.ChecksumMode, oi.ChecksumCRC32, oi.ChecksumCRC32C, oi.ChecksumSHA1, oi.ChecksumSHA256, oi.ChecksumCRC64NVME}, "|")
}
if checksums(src) == "|||||" || checksums(dst) != checksums(src) {
t.Errorf("replica checksum %q, source %q", checksums(dst), checksums(src))
}
})
}
}
+7
View File
@@ -2062,6 +2062,8 @@ const (
ReplicateIncoming = "replicate:incoming" ReplicateIncoming = "replicate:incoming"
// ReplicateIncomingDelete - audit trail of inline replication of deletes. // ReplicateIncomingDelete - audit trail of inline replication of deletes.
ReplicateIncomingDelete = "replicate:incoming:delete" ReplicateIncomingDelete = "replicate:incoming:delete"
// ReplicateILMDelete - audit trail of delete replication triggered by ILM expiry.
ReplicateILMDelete = "replicate:ilm:delete"
// ReplicateHeal - audit trail for healing of failed/pending replications // ReplicateHeal - audit trail for healing of failed/pending replications
ReplicateHeal = "replicate:heal" ReplicateHeal = "replicate:heal"
@@ -4250,6 +4252,11 @@ func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) (cs map[string]string
if cktype.IsSet() { if cktype.IsSet() {
meta[cktype.Key()] = v meta[cktype.Key()] = v
meta[xhttp.AmzChecksumAlgo] = cktype.String() meta[xhttp.AmzChecksumAlgo] = cktype.String()
// Without the type the target creates a COMPOSITE multipart
// upload and rejects the full object CRC on completion.
if isMP && cktype.CanMerge() && cs[xhttp.AmzChecksumType] == xhttp.AmzChecksumTypeFullObject {
meta[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject
}
} }
} }
return meta, isMP return meta, isMP
+1 -1
View File
@@ -883,7 +883,7 @@ func serverHandleEnvVars() {
} }
globalInplaceUpdateDisabled = true globalInplaceUpdateDisabled = true
if env.IsSet(api.EnvAPIMultipartListing) { if env.IsSet(api.EnvAPIMultipartListing) {
logger.Warning("%s is ignored: multipart listing always uses persistent metadata; uploads without identity fields are omitted", api.EnvAPIMultipartListing) logger.Warning("%s is ignored: multipart listing uses the legacy cache/exact-key behavior; persistent listing is deferred", api.EnvAPIMultipartListing)
} }
// Check if the supported credential env vars, // Check if the supported credential env vars,
+1 -1
View File
@@ -91,7 +91,7 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
return nil, err return nil, err
} }
data, err = decryptData(data, configFile) data, err = decryptData(ctx, data, configFile)
if err != nil { if err != nil {
return nil, err return nil, err
} }
+3 -3
View File
@@ -68,7 +68,7 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
continue continue
} }
data, err = decryptData(data, obj.Name) data, err = decryptData(ctx, data, obj.Name)
if err != nil { if err != nil {
// ignore history file that cannot be loaded. // ignore history file that cannot be loaded.
continue continue
@@ -110,7 +110,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str
return nil, err return nil, err
} }
return decryptData(data, historyFile) return decryptData(ctx, data, historyFile)
} }
func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error { func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
@@ -162,7 +162,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer, data []byte) (con
return nil, err return nil, err
} }
data, err = decryptData(data, configFile) data, err = decryptData(ctx, data, configFile)
if err != nil { if err != nil {
lookupConfigs(srvCfg, objAPI) lookupConfigs(srvCfg, objAPI)
return nil, err return nil, err
+67 -8
View File
@@ -776,7 +776,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
} }
// We got an entry which we should be able to heal. // We got an entry which we should be able to heal.
fiv, err := entry.fileInfoVersions(bucket) fiv, err := entry.allFileInfoVersions(bucket)
if err != nil { if err != nil {
err := bgSeq.queueHealTask(healSource{ err := bgSeq.queueHealTask(healSource{
bucket: bucket, bucket: bucket,
@@ -1130,15 +1130,9 @@ eventLoop:
applyExpiryRule(event, lcEventSrc_Scanner, oi) applyExpiryRule(event, lcEventSrc_Scanner, oi)
case lifecycle.DeleteVersionAction: // noncurrent versions expiration case lifecycle.DeleteVersionAction: // noncurrent versions expiration
opts := objOpts[idx]
remainingVersions-- remainingVersions--
size = 0 size = 0
toDel = append(toDel, ObjectToDelete{ toDel = append(toDel, ilmPurgeToDelete(ctx, i.bucket, oi, i.replication.Config != nil))
ObjectV: ObjectV{
ObjectName: opts.Name,
VersionID: opts.VersionID,
},
})
noncurrentEvents = append(noncurrentEvents, event) noncurrentEvents = append(noncurrentEvents, event)
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
@@ -1240,6 +1234,21 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
// use prefix delete on exact object (this is an optimization to avoid fan-out calls) // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
opts.DeletePrefixObject = true opts.DeletePrefixObject = true
} }
// ILM-created delete markers and version purges must carry the same
// replication decision as API-created deletes. DeleteAll is intentionally
// excluded: it is an object-local purge and has no single version to
// replicate.
if lcEvent.Action == lifecycle.DeleteAction || lcEvent.Action == lifecycle.DeleteVersionAction {
opts.SetEvalMetadataFn(func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
return checkReplicateDelete(ctx, obj.Bucket, ObjectToDelete{
ObjectV: ObjectV{
ObjectName: oi.Name,
VersionID: opts.VersionID,
},
}, *oi, opts, gerr), nil
})
}
var ( var (
dobj ObjectInfo dobj ObjectInfo
err error err error
@@ -1276,6 +1285,29 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
dobj = obj dobj = obj
} }
// Queue replication of the ILM-created delete marker or version purge,
// mirroring the DELETE handler's post-delete scheduling.
if dobj.ReplicationStatus == replication.Pending || dobj.VersionPurgeStatus == replication.VersionPurgePending {
dmVersionID, versionID := "", ""
if dobj.DeleteMarker && dobj.VersionPurgeStatus.Empty() {
dmVersionID = dobj.VersionID
} else {
versionID = dobj.VersionID
}
scheduleReplicationDelete(ctx, DeletedObjectReplicationInfo{
DeletedObject: DeletedObject{
ObjectName: dobj.Name,
VersionID: versionID,
DeleteMarkerVersionID: dmVersionID,
DeleteMarkerMTime: DeleteMarkerMTime{dobj.ModTime},
DeleteMarker: dobj.DeleteMarker,
ReplicationState: dobj.ReplicationState(),
},
Bucket: obj.Bucket,
EventType: ReplicateILMDelete,
}, objLayer)
}
tags := newLifecycleAuditEvent(src, lcEvent).Tags() tags := newLifecycleAuditEvent(src, lcEvent).Tags()
tags["version-id"] = dobj.VersionID tags["version-id"] = dobj.VersionID
@@ -1309,6 +1341,33 @@ func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) {
globalExpiryState.enqueueByDays(obj, event, src) globalExpiryState.enqueueByDays(obj, event, src)
} }
// ilmPurgeToDelete builds the bulk delete entry for an ILM-initiated removal
// of a noncurrent version. The entry carries a pending version-purge state so
// the purge is replicated and retried like an API version purge.
func ilmPurgeToDelete(ctx context.Context, bucket string, oi ObjectInfo, replicate bool) ObjectToDelete {
otd := ObjectToDelete{
ObjectV: ObjectV{
ObjectName: oi.Name,
VersionID: oi.VersionID,
},
}
if !replicate {
return otd
}
delOpts := ObjectOptions{
Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, oi.Name),
VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, oi.Name),
}
dsc := checkReplicateDelete(ctx, bucket, otd, oi, delOpts, nil)
if !dsc.ReplicateAny() {
return otd
}
otd.VersionPurgeStatus = replication.VersionPurgePending
otd.VersionPurgeStatuses = dsc.PendingStatus()
otd.ReplicateDecisionStr = dsc.String()
return otd
}
// objectPath returns the prefix and object name. // objectPath returns the prefix and object name.
func (i *scannerItem) objectPath() string { func (i *scannerItem) objectPath() string {
return pathJoin(i.prefix, i.objectName) return pathJoin(i.prefix, i.objectName)
+1
View File
@@ -27,6 +27,7 @@ import (
// Tests create endpoints with ellipses and without. // Tests create endpoints with ellipses and without.
func TestCreateServerEndpoints(t *testing.T) { func TestCreateServerEndpoints(t *testing.T) {
useStaticTestEndpoints(t)
testCases := []struct { testCases := []struct {
serverAddr string serverAddr string
args []string args []string
+37
View File
@@ -18,6 +18,7 @@
package cmd package cmd
import ( import (
"context"
"fmt" "fmt"
"net" "net"
"net/url" "net/url"
@@ -25,9 +26,42 @@ import (
"reflect" "reflect"
"strings" "strings"
"testing" "testing"
"github.com/minio/dnscache"
) )
type endpointTestResolver struct {
dnscache.DNSResolver
}
func (r endpointTestResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
// These endpoint fixtures test locality and parsing, not public DNS.
for i, name := range []string{"example.org", "example.com", "example.net", "example.edu"} {
if host == name {
return []string{fmt.Sprintf("203.0.113.%d", i+1)}, nil
}
}
return r.DNSResolver.LookupHost(ctx, host)
}
func useTestEndpointDNS(t *testing.T) {
t.Helper()
previous := globalDNSCache
globalDNSCache = &dnscache.Resolver{Resolver: endpointTestResolver{net.DefaultResolver}}
t.Cleanup(func() { globalDNSCache = previous })
}
func useStaticTestEndpoints(t *testing.T) {
t.Helper()
useTestEndpointDNS(t)
// These fixtures deliberately use localhost, including inside containers.
previous := orchestrated
orchestrated = false
t.Cleanup(func() { orchestrated = previous })
}
func TestNewEndpoint(t *testing.T) { func TestNewEndpoint(t *testing.T) {
useTestEndpointDNS(t)
u2, _ := url.Parse("https://example.org/path") u2, _ := url.Parse("https://example.org/path")
u4, _ := url.Parse("http://192.168.253.200/path") u4, _ := url.Parse("http://192.168.253.200/path")
rootSlashFoo, _ := filepath.Abs("/foo") rootSlashFoo, _ := filepath.Abs("/foo")
@@ -130,6 +164,7 @@ func TestNewEndpoints(t *testing.T) {
} }
func TestCreateEndpoints(t *testing.T) { func TestCreateEndpoints(t *testing.T) {
useStaticTestEndpoints(t)
tempGlobalMinioPort := globalMinioPort tempGlobalMinioPort := globalMinioPort
defer func() { defer func() {
globalMinioPort = tempGlobalMinioPort globalMinioPort = tempGlobalMinioPort
@@ -353,6 +388,7 @@ func TestCreateEndpoints(t *testing.T) {
// So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001 // So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
// is considered a remote service from localhost:9000 perspective. // is considered a remote service from localhost:9000 perspective.
func TestGetLocalPeer(t *testing.T) { func TestGetLocalPeer(t *testing.T) {
useStaticTestEndpoints(t)
tempGlobalMinioPort := globalMinioPort tempGlobalMinioPort := globalMinioPort
defer func() { defer func() {
globalMinioPort = tempGlobalMinioPort globalMinioPort = tempGlobalMinioPort
@@ -393,6 +429,7 @@ func TestGetLocalPeer(t *testing.T) {
} }
func TestGetRemotePeers(t *testing.T) { func TestGetRemotePeers(t *testing.T) {
useStaticTestEndpoints(t)
tempGlobalMinioPort := globalMinioPort tempGlobalMinioPort := globalMinioPort
defer func() { defer func() {
globalMinioPort = tempGlobalMinioPort globalMinioPort = tempGlobalMinioPort
+60 -3
View File
@@ -126,8 +126,8 @@ func (d *multipartCleanupDisk) ReadVersion(context.Context, string, string, stri
return FileInfo{ModTime: d.modTime}, d.readErr return FileInfo{ModTime: d.modTime}, d.readErr
} }
func (d *multipartCleanupDisk) StatVol(context.Context, string) (VolInfo, error) { func (d *multipartCleanupDisk) StatVol(ctx context.Context, volume string) (VolInfo, error) {
return VolInfo{Created: time.Now()}, nil return (&xlStorage{drivePath: d.root}).StatVol(ctx, volume)
} }
func TestCleanupStaleUploadsOnDisk(t *testing.T) { func TestCleanupStaleUploadsOnDisk(t *testing.T) {
@@ -157,7 +157,13 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) {
} }
disk := &multipartCleanupDisk{root: t.TempDir(), modTime: modTime, readErr: tc.readErr} disk := &multipartCleanupDisk{root: t.TempDir(), modTime: modTime, readErr: tc.readErr}
uploadDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha", id) uploadDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha", id)
if err := os.MkdirAll(uploadDir, 0o755); err != nil { for _, dir := range []string{uploadDir, filepath.Join(disk.root, minioMetaTmpDeletedBucket)} {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
// An old parent must not bypass a skipped upload's age or read error.
if err := os.Chtimes(filepath.Dir(uploadDir), modTime, modTime); err != nil {
t.Fatal(err) t.Fatal(err)
} }
erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
@@ -169,9 +175,60 @@ func TestCleanupStaleUploadsOnDisk(t *testing.T) {
if !os.IsNotExist(err) { if !os.IsNotExist(err) {
t.Errorf("expired upload must be removed: %v", err) t.Errorf("expired upload must be removed: %v", err)
} }
if _, err := os.Stat(filepath.Dir(uploadDir)); !os.IsNotExist(err) {
t.Errorf("empty hash directory must be removed in the same pass: %v", err)
}
} else if err != nil { } else if err != nil {
t.Errorf("upload must be preserved: %v", err) t.Errorf("upload must be preserved: %v", err)
} }
}) })
} }
} }
func TestCleanupStaleUploadsEmptyHashDirectory(t *testing.T) {
for _, age := range []time.Duration{0, 2 * globalAPIConfig.getStaleUploadsExpiry()} {
t.Run(age.String(), func(t *testing.T) {
disk := &multipartCleanupDisk{root: t.TempDir()}
shaDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha")
for _, dir := range []string{shaDir, filepath.Join(disk.root, minioMetaTmpDeletedBucket)} {
if err := os.MkdirAll(dir, 0o755); err != nil {
t.Fatal(err)
}
}
modTime := time.Now().Add(-age)
if err := os.Chtimes(shaDir, modTime, modTime); err != nil {
t.Fatal(err)
}
erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
if _, err := os.Stat(shaDir); !os.IsNotExist(err) {
t.Fatalf("empty hash directory must be removed regardless of age: %v", err)
}
})
}
}
func TestCleanupStaleUploadsSharedHashDirectory(t *testing.T) {
disk := &multipartCleanupDisk{root: t.TempDir(), readErr: errFileNotFound}
shaDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha")
now := time.Now()
expiredID := fmt.Sprintf("d16ad42a-6fe5-4a3b-8092-69983142238fx%d", now.Add(-2*globalAPIConfig.getStaleUploadsExpiry()).UnixNano())
recentID := fmt.Sprintf("4eacd477-e733-4851-b821-6fd42785ff87x%d", now.UnixNano())
for _, id := range []string{expiredID, recentID} {
if err := os.MkdirAll(filepath.Join(shaDir, id), 0o755); err != nil {
t.Fatal(err)
}
}
partPath := filepath.Join(shaDir, recentID, "part.1")
const part = "recent upload part"
if err := os.WriteFile(partPath, []byte(part), 0o600); err != nil {
t.Fatal(err)
}
erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk)
if _, err := os.Stat(filepath.Join(shaDir, expiredID)); !os.IsNotExist(err) {
t.Errorf("expired upload must be removed: %v", err)
}
if got, err := os.ReadFile(partPath); err != nil || string(got) != part {
t.Errorf("recent upload must remain intact: got %q, err %v", got, err)
}
}
@@ -14,7 +14,7 @@ import (
// Run explicitly with -benchtime=1x. Fixture creation is excluded from timing; // Run explicitly with -benchtime=1x. Fixture creation is excluded from timing;
// each request still discovers real on-disk metadata. The 16-drive/10k case // each request still discovers real on-disk metadata. The 16-drive/10k case
// deliberately exceeds the existing aggregate directory-entry budget. // deliberately exceeds the existing aggregate directory-entry budget.
func BenchmarkMultipartListingCapacity(b *testing.B) { func BenchmarkMultipartDiagnosticScanCapacity(b *testing.B) {
for _, tc := range []struct { for _, tc := range []struct {
drives, uploads int drives, uploads int
mixed, limited bool mixed, limited bool
@@ -67,7 +67,7 @@ func BenchmarkMultipartListingCapacity(b *testing.B) {
} }
b.ResetTimer() b.ResetTimer()
for range b.N { for range b.N {
got, err := z.ListMultipartUploads(b.Context(), bucket, "dir/", "", "", "", 1000) got, err := z.scanMultipartUploadsForTest(b.Context(), bucket, "dir/", "", "", "", 1000)
if tc.limited { if tc.limited {
var limited SlowDown var limited SlowDown
if !errors.As(err, &limited) || len(got.Uploads) != 0 { if !errors.As(err, &limited) || len(got.Uploads) != 0 {
@@ -83,3 +83,88 @@ func BenchmarkMultipartListingCapacity(b *testing.B) {
}) })
} }
} }
// The public path is measured with a populated cache as well as after cache
// loss. An empty post-restart response alone is not capacity acceptance.
func BenchmarkMultipartPublicListingCapacity(b *testing.B) {
for _, tc := range []struct {
drives, uploads int
sameKey bool
}{
{4, 10_000, false},
{16, 3_126, false},
{16, 10_000, true},
} {
b.Run(fmt.Sprintf("drives=%d/uploads=%d/same-key=%t", tc.drives, tc.uploads, tc.sameKey), func(b *testing.B) {
obj, dirs, err := prepareErasure(b.Context(), tc.drives)
if err != nil {
b.Fatal(err)
}
z := obj.(*erasureServerPools)
b.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
const bucket, other = "multipart-public-capacity", "multipart-public-other"
for _, name := range []string{bucket, other} {
if err := z.MakeBucket(b.Context(), name, MakeBucketOptions{}); err != nil {
b.Fatal(err)
}
}
wantCached, wantExact := 0, 1
for n := range tc.uploads {
target, key := bucket, fmt.Sprintf("dir/%05d", n)
if tc.sameKey {
key = "dir/00000"
} else if n%2 != 0 {
target = other
}
if target == bucket {
wantCached++
}
if _, err := z.NewMultipartUpload(b.Context(), target, key, ObjectOptions{}); err != nil {
b.Fatal(err)
}
}
if tc.sameKey {
wantExact = 1000
}
if tc.drives == 16 && !tc.sameKey {
_, err := z.scanMultipartUploadsForTest(b.Context(), bucket, "", "", "", "", 1000)
var limited SlowDown
if !errors.As(err, &limited) {
b.Fatalf("fixture did not reproduce the previous global budget failure: %v", err)
}
}
var directoryCalls, metadataCalls atomic.Int64
for _, set := range z.serverPools[0].sets {
original := set.getDisks
set.getDisks = func() []StorageAPI {
disks := append([]StorageAPI(nil), original()...)
for i, disk := range disks {
disks[i] = multipartListingCountingDisk{disk, &directoryCalls, &metadataCalls}
}
return disks
}
b.Cleanup(func() { set.getDisks = original })
}
measure := func(name, prefix string, want int) {
b.Run(name, func(b *testing.B) {
directoryCalls.Store(0)
metadataCalls.Store(0)
b.ResetTimer()
for range b.N {
got, err := z.ListMultipartUploads(b.Context(), bucket, prefix, "", "", "", 1000)
if err != nil || len(got.Uploads) != want {
b.Fatalf("public listing: count=%d want=%d err=%v", len(got.Uploads), want, err)
}
}
b.StopTimer()
b.ReportMetric(float64(directoryCalls.Load())/float64(b.N), "ListDir/op")
b.ReportMetric(float64(metadataCalls.Load())/float64(b.N), "ReadVersion/op")
})
}
measure("populated-cache", "", wantCached)
z.mpCache.Clear()
measure("lost-cache", "", 0)
measure("known-key", "dir/00000", wantExact)
})
}
}
+1 -1
View File
@@ -359,7 +359,7 @@ func (z *erasureServerPools) multipartPreflight(ctx context.Context) (multipartP
return multipartPreflightReport{}, err return multipartPreflightReport{}, err
} }
defer s.close() defer s.close()
report := multipartPreflightReport{Complete: true, Mode: "persistent"} report := multipartPreflightReport{Complete: true, Mode: "legacy"}
for p, pool := range z.serverPools { for p, pool := range z.serverPools {
for i, set := range pool.sets { for i, set := range pool.sets {
_, _ = set.scanMultipartUploads(s, "", p, i) _, _ = set.scanMultipartUploads(s, "", p, i)
+71 -28
View File
@@ -22,10 +22,53 @@ import (
"github.com/minio/minio/internal/config/storageclass" "github.com/minio/minio/internal/config/storageclass"
) )
// Check the real handler and storage path, including a successful abort between // The persistent scanner is retained for diagnostics. This test-only wrapper
// pages. Choose IDs increasing in both initiation time and lexical order so the // preserves its pagination coverage without exposing it through public listing.
// failure does not depend on differing interpretations of S3 marker ordering. func (z *erasureServerPools) scanMultipartUploadsForTest(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) { if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
return ListMultipartsInfo{}, err
}
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
return ListMultipartsInfo{}, toObjectErr(err, bucket)
}
scan, err := startMultipartScan(ctx, false)
if err != nil {
return ListMultipartsInfo{}, err
}
defer scan.close()
var uploads []MultipartInfo
for idx, pool := range z.serverPools {
if z.IsSuspended(idx) {
continue
}
poolUploads, err := pool.scanMultipartUploadsForTest(scan, bucket, idx)
if err != nil {
return ListMultipartsInfo{}, err
}
uploads = append(uploads, poolUploads...)
}
// Confirmed historical records lack a recoverable bucket/key and are
// omitted by the scan. They do not block listing identifiable uploads.
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
}
func (s *erasureSets) scanMultipartUploadsForTest(scan *multipartScan, bucket string, poolIdx int) ([]MultipartInfo, error) {
var uploads []MultipartInfo
for i, set := range s.sets {
setUploads, err := set.scanMultipartUploads(scan, bucket, poolIdx, i)
if err != nil {
return nil, err
}
uploads = append(uploads, setUploads...)
}
return uploads, nil
}
// Check that a successful HTTP abort removes the cached upload. The restored
// cache path still ignores page limits; this is a documented legacy limitation.
func TestMultipartListingAbortHTTPUpdatesCache(t *testing.T) {
z, _ := consistencyPools(t) z, _ := consistencyPools(t)
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads", "AbortMultipart"}, MakeBucketOptions{}) bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads", "AbortMultipart"}, MakeBucketOptions{})
if err != nil { if err != nil {
@@ -81,8 +124,8 @@ func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
return result return result
} }
first := list("", "", 1) first := list("", "", 1)
if len(first.Uploads) != 1 || first.Uploads[0].UploadID != firstID || !first.IsTruncated { if len(first.Uploads) != 3 || first.Uploads[0].UploadID != firstID || first.IsTruncated {
t.Fatalf("unexpected first page: %+v", first) t.Fatalf("unexpected legacy cache view: %+v", first)
} }
rec := request(http.MethodDelete, getAbortMultipartUploadURL("", bucket, "a", firstID)) rec := request(http.MethodDelete, getAbortMultipartUploadURL("", bucket, "a", firstID))
if rec.Code != http.StatusNoContent { if rec.Code != http.StatusNoContent {
@@ -93,12 +136,12 @@ func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
for _, upload := range rest.Uploads { for _, upload := range rest.Uploads {
keys = append(keys, upload.Key) keys = append(keys, upload.Key)
} }
t.Logf("GET first page=200; DELETE marker=204; GET next page=200 keys=%v truncated=%v", keys, rest.IsTruncated) t.Logf("GET cache=200; DELETE upload=204; GET cache=200 keys=%v truncated=%v", keys, rest.IsTruncated)
if _, err := z.GetMultipartInfo(t.Context(), bucket, "a", secondID, ObjectOptions{}); err != nil { if _, err := z.GetMultipartInfo(t.Context(), bucket, "a", secondID, ObjectOptions{}); err != nil {
t.Fatalf("remaining upload is not valid: %v", err) t.Fatalf("remaining upload is not valid: %v", err)
} }
if len(rest.Uploads) != 2 || rest.Uploads[0].UploadID != secondID { if len(rest.Uploads) != 2 || rest.Uploads[0].UploadID != secondID {
t.Fatalf("valid remaining upload for key a is missing from continuation: %+v", rest.Uploads) t.Fatalf("valid remaining upload for key a is missing from cache: %+v", rest.Uploads)
} }
} }
@@ -149,7 +192,7 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
} }
z.mpCache.Clear() z.mpCache.Clear()
for _, target := range []string{bucket, other} { for _, target := range []string{bucket, other} {
got, err := z.ListMultipartUploads(t.Context(), target, "", "", "", "", 10) got, err := z.scanMultipartUploadsForTest(t.Context(), target, "", "", "", "", 10)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -159,14 +202,14 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
requireMultipartUploadKeys(t, got) requireMultipartUploadKeys(t, got)
} }
} }
// Exact-looking prefixes now have the same durable semantics. // Diagnostic discovery cannot recover an unidentified historical key.
exact, err := z.ListMultipartUploads(t.Context(), other, "old", "", "", "", 10) exact, err := z.scanMultipartUploadsForTest(t.Context(), other, "old", "", "", "", 10)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
requireMultipartUploadKeys(t, exact) requireMultipartUploadKeys(t, exact)
report, err := z.multipartPreflight(t.Context()) report, err := z.multipartPreflight(t.Context())
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "persistent" { if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "legacy" {
t.Fatalf("legacy preflight: %+v %v", report, err) t.Fatalf("legacy preflight: %+v %v", report, err)
} }
if err = z.AbortMultipartUpload(t.Context(), other, "old", old.UploadID, ObjectOptions{}); err != nil { if err = z.AbortMultipartUpload(t.Context(), other, "old", old.UploadID, ObjectOptions{}); err != nil {
@@ -176,14 +219,14 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
if err != nil || !report.Ready || report.LegacyUploads != 0 { if err != nil || !report.Ready || report.LegacyUploads != 0 {
t.Fatalf("drained preflight: %+v %v", report, err) t.Fatalf("drained preflight: %+v %v", report, err)
} }
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
requireMultipartUploadKeys(t, got, "a") requireMultipartUploadKeys(t, got, "a")
} }
func TestMultipartListingIdentityFallback(t *testing.T) { func TestMultipartDiagnosticIdentityFallback(t *testing.T) {
for _, kind := range []string{"old", "partial", "corrupt", "read-failure", "wrong-bucket"} { for _, kind := range []string{"old", "partial", "corrupt", "read-failure", "wrong-bucket"} {
t.Run(kind, func(t *testing.T) { t.Run(kind, func(t *testing.T) {
z, set, bucket := multipartListingFixture(t) z, set, bucket := multipartListingFixture(t)
@@ -220,7 +263,7 @@ func TestMultipartListingIdentityFallback(t *testing.T) {
} }
return disks return disks
} }
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -469,7 +512,7 @@ func (d multipartListingCountingDisk) ReadVersion(ctx context.Context, original,
} }
// Counts storage API calls; this is not a deployment throughput benchmark. // Counts storage API calls; this is not a deployment throughput benchmark.
func TestMultipartListingScanCosts(t *testing.T) { func TestMultipartDiagnosticScanCosts(t *testing.T) {
obj, dirs, err := prepareErasure(t.Context(), 4) obj, dirs, err := prepareErasure(t.Context(), 4)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -507,7 +550,7 @@ func TestMultipartListingScanCosts(t *testing.T) {
t.Helper() t.Helper()
directoryCalls.Store(0) directoryCalls.Store(0)
metadataCalls.Store(0) metadataCalls.Store(0)
result, err := z.ListMultipartUploads(t.Context(), bucket, "dir/", "", "", "", 1) result, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "dir/", "", "", "", 1)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -589,7 +632,7 @@ func TestMultipartListingMissingMarkers(t *testing.T) {
} }
} }
func TestMultipartListingCoverage(t *testing.T) { func TestMultipartDiagnosticCoverage(t *testing.T) {
z, set, bucket := multipartListingFixture(t) z, set, bucket := multipartListingFixture(t)
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
if err != nil { if err != nil {
@@ -605,12 +648,12 @@ func TestMultipartListingCoverage(t *testing.T) {
} }
} }
set.getDisks = func() []StorageAPI { return []StorageAPI{disks[0], disks[1], nil, nil} } set.getDisks = func() []StorageAPI { return []StorageAPI{disks[0], disks[1], nil, nil} }
_, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) _, err = z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 { if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
t.Fatalf("incomplete discovery returned success: %v", err) t.Fatalf("incomplete discovery returned success: %v", err)
} }
set.getDisks = func() []StorageAPI { return []StorageAPI{nil, disks[1], disks[2], disks[3]} } set.getDisks = func() []StorageAPI { return []StorageAPI{nil, disks[1], disks[2], disks[3]} }
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -621,7 +664,7 @@ func TestMultipartListingCoverage(t *testing.T) {
} }
} }
func TestMultipartListingBudgetAndAdmission(t *testing.T) { func TestMultipartDiagnosticBudgetAndAdmission(t *testing.T) {
_, set, bucket := multipartListingFixture(t) _, set, bucket := multipartListingFixture(t)
if _, err := set.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil { if _, err := set.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -651,7 +694,7 @@ func TestMultipartListingBudgetAndAdmission(t *testing.T) {
} }
} }
func TestMultipartListingAdmissionHTTP(t *testing.T) { func TestMultipartPublicListingAdmissionHTTP(t *testing.T) {
z, _, _ := multipartListingFixture(t) z, _, _ := multipartListingFixture(t)
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{}) bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
if err != nil { if err != nil {
@@ -673,12 +716,12 @@ func TestMultipartListingAdmissionHTTP(t *testing.T) {
} }
rec := httptest.NewRecorder() rec := httptest.NewRecorder()
router.ServeHTTP(rec, req) router.ServeHTTP(rec, req)
var response APIErrorResponse var response ListMultipartUploadsResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil { if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if rec.Code != http.StatusServiceUnavailable || response.Code != "SlowDown" { if rec.Code != http.StatusOK || len(response.Uploads) != 0 {
t.Fatalf("admission returned %d %s", rec.Code, rec.Body.String()) t.Fatalf("diagnostic admission blocked public listing: %d %s", rec.Code, rec.Body.String())
} }
} }
@@ -705,12 +748,12 @@ func TestMultipartListingPreflightMinorityLegacy(t *testing.T) {
} }
} }
report, err := z.multipartPreflight(t.Context()) report, err := z.multipartPreflight(t.Context())
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "persistent" { if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "legacy" {
t.Fatalf("minority legacy copy must prevent readiness: %+v %v", report, err) t.Fatalf("minority legacy copy must prevent readiness: %+v %v", report, err)
} }
} }
func TestMultipartListingCancellationRetainsAdmission(t *testing.T) { func TestMultipartDiagnosticCancellationRetainsAdmission(t *testing.T) {
z, set, bucket := multipartListingFixture(t) z, set, bucket := multipartListingFixture(t)
for i := range 40 { for i := range 40 {
if _, err := z.NewMultipartUpload(t.Context(), bucket, fmt.Sprintf("key-%02d", i), ObjectOptions{}); err != nil { if _, err := z.NewMultipartUpload(t.Context(), bucket, fmt.Sprintf("key-%02d", i), ObjectOptions{}); err != nil {
@@ -749,7 +792,7 @@ func TestMultipartListingCancellationRetainsAdmission(t *testing.T) {
defer cancel() defer cancel()
done := make(chan error, 1) done := make(chan error, 1)
go func() { go func() {
_, err := z.ListMultipartUploads(ctx, bucket, "", "", "", "", 10) _, err := z.scanMultipartUploadsForTest(ctx, bucket, "", "", "", "", 10)
done <- err done <- err
}() }()
for range 16 { for range 16 {
+106
View File
@@ -0,0 +1,106 @@
// Copyright (c) 2026 Ruohang Feng
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"encoding/xml"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
)
type multipartRootScanGuard struct {
StorageAPI
roots *atomic.Int64
}
func (d multipartRootScanGuard) ListDir(ctx context.Context, original, volume, dir string, count int) ([]string, error) {
if volume == minioMetaMultipartBucket && dir == "" {
d.roots.Add(1)
return nil, errFaultyDisk
}
return d.StorageAPI.ListDir(ctx, original, volume, dir, count)
}
// A busy or unreadable diagnostic namespace must not become a prerequisite for
// ordinary listing, including the lower-level exact-object entry points.
func TestMultipartPublicListingIndependentOfDiagnostics(t *testing.T) {
for _, busy := range []bool{false, true} {
t.Run(map[bool]string{false: "unreadable-root", true: "busy-diagnostics"}[busy], func(t *testing.T) {
z, _ := consistencyPools(t)
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
if err != nil {
t.Fatal(err)
}
const key = "dir/known"
mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
var roots atomic.Int64
for _, pool := range z.serverPools {
for _, set := range pool.sets {
original := set.getDisks
set.getDisks = func() []StorageAPI {
disks := append([]StorageAPI(nil), original()...)
for i, disk := range disks {
if disk != nil {
disks[i] = multipartRootScanGuard{disk, &roots}
}
}
return disks
}
t.Cleanup(func() { set.getDisks = original })
}
}
if busy {
for range cap(multipartScanSlots) {
scan, err := startMultipartScan(t.Context(), true)
if err != nil {
t.Fatal(err)
}
defer scan.close()
}
}
for _, prefix := range []string{"", key, "dir/"} {
req, err := newTestSignedRequestV4(http.MethodGet,
getListMultipartUploadsURLWithParams("", bucket, prefix, "", "", "", "1"),
0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("prefix %q: HTTP %d %s", prefix, rec.Code, rec.Body.String())
}
var got ListMultipartUploadsResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if prefix == "dir/" {
if len(got.Uploads) != 0 {
t.Fatalf("partial prefix must retain the documented exact-key limitation: %+v", got)
}
} else if len(got.Uploads) != 1 || got.Uploads[0].UploadID != mp.UploadID {
t.Fatalf("known upload missing: %+v", got)
}
}
for _, pool := range z.serverPools {
for _, list := range []func(context.Context, string, string, string, string, string, int) (ListMultipartsInfo, error){
pool.ListMultipartUploads, pool.getHashedSet(key).ListMultipartUploads,
} {
if _, err := list(t.Context(), bucket, key, "", "", "", 10); err != nil {
t.Fatalf("lower-level exact listing: %v", err)
}
}
}
if roots.Load() != 0 {
t.Fatalf("public listing attempted %d global multipart scans", roots.Load())
}
})
}
}
+30 -30
View File
@@ -4,11 +4,10 @@
package cmd package cmd
import ( import (
"bytes"
"context" "context"
"encoding/xml"
"errors" "errors"
"net/http" "net/http"
"net/http/httptest"
"slices" "slices"
"testing" "testing"
) )
@@ -33,14 +32,10 @@ func removeMultipartIdentity(metadata map[string]string) {
delete(metadata, multipartMetaObject) delete(metadata, multipartMetaObject)
} }
// The actual S3 handler must use the same semantics for every prefix and page, // The retained diagnostic scanner handles mixed records independently of a
// even with old records and a stale, misleading node-local cache. // stale cache. Public HTTP behavior is tested separately.
func TestMultipartListingMixedHTTP(t *testing.T) { func TestMultipartDiagnosticMixed(t *testing.T) {
z, set, _ := multipartListingFixture(t) z, set, bucket := multipartListingFixture(t)
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
if err != nil {
t.Fatal(err)
}
for _, key := range []string{"dir/old", "dir/a", "dir/sub/b", "outside"} { for _, key := range []string{"dir/old", "dir/a", "dir/sub/b", "outside"} {
mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{}) mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
if err != nil { if err != nil {
@@ -54,22 +49,11 @@ func TestMultipartListingMixedHTTP(t *testing.T) {
z.mpCache.Store("stale", MultipartInfo{Bucket: bucket, Object: "ghost", UploadID: "stale"}) z.mpCache.Store("stale", MultipartInfo{Bucket: bucket, Object: "ghost", UploadID: "stale"})
list := func(prefix, keyMarker, uploadMarker, delimiter string) ListMultipartUploadsResponse { list := func(prefix, keyMarker, uploadMarker, delimiter string) ListMultipartUploadsResponse {
t.Helper() t.Helper()
req, err := newTestSignedRequestV4(http.MethodGet, got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, prefix, keyMarker, uploadMarker, delimiter, 1)
getListMultipartUploadsURLWithParams("", bucket, prefix, keyMarker, uploadMarker, delimiter, "1"),
0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
rec := httptest.NewRecorder() return generateListMultipartUploadsResponse(bucket, got, "")
router.ServeHTTP(rec, req)
var got ListMultipartUploadsResponse
if rec.Code != http.StatusOK {
t.Fatalf("list %q: HTTP %d %s", prefix, rec.Code, rec.Body.String())
}
if err := xml.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
return got
} }
var keys []string var keys []string
keyMarker, uploadMarker := "", "" keyMarker, uploadMarker := "", ""
@@ -103,7 +87,7 @@ func TestMultipartListingMixedHTTP(t *testing.T) {
} }
} }
func TestMultipartListingInvalidIdentity(t *testing.T) { func TestMultipartDiagnosticInvalidIdentity(t *testing.T) {
for _, kind := range []string{"missing-bucket", "missing-object", "empty-bucket", "empty-object", "empty-both", "wrong-hash", "read-failure"} { for _, kind := range []string{"missing-bucket", "missing-object", "empty-bucket", "empty-object", "empty-both", "wrong-hash", "read-failure"} {
t.Run(kind, func(t *testing.T) { t.Run(kind, func(t *testing.T) {
z, set, bucket := multipartListingFixture(t) z, set, bucket := multipartListingFixture(t)
@@ -142,7 +126,7 @@ func TestMultipartListingInvalidIdentity(t *testing.T) {
return disks return disks
} }
} }
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
if err == nil || len(got.Uploads) != 0 || toAPIError(t.Context(), err).HTTPStatusCode != http.StatusServiceUnavailable { if err == nil || len(got.Uploads) != 0 || toAPIError(t.Context(), err).HTTPStatusCode != http.StatusServiceUnavailable {
t.Fatalf("invalid/unreadable record returned a successful page: %+v %v", got, err) t.Fatalf("invalid/unreadable record returned a successful page: %+v %v", got, err)
} }
@@ -185,7 +169,14 @@ func TestMultipartListingRestart(t *testing.T) {
if _, cached := obj.(*erasureServerPools).mpCache.Load(mp.UploadID); cached { if _, cached := obj.(*erasureServerPools).mpCache.Load(mp.UploadID); cached {
t.Fatal("restart unexpectedly preserved the upload cache") t.Fatal("restart unexpectedly preserved the upload cache")
} }
got, err := obj.ListMultipartUploads(t.Context(), bucket, "dir/", "", "", "", 10) for _, prefix := range []string{"", "dir/"} {
got, err := obj.ListMultipartUploads(t.Context(), bucket, prefix, "", "", "", 10)
if err != nil {
t.Fatal(err)
}
requireMultipartUploadKeys(t, got)
}
got, err := obj.ListMultipartUploads(t.Context(), bucket, key, "", "", "", 10)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -193,9 +184,18 @@ func TestMultipartListingRestart(t *testing.T) {
if got.Uploads[0].UploadID != mp.UploadID { if got.Uploads[0].UploadID != mp.UploadID {
t.Fatal("restart changed upload identity") t.Fatal("restart changed upload identity")
} }
part, err := obj.PutObjectPart(t.Context(), bucket, key, mp.UploadID, 1,
mustGetPutObjReader(t, bytes.NewBufferString("part"), 4, "", ""), ObjectOptions{})
if err != nil {
t.Fatalf("continue known upload after restart: %v", err)
}
if _, err := obj.CompleteMultipartUpload(t.Context(), bucket, key, mp.UploadID,
[]CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{}); err != nil {
t.Fatalf("complete known upload after restart: %v", err)
}
} }
func TestMultipartListingConflictingIdentityCopies(t *testing.T) { func TestMultipartDiagnosticConflictingIdentityCopies(t *testing.T) {
for _, oldCopies := range []int{1, 3} { for _, oldCopies := range []int{1, 3} {
z, set, bucket := multipartListingFixture(t) z, set, bucket := multipartListingFixture(t)
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
@@ -213,14 +213,14 @@ func TestMultipartListingConflictingIdentityCopies(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
if !errors.Is(err, errMultipartListingIdentity) || len(got.Uploads) != 0 { if !errors.Is(err, errMultipartListingIdentity) || len(got.Uploads) != 0 {
t.Fatalf("%d contradictory copies returned a successful page: %+v %v", oldCopies, got, err) t.Fatalf("%d contradictory copies returned a successful page: %+v %v", oldCopies, got, err)
} }
} }
} }
func TestMultipartListingUnsupportedStoredID(t *testing.T) { func TestMultipartDiagnosticUnsupportedStoredID(t *testing.T) {
for _, historical := range []bool{false, true} { for _, historical := range []bool{false, true} {
z, set, bucket := multipartListingFixture(t) z, set, bucket := multipartListingFixture(t)
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
@@ -244,7 +244,7 @@ func TestMultipartListingUnsupportedStoredID(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
} }
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) got, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 10)
if historical { if historical {
if err != nil || len(got.Uploads) != 0 { if err != nil || len(got.Uploads) != 0 {
t.Fatalf("unidentified historical ID: %+v %v", got, err) t.Fatalf("unidentified historical ID: %+v %v", got, err)
+9 -24
View File
@@ -197,6 +197,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
drivePath := disk.Endpoint().Path drivePath := disk.Endpoint().Path
readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error { readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
hasSkippedUploads := false
readDirFn(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error { readDirFn(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error {
uploadIDPath := pathJoin(shaDir, uploadIDDir) uploadIDPath := pathJoin(shaDir, uploadIDDir)
modTime := extractModTimeFromUploadIDDir(uploadIDDir) modTime := extractModTimeFromUploadIDDir(uploadIDDir)
@@ -205,12 +206,14 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
wait := deleteMultipartCleanupSleeper.Timer(ctx) wait := deleteMultipartCleanupSleeper.Timer(ctx)
fi, err := disk.ReadVersion(ctx, "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{}) fi, err := disk.ReadVersion(ctx, "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{})
if err != nil { if err != nil {
hasSkippedUploads = true
return nil return nil
} }
modTime = fi.ModTime modTime = fi.ModTime
wait() wait()
} }
if time.Since(modTime) < globalAPIConfig.getStaleUploadsExpiry() { if time.Since(modTime) < globalAPIConfig.getStaleUploadsExpiry() {
hasSkippedUploads = true
return nil return nil
} }
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
@@ -223,24 +226,15 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
return nil return nil
}) })
}) })
// Get the modtime of the shaDir. if hasSkippedUploads {
vi, err := disk.StatVol(ctx, pathJoin(minioMetaMultipartBucket, shaDir))
if err != nil {
return nil
}
// Modtime is returned in the Created field. See (*xlStorage).StatVol
if time.Since(vi.Created) < globalAPIConfig.getStaleUploadsExpiry() {
return nil return nil
} }
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
return w.Run(func() error { return w.Run(func() error {
wait := deleteMultipartCleanupSleeper.Timer(ctx) wait := deleteMultipartCleanupSleeper.Timer(ctx)
pathUUID := mustGetUUID() // Removing an upload updates the parent's mtime. Remove the parent
targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID) // immediately if it is empty; other uploads keep it in place.
_ = Remove(pathJoin(drivePath, minioMetaMultipartBucket, shaDir))
// We are not deleting shaDir recursively here, if shaDir is empty
// and its older then we can happily delete it.
Rename(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), targetPath)
wait() wait()
return nil return nil
}) })
@@ -414,7 +408,7 @@ func paginateMultipartUploads(uploads []MultipartInfo, prefix, keyMarker, upload
} }
// listMultipartUploadsExact preserves the hashed exact-object lookup used by // listMultipartUploadsExact preserves the hashed exact-object lookup used by
// multipart write placement. Public listing always uses the durable scan. // multipart write placement and nonempty public listing prefixes.
func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) { func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er) auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er)
@@ -522,16 +516,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, prefi
if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil { if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
return ListMultipartsInfo{}, err return ListMultipartsInfo{}, err
} }
scan, err := startMultipartScan(ctx, false) return er.listMultipartUploadsExact(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
if err != nil {
return ListMultipartsInfo{}, err
}
defer scan.close()
uploads, err := er.scanMultipartUploads(scan, bucket, 0, 0)
if err != nil {
return ListMultipartsInfo{}, err
}
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
} }
// newMultipartUpload - wrapper for initializing a new multipart // newMultipartUpload - wrapper for initializing a new multipart
+24
View File
@@ -1891,6 +1891,23 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
if err == nil { if err == nil {
continue continue
} }
if err == errFileNotFound || err == errFileVersionNotFound {
// A retry after partial physical deletion may mix removals
// with already-absent replies. Both prove the desired outcome.
// Marker creation and replication-state updates still require
// successful writes; absence cannot acknowledge those operations.
physical := true
for _, v := range dedupVersions[i].Versions {
if v.Deleted || v.MarkDeleted || v.ReplicationState.ReplicationStatusInternal != "" ||
v.ReplicationState.VersionPurgeStatusInternal != "" || v.ReplicationState.ReplicaStatus != "" {
physical = false
break
}
}
if physical {
continue
}
}
for _, v := range dedupVersions[i].Versions { for _, v := range dedupVersions[i].Versions {
delObjErrs[index][v.Idx] = err delObjErrs[index][v.Idx] = err
} }
@@ -2195,6 +2212,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
} }
} }
if opts.DataMovement && opts.DeleteMarker {
// Moving a null marker replaces any older null data/marker. Moving
// a UUID marker preserves it on retries instead of purging it.
deleteMarker = true
markDelete = opts.VersionID != nullVersionID
}
purge := opts.isVersionPurge() purge := opts.isVersionPurge()
if purge { if purge {
// A delete marker describes the version being removed; it is not an // A delete marker describes the version being removed; it is not an
+309
View File
@@ -0,0 +1,309 @@
//go:build consistency_strict
// Preserved diagnostic for the deferred strict-read policy. This is not a
// maintenance release gate; failures record intentionally unresolved risks.
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"fmt"
"net/http"
"sync"
"testing"
"time"
madmin "github.com/minio/madmin-go/v3"
)
// All metadata is valid and readable. Only the completed generations differ.
// No timing, missing files, synthetic read error or malformed metadata is needed.
func TestReadGenerationConflictIsNotAbsence(t *testing.T) {
raw := make([]RawFileInfo, 8)
for i := range raw {
fi := newFileInfo("object", 4, 4)
fi.Erasure.Index = i + 1
fi.ModTime = time.Unix(100+int64(i%4), 0)
fi.DataDir = mustGetUUID()
fi.Metadata = map[string]string{"etag": fmt.Sprint(i % 4)}
var meta xlMetaV2
if err := meta.AddVersion(fi); err != nil {
t.Fatal(err)
}
buf, err := meta.AppendTo(nil)
if err != nil {
t.Fatal(err)
}
raw[i] = RawFileInfo{Buf: buf}
}
_, errs := pickLatestQuorumFilesInfo(t.Context(), raw, make([]error, len(raw)), "bucket", "object", true, false)
err := reduceReadQuorumErrs(t.Context(), errs, objectOpIgnoredErrs, 4)
if api := toAPIError(t.Context(), toObjectErr(err, "bucket", "object")); api.HTTPStatusCode != http.StatusServiceUnavailable {
t.Fatalf("readable conflicting generations reported as absence: %v (%+v)", err, api)
}
}
func TestPoolsReadGenerationConflictDoesNotExposeOldValue(t *testing.T) {
z, bucket := consistencyPools(t)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
router := initTestAPIEndPoints(z, []string{"GetObject", "HeadObject"})
for _, tc := range []struct {
pool int
mode string
}{
{0, "split"},
{1, "split"},
{0, "older-quorum"},
{1, "older-quorum"},
{0, "newer-quorum"},
{1, "newer-quorum"},
} {
t.Run(fmt.Sprintf("%s/pool-%d", tc.mode, tc.pool), func(t *testing.T) {
broken := tc.pool
object := fmt.Sprintf("conflict-%s-%d", tc.mode, broken)
base := UTCNow().Add(-time.Hour)
putConsistencyObject(t, z, bucket, object, 1-broken, "old-acknowledged", ObjectOptions{MTime: base})
set := z.serverPools[broken].getHashedSet(object)
disks := set.getDisks()
saved := make([][]byte, len(disks))
savedParts := make([]map[string][]byte, len(disks))
for generation := range 4 {
putConsistencyObject(t, z, bucket, object, broken, fmt.Sprintf("newer-%d", generation),
ObjectOptions{MTime: base.Add(time.Duration(generation+1) * time.Minute)})
for i, disk := range disks {
g := i % 4
switch tc.mode {
case "older-quorum":
g = 0
if i >= len(disks)-3 {
g = i - (len(disks) - 3) + 1
}
case "newer-quorum":
g = 3
if i < 3 {
g = i
}
}
if g != generation {
continue
}
buf, err := disk.ReadAll(ctx, bucket, pathJoin(object, xlStorageFormatFile))
if err != nil {
t.Fatal(err)
}
saved[i] = bytes.Clone(buf)
savedParts[i] = readGenerationParts(t, disk, bucket, object, buf)
}
}
for i, disk := range disks {
for path, data := range savedParts[i] {
if err := disk.WriteAll(ctx, bucket, path, data); err != nil {
t.Fatal(err)
}
}
if err := disk.WriteAll(ctx, bucket, pathJoin(object, xlStorageFormatFile), saved[i]); err != nil {
t.Fatal(err)
}
}
for _, version := range []string{"", nullVersionID} {
for _, method := range []string{http.MethodHead, http.MethodGet} {
rec := consistencyRequest(t, router, method, bucket, object, version)
want := http.StatusServiceUnavailable
if tc.mode == "newer-quorum" {
want = http.StatusOK
}
if rec.Code != want {
t.Errorf("%s version=%q: want status=%d, got=%d body=%q", method, version, want, rec.Code, rec.Body.String())
}
if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != "newer-3" {
t.Errorf("latest quorum did not return its payload: %q", rec.Body.String())
}
}
}
if tc.mode != "newer-quorum" {
for _, version := range []string{"", nullVersionID} {
_, err := set.CopyObject(ctx, bucket, object, bucket, object,
ObjectInfo{metadataOnly: true, UserDefined: map[string]string{"copy-test": "conflicting-source"}},
ObjectOptions{VersionID: version}, ObjectOptions{})
if !isErrReadQuorum(err) {
t.Errorf("COPY-to-self version=%q: want read-quorum error, got %v", version, err)
}
for _, scan := range []madmin.HealScanMode{madmin.HealNormalScan, madmin.HealDeepScan} {
_, err := set.HealObject(ctx, bucket, object, version, madmin.HealOpts{ScanMode: scan, Remove: true})
if !isErrReadQuorum(err) {
t.Errorf("heal version=%q scan=%v: want read-quorum error, got %v", version, scan, err)
}
}
}
}
for i, disk := range disks {
buf, err := disk.ReadAll(ctx, bucket, pathJoin(object, xlStorageFormatFile))
if err != nil || !bytes.Equal(buf, saved[i]) {
t.Errorf("read or heal destroyed conflicting generation on disk %d: %v", i, err)
}
if tc.mode != "newer-quorum" {
for path, data := range savedParts[i] {
got, err := disk.ReadAll(ctx, bucket, path)
if err != nil || !bytes.Equal(got, data) {
t.Errorf("read or heal destroyed generation shard on disk %d: %s: %v", i, path, err)
}
}
}
}
})
}
}
func readGenerationParts(t *testing.T, disk StorageAPI, bucket, object string, metadata []byte) map[string][]byte {
t.Helper()
fi, err := fileInfoFromRaw(RawFileInfo{Buf: metadata}, bucket, object, false, false)
if err != nil {
t.Fatal(err)
}
parts := make(map[string][]byte)
if fi.InlineData() || fi.Size == 0 {
return parts
}
// Later PUTs remove the previous data directory. Save its real bitrot
// shards too, so restoring metadata never fabricates missing-part damage.
for _, part := range fi.Parts {
path := pathJoin(object, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
data, err := disk.ReadAll(t.Context(), bucket, path)
if err != nil {
t.Fatal(err)
}
parts[path] = data
}
return parts
}
func TestPoolsReadUnavailablePoolDoesNotExposeOldValue(t *testing.T) {
z, bucket := consistencyPools(t)
for broken := range 2 {
t.Run(fmt.Sprintf("broken-pool-%d", broken), func(t *testing.T) {
object := fmt.Sprintf("unavailable-%d", broken)
putConsistencyObject(t, z, bucket, object, 1-broken, "old-acknowledged", ObjectOptions{})
set := z.serverPools[broken].getHashedSet(object)
disks, switchDisks := installListingDiskSwitch(set)
defer switchDisks(disks)
faulty := append([]StorageAPI(nil), disks...)
for i := range faulty {
faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
}
switchDisks(faulty)
if _, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err == nil || isErrObjectNotFound(err) {
t.Errorf("unknown pool state allowed an old value: %v", err)
}
})
}
}
type generationReadGateDisk struct {
StorageAPI
ready chan<- struct{}
release <-chan struct{}
}
func (d generationReadGateDisk) ReadXL(ctx context.Context, volume, path string, data bool) (RawFileInfo, error) {
fi, err := d.StorageAPI.ReadXL(ctx, volume, path, data)
d.ready <- struct{}{}
if d.release != nil {
<-d.release
}
return fi, err
}
func (d generationReadGateDisk) ReadVersion(ctx context.Context, orig, volume, path, version string, opts ReadOptions) (FileInfo, error) {
fi, err := d.StorageAPI.ReadVersion(ctx, orig, volume, path, version, opts)
d.ready <- struct{}{}
if d.release != nil {
<-d.release
}
return fi, err
}
func TestReadGenerationFastInfoWaitsForSlowDisk(t *testing.T) {
for _, version := range []string{"", nullVersionID} {
t.Run("version="+version, func(t *testing.T) {
z, bucket := consistencyPools(t)
const object = "slow-newer-generation"
set := z.serverPools[0].getHashedSet(object)
disks, switchDisks := installListingDiskSwitch(set)
defer switchDisks(disks)
base := UTCNow().Add(-time.Hour)
putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: base})
saved := make([][]byte, len(disks)-1)
for i := range saved {
saved[i] = mustReadNullQuorumMeta(t, disks[i], bucket, object)
}
putConsistencyObject(t, z, bucket, object, 0, "new", ObjectOptions{MTime: base.Add(time.Minute)})
for i, meta := range saved {
if err := disks[i].WriteAll(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), meta); err != nil {
t.Fatal(err)
}
}
ready, release := make(chan struct{}, len(disks)), make(chan struct{})
var once sync.Once
unblock := func() { once.Do(func() { close(release) }) }
defer unblock()
wrapped := append([]StorageAPI(nil), disks...)
for i, disk := range disks {
gate := generationReadGateDisk{StorageAPI: disk, ready: ready}
if i == len(disks)-1 {
gate.release = release
}
wrapped[i] = gate
}
switchDisks(wrapped)
done := make(chan error, 1)
go func() {
_, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: true, VersionID: version}, false)
done <- err
}()
for range disks {
select {
case <-ready:
case <-time.After(5 * time.Second):
t.Fatal("metadata reads did not reach the barrier")
}
}
early := false
select {
case err := <-done:
early = true
t.Errorf("fast read returned before the newer disk: %v", err)
case <-time.After(100 * time.Millisecond):
}
unblock()
if !early {
select {
case err := <-done:
if !isErrReadQuorum(err) {
t.Fatalf("fast read exposed the old generation: %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("fast read did not finish after the newer disk")
}
}
})
}
}
+364 -1
View File
@@ -27,6 +27,7 @@ import (
madmin "github.com/minio/madmin-go/v3" madmin "github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/lifecycle"
xhttp "github.com/minio/minio/internal/http" xhttp "github.com/minio/minio/internal/http"
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
) )
// objectPoolInfos reads the addressed version in every pool, including pools // objectPoolInfos reads the addressed version in every pool, including pools
@@ -62,6 +63,257 @@ func (z *erasureServerPools) objectPoolInfos(ctx context.Context, bucket, object
return copies, nil return copies, nil
} }
// checkDataMovementCopies rejects moving a version when another pool holds a
// newer copy of it, which reads return instead. A cross-pool overwrite can
// leave such a copy, and a client write can add one while the move copies.
// Equal timestamps normally identify an interrupted move. Conflicting ties
// retain the same pool-order winner as reads. While the caller holds the
// pools-layer object lock, the returned resolver serves the set layer's
// reconciliation of this version from the same reads. Without opts.NoLock
// it takes the read lock itself.
func (z *erasureServerPools) checkDataMovementCopies(ctx context.Context, bucket, object string, opts ObjectOptions) (GetObjectInfoFn, error) {
if opts.MTime.IsZero() {
// The set layer would stamp the moved copy as the newest one.
return nil, errInvalidArgument
}
if !opts.NoLock {
lk := z.NewNSLock(bucket, object)
lkctx, err := lk.GetRLock(ctx, globalOperationTimeout)
if err != nil {
return nil, err
}
ctx = lkctx.Context()
defer lk.RUnlock(lkctx)
}
versionID := opts.VersionID
if versionID == "" {
versionID = nullVersionID
}
copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{VersionID: versionID, NoAuditLog: true})
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
return nil, err
}
// The snapshot may have been purged or replaced after the reader opened.
// In that case the walker must leave the current source state alone.
sourcePresent := false
for _, copy := range copies {
if copy.Index == opts.SrcPoolIdx && copy.ObjInfo.ModTime.Equal(opts.MTime) && copy.ObjInfo.DeleteMarker == opts.DeleteMarker {
sourcePresent = true
}
}
if !sourcePresent {
return nil, VersionNotFound{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
}
for _, copy := range copies {
if copy.Index != opts.SrcPoolIdx && copy.ObjInfo.ModTime.After(opts.MTime) {
return nil, DataMovementOverwriteErr{Bucket: bucket, Object: object, VersionID: opts.VersionID, Err: errDataMovementSuperseded}
}
}
if winner := copies[0]; winner.Index != opts.SrcPoolIdx && winner.ObjInfo.ModTime.Equal(opts.MTime) {
for _, copy := range copies {
if copy.Index == opts.SrcPoolIdx && (copy.ObjInfo.ETag != winner.ObjInfo.ETag || copy.ObjInfo.DeleteMarker != winner.ObjInfo.DeleteMarker || copy.ObjInfo.Size != winner.ObjInfo.Size) {
return nil, DataMovementOverwriteErr{Bucket: bucket, Object: object, VersionID: opts.VersionID, Err: errDataMovementSuperseded}
}
}
}
return func(ctx context.Context, b, o string, ropts ObjectOptions) (ObjectInfo, error) {
if ropts.VersionID == "" {
ropts.VersionID = nullVersionID
}
if b != bucket || o != object || ropts.VersionID != versionID {
return z.replicaObjectInfo(ctx, b, o, ropts)
}
return replicaCopiesInfo(copies, err, bucket, object, versionID)
}, nil
}
type migrationExpiredVersion struct {
version FileInfo
event lifecycle.Event
lifecycleUpdatedAt time.Time
source lcEventSrc
}
// applyMigrationExpiry runs the logical expiry path before migration retires
// any remaining source copies. This keeps replication and tier cleanup under
// their normal ownership and makes lifecycle configuration changes fail safe.
func (z *erasureServerPools) applyMigrationExpiry(ctx context.Context, bucket, object string, source int, expired []migrationExpiredVersion) error {
if len(expired) == 0 {
return nil
}
lc, lifecycleUpdatedAt, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
if err != nil {
return PreConditionFailed{}
}
for _, pending := range expired {
if !pending.lifecycleUpdatedAt.Equal(lifecycleUpdatedAt) {
return PreConditionFailed{}
}
}
lr, err := globalBucketObjectLockSys.Get(bucket)
if err != nil {
return err
}
rcfg, err := getReplicationConfig(ctx, bucket)
if err != nil {
return err
}
encodedObject := encodeDirObject(object)
seenDeleteAll := false
eventSource := lcEventSrc_Decom
for _, pending := range expired {
if pending.source != lcEventSrc_None {
eventSource = pending.source
break
}
}
for _, pending := range expired {
if pending.event.Action.DeleteAll() {
if seenDeleteAll {
continue
}
seenDeleteAll = true
}
versionID := pending.version.VersionID
if versionID == "" {
versionID = nullVersionID
}
copies, err := z.objectPoolInfos(ctx, bucket, encodedObject, ObjectOptions{
VersionID: versionID,
NoAuditLog: true,
})
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
continue
}
if err != nil {
return err
}
current := copies[0].ObjInfo
for _, copy := range copies {
if copy.Index == source {
current = copy.ObjInfo
break
}
}
current.Bucket = bucket
current.Name = object
currentEvent := evalActionFromLifecycle(ctx, *lc, lr, rcfg, current)
if currentEvent.Action != pending.event.Action {
return PreConditionFailed{}
}
var ok bool
if current.TransitionedObject.Status != "" && !pending.event.Action.DeleteAll() {
ok = applyExpiryOnTransitionedObject(ctx, z, current, pending.event, eventSource)
} else {
ok = applyExpiryOnNonTransitionedObjects(ctx, z, current, pending.event, eventSource)
}
if ok {
continue
}
// A concurrent expiry may have removed the addressed version after
// the first lookup. Anything still present means the logical delete
// was not committed and migration must stop before purging its source.
if _, err := z.objectPoolInfos(ctx, bucket, encodedObject, ObjectOptions{VersionID: versionID, NoAuditLog: true}); err == nil {
return PreConditionFailed{}
} else if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
return err
}
}
return nil
}
// removeMovedVersions retires only the source versions in the walker's
// snapshot. A concurrent delete can create a new marker in the source pool;
// deleting the whole prefix would remove that marker and expose moved data.
// Versions in expired were not moved because ILM deletes them. They must stay
// in the source until the ordinary expiry path has deleted them: that path
// owns replication state and tier cleanup, and it re-evaluates the current
// lifecycle rule. A migration cleanup must never turn an asynchronous ILM
// decision into an unconditional physical purge.
func (z *erasureServerPools) removeMovedVersions(ctx context.Context, bucket, object string, source int, versions []FileInfo, expired []migrationExpiredVersion) error {
if err := z.applyMigrationExpiry(ctx, bucket, object, source, expired); err != nil {
return err
}
object = encodeDirObject(object)
lk := z.NewNSLock(bucket, object)
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
if err != nil {
return err
}
ctx = lkctx.Context()
defer lk.Unlock(lkctx)
for _, version := range versions {
vid := version.VersionID
if vid == "" {
vid = nullVersionID
}
copies, err := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{VersionID: vid, NoAuditLog: true})
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
continue
}
if err != nil {
return err
}
var src *ObjectInfo
var retained []PoolObjInfo
for _, copy := range copies {
if copy.Index == source {
src = &copy.ObjInfo
} else {
retained = append(retained, copy)
}
}
if src == nil {
continue
}
if !src.ModTime.Equal(version.ModTime) || src.DeleteMarker != version.Deleted {
return PreConditionFailed{}
}
if isExpiredVersion(expired, version) {
// The logical expiry path owns this version. Keeping the source is
// also the retry point if that path failed or the pool check races it.
continue
}
// Successful movement is not enough if the destination changed again.
// Keep the source unless another copy still preserves its read result.
if len(retained) == 0 || retained[0].ObjInfo.ModTime.Before(src.ModTime) {
return PreConditionFailed{}
}
_, err = z.serverPools[source].DeleteObject(ctx, bucket, object, ObjectOptions{
VersionID: vid, NoLock: true, NoAuditLog: true,
SkipFreeVersion: true, // preserve the old prefix cleanup's tier-GC behavior
})
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
return err
}
}
return nil
}
// ilmDeletesVersionOnly reports whether the ILM action queued for a version
// that data movement skipped deletes that version and nothing else, rather
// than every version of the object or only the restored copy of a
// transitioned version. Such a version is passed to removeMovedVersions.
func ilmDeletesVersionOnly(action lifecycle.Action) bool {
return action == lifecycle.DeleteAction || action == lifecycle.DeleteVersionAction
}
func isExpiredVersion(expired []migrationExpiredVersion, version FileInfo) bool {
for _, pending := range expired {
fi := pending.version
if fi.VersionID == version.VersionID && fi.ModTime.Equal(version.ModTime) && fi.Deleted == version.Deleted {
return true
}
}
return false
}
// Healing also commits object metadata. Both queued healing and drive healing // Healing also commits object metadata. Both queued healing and drive healing
// must use the same namespace as pooled writes, rather than racing them under // must use the same namespace as pooled writes, rather than racing them under
// a destination set's independent namespace. // a destination set's independent namespace.
@@ -137,11 +389,15 @@ func (z *erasureServerPools) replicaObjectInfo(ctx context.Context, bucket, obje
opts.VersionID = nullVersionID opts.VersionID = nullVersionID
} }
copies, err := z.objectPoolInfos(ctx, bucket, object, opts) copies, err := z.objectPoolInfos(ctx, bucket, object, opts)
return replicaCopiesInfo(copies, err, bucket, object, opts.VersionID)
}
func replicaCopiesInfo(copies []PoolObjInfo, err error, bucket, object, versionID string) (ObjectInfo, error) {
if err != nil { if err != nil {
return ObjectInfo{}, err return ObjectInfo{}, err
} }
if copies[0].ObjInfo.DeleteMarker { if copies[0].ObjInfo.DeleteMarker {
return copies[0].ObjInfo, MethodNotAllowed{Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID} return copies[0].ObjInfo, MethodNotAllowed{Bucket: bucket, Object: decodeDirObject(object), VersionID: versionID}
} }
return mergedPoolObjectInfo(copies), nil return mergedPoolObjectInfo(copies), nil
} }
@@ -410,3 +666,110 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
oi.replicationDecision = opts.DeleteReplication.ReplicateDecisionStr oi.replicationDecision = opts.DeleteReplication.ReplicateDecisionStr
return oi, err return oi, err
} }
// deleteObjectsReconciled removes data while retaining each item's authoritative
// copy until every other pool has completed its cleanup. The caller holds the
// pools-layer write locks and separates marker creation from this batch.
func (z *erasureServerPools) deleteObjectsReconciled(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
deleted := make([]DeletedObject, len(objects))
errs := make([]error, len(objects))
primary := make([]int, len(objects))
tiered := make([]bool, len(objects))
lookup := errgroup.WithNErrs(len(objects)).WithConcurrency(32)
for i, obj := range objects {
// Preserve the full request identity even if lookup or cleanup fails.
// The HTTP handler includes replication state in its response-index key.
deleted[i] = DeletedObject{
ObjectName: obj.ObjectName,
VersionID: obj.VersionID,
ReplicationState: obj.ReplicationState(),
}
primary[i] = -1
lookup.Go(func() error {
copies, err := z.objectPoolInfos(ctx, bucket, obj.ObjectName, ObjectOptions{VersionID: obj.VersionID, NoAuditLog: true})
if err == nil {
primary[i] = copies[0].Index
for _, copy := range copies {
tiered[i] = tiered[i] || copy.ObjInfo.TransitionedObject.Status == lifecycle.TransitionComplete
}
} else if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
if isErrReadQuorum(err) {
err = InsufficientWriteQuorum{Bucket: bucket, Object: decodeDirObject(obj.ObjectName)}
}
errs[i] = err
}
return nil
}, i)
}
lookup.Wait()
type poolBatch struct {
objects []ObjectToDelete
indices []int
deleted []DeletedObject
errs []error
}
for phase := 0; phase < 2; phase++ {
batches := make([]poolBatch, len(z.serverPools))
for i, obj := range objects {
if errs[i] != nil || tiered[i] {
continue
}
for p := range z.serverPools {
if (p == primary[i]) != (phase == 1) {
continue
}
batches[p].objects = append(batches[p].objects, obj)
batches[p].indices = append(batches[p].indices, i)
}
}
// Each phase still uses the set layer's bulk disk I/O and runs pools
// concurrently. Include apparently absent pools in cleanup: read-quorum
// absence is not proof that an offline drive holds no remaining copy.
group := errgroup.WithNErrs(len(z.serverPools))
for p, pool := range z.serverPools {
if len(batches[p].objects) == 0 {
continue
}
group.Go(func() error {
batches[p].deleted, batches[p].errs = pool.DeleteObjects(ctx, bucket, batches[p].objects, opts)
return nil
}, p)
}
group.Wait()
for _, batch := range batches {
for j, i := range batch.indices {
if errs[i] != nil {
continue
}
if batch.errs[j] != nil {
errs[i] = batch.errs[j]
} else if phase == 1 || primary[i] == -1 {
deleted[i] = batch.deleted[j]
}
}
}
}
// Tiered copies can share one remote object, including restored copies.
// Reuse the existing reconciliation which suppresses premature tier GC.
// Serializing this uncommon path also keeps duplicate key/version entries
// in one request from racing their metadata updates under the shared lock.
for i, obj := range objects {
if !tiered[i] || errs[i] != nil {
continue
}
deleteOpts := opts
deleteOpts.VersionID, deleteOpts.NoLock = obj.VersionID, true
deleteOpts.DeleteReplication = obj.ReplicationState()
if opts.PrefixEnabledFn != nil {
deleteOpts.Versioned = opts.PrefixEnabledFn(obj.ObjectName)
}
_, err := z.deleteObjectReconciled(ctx, bucket, obj.ObjectName, deleteOpts)
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
errs[i] = err
} else {
deleted[i].found = true
}
}
return deleted, errs
}
+205 -5
View File
@@ -1071,6 +1071,8 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
restored bool restored bool
unconditional bool unconditional bool
skipFreeVersion bool skipFreeVersion bool
bulk bool
unversioned bool
}{ }{
{name: "metadata-copy"}, {name: "metadata-copy"},
{name: "restored-metadata-copy", restored: true}, {name: "restored-metadata-copy", restored: true},
@@ -1081,6 +1083,14 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
{name: "ordinary-failed-primary-delete", deleting: true, unconditional: true, failPrimaryDelete: true}, {name: "ordinary-failed-primary-delete", deleting: true, unconditional: true, failPrimaryDelete: true},
{name: "ordinary-successful-delete", deleting: true, unconditional: true}, {name: "ordinary-successful-delete", deleting: true, unconditional: true},
{name: "ordinary-skip-free-version", deleting: true, unconditional: true, skipFreeVersion: true}, {name: "ordinary-skip-free-version", deleting: true, unconditional: true, skipFreeVersion: true},
{name: "bulk-failed-primary-delete", deleting: true, bulk: true, failPrimaryDelete: true},
{name: "bulk-restored-failed-primary-delete", deleting: true, bulk: true, restored: true, failPrimaryDelete: true},
{name: "bulk-successful-delete", deleting: true, bulk: true},
{name: "bulk-distinct-reference", deleting: true, bulk: true, differentRemote: true},
{name: "bulk-skip-free-version", deleting: true, bulk: true, skipFreeVersion: true},
{name: "unversioned-failed-primary-delete", deleting: true, unconditional: true, unversioned: true, failPrimaryDelete: true},
{name: "unversioned-successful-delete", deleting: true, unconditional: true, unversioned: true},
{name: "bulk-unversioned-failed-primary-delete", deleting: true, bulk: true, unversioned: true, failPrimaryDelete: true},
} { } {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
z, bucket := consistencyPools(t) z, bucket := consistencyPools(t)
@@ -1094,13 +1104,13 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
if test.restored { if test.restored {
metadata[xhttp.AmzRestore] = completedRestoreObj(time.Now().Add(time.Hour)).String() metadata[xhttp.AmzRestore] = completedRestoreObj(time.Now().Add(time.Hour)).String()
} }
oi := putConsistencyObject(t, z, bucket, object, 0, "data", ObjectOptions{Versioned: true, UserDefined: metadata}) oi := putConsistencyObject(t, z, bucket, object, 0, "data", ObjectOptions{Versioned: !test.unversioned, UserDefined: metadata})
secondaryMetadata := maps.Clone(metadata) secondaryMetadata := maps.Clone(metadata)
if test.differentRemote { if test.differentRemote {
secondaryMetadata[ReservedMetadataPrefixLower+TransitionedObjectName] = "other-remote-object" secondaryMetadata[ReservedMetadataPrefixLower+TransitionedObjectName] = "other-remote-object"
} }
putConsistencyObject(t, z, bucket, object, 1, "data", ObjectOptions{ putConsistencyObject(t, z, bucket, object, 1, "data", ObjectOptions{
Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime, UserDefined: secondaryMetadata, Versioned: !test.unversioned, VersionID: oi.VersionID, MTime: oi.ModTime, UserDefined: secondaryMetadata,
}) })
current, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: oi.VersionID}) current, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: oi.VersionID})
if err != nil || current.TransitionedObject.Status != "complete" || current.IsRemote() == test.restored { if err != nil || current.TransitionedObject.Status != "complete" || current.IsRemote() == test.restored {
@@ -1113,7 +1123,11 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
set := z.serverPools[0].getHashedSet(object) set := z.serverPools[0].getHashedSet(object)
getDisks := set.getDisks getDisks := set.getDisks
faulty := append([]StorageAPI(nil), getDisks()...) faulty := append([]StorageAPI(nil), getDisks()...)
var failed, attempted atomic.Int32
for i := range faulty { for i := range faulty {
if test.bulk {
faulty[i] = bulkDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, mode: "eio", fail: true, failed: &failed, attempted: &attempted}
}
faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID} faulty[i] = consistencyDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object, version: oi.VersionID}
} }
set.getDisks = func() []StorageAPI { return faulty } set.getDisks = func() []StorageAPI { return faulty }
@@ -1121,13 +1135,20 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
} }
if test.deleting { if test.deleting {
opts := ObjectOptions{ opts := ObjectOptions{
Versioned: true, VersionID: oi.VersionID, SkipFreeVersion: test.skipFreeVersion, Versioned: !test.unversioned, VersionID: oi.VersionID, SkipFreeVersion: test.skipFreeVersion,
CheckPrecondFn: func(info ObjectInfo) bool { return info.ETag != current.ETag }, CheckPrecondFn: func(info ObjectInfo) bool { return info.ETag != current.ETag },
} }
if test.unconditional { if test.unconditional {
opts.CheckPrecondFn = nil opts.CheckPrecondFn = nil
} }
_, err := z.DeleteObject(t.Context(), bucket, object, opts) var err error
if test.bulk {
opts.CheckPrecondFn = nil
_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object, VersionID: oi.VersionID}}}, opts)
err = errs[0]
} else {
_, err = z.DeleteObject(t.Context(), bucket, object, opts)
}
if (err != nil) != test.failPrimaryDelete { if (err != nil) != test.failPrimaryDelete {
t.Fatalf("unexpected authoritative delete result: %v", err) t.Fatalf("unexpected authoritative delete result: %v", err)
} }
@@ -1145,7 +1166,7 @@ func TestPoolsRetiringCopyPreservesSharedTierObject(t *testing.T) {
_, err = z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: oi.VersionID}) _, err = z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: oi.VersionID})
primaryDeleted := test.deleting && !test.failPrimaryDelete primaryDeleted := test.deleting && !test.failPrimaryDelete
if primaryDeleted { if primaryDeleted {
if !isErrVersionNotFound(err) { if !isErrVersionNotFound(err) && (!test.unversioned || !isErrObjectNotFound(err)) {
t.Fatalf("authoritative copy survived successful delete: %v", err) t.Fatalf("authoritative copy survived successful delete: %v", err)
} }
} else if err != nil { } else if err != nil {
@@ -1306,3 +1327,182 @@ func TestPoolsDeleteDirectoryMarker(t *testing.T) {
} }
} }
} }
// An unconditional DELETE of an object without versions removes its copies in
// every pool. When the delete fails in one pool, the latest copy must stay in
// place: removing it anyway serves an older write of the object.
func TestPoolsDeleteUnversionedReportsOtherPoolFailure(t *testing.T) {
for _, offline := range []bool{false, true} {
for latest := range 2 {
t.Run(fmt.Sprintf("offline=%v/latest=%d", offline, latest), func(t *testing.T) {
z, bucket := consistencyPools(t)
const object = "unversioned-failure"
older := 1 - latest
putConsistencyObject(t, z, bucket, object, older, "older", ObjectOptions{MaxParity: true, MTime: time.Now().Add(-time.Hour)})
oi := putConsistencyObject(t, z, bucket, object, latest, "latest", ObjectOptions{MaxParity: true})
set := z.serverPools[older].getHashedSet(object)
getDisks := set.getDisks
disks := append([]StorageAPI(nil), getDisks()...)
for i := range disks {
switch {
case !offline:
disks[i] = consistencyDeleteFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object}
case i < len(disks)/2:
// An EC 8+8 copy stays readable, but a delete needs 9 drives.
disks[i] = nil
}
}
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = getDisks }()
if _, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{}); err == nil {
t.Error("delete reported success despite the other pool's write failure")
}
set.getDisks = getDisks
if got, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err != nil || got.ETag != oi.ETag {
t.Errorf("after the failed delete the object reads ETag %q, %v; want the latest write %q", got.ETag, err, oi.ETag)
}
})
}
}
}
// consistencyBulkDeleteFaultDisk fails the multi-object deletes of one object.
type consistencyBulkDeleteFaultDisk struct {
StorageAPI
bucket, object string
}
func (d consistencyBulkDeleteFaultDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error {
errs := make([]error, len(versions))
var rest []FileInfoVersions
var restIdx []int
for i, v := range versions {
if volume == d.bucket && v.Name == d.object {
errs[i] = errDiskFull
continue
}
rest = append(rest, v)
restIdx = append(restIdx, i)
}
if len(rest) > 0 {
for i, err := range d.StorageAPI.DeleteVersions(ctx, volume, rest, opts) {
errs[restIdx[i]] = err
}
}
return errs
}
// A multi-object delete must not report an object or version as deleted while
// a pool that failed to delete it still holds a copy.
func TestPoolsDeleteObjectsReportsOtherPoolFailure(t *testing.T) {
for _, versioned := range []bool{false, true} {
t.Run(fmt.Sprintf("versioned=%v", versioned), func(t *testing.T) {
z, bucket := consistencyPools(t)
const object = "bulk-delete-failure"
opts := ObjectOptions{Versioned: versioned}
oi := putConsistencyObject(t, z, bucket, object, 0, "latest", opts)
if versioned {
// The copy a data movement leaves until it removes the source.
putConsistencyObject(t, z, bucket, object, 1, "latest", ObjectOptions{Versioned: true, VersionID: oi.VersionID, MTime: oi.ModTime})
} else {
putConsistencyObject(t, z, bucket, object, 1, "older", ObjectOptions{MTime: time.Now().Add(-time.Hour)})
}
set := z.serverPools[1].getHashedSet(object)
getDisks := set.getDisks
faulty := append([]StorageAPI(nil), getDisks()...)
for i := range faulty {
faulty[i] = consistencyBulkDeleteFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
}
set.getDisks = func() []StorageAPI { return faulty }
defer func() { set.getDisks = getDisks }()
_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object, VersionID: oi.VersionID}}}, opts)
if errs[0] == nil {
t.Error("multi-object delete reported success despite the other pool's write failure")
}
})
}
}
// Pools delete a multi-object batch in parallel. With half the drives of the
// older copy's set offline, an EC 8+8 copy stays readable but cannot be
// deleted: the batch must fail the object and keep its latest copy, instead of
// deleting it and serving the older write once the drives are back.
func TestPoolsDeleteObjectsKeepsLatestWithoutDeleteQuorum(t *testing.T) {
for latest := range 2 {
t.Run(fmt.Sprintf("latest=%d", latest), func(t *testing.T) {
z, bucket := consistencyPools(t)
const object = "bulk-delete-quorum"
older := 1 - latest
putConsistencyObject(t, z, bucket, object, older, "older", ObjectOptions{MaxParity: true, MTime: time.Now().Add(-time.Hour)})
oi := putConsistencyObject(t, z, bucket, object, latest, "latest", ObjectOptions{MaxParity: true})
set := z.serverPools[older].getHashedSet(object)
getDisks := set.getDisks
disks := append([]StorageAPI(nil), getDisks()...)
for i := range len(disks) / 2 {
disks[i] = nil
}
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = getDisks }()
_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}}, ObjectOptions{})
if !errors.As(errs[0], &InsufficientWriteQuorum{}) {
t.Errorf("multi-object delete: got %v, want InsufficientWriteQuorum", errs[0])
}
set.getDisks = getDisks
if got, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err != nil || got.ETag != oi.ETag {
t.Errorf("after the failed delete the object reads ETag %q, %v; want the latest write %q", got.ETag, err, oi.ETag)
}
// Nothing was deleted, so the retry removes every copy.
if _, errs = z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}}, ObjectOptions{}); errs[0] != nil {
t.Fatal(errs[0])
}
for i, pool := range z.serverPools {
if _, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
t.Errorf("pool %d retained the object after the retry: %v", i, err)
}
}
})
}
}
// A delete marker hides the object in every pool. When the versions of an
// object are in two pools and the set of one of them cannot take a write, the
// marker goes to the other pool instead of failing the delete. The versions
// stay below the marker.
func TestPoolsDeleteMarkerSkipsSetWithoutQuorum(t *testing.T) {
for _, tc := range []struct {
name string
degraded int // pool whose set loses drives
offline int
}{
{"older version readable", 0, 8},
{"older version unreadable", 0, 9},
{"latest version readable", 1, 8},
} {
t.Run(tc.name, func(t *testing.T) {
z, bucket := consistencyPools(t)
const object = "marker-quorum"
v1 := putConsistencyObject(t, z, bucket, object, 0, "v1", ObjectOptions{Versioned: true, MaxParity: true, MTime: time.Now().Add(-time.Hour)})
v2 := putConsistencyObject(t, z, bucket, object, 1, "v2", ObjectOptions{Versioned: true, MaxParity: true})
set := z.serverPools[tc.degraded].getHashedSet(object)
getDisks := set.getDisks
disks := append([]StorageAPI(nil), getDisks()...)
for i := range tc.offline {
disks[i] = nil
}
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = getDisks }()
if _, err := z.DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true}); err != nil {
t.Fatalf("delete with the set of pool %d without quorum: %v", tc.degraded, err)
}
set.getDisks = getDisks
if _, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) && !isErrMethodNotAllowed(err) {
t.Errorf("object still readable after the delete: %v", err)
}
for pool, v := range []ObjectInfo{v1, v2} {
if _, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: v.VersionID}); err != nil {
t.Errorf("pool %d lost version %s: %v", pool, v.VersionID, err)
}
}
})
}
}
+138 -129
View File
@@ -616,6 +616,12 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
} }
if objInfo.isMultipart() { if objInfo.isMultipart() {
// Completion repeats this check under the object write lock.
if _, err := z.checkDataMovementCopies(ctx, bucket, encodeDirObject(objInfo.Name), ObjectOptions{
VersionID: objInfo.VersionID, MTime: objInfo.ModTime, SrcPoolIdx: idx,
}); err != nil {
return err
}
res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{ res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
VersionID: objInfo.VersionID, VersionID: objInfo.VersionID,
UserDefined: migrationObjectMetadata(objInfo), UserDefined: migrationObjectMetadata(objInfo),
@@ -656,6 +662,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bu
} }
} }
_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{ _, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{
VersionID: objInfo.VersionID,
SrcPoolIdx: idx, SrcPoolIdx: idx,
DataMovement: true, DataMovement: true,
MTime: objInfo.ModTime, MTime: objInfo.ModTime,
@@ -762,6 +769,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
var vc *versioning.Versioning var vc *versioning.Versioning
var lc *lifecycle.Lifecycle var lc *lifecycle.Lifecycle
var lifecycleUpdatedAt time.Time
var lr objectlock.Retention var lr objectlock.Retention
var rcfg *replication.Config var rcfg *replication.Config
if bi.Name != minioMetaBucket { if bi.Name != minioMetaBucket {
@@ -771,7 +779,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
} }
// Check if the current bucket has a configured lifecycle policy // Check if the current bucket has a configured lifecycle policy
lc, err = globalLifecycleSys.Get(bi.Name) lc, lifecycleUpdatedAt, err = globalBucketMetadataSys.GetLifecycleConfig(bi.Name)
if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) { if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) {
return err return err
} }
@@ -789,9 +797,11 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
} }
for setIdx, set := range pool.sets { for setIdx, set := range pool.sets {
filterLifecycle := func(bucket, object string, fi FileInfo) bool { // filterLifecycle returns an ILM deletion that migration must handle
// before retiring the source copy, or NoneAction.
filterLifecycle := func(bucket, object string, fi FileInfo) lifecycle.Action {
if lc == nil { if lc == nil {
return false return lifecycle.NoneAction
} }
versioned := vc != nil && vc.Versioned(object) versioned := vc != nil && vc.Versioned(object)
objInfo := fi.ToObjectInfo(bucket, object, versioned) objInfo := fi.ToObjectInfo(bucket, object, versioned)
@@ -799,13 +809,20 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo) evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
switch { switch {
case evt.Action.DeleteRestored(): // if restored copy has expired, delete it synchronously case evt.Action.DeleteRestored(): // if restored copy has expired, delete it synchronously
applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom) if !applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom) {
return false return evt.Action
}
return lifecycle.NoneAction
case evt.Action == lifecycle.DeleteAction && versioned:
// Expiring the latest version of a versioned bucket only puts
// a delete marker on top of it. The version itself survives as
// a noncurrent version and has to be moved like any other. The
// scanner applies the expiry once it lives in another pool.
return lifecycle.NoneAction
case evt.Action.Delete(): case evt.Action.Delete():
globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Decom) return evt.Action
return true
default: default:
return false return lifecycle.NoneAction
} }
} }
@@ -825,27 +842,30 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
// to create the appropriate stack. // to create the appropriate stack.
versionsSorter(fivs.Versions).reverse() versionsSorter(fivs.Versions).reverse()
var decommissioned, expired int var decommissioned int
var expired []migrationExpiredVersion
for _, version := range fivs.Versions { for _, version := range fivs.Versions {
stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID) stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
// Apply lifecycle rules on the objects that are expired. // Apply lifecycle rules on the objects that are expired.
if filterLifecycle(bi.Name, version.Name, version) { if action := filterLifecycle(bi.Name, version.Name, version); action.Delete() {
expired++
decommissioned++ decommissioned++
if ilmDeletesVersionOnly(action) || (action.DeleteAll() && !version.IsRemote()) {
// Run the logical expiry before retiring the source.
// A full-object expiry is safe here only for hot-tier
// metadata; transitioned objects retain their tier cleanup
// ownership in the ordinary expiry worker.
expired = append(expired, migrationExpiredVersion{
version: version,
event: lifecycle.Event{Action: action},
lifecycleUpdatedAt: lifecycleUpdatedAt,
source: lcEventSrc_Decom,
})
}
stopFn(version.Size, errors.New("ILM expired object/version will be skipped")) stopFn(version.Size, errors.New("ILM expired object/version will be skipped"))
continue continue
} }
// any object with only single DEL marker we don't need // Preserve every marker: older versions may live in another pool.
// to decommission, just skip it, this also includes
// any other versions that have already expired.
remainingVersions := len(fivs.Versions) - expired
if version.Deleted && remainingVersions == 1 && rcfg == nil {
decommissioned++
stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped"))
continue
}
versionID := version.VersionID versionID := version.VersionID
if versionID == "" { if versionID == "" {
versionID = nullVersionID versionID = nullVersionID
@@ -870,6 +890,9 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
SkipDecommissioned: true, // make sure we skip the decommissioned pool SkipDecommissioned: true, // make sure we skip the decommissioned pool
NoAuditLog: true, NoAuditLog: true,
}) })
if isDataMovementSuperseded(err) {
err = nil
}
if err != nil { if err != nil {
// This can happen when rebalance stop races with ongoing rebalance workers. // This can happen when rebalance stop races with ongoing rebalance workers.
// These rebalance failures can be ignored. // These rebalance failures can be ignored.
@@ -898,17 +921,19 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
// gr.Close() is ensured by decommissionObject(). // gr.Close() is ensured by decommissionObject().
for range 3 { for range 3 {
if version.IsRemote() { if version.IsRemote() {
if err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{ err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
VersionID: versionID, VersionID: versionID,
MTime: version.ModTime, MTime: version.ModTime,
UserDefined: version.Metadata, UserDefined: version.Metadata,
SrcPoolIdx: idx, SrcPoolIdx: idx,
DataMovement: true, DataMovement: true,
}); err != nil { })
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) { if isDataMovementSuperseded(err) {
ignore = true err = nil
stopFn(0, nil) }
} if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
ignore = true
stopFn(0, nil)
} }
if !ignore { if !ignore {
stopFn(version.Size, err) stopFn(version.Size, err)
@@ -949,7 +974,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
stopFn(version.Size, err) stopFn(version.Size, err)
continue continue
} }
if err = z.decommissionObject(ctx, idx, bi.Name, gr); err != nil { if err = z.decommissionObject(ctx, idx, bi.Name, gr); err != nil && !isDataMovementSuperseded(err) {
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) { if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
ignore = true ignore = true
stopFn(0, nil) stopFn(0, nil)
@@ -979,15 +1004,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
// if all versions were decommissioned, then we can delete the object versions. // if all versions were decommissioned, then we can delete the object versions.
if decommissioned == len(fivs.Versions) { if decommissioned == len(fivs.Versions) {
stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name) stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name)
_, err := set.DeleteObject(ctx, err := z.removeMovedVersions(ctx, bi.Name, entry.name, idx, fivs.Versions, expired)
bi.Name, if err == nil {
encodeDirObject(entry.name), err = z.removeFreeVersions(ctx, bi.Name, entry.name, idx, fivs.FreeVersions)
ObjectOptions{ }
DeletePrefix: true, // use prefix delete to delete all versions at once.
DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
NoAuditLog: true,
},
)
stopFn(0, err) stopFn(0, err)
auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err) auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
if err != nil { if err != nil {
@@ -1103,6 +1123,15 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
stopFn(0, err) stopFn(0, err)
return err return err
} }
// Do not retire the bucket from the resumable queue until every
// source version and free version is gone. In particular, an ILM
// action queued during the walk may still be pending; marking the
// bucket complete here would allow the source pool to be removed
// before that action performs its logical and tier cleanup.
if err := z.checkAfterDecomBucket(ctx, idx, bucket); err != nil {
stopFn(0, err)
return err
}
stopFn(0, nil) stopFn(0, nil)
z.poolMetaMutex.Lock() z.poolMetaMutex.Lock()
@@ -1114,105 +1143,85 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
return nil return nil
} }
// removeFreeVersions removes the free versions fvs of object from the pool
// idx being decommissioned, as the scanner removes free versions: first the
// remote object of each, then the free version. A free version is not moved.
// In another pool it would be a delete marker on top of the object, and once
// this pool is gone nothing would remove its remote object. A remote object
// that is already gone is not an error.
func (z *erasureServerPools) removeFreeVersions(ctx context.Context, bucket, object string, idx int, fvs []FileInfo) error {
if len(fvs) == 0 {
return nil
}
for _, fv := range fvs {
err := deleteObjectFromRemoteTier(ctx, fv.TransitionedObjName, fv.TransitionVersionID, fv.TransitionTier)
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
return fmt.Errorf("unable to remove the remote object %s of a free version of %s/%s from tier %s: %w",
fv.TransitionedObjName, bucket, object, fv.TransitionTier, err)
}
}
object = encodeDirObject(object)
lk := z.NewNSLock(bucket, object)
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
if err != nil {
return err
}
ctx = lkctx.Context()
defer lk.Unlock(lkctx)
for _, fv := range fvs {
_, err := z.serverPools[idx].DeleteObject(ctx, bucket, object, ObjectOptions{
VersionID: fv.VersionID,
InclFreeVersions: true,
NoLock: true,
NoAuditLog: true,
})
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
return err
}
}
return nil
}
// checkAfterDecomBucket verifies that a source bucket is physically empty
// before it is removed from the resumable decommission queue. ILM decisions
// are deliberately not treated as an exemption: an expired version remains
// until the expiry path has performed its logical delete, replication, and
// tier cleanup.
func (z *erasureServerPools) checkAfterDecomBucket(ctx context.Context, idx int, bi decomBucketInfo) error {
versionsFound := 0
for _, set := range z.serverPools[idx].sets {
if err := set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) {
if !entry.isObject() {
return
}
if bi.Name == minioMetaBucket && strings.Contains(entry.name, dataUsageCacheName) {
return
}
fivs, err := entry.fileInfoVersions(bi.Name)
if err != nil {
return
}
versionsFound += len(fivs.Versions) + len(fivs.FreeVersions)
}); err != nil {
return err
}
}
if versionsFound > 0 {
return fmt.Errorf("at least %d object(s)/version(s) were found in bucket `%s` after decommissioning", versionsFound, bi.Name)
}
return nil
}
func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error { func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error {
buckets, err := z.getBucketsToDecommission(ctx) buckets, err := z.getBucketsToDecommission(ctx)
if err != nil { if err != nil {
return err return err
} }
for _, bi := range buckets {
pool := z.serverPools[idx] if err := z.checkAfterDecomBucket(ctx, idx, bi); err != nil {
for _, set := range pool.sets { return err
for _, bi := range buckets {
var vc *versioning.Versioning
var lc *lifecycle.Lifecycle
var lr objectlock.Retention
var rcfg *replication.Config
if bi.Name != minioMetaBucket {
vc, err = globalBucketVersioningSys.Get(bi.Name)
if err != nil {
return err
}
// Check if the current bucket has a configured lifecycle policy
lc, err = globalLifecycleSys.Get(bi.Name)
if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) {
return err
}
// Check if bucket is object locked.
lr, err = globalBucketObjectLockSys.Get(bi.Name)
if err != nil {
return err
}
rcfg, err = getReplicationConfig(ctx, bi.Name)
if err != nil {
return err
}
}
filterLifecycle := func(bucket, object string, fi FileInfo) bool {
if lc == nil {
return false
}
versioned := vc != nil && vc.Versioned(object)
objInfo := fi.ToObjectInfo(bucket, object, versioned)
evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
switch {
case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
return false
case evt.Action.Delete():
globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Decom)
return true
default:
return false
}
}
var versionsFound int
if err = set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) {
if !entry.isObject() {
return
}
// `.usage-cache.bin` still exists, must be not readable ignore it.
if bi.Name == minioMetaBucket && strings.Contains(entry.name, dataUsageCacheName) {
// skipping bucket usage cache name, as its autogenerated.
return
}
fivs, err := entry.fileInfoVersions(bi.Name)
if err != nil {
return
}
var ignored int
for _, version := range fivs.Versions {
// Apply lifecycle rules on the objects that are expired.
if filterLifecycle(bi.Name, version.Name, version) {
ignored++
continue
}
if version.Deleted {
ignored++
continue
}
}
versionsFound += len(fivs.Versions) - ignored
}); err != nil {
return err
}
if versionsFound > 0 {
return fmt.Errorf("at least %d object(s)/version(s) were found in bucket `%s` after decommissioning", versionsFound, bi.Name)
}
} }
} }
return nil return nil
} }
+582
View File
@@ -0,0 +1,582 @@
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"encoding/xml"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"testing"
"time"
minio "github.com/minio/minio-go/v7"
)
func bulkDeletePools(t *testing.T, n int) (*erasureServerPools, string) {
t.Helper()
ctx, cancel := context.WithCancel(t.Context())
dirs, err := getRandomDisks(4 * n)
if err != nil {
cancel()
t.Fatal(err)
}
var endpoints EndpointServerPools
for i := range n {
endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
}
obj, _, err := initObjectLayer(ctx, endpoints)
if err != nil {
cancel()
removeRoots(dirs)
t.Fatal(err)
}
z := obj.(*erasureServerPools)
previous := newObjectLayerFn()
setObjectLayer(z)
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) })
bucket := "bulk-delete-consistency"
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
return z, bucket
}
func bulkDeleteBody(t *testing.T, z *erasureServerPools, bucket, object, version string) string {
t.Helper()
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: version})
if err != nil {
t.Fatal(err)
}
defer r.Close()
body, err := io.ReadAll(r)
if err != nil {
t.Fatal(err)
}
return string(body)
}
// All drives are online and all metadata reads succeed. Failure starts inside
// DeleteVersions, after any preflight. Count attempts to delete the latest copy
// instead of waiting for them: a safe implementation must never make that call.
type bulkDeleteFaultDisk struct {
StorageAPI
bucket, object, mode string
fail, latest bool
failed, attempted *atomic.Int32
}
func (d bulkDeleteFaultDisk) DeleteVersions(ctx context.Context, bucket string, versions []FileInfoVersions, opts DeleteOptions) []error {
if bucket != d.bucket {
return d.StorageAPI.DeleteVersions(ctx, bucket, versions, opts)
}
errs := make([]error, len(versions))
for i, v := range versions {
if v.Name != d.object {
errs[i] = d.StorageAPI.DeleteVersions(ctx, bucket, []FileInfoVersions{v}, opts)[0]
continue
}
if d.latest {
d.attempted.Add(1)
}
if !d.fail {
errs[i] = d.StorageAPI.DeleteVersions(ctx, bucket, []FileInfoVersions{v}, opts)[0]
continue
}
d.failed.Add(1)
switch d.mode {
case "eio":
errs[i] = syscall.EIO
case "disconnect":
errs[i] = errDiskNotFound
case "permission":
meta := filepath.Join(d.String(), bucket, v.Name, xlStorageFormatFile)
info, err := os.Stat(meta)
if err == nil {
err = os.Chmod(meta, 0)
}
if err != nil {
errs[i] = err
continue
}
errs[i] = d.StorageAPI.DeleteVersions(ctx, bucket, []FileInfoVersions{v}, opts)[0]
if err := os.Chmod(meta, info.Mode()); err != nil {
panic(err)
}
}
}
return errs
}
func requireBulkDeletePermissionFailure(t *testing.T) {
t.Helper()
probe := filepath.Join(t.TempDir(), "permission-probe")
if err := os.WriteFile(probe, []byte("metadata"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.Chmod(probe, 0); err != nil {
t.Skipf("cannot inject filesystem permissions: %v", err)
}
_, err := os.ReadFile(probe)
if restoreErr := os.Chmod(probe, 0o600); restoreErr != nil {
t.Fatal(restoreErr)
}
if !os.IsPermission(err) {
t.Skip("this user/filesystem can read mode-000 files")
}
}
func TestPoolsBulkDeleteKeepsLatestOnIOFailure(t *testing.T) {
for _, mode := range []string{"eio", "disconnect", "permission"} {
for _, n := range []int{2, 3} {
for primary := range n {
for _, kind := range []string{"unversioned", "null", "uuid"} {
t.Run(fmt.Sprintf("%s/pools=%d/primary=%d/%s", mode, n, primary, kind), func(t *testing.T) {
if mode == "permission" {
requireBulkDeletePermissionFailure(t)
}
z, bucket := bulkDeletePools(t, n)
object, old := "target", (primary+1)%n
opts := ObjectOptions{MTime: UTCNow().Add(-time.Hour), MaxParity: true}
oldBody := "old"
if kind == "uuid" {
opts.Versioned, opts.VersionID, oldBody = true, mustGetUUID(), "latest"
}
putConsistencyObject(t, z, bucket, object, old, oldBody, opts)
opts.MTime = opts.MTime.Add(time.Minute)
latest := putConsistencyObject(t, z, bucket, object, primary, "latest", opts)
version := opts.VersionID
if kind == "null" {
version = nullVersionID
}
for _, key := range []string{"before", "after"} {
putConsistencyObject(t, z, bucket, key, primary, "unrelated", ObjectOptions{})
}
var failed, attempted atomic.Int32
var restore []func()
for _, p := range []int{old, primary} {
set := z.serverPools[p].getHashedSet(object)
original := set.getDisks
disks := append([]StorageAPI(nil), original()...)
for i, disk := range disks {
disks[i] = bulkDeleteFaultDisk{StorageAPI: disk, bucket: bucket, object: object, mode: mode, fail: p == old, latest: p == primary, failed: &failed, attempted: &attempted}
}
set.getDisks = func() []StorageAPI { return disks }
restore = append(restore, func() { set.getDisks = original })
}
defer func() {
for _, f := range restore {
f()
}
}()
objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: "before"}}, {ObjectV: ObjectV{ObjectName: object, VersionID: version}}, {ObjectV: ObjectV{ObjectName: "after"}}}
deleted, errs := z.DeleteObjects(t.Context(), bucket, objects, ObjectOptions{})
if failed.Load() != 4 || attempted.Load() != 0 {
t.Errorf("cleanup failures=%d; latest deletion attempts=%d, want 4/0", failed.Load(), attempted.Load())
}
if errs[1] == nil {
t.Error("cleanup failure was reported as success")
}
for _, i := range []int{0, 2} {
if errs[i] != nil || deleted[i].ObjectName != objects[i].ObjectName {
t.Errorf("unrelated item %d: %+v, %v", i, deleted[i], errs[i])
}
}
if body := bulkDeleteBody(t, z, bucket, object, version); body != "latest" {
t.Errorf("GET during failure=%q", body)
}
for _, f := range restore {
f()
}
if body := bulkDeleteBody(t, z, bucket, object, version); body != "latest" {
t.Errorf("GET after recovery=%q", body)
}
listed, err := z.ListObjectVersions(t.Context(), bucket, object, "", "", "", 100)
if err != nil || len(listed.Objects) != 1 || listed.Objects[0].ETag != latest.ETag {
t.Errorf("recovered versions=%+v, %v", listed.Objects, err)
}
_, retry := z.DeleteObjects(t.Context(), bucket, objects[1:2], ObjectOptions{})
if retry[0] != nil {
t.Fatal(retry[0])
}
_, err = z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: version})
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
t.Errorf("retry retained the object: %v", err)
}
})
}
}
}
}
}
func TestPoolsBulkDeleteReplicationIdentity(t *testing.T) {
z, _ := bulkDeletePools(t, 2)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: true})
if err != nil {
t.Fatal(err)
}
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t.Error("unexpected outbound replication for a failed deletion")
w.WriteHeader(http.StatusNoContent)
}))
defer remote.Close()
client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
if err != nil {
t.Fatal(err)
}
_, restoreReplication := r5ReplicationFixture(t, z, bucket, client)
defer restoreReplication()
versions := make(map[string]string)
for _, key := range []string{"first", "second"} {
v := putConsistencyObject(t, z, bucket, key, 0, "payload", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
putConsistencyObject(t, z, bucket, key, 1, "payload", ObjectOptions{Versioned: true, VersionID: v.VersionID, MTime: v.ModTime})
versions[key] = v.VersionID
}
set := z.serverPools[0].getHashedSet("first")
original := set.getDisks
disks := append([]StorageAPI(nil), original()...)
disks[0], disks[1] = nil, nil
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = original }()
body := []byte(fmt.Sprintf("<Delete><Object><Key>first</Key><VersionId>%s</VersionId></Object><Object><Key>second</Key><VersionId>%s</VersionId></Object></Delete>", versions["first"], versions["second"]))
req, err := newTestSignedRequestV4(http.MethodPost, getMultiDeleteObjectURL("", bucket), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
var result struct {
Errors []struct {
Key, Code string
VersionID string `xml:"VersionId"`
} `xml:"Error"`
Deleted []struct{ Key string } `xml:"Deleted"`
}
if err := xml.Unmarshal(rec.Body.Bytes(), &result); err != nil {
t.Fatal(err)
}
if rec.Code != http.StatusOK || len(result.Errors) != 2 || len(result.Deleted) != 0 {
t.Fatalf("lost response identity: HTTP %d %s", rec.Code, rec.Body.String())
}
for _, e := range result.Errors {
if versions[e.Key] == "" || e.VersionID != versions[e.Key] || e.Code != "SlowDownWrite" {
t.Errorf("incorrect error identity: %+v", e)
}
delete(versions, e.Key)
}
if len(versions) != 0 {
t.Errorf("missing errors: %v", versions)
}
}
func TestPoolsBulkDeleteNullBeforeMarker(t *testing.T) {
for _, n := range []int{1, 2, 3} {
t.Run(fmt.Sprintf("pools=%d", n), func(t *testing.T) {
z, bucket := bulkDeletePools(t, n)
object := "suspended"
history := putConsistencyObject(t, z, bucket, object, 0, "history", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)})
for p := range n {
putConsistencyObject(t, z, bucket, object, p, "null", ObjectOptions{MTime: UTCNow().Add(-time.Duration(p+1) * time.Minute)})
}
objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}, {ObjectV: ObjectV{ObjectName: object, VersionID: nullVersionID}}}
deleted, errs := z.DeleteObjects(t.Context(), bucket, objects, ObjectOptions{VersionSuspended: true})
if errs[0] != nil || errs[1] != nil || !deleted[0].DeleteMarker || deleted[1].VersionID != nullVersionID {
t.Fatalf("results=%+v errors=%v", deleted, errs)
}
info, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
if !info.DeleteMarker || !isErrObjectNotFound(err) {
t.Errorf("replacement null marker lost: %+v, %v", info, err)
}
if body := bulkDeleteBody(t, z, bucket, object, history.VersionID); body != "history" {
t.Errorf("UUID history changed: %q", body)
}
})
}
}
func TestPoolsBulkDeletePartialCleanupRetry(t *testing.T) {
for _, kind := range []string{"unversioned", "null", "uuid"} {
t.Run(kind, func(t *testing.T) {
z, bucket := bulkDeletePools(t, 2)
object := "partial-cleanup"
opts := ObjectOptions{MTime: UTCNow().Add(-time.Hour)}
oldBody := "old"
if kind == "uuid" {
opts.Versioned, opts.VersionID, oldBody = true, mustGetUUID(), "latest"
}
putConsistencyObject(t, z, bucket, object, 0, oldBody, opts)
opts.MTime = UTCNow()
putConsistencyObject(t, z, bucket, object, 1, "latest", opts)
version := opts.VersionID
if kind == "null" {
version = nullVersionID
}
set := z.serverPools[0].getHashedSet(object)
original := set.getDisks
disks := append([]StorageAPI(nil), original()...)
var failed, attempted atomic.Int32
for i := range 2 {
disks[i] = bulkDeleteFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object, mode: "eio", fail: true, failed: &failed, attempted: &attempted}
}
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = original }()
objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object, VersionID: version}}}
_, errs := z.DeleteObjects(t.Context(), bucket, objects, ObjectOptions{})
if errs[0] == nil || failed.Load() != 2 {
t.Fatalf("did not partially fail cleanup: %v, failures=%d", errs, failed.Load())
}
set.getDisks = original
if got := bulkDeleteBody(t, z, bucket, object, version); got != "latest" {
t.Fatalf("latest lost after partial cleanup: %q", got)
}
_, errs = z.DeleteObjects(t.Context(), bucket, objects, ObjectOptions{})
if errs[0] != nil {
t.Fatalf("recovered retry must combine successful removals and already-absent replies: %v", errs[0])
}
_, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: version})
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
t.Errorf("retry retained object: %v", err)
}
})
}
}
func TestPoolsBulkDeleteDifferentPrimaries(t *testing.T) {
for _, n := range []int{2, 3} {
t.Run(fmt.Sprintf("pools=%d", n), func(t *testing.T) {
z, bucket := bulkDeletePools(t, n)
objects := make([]ObjectToDelete, n)
for primary := range n {
object := fmt.Sprintf("key-%d", primary)
objects[primary].ObjectName = object
putConsistencyObject(t, z, bucket, object, (primary+1)%n, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
putConsistencyObject(t, z, bucket, object, primary, "latest", ObjectOptions{})
}
set := z.serverPools[0].getHashedSet(objects[n-1].ObjectName)
original := set.getDisks
disks := append([]StorageAPI(nil), original()...)
var failed, attempted atomic.Int32
for i, disk := range disks {
disks[i] = bulkDeleteFaultDisk{StorageAPI: disk, bucket: bucket, object: objects[n-1].ObjectName, mode: "eio", fail: true, failed: &failed, attempted: &attempted}
}
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = original }()
deleted, errs := z.DeleteObjects(t.Context(), bucket, objects, ObjectOptions{})
for i := range objects {
if deleted[i].ObjectName != objects[i].ObjectName {
t.Errorf("wrong result index %d: %+v", i, deleted[i])
}
if i == n-1 {
if errs[i] == nil || bulkDeleteBody(t, z, bucket, objects[i].ObjectName, "") != "latest" {
t.Errorf("failed item lost latest: %v", errs[i])
}
continue
}
_, err := z.GetObjectInfo(t.Context(), bucket, objects[i].ObjectName, ObjectOptions{})
if errs[i] != nil || !isErrObjectNotFound(err) {
t.Errorf("healthy item %d: delete=%v GET=%v", i, errs[i], err)
}
}
})
}
}
func TestPoolsBulkDeleteTieAndUnreadable(t *testing.T) {
for _, unreadable := range []bool{false, true} {
t.Run(fmt.Sprintf("unreadable=%t", unreadable), func(t *testing.T) {
z, bucket := bulkDeletePools(t, 3)
object := "same-time"
stamp := UTCNow().Add(-time.Hour)
for p := range 3 {
putConsistencyObject(t, z, bucket, object, p, fmt.Sprintf("pool-%d", p), ObjectOptions{MTime: stamp})
}
if got := bulkDeleteBody(t, z, bucket, object, ""); got != "pool-0" {
t.Fatalf("unexpected read winner: %q", got)
}
set := z.serverPools[2].getHashedSet(object)
original := set.getDisks
disks := append([]StorageAPI(nil), original()...)
var failed, attempted atomic.Int32
for i, disk := range disks {
if unreadable {
disks[i] = consistencyReadFaultDisk{StorageAPI: disk, bucket: bucket, object: object}
} else {
disks[i] = bulkDeleteFaultDisk{StorageAPI: disk, bucket: bucket, object: object, mode: "eio", fail: true, failed: &failed, attempted: &attempted}
}
}
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = original }()
_, errs := z.DeleteObjects(t.Context(), bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}}, ObjectOptions{})
set.getDisks = original
if errs[0] == nil || bulkDeleteBody(t, z, bucket, object, "") != "pool-0" {
t.Errorf("lost tie winner or unreadable-pool error: %v", errs[0])
}
})
}
}
type bulkDeletePauseDisk struct {
StorageAPI
bucket, object string
entered chan<- struct{}
release <-chan struct{}
}
func (d bulkDeletePauseDisk) DeleteVersions(ctx context.Context, bucket string, versions []FileInfoVersions, opts DeleteOptions) []error {
for _, v := range versions {
if bucket != d.bucket || v.Name != d.object {
continue
}
select {
case d.entered <- struct{}{}:
default:
}
select {
case <-d.release:
case <-ctx.Done():
}
errs := make([]error, len(versions))
for i := range errs {
errs[i] = syscall.EIO
}
return errs
}
return d.StorageAPI.DeleteVersions(ctx, bucket, versions, opts)
}
func TestPoolsBulkDeleteLockAndCancellation(t *testing.T) {
for _, canceled := range []bool{false, true} {
t.Run(fmt.Sprintf("cancel=%t", canceled), func(t *testing.T) {
z, bucket := bulkDeletePools(t, 2)
object := "locked"
putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)})
putConsistencyObject(t, z, bucket, object, 1, "latest", ObjectOptions{})
set := z.serverPools[0].getHashedSet(object)
original := set.getDisks
disks := append([]StorageAPI(nil), original()...)
entered, release := make(chan struct{}, 4), make(chan struct{})
for i, disk := range disks {
disks[i] = bulkDeletePauseDisk{StorageAPI: disk, bucket: bucket, object: object, entered: entered, release: release}
}
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = original }()
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
done := make(chan error, 1)
go func() {
_, errs := z.DeleteObjects(ctx, bucket, []ObjectToDelete{{ObjectV: ObjectV{ObjectName: object}}}, ObjectOptions{})
done <- errs[0]
}()
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("old-copy cleanup did not start")
}
if canceled {
cancel()
if err := <-done; err == nil {
t.Fatal("canceled cleanup succeeded")
}
set.getDisks = original
if got := bulkDeleteBody(t, z, bucket, object, ""); got != "latest" {
t.Errorf("cancellation lost latest: %q", got)
}
return
}
putDone := make(chan error, 1)
reader := mustGetPutObjReader(t, bytes.NewBufferString("after-put"), 9, "", "")
go func() {
_, err := z.PutObject(t.Context(), bucket, object, reader, ObjectOptions{})
putDone <- err
}()
select {
case err := <-putDone:
t.Fatalf("PUT passed the held DELETE lock: %v", err)
case <-time.After(50 * time.Millisecond):
}
close(release)
if err := <-done; err == nil {
t.Error("cleanup failure was lost")
}
if err := <-putDone; err != nil {
t.Fatal(err)
}
if got := bulkDeleteBody(t, z, bucket, object, ""); got != "after-put" {
t.Errorf("concurrent PUT lost: %q", got)
}
})
}
}
type bulkDeleteReplyDisk struct {
StorageAPI
err error
}
func (d bulkDeleteReplyDisk) IsOnline() bool { return true }
func (d bulkDeleteReplyDisk) DeleteVersions(_ context.Context, _ string, versions []FileInfoVersions, _ DeleteOptions) []error {
errs := make([]error, len(versions))
for i := range errs {
errs[i] = d.err
}
return errs
}
func TestBulkDeleteQuorumEvidence(t *testing.T) {
for _, tc := range []struct {
name string
obj ObjectToDelete
opts ObjectOptions
ok bool
}{
{name: "unversioned", ok: true},
{name: "null", obj: ObjectToDelete{ObjectV: ObjectV{VersionID: nullVersionID}}, ok: true},
{name: "uuid", obj: ObjectToDelete{ObjectV: ObjectV{VersionID: mustGetUUID()}}, ok: true},
{name: "marker", opts: ObjectOptions{Versioned: true}},
{name: "suspended-marker", opts: ObjectOptions{VersionSuspended: true}},
{name: "pending-purge", obj: ObjectToDelete{ObjectV: ObjectV{VersionID: mustGetUUID()}, VersionPurgeStatuses: "target=PENDING;"}},
{name: "replicated-marker", obj: ObjectToDelete{DeleteMarkerReplicationStatus: "target=PENDING;"}, opts: ObjectOptions{Versioned: true}},
} {
t.Run(tc.name, func(t *testing.T) {
disks := []StorageAPI{
bulkDeleteReplyDisk{},
bulkDeleteReplyDisk{},
bulkDeleteReplyDisk{err: errFileNotFound},
bulkDeleteReplyDisk{err: errFileVersionNotFound},
}
er := erasureObjects{getDisks: func() []StorageAPI { return disks }}
tc.obj.ObjectName, tc.opts.NoAuditLog = "object", true
_, errs := er.DeleteObjects(t.Context(), "bucket", []ObjectToDelete{tc.obj}, tc.opts)
if (errs[0] == nil) != tc.ok {
t.Errorf("two writes + two absent replies: error=%v, want success=%t", errs[0], tc.ok)
}
})
}
}
@@ -0,0 +1,526 @@
// Copyright (c) 2026 Denis Babchuk
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"errors"
"slices"
"testing"
"time"
madmin "github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/bucket/lifecycle"
objectlock "github.com/minio/minio/internal/bucket/object/lock"
)
// Deleting a version whose data is in a remote tier leaves a free version in
// its place: a record of the remote object, kept until the scanner removes
// both. A free version is not a version of the object. There is no delete
// marker, and the version below it is the latest one; reads already skip it.
// freeVersionNoncurrent expires versions one day after they become noncurrent.
const freeVersionNoncurrent = "<Rule><ID>noncurrent</ID><Status>Enabled</Status><Filter><Prefix></Prefix></Filter>" +
"<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration></Rule>"
// freeVersionTier is a remote tier held in memory whose removals can fail.
type freeVersionTier struct {
*tierExpiryTestBackend
removeErr error
}
func (b *freeVersionTier) Remove(ctx context.Context, object string, rv remoteVersionID) error {
if b.removeErr != nil {
return b.removeErr
}
return b.tierExpiryTestBackend.Remove(ctx, object, rv)
}
type freeVersionTest struct {
t *testing.T
z *erasureServerPools
bucket string
tier *freeVersionTier
labels map[string]string // version ID -> label
}
// newFreeVersionTest returns two pools with a versioned bucket under rules and
// a remote tier WARM. In pool 0, obj-01 has v1 and v2 and a tiered v3, written
// five, four and three days ago; the client deleted v3. obj-02 had only a
// tiered v1, deleted as well.
func newFreeVersionTest(t *testing.T, rules string) *freeVersionTest {
t.Helper()
z, _ := migrationTestPools(t)
ctx := t.Context()
f := &freeVersionTest{t: t, z: z, bucket: "free-version", labels: map[string]string{}}
if err := z.MakeBucket(ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
t.Fatal(err)
}
if rules != "" {
meta, err := globalBucketMetadataSys.Get(f.bucket)
if err != nil {
t.Fatal(err)
}
meta.LifecycleConfigXML = []byte("<LifecycleConfiguration>" + rules + "</LifecycleConfiguration>")
if err := meta.Save(ctx, z); err != nil {
t.Fatal(err)
}
globalBucketMetadataSys.Set(f.bucket, meta)
}
f.tier = &freeVersionTier{tierExpiryTestBackend: &tierExpiryTestBackend{objects: map[string][]byte{}}}
prevTier := globalTierConfigMgr
globalTierConfigMgr = NewTierConfigMgr()
globalTierConfigMgr.drivercache["WARM"] = f.tier
t.Cleanup(func() { globalTierConfigMgr = prevTier })
now := UTCNow()
f.put("obj-01", "v1", now.AddDate(0, 0, -5))
f.put("obj-01", "v2", now.AddDate(0, 0, -4))
f.deleteVersion(f.tiered("obj-01", "v3", now.AddDate(0, 0, -3)))
f.deleteVersion(f.tiered("obj-02", "v1", now.AddDate(0, 0, -3)))
return f
}
// put writes a version of object to pool 0.
func (f *freeVersionTest) put(object, label string, mtime time.Time) ObjectInfo {
f.t.Helper()
oi := putConsistencyObject(f.t, f.z, f.bucket, object, 0, object+" "+label, ObjectOptions{Versioned: true, MTime: mtime})
f.labels[oi.VersionID] = label
return oi
}
// tiered writes a version of object to pool 0 whose data is in the tier.
func (f *freeVersionTest) tiered(object, label string, mtime time.Time) ObjectInfo {
f.t.Helper()
oi := f.put(object, label, mtime)
infos, errs := readAllFileInfo(f.t.Context(), f.z.serverPools[0].getHashedSet(object).getDisks(), "", f.bucket, object, oi.VersionID, false, false)
if errs[0] != nil {
f.t.Fatal(errs[0])
}
fi := infos[0]
fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", f.remote(object, label)
f.tier.objects[fi.TransitionedObjName] = []byte(object + " " + label)
if err := f.z.serverPools[0].DecomTieredObject(f.t.Context(), f.bucket, object, fi, ObjectOptions{}); err != nil {
f.t.Fatal(err)
}
return oi
}
// remote is the name of the remote object of a tiered version.
func (f *freeVersionTest) remote(object, label string) string {
return f.bucket + "/" + object + "/" + label
}
// deleteVersion deletes a tiered version by its ID, as a client does, which
// leaves a free version in its place.
func (f *freeVersionTest) deleteVersion(oi ObjectInfo) {
f.t.Helper()
if _, err := f.z.DeleteObject(f.t.Context(), f.bucket, oi.Name, ObjectOptions{Versioned: true, VersionID: oi.VersionID}); err != nil {
f.t.Fatal(err)
}
if got := f.stored(0, oi.Name); len(got) == 0 || got[0] != "free" {
f.t.Fatalf("fixture: %s holds %v, want a free version on top", oi.Name, got)
}
}
func (f *freeVersionTest) label(fi FileInfo) string {
switch {
case fi.TierFreeVersion():
return "free"
case fi.Deleted:
return "DM"
case f.labels[fi.VersionID] != "":
return f.labels[fi.VersionID]
}
return "?"
}
// xlMeta reads the metadata of object in pool from its first drive, nil if
// the pool has none.
func (f *freeVersionTest) xlMeta(pool int, object string) []byte {
f.t.Helper()
disk := f.z.serverPools[pool].getHashedSet(object).getDisks()[0]
buf, err := disk.ReadAll(f.t.Context(), f.bucket, pathJoin(object, xlStorageFormatFile))
if errors.Is(err, errFileNotFound) {
return nil
}
if err != nil {
f.t.Fatal(err)
}
return buf
}
// stored returns what pool holds of object, newest first, free versions
// included.
func (f *freeVersionTest) stored(pool int, object string) []string {
f.t.Helper()
buf := f.xlMeta(pool, object)
if buf == nil {
return nil
}
fivs, err := getFileInfoVersions(buf, f.bucket, object, true)
if err != nil {
f.t.Fatal(err)
}
var got []string
for _, fi := range fivs.Versions {
got = append(got, f.label(fi))
}
return got
}
// versions labels the versions of a listing as "<object> <label>[ latest]".
func (f *freeVersionTest) versions(objects []ObjectInfo) []string {
var got []string
for _, oi := range objects {
label := oi.Name + " " + f.label(FileInfo{VersionID: oi.VersionID, Deleted: oi.DeleteMarker})
if oi.IsLatest {
label += " latest"
}
got = append(got, label)
}
return got
}
// listVersions returns the versions ListObjectVersions lists in one page.
func (f *freeVersionTest) listVersions() []string {
f.t.Helper()
lv, err := f.z.ListObjectVersions(f.t.Context(), f.bucket, "", "", "", "", 100)
if err != nil {
f.t.Fatal(err)
}
return f.versions(lv.Objects)
}
// expiryQueue replaces the expiry workers with a queue for the test.
func (f *freeVersionTest) expiryQueue() chan expiryOp {
queue := make(chan expiryOp, 100)
es := &expiryState{ctx: f.t.Context(), objAPI: f.z}
workers := []chan expiryOp{queue}
es.workers.Store(&workers)
prev := globalExpiryState
globalExpiryState = es
f.t.Cleanup(func() { globalExpiryState = prev })
return queue
}
// expired returns the labels of the versions queued for expiry.
func (f *freeVersionTest) expired(queue chan expiryOp) []string {
var got []string
for len(queue) > 0 {
switch task := (<-queue).(type) {
case expiryTask:
got = append(got, f.labels[task.objInfo.VersionID])
case noncurrentVersionsTask:
for _, v := range task.versions {
got = append(got, f.labels[v.VersionID])
}
}
}
return got
}
// The scanner reads the versions of obj-01 without the free version. v2 is the
// latest version, and a noncurrent expiry must not count it as noncurrent
// since the deleted v3 was written. Otherwise the scanner deletes it.
func TestFreeVersionScannerLatestVersion(t *testing.T) {
f := newFreeVersionTest(t, freeVersionNoncurrent)
fivs, err := getFileInfoVersions(f.xlMeta(0, "obj-01"), f.bucket, "obj-01", false)
if err != nil {
t.Fatal(err)
}
var got []string
for _, fi := range fivs.Versions {
got = append(got, f.label(fi))
}
if want := []string{"v2", "v1"}; !slices.Equal(got, want) || len(fivs.FreeVersions) != 1 {
t.Fatalf("scanner versions %v with %d free versions, want %v with 1", got, len(fivs.FreeVersions), want)
}
v2, v1 := fivs.Versions[0], fivs.Versions[1]
if !v2.IsLatest || !v2.SuccessorModTime.IsZero() {
t.Errorf("v2: latest %v, successor mod time %v; want latest without a successor", v2.IsLatest, v2.SuccessorModTime)
}
if v1.IsLatest || !v1.SuccessorModTime.Equal(v2.ModTime) {
t.Errorf("v1: latest %v, successor mod time %v; want noncurrent since v2 at %v", v1.IsLatest, v1.SuccessorModTime, v2.ModTime)
}
if !fivs.LatestModTime.Equal(v2.ModTime) {
t.Errorf("latest mod time %v, want that of v2 %v", fivs.LatestModTime, v2.ModTime)
}
// Apply the rules to these versions as the scanner does.
lc, err := globalLifecycleSys.Get(f.bucket)
if err != nil {
t.Fatal(err)
}
objInfos := make([]ObjectInfo, len(fivs.Versions))
for i, fi := range fivs.Versions {
objInfos[i] = fi.ToObjectInfo(f.bucket, "obj-01", true)
}
queue := f.expiryQueue()
item := scannerItem{Path: "obj-01", bucket: f.bucket, objectName: "obj-01", lifeCycle: lc}
item.applyActions(t.Context(), f.z, objInfos, objectlock.Retention{}, &sizeSummary{}, nil)
if expired, want := f.expired(queue), []string{"v1"}; !slices.Equal(expired, want) {
t.Errorf("scanner queued %v for expiry, want %v", expired, want)
}
}
// ListObjectsV2 lists obj-01: its latest version is v2, not a delete marker.
// obj-02 has no versions left.
func TestFreeVersionListObjects(t *testing.T) {
f := newFreeVersionTest(t, "")
lo, err := f.z.ListObjectsV2(t.Context(), f.bucket, "", "", "", 100, false, "")
if err != nil {
t.Fatal(err)
}
var got []string
for _, oi := range lo.Objects {
got = append(got, oi.Name)
}
if want := []string{"obj-01"}; !slices.Equal(got, want) {
t.Errorf("ListObjectsV2: %v, want %v", got, want)
}
}
// The listing checks of the latest version skip the free version: obj-01 is
// shown, obj-02 has nothing to show. Both the encoded metadata and the
// decoded form, which the listing caches, are checked.
func TestFreeVersionLatestDeleteMarker(t *testing.T) {
f := newFreeVersionTest(t, "")
for _, tc := range []struct {
object string
hidden bool
}{{"obj-01", false}, {"obj-02", true}} {
entry := metaCacheEntry{name: tc.object, metadata: f.xlMeta(0, tc.object)}
meta, _, err := isIndexedMetaV2(entry.metadata)
if err != nil || meta == nil {
t.Fatalf("%s: metadata is not indexed: %v", tc.object, err)
}
if got := meta.IsLatestDeleteMarker(); got != tc.hidden {
t.Errorf("%s: IsLatestDeleteMarker %v, want %v", tc.object, got, tc.hidden)
}
if got := meta.AllHidden(true); got != tc.hidden {
t.Errorf("%s: AllHidden(true) %v, want %v", tc.object, got, tc.hidden)
}
if got := meta.AllHidden(false); got != tc.hidden {
t.Errorf("%s: AllHidden(false) %v, want %v", tc.object, got, tc.hidden)
}
if got := entry.isLatestDeletemarker(); got != tc.hidden {
t.Errorf("%s: isLatestDeletemarker %v, want %v", tc.object, got, tc.hidden)
}
if _, err := entry.xlmeta(); err != nil {
t.Fatal(err)
}
if got := entry.isLatestDeletemarker(); got != tc.hidden {
t.Errorf("%s: isLatestDeletemarker of the decoded metadata %v, want %v", tc.object, got, tc.hidden)
}
}
}
// ListObjectVersions lists v2 as the latest version of obj-01 and no delete
// markers. Keys with nothing but a free version have no versions to list and
// do not take up a page. Walk returns the same versions.
func TestFreeVersionListObjectVersions(t *testing.T) {
f := newFreeVersionTest(t, "")
f.deleteVersion(f.tiered("obj-00", "v1", UTCNow().AddDate(0, 0, -3)))
want := []string{"obj-01 v2 latest", "obj-01 v1"}
if got := f.listVersions(); !slices.Equal(got, want) {
t.Errorf("ListObjectVersions: %v, want %v", got, want)
}
results := make(chan itemOrErr[ObjectInfo], 100)
if err := f.z.Walk(t.Context(), f.bucket, "", results, WalkOptions{}); err != nil {
t.Fatal(err)
}
var walked []ObjectInfo
for r := range results {
if r.Err != nil {
t.Fatal(r.Err)
}
walked = append(walked, r.Item)
}
if got, want := f.versions(walked), []string{"obj-01 v1", "obj-01 v2 latest"}; !slices.Equal(got, want) {
t.Errorf("Walk: %v, want %v", got, want)
}
// Page by page, one version a page.
for _, delimiter := range []string{"", "/"} {
var got []string
marker, versionMarker := "", ""
for range 10 {
lv, err := f.z.ListObjectVersions(t.Context(), f.bucket, "", marker, versionMarker, delimiter, 1)
if err != nil {
t.Fatal(err)
}
got = append(got, f.versions(lv.Objects)...)
if !lv.IsTruncated {
break
}
marker, versionMarker = lv.NextMarker, lv.NextVersionIDMarker
}
if !slices.Equal(got, want) {
t.Errorf("ListObjectVersions page by page, delimiter %q: %v, want %v", delimiter, got, want)
}
}
}
// Healing still visits the free versions, so a free version lost on some
// drives is healed and the scanner still finds it.
func TestFreeVersionHealObjects(t *testing.T) {
f := newFreeVersionTest(t, "")
var got []string
err := f.z.HealObjects(t.Context(), f.bucket, "", madmin.HealOpts{Recursive: true}, func(_, object, versionID string, _ madmin.HealScanMode) error {
label := f.labels[versionID]
if label == "" {
label = "free"
}
got = append(got, object+" "+label)
return nil
})
if err != nil {
t.Fatal(err)
}
slices.Sort(got)
if want := []string{"obj-01 free", "obj-01 v1", "obj-01 v2", "obj-02 free"}; !slices.Equal(got, want) {
t.Errorf("healed versions %v, want %v", got, want)
}
}
// Decommission and rebalance do not move a free version: in another pool it
// would be a delete marker that hides v2. Decommission removes the remote
// object and the free version, as the scanner would before the pool is
// removed. Rebalance leaves both to the scanner of the source pool.
func TestFreeVersionDataMovement(t *testing.T) {
for _, kind := range []string{"decommission", "rebalance"} {
for _, rules := range []string{"", freeVersionNoncurrent} {
name := kind
if rules != "" {
name += "/noncurrent-expiry"
}
t.Run(name, func(t *testing.T) {
f := newFreeVersionTest(t, rules)
queue := f.expiryQueue()
if failed := migrationTestWalk(t, f.z, f.bucket, 0, kind); failed != 0 {
t.Fatalf("%d versions failed to move", failed)
}
if expired := f.expired(queue); slices.Contains(expired, "v2") {
t.Errorf("%s queued the latest version v2 for expiry: %v", kind, expired)
}
// v1 has been noncurrent for days; whether a move skips an
// expired version is out of scope here.
if got := f.stored(1, "obj-01"); len(got) == 0 || got[0] != "v2" || slices.Contains(got, "DM") {
t.Errorf("pool 1 holds %v of obj-01, want v2 on top and no delete marker", got)
}
if got := f.stored(1, "obj-02"); len(got) != 0 {
t.Errorf("pool 1 holds %v of obj-02, want nothing", got)
}
if body := migrationTestBody(t, f.z.GetObjectNInfo, f.bucket, "obj-01", ""); body != "obj-01 v2" {
t.Errorf("obj-01 reads %q, want %q", body, "obj-01 v2")
}
if got := f.listVersions(); len(got) == 0 || got[0] != "obj-01 v2 latest" || slices.Contains(got, "obj-02 DM latest") {
t.Errorf("ListObjectVersions after %s: %v", kind, got)
}
if rules != "" {
// The expired v1 is not moved, so its source is kept.
return
}
var wantSource []string // decommission
removed := true
if kind == "rebalance" {
wantSource, removed = []string{"free"}, false
}
for _, object := range []string{"obj-01", "obj-02"} {
if got := f.stored(0, object); !slices.Equal(got, wantSource) {
t.Errorf("pool 0 holds %v of %s, want %v", got, object, wantSource)
}
}
for _, remote := range []string{f.remote("obj-01", "v3"), f.remote("obj-02", "v1")} {
if slices.Contains(f.tier.removed, remote) != removed {
t.Errorf("remote object %s of a deleted version removed: %v, want %v", remote, !removed, removed)
}
}
})
}
}
}
// decommission decommissions pool 0 as the admin API does, including the check
// for versions left behind, and returns its state.
func (f *freeVersionTest) decommission() PoolDecommissionInfo {
f.t.Helper()
ctx := f.t.Context()
if err := f.z.StartDecommission(ctx, 0); err != nil {
f.t.Fatal(err)
}
f.z.doDecommissionInRoutine(ctx, 0)
f.z.poolMetaMutex.RLock()
defer f.z.poolMetaMutex.RUnlock()
return *f.z.poolMeta.Pools[0].Decommission
}
// While the tier is unavailable, the decommission moves the versions but keeps
// the free versions, and the check for versions left behind fails it. A retry
// with the tier back completes it.
func TestFreeVersionDecommissionTierUnavailable(t *testing.T) {
f := newFreeVersionTest(t, "")
f.tier.removeErr = errors.New("tier unavailable")
if info := f.decommission(); info.Complete || !info.Failed {
t.Fatalf("decommission with the tier unavailable: complete %v, failed %v; want failed", info.Complete, info.Failed)
}
if got, want := f.stored(1, "obj-01"), []string{"v2", "v1"}; !slices.Equal(got, want) {
t.Errorf("pool 1 holds %v of obj-01, want %v", got, want)
}
for _, object := range []string{"obj-01", "obj-02"} {
if got, want := f.stored(0, object), []string{"free"}; !slices.Equal(got, want) {
t.Errorf("pool 0 holds %v of %s, want %v", got, object, want)
}
}
if body := migrationTestBody(t, f.z.GetObjectNInfo, f.bucket, "obj-01", ""); body != "obj-01 v2" {
t.Errorf("obj-01 reads %q, want %q", body, "obj-01 v2")
}
if len(f.tier.removed) != 0 {
t.Errorf("remote objects removed: %v", f.tier.removed)
}
// The failed decommission moved the bucket metadata, so pool 0 has no
// .minio.sys/buckets left. On a server the scanner recreates it when it
// saves its usage cache there; it does not run here.
for _, set := range f.z.serverPools[0].sets {
for _, disk := range set.getDisks() {
if err := disk.MakeVol(t.Context(), pathJoin(minioMetaBucket, bucketMetaPrefix)); err != nil && !errors.Is(err, errVolumeExists) {
t.Fatal(err)
}
}
}
f.tier.removeErr = nil
if info := f.decommission(); !info.Complete || info.Failed {
t.Fatalf("retry: complete %v, failed %v; want complete", info.Complete, info.Failed)
}
if got, want := f.stored(1, "obj-01"), []string{"v2", "v1"}; !slices.Equal(got, want) {
t.Errorf("pool 1 holds %v of obj-01 after the retry, want %v", got, want)
}
for _, object := range []string{"obj-01", "obj-02"} {
if got := f.stored(0, object); len(got) != 0 {
t.Errorf("pool 0 holds %v of %s after the retry", got, object)
}
}
if want := []string{f.remote("obj-01", "v3"), f.remote("obj-02", "v1")}; !slices.Equal(slices.Sorted(slices.Values(f.tier.removed)), want) {
t.Errorf("remote objects removed: %v, want %v", f.tier.removed, want)
}
}
@@ -0,0 +1,169 @@
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"fmt"
"testing"
"time"
)
// Both walkers must preserve markers even when all older versions reside in
// another pool. Local version counts cannot prove that a marker is redundant.
func TestPoolsMigrationDeleteMarkers(t *testing.T) {
for _, kind := range []string{"rebalance", "decommission"} {
for _, pools := range []int{2, 3} {
for _, null := range []bool{false, true} {
for _, history := range []string{"none", "source", "destination"} {
t.Run(fmt.Sprintf("%s/pools=%d/null=%t/history=%s", kind, pools, null, history), func(t *testing.T) {
var z *erasureServerPools
var bucket string
if pools == 2 {
z, bucket = migrationTestPools(t)
} else {
z, bucket = migrationTestThreePools(t)
}
const object = "deleted"
old, stamp := UTCNow().Add(-time.Hour), UTCNow().Add(-time.Minute)
var prior ObjectInfo
if history != "none" {
pool := 0
if history == "destination" {
pool = 1
}
prior = putConsistencyObject(t, z, bucket, object, pool, "history", ObjectOptions{Versioned: true, MTime: old})
}
// A stale null copy is a normal consequence of cross-pool overwrites.
putConsistencyObject(t, z, bucket, object, 1, "stale", ObjectOptions{MTime: old})
vid := mustGetUUID()
if null {
vid = nullVersionID
}
opts := ObjectOptions{Versioned: true, VersionID: vid, DeleteMarker: true, MTime: stamp}
marker, err := z.serverPools[0].DeleteObject(t.Context(), bucket, object, opts)
mustIAM(t, err)
if !marker.DeleteMarker {
t.Fatal("fixture did not create marker")
}
if oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) || !oi.DeleteMarker {
t.Fatalf("fixture visible: %+v %v", oi, err)
}
if failed := migrationTestWalk(t, z, bucket, 0, kind); failed != 0 {
t.Fatalf("failed moves: %d", failed)
}
for attempt := 0; attempt < 2; attempt++ {
if oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) || !oi.DeleteMarker || !oi.ModTime.Equal(stamp) {
t.Fatalf("marker lost: %+v %v", oi, err)
}
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: migrationTestVersion(marker)}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
t.Fatalf("source marker retained: %v", err)
}
if prior.VersionID != "" && migrationTestBody(t, z.GetObjectNInfo, bucket, object, prior.VersionID) != "history" {
t.Fatal("lost historical version")
}
// Repeating a completed walk must preserve the same state.
migrationTestWalk(t, z, bucket, 0, kind)
}
})
}
}
}
}
}
// A reader opened before a public purge must not recreate the purged version.
func TestPoolsMigrationDoesNotResurrectPurgedSource(t *testing.T) {
for _, kind := range []string{"rebalance", "decommission"} {
for _, multipart := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/multipart=%t", kind, multipart), func(t *testing.T) {
z, bucket := migrationTestPools(t)
move := migrationTestMover(t, z, 0, kind)
oi := migrationTestObject(t, z, bucket, "purged", 0, multipart, []string{"payload"}, ObjectOptions{Versioned: true})
gr := migrationTestReader(t, z, 0, oi)
_, err := z.DeleteObject(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID, Versioned: true})
mustIAM(t, err)
err = move(t.Context(), 0, bucket, gr)
if err == nil {
t.Error("move accepted a purged source")
}
if oi, err := z.GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{}); !isErrObjectNotFound(err) {
t.Fatalf("purged version resurrected: %+v %v", oi, err)
}
})
}
}
}
// Cleanup must not erase a marker created after the data was copied. It must
// also fail closed if the destination becomes unreadable before source removal.
func TestPoolsMigrationCleanupRechecksSource(t *testing.T) {
for _, change := range []string{"delete-null", "delete-uuid", "unreadable-destination", "missing-destination"} {
t.Run(change, func(t *testing.T) {
z, bucket := migrationTestPools(t)
move := migrationTestMover(t, z, 0, "decommission")
oi := migrationTestObject(t, z, bucket, "cleanup", 0, false, []string{"payload"}, ObjectOptions{})
snapshot := []FileInfo{{VersionID: oi.VersionID, ModTime: oi.ModTime}}
mustIAM(t, move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, oi)))
switch change {
case "delete-null", "delete-uuid":
_, err := z.DeleteObject(t.Context(), bucket, oi.Name, ObjectOptions{VersionSuspended: change == "delete-null", Versioned: change == "delete-uuid"})
mustIAM(t, err)
case "unreadable-destination":
restore := conditionalPutSwapDisks(z.serverPools[1], oi.Name, func(d StorageAPI) StorageAPI {
return consistencyReadFaultDisk{StorageAPI: d, bucket: bucket, object: oi.Name}
})
defer restore()
case "missing-destination":
_, err := z.serverPools[1].DeleteObject(t.Context(), bucket, oi.Name, ObjectOptions{VersionID: nullVersionID})
mustIAM(t, err)
}
err := z.removeMovedVersions(t.Context(), bucket, oi.Name, 0, snapshot, nil)
if change != "delete-uuid" && err == nil {
t.Fatal("cleanup accepted changed or uncertain state")
}
if change == "delete-null" || change == "delete-uuid" {
if got, err := z.GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{}); !isErrObjectNotFound(err) || !got.DeleteMarker {
t.Fatalf("cleanup resurrected data: %v", err)
}
} else if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, oi.Name, ObjectOptions{}); err != nil {
t.Fatalf("cleanup lost source: %v", err)
}
})
}
}
func TestPoolsMigrationConflictingTimestampTie(t *testing.T) {
for _, kind := range []string{"rebalance", "decommission"} {
for source := range 2 {
t.Run(fmt.Sprintf("%s/source=%d", kind, source), func(t *testing.T) {
z, bucket := migrationTestPools(t)
stamp := UTCNow().Add(-time.Minute)
for i := range 2 {
putConsistencyObject(t, z, bucket, "tie", i, fmt.Sprintf("pool-%d", i), ObjectOptions{MTime: stamp})
}
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, "tie", ""); got != "pool-0" {
t.Fatalf("fixture winner: %s", got)
}
migrationTestWalk(t, z, bucket, source, kind)
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, "tie", ""); got != "pool-0" {
t.Fatalf("changed tie winner: %s", got)
}
})
}
}
}
@@ -0,0 +1,588 @@
// Copyright (c) 2026 Denis Babchuk
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"errors"
"net/http"
"slices"
"strings"
"testing"
"time"
"github.com/minio/minio/internal/amztime"
"github.com/minio/minio/internal/bucket/lifecycle"
objectlock "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/replication"
"github.com/minio/minio/internal/bucket/versioning"
xhttp "github.com/minio/minio/internal/http"
)
// Decommission and rebalance leave a version that ILM expires to the expiry
// workers. Expiring the current version of a versioned object only puts a
// delete marker on top of it: the version stays as a noncurrent version and
// has to be moved like any other. A version that ILM does delete is not moved,
// and the source copies of the versions that were moved must still be removed.
// migrationILM is a bucket in pool 0 of two pools. The ILM expiry workers are
// replaced by a queue that nothing drains, so the expiries a decommission or
// rebalance queues run after it, as when the workers lag behind the walk.
type migrationILM struct {
t *testing.T
z *erasureServerPools
bucket string
queue chan expiryOp
objects []string
labels map[string]string // object and version ID -> label of a version
markers map[string]string // object and version ID -> label of a delete marker
}
func newMigrationILM(t *testing.T, status versioning.State, lock bool, rules string) *migrationILM {
t.Helper()
z, _ := migrationTestPools(t)
ctx := t.Context()
m := &migrationILM{
t: t, z: z, bucket: "migration-ilm", queue: make(chan expiryOp, 100),
labels: map[string]string{}, markers: map[string]string{},
}
err := z.MakeBucket(ctx, m.bucket, MakeBucketOptions{LockEnabled: lock, VersioningEnabled: status == versioning.Enabled})
if err != nil {
t.Fatal(err)
}
meta, err := globalBucketMetadataSys.Get(m.bucket)
if err != nil {
t.Fatal(err)
}
if status == versioning.Suspended {
meta.VersioningConfigXML = []byte("<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>")
}
if rules != "" {
meta.LifecycleConfigXML = []byte("<LifecycleConfiguration>" + rules + "</LifecycleConfiguration>")
}
if err := meta.Save(ctx, z); err != nil {
t.Fatal(err)
}
globalBucketMetadataSys.Set(m.bucket, meta)
es := &expiryState{ctx: ctx, objAPI: z}
workers := []chan expiryOp{m.queue}
es.workers.Store(&workers)
prev := globalExpiryState
globalExpiryState = es
t.Cleanup(func() { globalExpiryState = prev })
return m
}
func (m *migrationILM) key(object, versionID string) string {
if versionID == "" {
versionID = nullVersionID
}
return object + " " + versionID
}
func (m *migrationILM) track(object string) {
if !slices.Contains(m.objects, object) {
m.objects = append(m.objects, object)
}
}
// put writes a version of object to pool 0.
func (m *migrationILM) put(object, label string, opts ObjectOptions) ObjectInfo {
m.t.Helper()
oi := putConsistencyObject(m.t, m.z, m.bucket, object, 0, object+" "+label, opts)
m.labels[m.key(object, oi.VersionID)] = label
m.track(object)
return oi
}
// marker writes a delete marker of object to pool 0.
func (m *migrationILM) marker(object, label string, mtime time.Time) {
m.t.Helper()
versionID := mustGetUUID()
_, err := m.z.serverPools[0].DeleteObject(m.t.Context(), m.bucket, object,
ObjectOptions{Versioned: true, VersionID: versionID, DeleteMarker: true, MTime: mtime})
if err != nil {
m.t.Fatal(err)
}
m.markers[m.key(object, versionID)] = label
m.track(object)
}
// tiered writes a version of object to pool 0 whose data is in a remote tier.
func (m *migrationILM) tiered(object, label string) ObjectInfo {
m.t.Helper()
ctx := m.t.Context()
oi := m.put(object, label, ObjectOptions{Versioned: true})
infos, errs := readAllFileInfo(ctx, m.z.serverPools[0].getHashedSet(object).getDisks(), "", m.bucket, object, oi.VersionID, false, false)
if errs[0] != nil {
m.t.Fatal(errs[0])
}
fi := infos[0]
fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", "remote"
if err := m.z.serverPools[0].DecomTieredObject(ctx, m.bucket, object, fi, ObjectOptions{}); err != nil {
m.t.Fatal(err)
}
stored, err := m.z.serverPools[0].GetObjectInfo(ctx, m.bucket, object, ObjectOptions{VersionID: oi.VersionID})
if err != nil || stored.TransitionedObject.Status != lifecycle.TransitionComplete {
m.t.Fatalf("fixture is not a transitioned version: %q, %v", stored.TransitionedObject.Status, err)
}
return oi
}
func (m *migrationILM) label(object, versionID string, deleted bool) string {
key := m.key(object, versionID)
if l, ok := m.markers[key]; ok && deleted {
return l
}
if l, ok := m.labels[key]; ok && !deleted {
return l
}
if deleted {
return "DM"
}
return "?"
}
// move moves the contents of pool 0 out with a decommission, as the admin API
// runs it, including the check for versions left behind, or with a rebalance
// pass, with the walker's accounting and the removal of the source copies.
// The rebalance goal is never reached, so the walk visits every object. The
// test fails unless a decommission completes. ILM has not caught up yet.
func (m *migrationILM) move(kind string) {
m.t.Helper()
ctx := m.t.Context()
if kind == "rebalance" {
migrationTestWalk(m.t, m.z, m.bucket, 0, kind)
m.z.rebalMu.Lock()
m.z.rebalMeta.PoolStats[0].Info.Status = rebalCompleted
m.z.rebalMu.Unlock()
return
}
if err := m.z.StartDecommission(ctx, 0); err != nil {
m.t.Fatal(err)
}
m.z.doDecommissionInRoutine(ctx, 0)
m.z.poolMetaMutex.RLock()
info := *m.z.poolMeta.Pools[0].Decommission
m.z.poolMetaMutex.RUnlock()
if !info.Complete || info.Failed || info.ItemsDecommissionFailed > 0 {
m.t.Fatalf("decommission did not complete: complete=%t failed=%t itemsFailed=%d; pool 0 holds %v",
info.Complete, info.Failed, info.ItemsDecommissionFailed, m.view(0))
}
}
// migrate moves the contents of pool 0 out and lets ILM catch up.
func (m *migrationILM) migrate(kind string) {
m.t.Helper()
m.move(kind)
m.catchUp()
}
// catchUp lets ILM catch up with data movement out of pool 0: the queued
// expiries run as the expiry workers run them, and the rules are applied to
// the current version of every object as the scanner applies them in the pool
// that holds it now. A transitioned version is not expired: its remote tier is
// not configured here.
func (m *migrationILM) catchUp() {
m.t.Helper()
ctx := m.t.Context()
for len(m.queue) > 0 {
op := <-m.queue
task, ok := op.(expiryTask)
if !ok {
m.t.Fatalf("unexpected expiry operation %T", op)
}
oi := task.objInfo
m.t.Logf("%s queued %s of %s %s", task.src, task.event.Action, oi.Name, m.label(oi.Name, oi.VersionID, oi.DeleteMarker))
if oi.TransitionedObject.Status == "" {
applyExpiryOnNonTransitionedObjects(ctx, m.z, oi, task.event, task.src)
}
}
lc, err := globalLifecycleSys.Get(m.bucket)
if err != nil {
return // no rules
}
lr, err := globalBucketObjectLockSys.Get(m.bucket)
if err != nil {
m.t.Fatal(err)
}
for _, object := range m.objects {
oi, err := m.z.GetObjectInfo(ctx, m.bucket, object, ObjectOptions{})
if isErrObjectNotFound(err) || isErrMethodNotAllowed(err) {
continue // deleted
}
if err != nil {
m.t.Fatal(err)
}
evt := evalActionFromLifecycle(ctx, *lc, lr, nil, oi)
if evt.Action.Delete() && oi.TransitionedObject.Status == "" {
applyExpiryOnNonTransitionedObjects(ctx, m.z, oi, evt, lcEventSrc_Scanner)
}
}
}
// view returns the versions of each object in a pool, newest first. Delete
// markers and versions the test did not write are "DM" and "?".
func (m *migrationILM) view(pool int) map[string][]string {
m.t.Helper()
view := map[string][]string{}
for _, set := range m.z.serverPools[pool].sets {
err := set.listObjectsToDecommission(m.t.Context(), decomBucketInfo{Name: m.bucket}, func(entry metaCacheEntry) {
fivs, err := entry.fileInfoVersions(m.bucket)
if err != nil {
m.t.Errorf("%s: %v", entry.name, err)
return
}
for _, fi := range fivs.Versions {
view[entry.name] = append(view[entry.name], m.label(entry.name, fi.VersionID, fi.Deleted))
}
})
if err != nil {
m.t.Fatal(err)
}
}
return view
}
// check fails the test unless pool 0 is empty and pool 1 holds the versions
// of want.
func (m *migrationILM) check(want map[string][]string) {
m.t.Helper()
if left := m.view(0); len(left) > 0 {
m.t.Errorf("pool 0 still holds %v", left)
}
got := m.view(1)
for _, object := range m.objects {
if !slices.Equal(got[object], want[object]) {
m.t.Errorf("%s: pool 1 holds %v, want %v", object, got[object], want[object])
}
}
}
// versions writes v1, v2 and v3 of each object.
func (m *migrationILM) versions(opts ObjectOptions, objects ...string) {
for _, object := range objects {
for _, label := range []string{"v1", "v2", "v3"} {
m.put(object, label, opts)
}
}
}
// aged writes v1, v2 and v3 of each object, five, four and age days ago.
func (m *migrationILM) aged(age int, objects ...string) {
for _, object := range objects {
m.put(object, "v1", ObjectOptions{Versioned: true, MTime: UTCNow().AddDate(0, 0, -5)})
m.put(object, "v2", ObjectOptions{Versioned: true, MTime: UTCNow().AddDate(0, 0, -4)})
m.put(object, "v3", ObjectOptions{Versioned: true, MTime: UTCNow().AddDate(0, 0, -age)})
}
}
// migrationILMRule returns a rule for the objects under obj-.
func migrationILMRule(action string) string {
return "<Rule><ID>rule</ID><Status>Enabled</Status><Filter><Prefix>obj-</Prefix></Filter>" + action + "</Rule>"
}
// migrationILMExpireCurrent expires the current versions at a past date (S3
// expiration dates are midnights).
func migrationILMExpireCurrent() string {
date := UTCNow().Truncate(24*time.Hour).AddDate(0, 0, -2).Format(time.RFC3339)
return migrationILMRule("<Expiration><Date>" + date + "</Date></Expiration>")
}
// testMigrationLifecycleExpiry runs the cases that decommission and rebalance
// share.
func testMigrationLifecycleExpiry(t *testing.T, kind string) {
versioned := ObjectOptions{Versioned: true}
t.Run("versioned/no-lifecycle", func(t *testing.T) {
m := newMigrationILM(t, versioning.Enabled, false, "")
m.versions(versioned, "obj-01", "keep-01")
m.migrate(kind)
m.check(map[string][]string{"obj-01": {"v3", "v2", "v1"}, "keep-01": {"v3", "v2", "v1"}})
})
// The expiry puts a delete marker on top of v3, which must be moved.
t.Run("versioned/expire-current", func(t *testing.T) {
m := newMigrationILM(t, versioning.Enabled, false, migrationILMExpireCurrent())
m.versions(versioned, "obj-01", "keep-01")
m.migrate(kind)
m.check(map[string][]string{"obj-01": {"DM", "v3", "v2", "v1"}, "keep-01": {"v3", "v2", "v1"}})
})
// Object lock does not hold back the delete marker, and v3 is under
// retention.
t.Run("versioned/expire-current-locked", func(t *testing.T) {
m := newMigrationILM(t, versioning.Enabled, true, migrationILMExpireCurrent())
m.put("obj-01", "v1", versioned)
m.put("obj-01", "v2", versioned)
v3 := m.put("obj-01", "v3", ObjectOptions{Versioned: true, UserDefined: map[string]string{
strings.ToLower(xhttp.AmzObjectLockMode): string(objectlock.RetCompliance),
strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): amztime.ISO8601Format(UTCNow().Add(24 * time.Hour)),
}})
m.migrate(kind)
m.check(map[string][]string{"obj-01": {"DM", "v3", "v2", "v1"}})
oi, err := m.z.serverPools[1].GetObjectInfo(t.Context(), m.bucket, "obj-01", ObjectOptions{VersionID: v3.VersionID})
if mode := objectlock.GetObjectRetentionMeta(oi.UserDefined).Mode; err != nil || mode != objectlock.RetCompliance {
t.Errorf("v3 retention in pool 1: %q, %v", mode, err)
}
})
// In a suspended bucket the expiry puts a null delete marker on top: a
// versioned v3 stays, a null current version is replaced.
t.Run("suspended/expire-current", func(t *testing.T) {
m := newMigrationILM(t, versioning.Suspended, false, migrationILMExpireCurrent())
m.versions(versioned, "obj-01")
m.put("obj-null", "v1", versioned)
m.put("obj-null", "null", ObjectOptions{VersionSuspended: true})
m.migrate(kind)
m.check(map[string][]string{"obj-01": {"DM", "v3", "v2", "v1"}, "obj-null": {"DM", "v1"}})
})
// v1 has been noncurrent for four days and is deleted by the queued
// expiry. The source copies of v2 and v3 must be removed although v1 has
// not been deleted yet.
t.Run("versioned/noncurrent-expiration", func(t *testing.T) {
m := newMigrationILM(t, versioning.Enabled, false,
migrationILMRule("<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration>"))
m.aged(0, "obj-01", "keep-01")
m.migrate(kind)
m.check(map[string][]string{"obj-01": {"v3", "v2"}, "keep-01": {"v3", "v2", "v1"}})
})
// Only a delete marker without noncurrent versions is expired.
t.Run("versioned/expired-delete-marker", func(t *testing.T) {
m := newMigrationILM(t, versioning.Enabled, false,
migrationILMRule("<Expiration><ExpiredObjectDeleteMarker>true</ExpiredObjectDeleteMarker></Expiration>"))
m.marker("obj-dm", "dm", UTCNow())
m.put("obj-top", "v1", versioned)
m.marker("obj-top", "dm", UTCNow())
m.marker("keep-dm", "dm", UTCNow())
m.migrate(kind)
m.check(map[string][]string{"obj-top": {"dm", "v1"}, "keep-dm": {"dm"}})
})
// Each rule deletes every version of one object. Migration must execute the
// logical DeleteAll action before it retires the source pool, so the object
// selected by the rule is gone from both pools before any later scan.
for _, tc := range []struct {
name, rule, deleted string
}{
{"expire-all-versions", "<Expiration><Days>1</Days><ExpiredObjectAllVersions>true</ExpiredObjectAllVersions></Expiration>", "obj-01"},
{"expire-marker-all-versions", "<DelMarkerExpiration><Days>1</Days></DelMarkerExpiration>", "obj-dm"},
} {
t.Run("versioned/"+tc.name, func(t *testing.T) {
m := newMigrationILM(t, versioning.Enabled, false, migrationILMRule(tc.rule))
m.aged(3, "obj-01", "keep-01")
m.put("obj-dm", "v1", ObjectOptions{Versioned: true, MTime: UTCNow().AddDate(0, 0, -5)})
m.marker("obj-dm", "dm", UTCNow().AddDate(0, 0, -3))
m.move(kind)
if len(m.queue) != 0 {
t.Fatalf("migration left stale expiry tasks queued: %d", len(m.queue))
}
want := map[string][]string{
"obj-01": {"v3", "v2", "v1"},
"keep-01": {"v3", "v2", "v1"},
"obj-dm": {"dm", "v1"},
}
delete(want, tc.deleted)
m.check(want)
m.catchUp()
m.check(want)
})
}
t.Run("unversioned/expire-current", func(t *testing.T) {
m := newMigrationILM(t, "", false, migrationILMExpireCurrent())
m.put("obj-01", "null", ObjectOptions{})
m.put("keep-01", "null", ObjectOptions{})
m.migrate(kind)
m.check(map[string][]string{"keep-01": {"null"}})
})
}
// A lifecycle action found during the migration walk must not be applied
// after the bucket's lifecycle configuration changes. The source copy stays
// available for a fresh scan under the new policy.
func TestPoolsMigrationLifecycleChangePreservesSource(t *testing.T) {
m := newMigrationILM(t, versioning.Enabled, false, migrationILMExpireCurrent())
oi := m.put("obj-01", "v1", ObjectOptions{Versioned: true})
_, updatedAt, err := globalBucketMetadataSys.GetLifecycleConfig(m.bucket)
if err != nil {
t.Fatal(err)
}
meta, err := globalBucketMetadataSys.Get(m.bucket)
if err != nil {
t.Fatal(err)
}
meta.LifecycleConfigXML = []byte(`<LifecycleConfiguration><Rule><ID>changed</ID><Status>Enabled</Status><Filter><Prefix>other-</Prefix></Filter><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>`)
meta.LifecycleConfigUpdatedAt = updatedAt.Add(time.Second)
if err := meta.Save(t.Context(), m.z); err != nil {
t.Fatal(err)
}
globalBucketMetadataSys.Set(m.bucket, meta)
err = m.z.applyMigrationExpiry(t.Context(), m.bucket, oi.Name, 0, []migrationExpiredVersion{{
version: FileInfo{
Name: oi.Name,
VersionID: oi.VersionID,
ModTime: oi.ModTime,
},
event: lifecycle.Event{Action: lifecycle.DeleteAction},
lifecycleUpdatedAt: updatedAt,
source: lcEventSrc_Decom,
}})
if err == nil {
t.Fatal("migration expiry succeeded after lifecycle configuration changed")
}
if _, err := m.z.serverPools[0].GetObjectInfo(t.Context(), m.bucket, oi.Name, ObjectOptions{VersionID: oi.VersionID}); err != nil {
t.Fatalf("source version was removed after lifecycle configuration changed: %v", err)
}
}
// A migration must route an ILM version purge through the normal delete
// replication path before it can retire the source copy. Otherwise the
// source pool can disappear while the replica still retains the version.
func TestPoolsMigrationLifecycleExpiryReplication(t *testing.T) {
m := newMigrationILM(t, versioning.Enabled, false,
migrationILMRule("<NoncurrentVersionExpiration><NoncurrentDays>1</NoncurrentDays></NoncurrentVersionExpiration>"))
m.aged(0, "obj-01")
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
stats := NewReplicationStats(ctx, nil)
oldStats := globalReplicationStats.Swap(stats)
defer globalReplicationStats.Store(oldStats)
target := newILMTestTarget(t)
registerILMTestTarget(t, m.z, m.bucket, target, func(w http.ResponseWriter, r *http.Request) {
opts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true}
switch r.Method {
case http.MethodHead:
oi, err := m.z.GetObjectInfo(r.Context(), target.bucket, "obj-01", opts)
if err != nil {
writeErrorResponseHeadersOnly(w, toAPIError(r.Context(), err))
return
}
if oi.DeleteMarker {
w.Header().Set(xhttp.AmzDeleteMarker, "true")
w.Header().Set(xhttp.AmzVersionID, oi.VersionID)
}
w.WriteHeader(http.StatusOK)
case http.MethodDelete:
target.deletes.Add(1)
opts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"
opts.ReplicationRequest = true
opts.SetReplicaStatus(replication.Replica)
if _, err := m.z.DeleteObject(r.Context(), target.bucket, "obj-01", opts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
return
}
w.WriteHeader(http.StatusNoContent)
default:
t.Errorf("unexpected remote method %s", r.Method)
}
})
setILMTestReplicationConfig(t, m.bucket, target.arn, target.bucket)
restorePool, receive := newILMTestReplicationPool(ctx, m.z, stats)
defer restorePool()
if err := m.z.StartDecommission(ctx, 0); err != nil {
t.Fatal(err)
}
m.z.doDecommissionInRoutine(ctx, 0)
m.z.poolMetaMutex.RLock()
info := *m.z.poolMeta.Pools[0].Decommission
m.z.poolMetaMutex.RUnlock()
if info.Complete || !info.Failed {
t.Fatalf("decommission completed before ILM purge replication: complete=%t failed=%t", info.Complete, info.Failed)
}
deletion := receive()
if deletion.ObjectName != "obj-01" || deletion.VersionID == "" || deletion.VersionPurgeStatus() != replication.VersionPurgePending {
t.Fatalf("unexpected migration purge: %+v", deletion)
}
if result := replicateDelete(ctx, deletion, m.z); result.VersionPurgeStatus() != replication.VersionPurgeComplete {
t.Fatalf("migration purge replication result: %+v", result)
}
if target.deletes.Load() != 1 {
t.Fatalf("replication target saw %d deletes, want 1", target.deletes.Load())
}
// A failed decommission may already have moved the source pool's bucket
// metadata. Recreate the metadata directory before the resumable retry,
// as the server's scanner would do in production.
for _, set := range m.z.serverPools[0].sets {
for _, disk := range set.getDisks() {
if err := disk.MakeVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix)); err != nil && !errors.Is(err, errVolumeExists) {
t.Fatal(err)
}
}
}
if err := m.z.StartDecommission(ctx, 0); err != nil {
t.Fatal(err)
}
m.z.doDecommissionInRoutine(ctx, 0)
m.z.poolMetaMutex.RLock()
info = *m.z.poolMeta.Pools[0].Decommission
m.z.poolMetaMutex.RUnlock()
if !info.Complete || info.Failed {
t.Fatalf("decommission did not complete after ILM purge replication: complete=%t failed=%t", info.Complete, info.Failed)
}
}
func TestPoolsDecommissionLifecycleExpiry(t *testing.T) {
testMigrationLifecycleExpiry(t, "decommission")
// A transitioned v3 is moved as metadata; its remote data stays.
// Rebalance does not move transitioned versions yet.
t.Run("versioned/expire-current-tiered", func(t *testing.T) {
m := newMigrationILM(t, versioning.Enabled, false, migrationILMExpireCurrent())
m.put("obj-01", "v1", ObjectOptions{Versioned: true})
m.put("obj-01", "v2", ObjectOptions{Versioned: true})
v3 := m.tiered("obj-01", "v3")
m.migrate("decommission")
m.check(map[string][]string{"obj-01": {"v3", "v2", "v1"}})
oi, err := m.z.serverPools[1].GetObjectInfo(t.Context(), m.bucket, "obj-01", ObjectOptions{VersionID: v3.VersionID})
if err != nil || oi.TransitionedObject.Status != lifecycle.TransitionComplete {
t.Errorf("v3 in pool 1 is not transitioned: %q, %v", oi.TransitionedObject.Status, err)
}
})
}
// A current version that ILM expires in a versioned bucket stays as a
// noncurrent version. The check after decommission must count it when it is
// left in the pool.
func TestPoolsDecommissionCountsExpiredCurrentVersions(t *testing.T) {
m := newMigrationILM(t, versioning.Enabled, false, migrationILMExpireCurrent())
m.migrate("decommission")
putConsistencyObject(t, m.z, m.bucket, "obj-01", 0, "v1", ObjectOptions{Versioned: true})
if err := m.z.checkAfterDecom(t.Context(), 0); err == nil {
t.Fatal("decommission check passed with an expired current version left in the pool")
}
}
// Rebalance removes the source copies of an object only when it has counted
// all of its versions. A version that ILM deletes must be counted, or the
// moved versions stay in pool 0 as duplicates.
func TestPoolsRebalanceLifecycleExpiry(t *testing.T) {
testMigrationLifecycleExpiry(t, "rebalance")
}
+427 -8
View File
@@ -24,11 +24,14 @@ import (
"fmt" "fmt"
"io" "io"
"maps" "maps"
"net/http"
"slices"
"strings" "strings"
"sync" "sync"
"testing" "testing"
"time" "time"
"github.com/minio/minio/internal/bucket/lifecycle"
xhttp "github.com/minio/minio/internal/http" xhttp "github.com/minio/minio/internal/http"
) )
@@ -114,15 +117,18 @@ func migrationTestPools(t *testing.T) (*erasureServerPools, string) {
t.Helper() t.Helper()
z, bucket := consistencyPools(t) z, bucket := consistencyPools(t)
for _, pool := range z.serverPools { for _, pool := range z.serverPools {
for _, set := range pool.sets { pool.erasureDisksMu.Lock()
original := set.getDisks previous := pool.erasureDisks
disks := append([]StorageAPI(nil), original()...) wrapped := make([][]StorageAPI, len(previous))
for i := range disks { for i, disks := range previous {
disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]} wrapped[i] = append([]StorageAPI(nil), disks...)
for j := range wrapped[i] {
wrapped[i][j] = tagTestCapacityDisk{StorageAPI: wrapped[i][j]}
} }
set.getDisks = func() []StorageAPI { return disks }
t.Cleanup(func() { set.getDisks = original })
} }
pool.erasureDisks = wrapped
pool.erasureDisksMu.Unlock()
t.Cleanup(func() { pool.erasureDisksMu.Lock(); pool.erasureDisks = previous; pool.erasureDisksMu.Unlock() })
} }
return z, bucket return z, bucket
} }
@@ -131,7 +137,10 @@ func migrationTestMover(t *testing.T, z *erasureServerPools, source int, kind st
t.Helper() t.Helper()
if kind == "rebalance" { if kind == "rebalance" {
z.rebalMu.Lock() z.rebalMu.Lock()
z.rebalMeta = &rebalanceMeta{PoolStats: []*rebalanceStats{{}, {}}} z.rebalMeta = &rebalanceMeta{PoolStats: make([]*rebalanceStats, len(z.serverPools))}
for i := range z.rebalMeta.PoolStats {
z.rebalMeta.PoolStats[i] = &rebalanceStats{}
}
z.rebalMeta.PoolStats[source] = &rebalanceStats{Participating: true, Info: rebalanceInfo{Status: rebalStarted}} z.rebalMeta.PoolStats[source] = &rebalanceStats{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}
z.rebalMu.Unlock() z.rebalMu.Unlock()
t.Cleanup(func() { z.rebalMu.Lock(); z.rebalMeta = nil; z.rebalMu.Unlock() }) t.Cleanup(func() { z.rebalMu.Lock(); z.rebalMeta = nil; z.rebalMu.Unlock() })
@@ -590,3 +599,413 @@ func TestPoolsMigrationUnreadablePool(t *testing.T) {
} }
} }
} }
// Three pools give a move two candidate destinations.
func migrationTestThreePools(t *testing.T) (*erasureServerPools, string) {
t.Helper()
ctx, cancel := context.WithCancel(t.Context())
dirs, err := getRandomDisks(12)
if err != nil {
cancel()
t.Fatal(err)
}
var endpoints EndpointServerPools
for i := range 3 {
endpoints = append(endpoints, mustGetPoolEndpoints(i, dirs[4*i:4*i+4]...)...)
}
obj, _, err := initObjectLayer(ctx, endpoints)
if err != nil {
cancel()
removeRoots(dirs)
t.Fatal(err)
}
z := obj.(*erasureServerPools)
previous := newObjectLayerFn()
setObjectLayer(z)
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) })
bucket := "pool-migration"
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
return z, bucket
}
type migrationTestGetter func(context.Context, string, string, *HTTPRangeSpec, http.Header, ObjectOptions) (*GetObjectReader, error)
// migrationTestBody returns the content read for a version, or the read error.
func migrationTestBody(t *testing.T, get migrationTestGetter, bucket, object, versionID string) string {
t.Helper()
gr, err := get(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: versionID})
if err != nil {
return fmt.Sprintf("<%v>", err)
}
defer gr.Close()
data, err := io.ReadAll(gr)
if err != nil {
t.Fatal(err)
}
return string(data)
}
// migrationTestWalk runs one bucket of a decommission or rebalance of source,
// including the walker's accounting and source cleanup. It returns the number
// of versions decommission counted as failed.
func migrationTestWalk(t *testing.T, z *erasureServerPools, bucket string, source int, kind string) int64 {
t.Helper()
migrationTestMover(t, z, source, kind)
if kind == "rebalance" {
if err := z.rebalanceBucket(t.Context(), bucket, source); err != nil {
t.Fatal(err)
}
return 0
}
z.poolMetaMutex.Lock()
z.poolMeta.Pools[source].LastUpdate = UTCNow() // keep pool.bin and peers out of the walk
z.poolMetaMutex.Unlock()
if err := z.decommissionPool(t.Context(), source, z.serverPools[source], decomBucketInfo{Name: bucket}); err != nil {
t.Fatal(err)
}
z.poolMetaMutex.RLock()
defer z.poolMetaMutex.RUnlock()
return z.poolMeta.Pools[source].Decommission.ItemsDecommissionFailed
}
// migrationTestTiered stores a null version whose data is in a remote tier, in
// the form decommission moves it.
func migrationTestTiered(t *testing.T, z *erasureServerPools, bucket, object string, pool int, mtime time.Time) ObjectInfo {
t.Helper()
oi := putConsistencyObject(t, z, bucket, object, pool, "old", ObjectOptions{MTime: mtime})
infos, errs := readAllFileInfo(t.Context(), z.serverPools[pool].getHashedSet(oi.Name).getDisks(), "", bucket, oi.Name, nullVersionID, false, false)
if errs[0] != nil {
t.Fatal(errs[0])
}
fi := infos[0]
fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName = lifecycle.TransitionComplete, "WARM", "remote"
if err := z.serverPools[pool].DecomTieredObject(t.Context(), bucket, oi.Name, fi, ObjectOptions{}); err != nil {
t.Fatal(err)
}
return oi
}
// A cross-pool overwrite can leave copies of one version in several pools.
// Reads return the newest copy, and moving an older one must not replace it.
// An equal ModTime marks the same write, as an interrupted move leaves it.
func TestPoolsMigrationKeepsCurrentCopy(t *testing.T) {
z, bucket := migrationTestPools(t)
base := time.Date(2026, 9, 16, 8, 0, 0, 0, time.UTC)
for _, kind := range []string{"rebalance", "decommission"} {
for _, method := range []string{"put", "multipart"} {
for _, versioned := range []bool{false, true} {
for source := range 2 {
for _, offset := range []time.Duration{-time.Minute, 0, time.Minute} {
t.Run(fmt.Sprintf("%s/%s/versioned=%t/source=%d/offset=%s", kind, method, versioned, source, offset), func(t *testing.T) {
move := migrationTestMover(t, z, source, kind)
opts := ObjectOptions{Versioned: versioned, MTime: base.Add(offset)}
if versioned {
opts.VersionID = mustGetUUID()
}
oi := migrationTestObject(t, z, bucket, t.Name(), source, method == "multipart", []string{"source"}, opts)
opts.MTime = base
want, other := "source", "other"
if offset == 0 {
other = "source"
opts.PreserveETag = oi.ETag // interrupted copies preserve the original multipart ETag
} else if offset < 0 {
want = "other"
}
putConsistencyObject(t, z, bucket, t.Name(), 1-source, other, opts)
version := migrationTestVersion(oi)
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, t.Name(), version); got != want {
t.Fatalf("fixture reads %q, want %q", got, want)
}
err := move(t.Context(), source, bucket, migrationTestReader(t, z, source, oi))
if want == "other" && !isDataMovementSuperseded(err) || want == "source" && err != nil {
t.Fatalf("move: %v", err)
}
if err != nil && strings.Contains(err.Error(), "CompleteMultipartUpload") {
t.Error("copied the parts of a superseded version")
}
if got := migrationTestBody(t, z.serverPools[1-source].GetObjectNInfo, bucket, t.Name(), version); got != want {
t.Errorf("destination holds %q, want %q", got, want)
}
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, t.Name(), version); got != want {
t.Errorf("move changed the read result to %q, want %q", got, want)
}
})
}
}
}
}
}
}
// A client overwrite, or a replica of the same version, committed while a move
// copies the old version is newer than the moved copy. The move must yield to
// it, also at multipart completion.
func TestPoolsMigrationYieldsToNewerWrite(t *testing.T) {
z, bucket := migrationTestPools(t)
for _, kind := range []string{"rebalance", "decommission"} {
for _, method := range []string{"put", "multipart"} {
for _, versioned := range []bool{false, true} {
for source := range 2 {
t.Run(fmt.Sprintf("%s/%s/versioned=%t/source=%d", kind, method, versioned, source), func(t *testing.T) {
migrationTestNewerWrite(t, z, bucket, kind, method, versioned, source)
})
}
}
}
}
}
func migrationTestNewerWrite(t *testing.T, z *erasureServerPools, bucket, kind, method string, versioned bool, source int) {
move := migrationTestMover(t, z, source, kind)
oi := migrationTestObject(t, z, bucket, t.Name(), source, method == "multipart", []string{"moved"}, ObjectOptions{Versioned: versioned, MTime: UTCNow().Add(-time.Hour)})
gr := migrationTestReader(t, z, source, oi)
ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
defer cancel()
gate := newMigrationTestGate()
done, finished := make(chan error, 1), make(chan struct{})
if method == "multipart" {
// Hold the move after its initial check, while it copies parts.
gr.Reader = migrationTestGateReader{Reader: gr.Reader, ctx: ctx, gate: gate}
go func() { defer close(finished); done <- move(ctx, source, bucket, gr) }()
defer func() { gate.unblock(); cancel(); <-finished }()
select {
case <-gate.entered:
case err := <-done:
t.Fatalf("migration finished before the part barrier: %v", err)
case <-ctx.Done():
t.Fatal(ctx.Err())
}
}
// An ordinary PUT copies under the object lock, so this write
// lands between the mover's source read and its write.
if versioned {
putConsistencyObject(t, z, bucket, t.Name(), 1-source, "newer", ObjectOptions{Versioned: true, VersionID: oi.VersionID, MTime: UTCNow()})
} else if _, err := z.PutObject(ctx, bucket, t.Name(), mustGetPutObjReader(t, strings.NewReader("newer"), 5, "", ""), ObjectOptions{}); err != nil {
t.Fatal(err)
}
var err error
if method == "multipart" {
gate.unblock()
err = <-done
} else {
err = move(ctx, source, bucket, gr)
}
if !isDataMovementSuperseded(err) {
t.Fatalf("move over a newer write: %v", err)
}
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, t.Name(), migrationTestVersion(oi)); got != "newer" {
t.Fatalf("newer write replaced by %q", got)
}
}
// With two candidate destinations a superseded copy must neither replace the
// newer copy nor become a new duplicate in the third pool.
func TestPoolsMigrationThreePools(t *testing.T) {
z, bucket := migrationTestThreePools(t)
stale, recent := UTCNow().Add(-time.Hour), UTCNow().Add(-time.Minute)
for _, kind := range []string{"rebalance", "decommission"} {
for _, newest := range []int{0, 1} {
t.Run(fmt.Sprintf("%s/newest=%d", kind, newest), func(t *testing.T) {
move := migrationTestMover(t, z, 0, kind)
mtime := func(pool int) time.Time {
if pool == newest {
return recent
}
return stale
}
oi := migrationTestObject(t, z, bucket, t.Name(), 0, false, []string{"pool-0"}, ObjectOptions{MTime: mtime(0)})
putConsistencyObject(t, z, bucket, t.Name(), 1, "pool-1", ObjectOptions{MTime: mtime(1)})
if newest == 0 {
putConsistencyObject(t, z, bucket, t.Name(), 2, "pool-2", ObjectOptions{MTime: stale})
}
want := fmt.Sprintf("pool-%d", newest)
err := move(t.Context(), 0, bucket, migrationTestReader(t, z, 0, oi))
if newest == 1 && !isDataMovementSuperseded(err) || newest == 0 && err != nil {
t.Fatalf("move: %v", err)
}
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, t.Name(), ""); got != want {
t.Errorf("reads return %q, want %q", got, want)
}
if newest == 1 {
if _, err := z.serverPools[2].GetObjectInfo(t.Context(), bucket, t.Name(), ObjectOptions{}); !isErrObjectNotFound(err) {
t.Errorf("superseded copy duplicated into pool 2: %v", err)
}
}
})
}
}
}
// The walkers count a superseded version as moved, so its stale source copy is
// removed with the object and the source pool drains.
func TestPoolsMigrationRetiresSupersededCopy(t *testing.T) {
for _, kind := range []string{"rebalance", "decommission"} {
t.Run(kind, func(t *testing.T) {
z, bucket := migrationTestPools(t)
ctx := t.Context()
stale, recent := UTCNow().Add(-time.Hour), UTCNow().Add(-time.Minute)
put := func(object string, pool int, body string, multipart bool, opts ObjectOptions) ObjectInfo {
return migrationTestObject(t, z, bucket, object, pool, multipart, []string{body}, opts)
}
put("stale", 0, "old", false, ObjectOptions{MTime: stale})
put("stale", 1, "new", false, ObjectOptions{MTime: recent})
put("stale-multipart", 0, "old", true, ObjectOptions{MTime: stale})
put("stale-multipart", 1, "new", false, ObjectOptions{MTime: recent})
put("newer", 0, "new", false, ObjectOptions{MTime: recent})
put("newer", 1, "old", false, ObjectOptions{MTime: stale})
oldest := put("history", 0, "old", false, ObjectOptions{Versioned: true, MTime: stale})
put("history", 1, "new", false, ObjectOptions{Versioned: true, MTime: recent})
// A suspended bucket's older null delete marker must not hide a newer
// null version.
hidden := put("stale-marker", 0, "old", false, ObjectOptions{Versioned: true, MTime: stale.Add(-time.Minute)})
if _, err := z.serverPools[0].DeleteObject(ctx, bucket, "stale-marker", ObjectOptions{VersionSuspended: true, MTime: stale}); err != nil {
t.Fatal(err)
}
put("stale-marker", 1, "new", false, ObjectOptions{VersionSuspended: true, MTime: recent})
objects := []string{"stale", "stale-multipart", "newer", "history", "stale-marker"}
if kind == "decommission" {
// Rebalance leaves tiered versions in place; decommission moves
// their metadata.
migrationTestTiered(t, z, bucket, "stale-tiered", 0, stale)
put("stale-tiered", 1, "new", false, ObjectOptions{MTime: recent})
objects = append(objects, "stale-tiered")
}
want := map[string]string{}
for _, object := range objects {
want[object] = migrationTestBody(t, z.GetObjectNInfo, bucket, object, "")
}
if want["stale"] != "new" || want["stale-marker"] != "new" {
t.Fatalf("fixture reads %v", want)
}
if failed := migrationTestWalk(t, z, bucket, 0, kind); failed != 0 {
t.Fatalf("%d versions failed to move", failed)
}
for _, object := range objects {
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, object, ""); got != want[object] {
t.Errorf("%s: reads %q after the move, want %q", object, got, want[object])
}
}
var retained []string
for _, set := range z.serverPools[0].sets {
if err := set.listObjectsToDecommission(ctx, decomBucketInfo{Name: bucket}, func(entry metaCacheEntry) {
retained = append(retained, entry.name)
}); err != nil {
t.Fatal(err)
}
}
if len(retained) > 0 {
t.Errorf("source pool retained %v", retained)
}
for _, oi := range []ObjectInfo{oldest, hidden} {
if got := migrationTestBody(t, z.GetObjectNInfo, bucket, oi.Name, oi.VersionID); got != "old" {
t.Errorf("%s: lost version %s: %q", oi.Name, oi.VersionID, got)
}
}
})
}
}
func TestPoolsMigrationAfterCompleteRemoval(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
dirs, err := getRandomDisks(32)
if err != nil {
t.Fatal(err)
}
previousEndpoints := globalEndpoints
endpoints := mustGetPoolEndpoints(0, dirs[:16]...)
endpoints = append(endpoints, mustGetPoolEndpoints(1, dirs[16:]...)...)
globalEndpoints = endpoints
obj, _, err := initObjectLayer(ctx, endpoints)
if err != nil {
globalEndpoints = previousEndpoints
removeRoots(dirs)
t.Fatal(err)
}
z := obj.(*erasureServerPools)
previous := newObjectLayerFn()
setObjectLayer(z)
var reopened *erasureServerPools
oldShutdown := false
t.Cleanup(func() {
if reopened != nil {
_ = reopened.Shutdown(context.Background())
}
if !oldShutdown {
_ = z.Shutdown(context.Background())
}
removeRoots(dirs)
globalEndpoints = previousEndpoints
setObjectLayer(previous)
})
bucket, _, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{})
if err != nil {
t.Fatal(err)
}
const object = "object"
mtime := time.Now().UTC().Add(-time.Hour)
oldBody := bytes.Repeat([]byte{'a'}, 64<<10)
newBody := bytes.Repeat([]byte{'b'}, 256<<10)
nullQuorumCrossParityOverwrite(t, z, 0, bucket, object, oldBody, newBody, mtime, ObjectOptions{})
z.poolMetaMutex.Lock()
z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
z.poolMetaMutex.Unlock()
if err := z.decommissionPool(ctx, 0, z.serverPools[0], decomBucketInfo{Name: bucket}); err != nil {
t.Fatal(err)
}
if err := z.CompleteDecommission(ctx, 0); err != nil {
t.Fatal(err)
}
if err := z.Shutdown(context.Background()); err != nil {
t.Fatal(err)
}
oldShutdown = true
globalEndpoints = mustGetPoolEndpoints(0, dirs[16:]...)
reopenedObj, _, err := initObjectLayer(t.Context(), globalEndpoints)
if err != nil {
t.Fatal(err)
}
reopened = reopenedObj.(*erasureServerPools)
setObjectLayer(reopened)
_, router, err := initAPIHandlerTest(t.Context(), reopened, nil, MakeBucketOptions{})
if err != nil {
t.Fatal(err)
}
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object))
if get.Code != http.StatusOK || !bytes.Equal(get.Body.Bytes(), newBody) {
t.Fatalf("GET after pool removal: status=%d bytes=%d", get.Code, get.Body.Len())
}
if code, keys := nullQuorumListV2Keys(t, router, bucket); code != http.StatusOK || !slices.Equal(keys, []string{object}) {
t.Fatalf("ListObjectsV2 after pool removal: status=%d keys=%v", code, keys)
}
if code, versions := nullQuorumListVersions(t, router, bucket); code != http.StatusOK || len(versions) != 1 || versions[0].Key != object || versions[0].VersionID != nullVersionID || versions[0].Size != int64(len(newBody)) {
t.Fatalf("ListObjectVersions after pool removal: status=%d versions=%+v", code, versions)
}
}
// A tiered move that fails, here because another pool cannot be read, must keep
// its source like any other failed version.
func TestPoolsMigrationTieredFailureKeepsSource(t *testing.T) {
z, bucket := migrationTestPools(t)
tiered := migrationTestTiered(t, z, bucket, t.Name(), 0, UTCNow().Add(-time.Hour))
set := z.serverPools[1].getHashedSet(tiered.Name)
original := set.getDisks
disks := append([]StorageAPI(nil), original()...)
for i := range disks {
disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: tiered.Name}
}
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = original }()
if failed := migrationTestWalk(t, z, bucket, 0, "decommission"); failed != 1 {
t.Fatalf("%d failed versions, want 1", failed)
}
if _, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, tiered.Name, ObjectOptions{VersionID: nullVersionID}); err != nil {
t.Fatalf("failed move removed its source: %v", err)
}
}
@@ -355,7 +355,7 @@ func TestPoolsConditionalPutVersionSelection(t *testing.T) {
opts.ReplicaLockReconcile, opts.ReplicationRequest = true, true opts.ReplicaLockReconcile, opts.ReplicationRequest = true, true
want = addressed want = addressed
case "movement": case "movement":
opts.DataMovement, opts.SrcPoolIdx = true, 1 opts.DataMovement, opts.SrcPoolIdx, opts.MTime = true, 1, addressed.ModTime
want = addressed want = addressed
case "tie": case "tie":
want = putConsistencyObject(t, z, bucket, object, 0, "tie-winner", ObjectOptions{Versioned: true, MTime: current.ModTime}) want = putConsistencyObject(t, z, bucket, object, 0, "tie-winner", ObjectOptions{Versioned: true, MTime: current.ModTime})
+50 -31
View File
@@ -542,6 +542,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
var vc *versioning.Versioning var vc *versioning.Versioning
var lc *lifecycle.Lifecycle var lc *lifecycle.Lifecycle
var lifecycleUpdatedAt time.Time
var lr objectlock.Retention var lr objectlock.Retention
var rcfg *replication.Config var rcfg *replication.Config
if bucket != minioMetaBucket { if bucket != minioMetaBucket {
@@ -551,7 +552,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
} }
// Check if the current bucket has a configured lifecycle policy // Check if the current bucket has a configured lifecycle policy
lc, err = globalLifecycleSys.Get(bucket) lc, lifecycleUpdatedAt, err = globalBucketMetadataSys.GetLifecycleConfig(bucket)
if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) { if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
return err return err
} }
@@ -587,19 +588,27 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
} }
for setIdx, set := range pool.sets { for setIdx, set := range pool.sets {
filterLifecycle := func(bucket, object string, fi FileInfo) bool { // filterLifecycle returns an ILM deletion that migration must handle
// before retiring the source copy, or NoneAction.
filterLifecycle := func(bucket, object string, fi FileInfo) lifecycle.Action {
if lc == nil { if lc == nil {
return false return lifecycle.NoneAction
} }
versioned := vc != nil && vc.Versioned(object) versioned := vc != nil && vc.Versioned(object)
objInfo := fi.ToObjectInfo(bucket, object, versioned) objInfo := fi.ToObjectInfo(bucket, object, versioned)
evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo) evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
if evt.Action.Delete() { switch {
globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Rebal) case evt.Action == lifecycle.DeleteAction && versioned:
return true // Expiring the latest version of a versioned bucket only puts
// a delete marker on top of it. The version itself survives as
// a noncurrent version and has to be moved like any other. The
// scanner applies the expiry once it lives in another pool.
return lifecycle.NoneAction
case evt.Action.Delete():
return evt.Action
} }
return false return lifecycle.NoneAction
} }
rebalanceEntry := func(entry metaCacheEntry) { rebalanceEntry := func(entry metaCacheEntry) {
@@ -623,7 +632,8 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
// to create the appropriate stack. // to create the appropriate stack.
versionsSorter(fivs.Versions).reverse() versionsSorter(fivs.Versions).reverse()
var rebalanced, expired int var rebalanced int
var expired []migrationExpiredVersion
for _, version := range fivs.Versions { for _, version := range fivs.Versions {
stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name, version.VersionID) stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name, version.VersionID)
@@ -634,22 +644,25 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
} }
// Apply lifecycle rules on the objects that are expired. // Apply lifecycle rules on the objects that are expired.
if filterLifecycle(bucket, version.Name, version) { if action := filterLifecycle(bucket, version.Name, version); action.Delete() {
expired++ // Count the version, or the source copies of the moved
// ones stay behind as duplicates. Its own source copy
// goes with them when ILM deletes it alone; any other
// version stays until the expiry has run.
rebalanced++
if ilmDeletesVersionOnly(action) || action.DeleteAll() {
expired = append(expired, migrationExpiredVersion{
version: version,
event: lifecycle.Event{Action: action},
lifecycleUpdatedAt: lifecycleUpdatedAt,
source: lcEventSrc_Rebal,
})
}
stopFn(version.Size, errors.New("ILM expired object/version will be skipped")) stopFn(version.Size, errors.New("ILM expired object/version will be skipped"))
continue continue
} }
// any object with only single DEL marker we don't need // Preserve every marker: older versions may live in another pool.
// to rebalance, just skip it, this also includes
// any other versions that have already expired.
remainingVersions := len(fivs.Versions) - expired
if version.Deleted && remainingVersions == 1 {
rebalanced++
stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped"))
continue
}
versionID := version.VersionID versionID := version.VersionID
if versionID == "" { if versionID == "" {
versionID = nullVersionID versionID = nullVersionID
@@ -671,6 +684,9 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
SkipRebalancing: true, // make sure we skip the decommissioned pool SkipRebalancing: true, // make sure we skip the decommissioned pool
NoAuditLog: true, NoAuditLog: true,
}) })
if isDataMovementSuperseded(err) {
err = nil
}
// This can happen when rebalance stop races with ongoing rebalance workers. // This can happen when rebalance stop races with ongoing rebalance workers.
// These rebalance failures can be ignored. // These rebalance failures can be ignored.
if err != nil { if err != nil {
@@ -719,7 +735,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
continue continue
} }
if err = z.rebalanceObject(ctx, poolIdx, bucket, gr); err != nil { if err = z.rebalanceObject(ctx, poolIdx, bucket, gr); err != nil && !isDataMovementSuperseded(err) {
// This can happen when rebalance stop races with ongoing rebalance workers. // This can happen when rebalance stop races with ongoing rebalance workers.
// These rebalance failures can be ignored. // These rebalance failures can be ignored.
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) { if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
@@ -750,15 +766,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
// if all versions were rebalanced, we can delete the object versions. // if all versions were rebalanced, we can delete the object versions.
if rebalanced == len(fivs.Versions) { if rebalanced == len(fivs.Versions) {
stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceRemoveObject, poolIdx, bucket, entry.name) stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceRemoveObject, poolIdx, bucket, entry.name)
_, err := set.DeleteObject(ctx, err := z.removeMovedVersions(ctx, bucket, entry.name, poolIdx, fivs.Versions, expired)
bucket,
encodeDirObject(entry.name),
ObjectOptions{
DeletePrefix: true, // use prefix delete to delete all versions at once.
DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
NoAuditLog: true,
},
)
stopFn(0, err) stopFn(0, err)
auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err) auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err)
if err != nil { if err != nil {
@@ -863,6 +871,12 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, b
} }
if oi.isMultipart() { if oi.isMultipart() {
// Completion repeats this check under the object write lock.
if _, err := z.checkDataMovementCopies(ctx, bucket, encodeDirObject(oi.Name), ObjectOptions{
VersionID: oi.VersionID, MTime: oi.ModTime, SrcPoolIdx: poolIdx,
}); err != nil {
return err
}
res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{ res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{
VersionID: oi.VersionID, VersionID: oi.VersionID,
UserDefined: migrationObjectMetadata(oi), UserDefined: migrationObjectMetadata(oi),
@@ -900,6 +914,8 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, b
} }
} }
_, err = z.CompleteMultipartUpload(ctx, bucket, oi.Name, res.UploadID, parts, ObjectOptions{ _, err = z.CompleteMultipartUpload(ctx, bucket, oi.Name, res.UploadID, parts, ObjectOptions{
VersionID: oi.VersionID,
SrcPoolIdx: poolIdx,
DataMovement: true, DataMovement: true,
MTime: oi.ModTime, MTime: oi.ModTime,
NoAuditLog: true, NoAuditLog: true,
@@ -1010,7 +1026,10 @@ const (
rebalanceMetricSaveMetadata rebalanceMetricSaveMetadata
) )
var errDataMovementSrcDstPoolSame = errors.New("source and destination pool are the same") var (
errDataMovementSrcDstPoolSame = errors.New("source and destination pool are the same")
errDataMovementSuperseded = errors.New("another pool holds a newer copy")
)
func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string, sz int64) madmin.TraceInfo { func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string, sz int64) madmin.TraceInfo {
var errStr string var errStr string
+289
View File
@@ -0,0 +1,289 @@
//go:build consistency_strict
// Preserved diagnostic for the deferred strict-read policy. This is not a
// maintenance release gate; failures record intentionally unresolved risks.
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"crypto/md5"
"crypto/sha256"
"encoding/base64"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
)
// An acknowledged recovery overwrite must resolve the unreadable pool, even
// when another pool already has a readable copy of the same object.
// Real inline metadata is combined in the same 15-old/1-new state used by
// that regression. Capacity adapters only make destination choice deterministic.
func TestPoolsMinorityOverwriteRecovery(t *testing.T) {
for _, versioning := range []string{"unversioned", "versioned", "suspended"} {
for _, mode := range []string{"single-pool", "dual-pool-only-inconsistent-copy", "dual-pool-readable-other-copy"} {
t.Run(versioning+"/"+mode, func(t *testing.T) {
testPoolsMinorityOverwriteRecovery(t, mode, versioning, "")
})
}
}
for _, encryption := range []string{"SSE-C", "SSE-S3", "SSE-KMS"} {
t.Run(encryption, func(t *testing.T) {
testPoolsMinorityOverwriteRecovery(t, "dual-pool-readable-other-copy", "unversioned", encryption)
})
}
}
func testPoolsMinorityOverwriteRecovery(t *testing.T, mode, versioning, encryption string) {
t.Helper()
var z *erasureServerPools
if mode == "single-pool" {
z, _ = mutationPools(t)
} else {
z, _ = consistencyPools(t)
}
const object = "review-retry"
for _, pool := range z.serverPools {
restore := conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI { return conditionalPutCapacityDisk{StorageAPI: d} })
defer restore()
}
bucket, router := conditionalPutBucket(t, z, versioning)
oldTime := time.Now().UTC().Add(-time.Hour)
oldBody := bytes.Repeat([]byte{'a'}, 8192)
pendingBody := bytes.Repeat([]byte{'b'}, 8192)
recoveryBody := bytes.Repeat([]byte{'c'}, 8192)
seed := func(pool int, body []byte, when time.Time) {
t.Helper()
putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{MTime: when})
}
seed(0, oldBody, oldTime)
disks := z.serverPools[0].getHashedSet(object).getDisks()
oldMeta := make([][]byte, len(disks))
for i, disk := range disks {
oldMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
}
seed(0, pendingBody, oldTime.Add(time.Minute))
for i := 0; i < len(disks)-1; i++ {
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
t.Fatal(err)
}
}
if mode == "dual-pool-readable-other-copy" {
seed(1, oldBody, oldTime.Add(-time.Minute))
// Ordinary allocation can select this healthy other pool.
// Force that permitted choice without altering metadata reads.
restore := conditionalPutPool(t, z, object, 1)
defer restore()
}
before := consistencyRequest(t, router, http.MethodGet, bucket, object, "")
if before.Code != http.StatusServiceUnavailable {
t.Fatalf("fixture must refuse ambiguous read; got %d", before.Code)
}
var headers, readHeaders map[string]string
if encryption != "" {
oldKMS, oldTLS := GlobalKMS, globalIsTLS
GlobalKMS, globalIsTLS = kms.NewStub("recovery-key"), true
defer func() { GlobalKMS, globalIsTLS = oldKMS, oldTLS }()
headers = map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
switch encryption {
case "SSE-C":
key := bytes.Repeat([]byte{0x42}, 32)
digest := md5.Sum(key)
headers = map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(digest[:]),
}
readHeaders = headers
case "SSE-KMS":
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
headers[xhttp.AmzServerSideEncryptionKmsID] = "recovery-key"
}
}
url := getPutObjectURL("", bucket, object)
result := multipartConditionRequest(t, router, http.MethodPut, url, string(recoveryBody), headers)
t.Logf("recovery PUT HTTP=%d response=%s", result.Code, result.Body.String())
if result.Code != http.StatusOK {
t.Fatalf("unconditional recovery PUT: %d %s", result.Code, result.Body.String())
}
after := multipartConditionRequest(t, router, http.MethodGet, url, "", readHeaders)
t.Logf("after acknowledged recovery PUT: GET HTTP=%d matches_new=%t", after.Code, bytes.Equal(after.Body.Bytes(), recoveryBody))
for i, pool := range z.serverPools {
oi, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
t.Logf("pool=%d mtime=%s err=%v", i, oi.ModTime, err)
}
if after.Code != http.StatusOK || !bytes.Equal(after.Body.Bytes(), recoveryBody) {
t.Fatalf("acknowledged recovery overwrite did not restore reads: HTTP=%d body=%s", after.Code, after.Body.String())
}
}
// Read failures activated by a completed disk mutation verify that a recovery
// ACK is not issued from preflight information alone.
type recoveryReadFaultDisk struct {
StorageAPI
bucket, object string
fail *atomic.Bool
afterRename bool
}
func (d recoveryReadFaultDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
if bucket == d.bucket && object == d.object && d.fail.Load() {
return RawFileInfo{}, errDiskNotFound
}
return d.StorageAPI.ReadXL(ctx, bucket, object, data)
}
func (d recoveryReadFaultDisk) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
if bucket == d.bucket && object == d.object && d.fail.Load() {
return FileInfo{}, errDiskNotFound
}
return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
}
func (d recoveryReadFaultDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
res, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
if err == nil && d.afterRename && dv == d.bucket && dp == d.object {
d.fail.Store(true)
}
return res, err
}
func TestPoolsRecoveryRefusesUnresolvedPools(t *testing.T) {
for _, mode := range []string{"two-inconsistent", "suspended", "rebalancing", "offline", "read-fails-after-commit", "newer-readable-other-copy"} {
t.Run(mode, func(t *testing.T) {
z, _ := consistencyPools(t)
const object = "refused-recovery"
bucket, router := conditionalPutBucket(t, z, "unversioned")
for _, pool := range z.serverPools {
restore := conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI { return conditionalPutCapacityDisk{StorageAPI: d} })
defer restore()
}
base := time.Now().UTC().Add(-time.Hour)
old := bytes.Repeat([]byte{'a'}, 8192)
pending := bytes.Repeat([]byte{'b'}, 8192)
replacement := bytes.Repeat([]byte{'c'}, 8192)
inconsistent := func(pool int) {
t.Helper()
putConsistencyObject(t, z, bucket, object, pool, string(old), ObjectOptions{MTime: base})
disks := z.serverPools[pool].getHashedSet(object).getDisks()
saved := make([][]byte, len(disks))
for i, d := range disks {
saved[i] = mustReadNullQuorumMeta(t, d, bucket, object)
}
putConsistencyObject(t, z, bucket, object, pool, string(pending), ObjectOptions{MTime: base.Add(time.Minute)})
for i := 0; i < len(disks)-1; i++ {
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, saved[i]); err != nil {
t.Fatal(err)
}
}
}
inconsistent(0)
otherTime := base.Add(-time.Minute)
if mode == "newer-readable-other-copy" {
otherTime = base.Add(2 * time.Hour)
}
putConsistencyObject(t, z, bucket, object, 1, string(old), ObjectOptions{MTime: otherTime})
if mode == "two-inconsistent" {
inconsistent(1)
}
// Select a healthy destination if the resolver incorrectly ignores pool0.
postCommitFailure := mode == "read-fails-after-commit" || mode == "newer-readable-other-copy"
if !postCommitFailure {
restore := conditionalPutPool(t, z, object, 1)
defer restore()
}
if mode == "suspended" {
z.poolMetaMutex.Lock()
z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
z.poolMetaMutex.Unlock()
}
if mode == "rebalancing" {
z.rebalMu.Lock()
z.rebalMeta = &rebalanceMeta{PoolStats: []*rebalanceStats{{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}, {}}}
z.rebalMu.Unlock()
}
var fail atomic.Bool
if mode == "offline" || mode == "read-fails-after-commit" {
fail.Store(mode == "offline")
restore := conditionalPutSwapDisks(z.serverPools[0], object, func(d StorageAPI) StorageAPI {
return recoveryReadFaultDisk{StorageAPI: d, bucket: bucket, object: object, fail: &fail, afterRename: mode == "read-fails-after-commit"}
})
defer restore()
}
snapshot := func() [][32]byte {
var result [][32]byte
for _, pool := range z.serverPools {
for _, disk := range pool.getHashedSet(object).getDisks() {
b, err := disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
if err != nil {
t.Fatal(err)
}
result = append(result, sha256.Sum256(b))
}
}
return result
}
before := snapshot()
req, err := newTestSignedRequestV4(http.MethodPut, getGetObjectURL("", bucket, object), int64(len(replacement)), bytes.NewReader(replacement), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
result := httptest.NewRecorder()
router.ServeHTTP(result, req.WithContext(t.Context()))
if result.Code != http.StatusServiceUnavailable {
t.Errorf("unresolved recovery returned HTTP=%d: %s", result.Code, result.Body.String())
}
switch {
case !postCommitFailure:
after := snapshot()
for i := range before {
if before[i] != after[i] {
t.Errorf("refused preflight modified shard %d", i)
}
}
case mode == "read-fails-after-commit":
fail.Store(false)
got := consistencyRequest(t, router, http.MethodGet, bucket, object, "")
if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), replacement) {
t.Errorf("read validation failure must not undo committed recovery: HTTP=%d", got.Code)
}
default:
// A readable but different latest copy cannot confirm recovery.
// The committed replacement must still remain in its target pool.
r, err := z.serverPools[0].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(r)
r.Close()
if err != nil || !bytes.Equal(body, replacement) {
t.Fatalf("failed recovery validation must preserve committed data: %v", err)
}
}
})
}
}
+217 -71
View File
@@ -39,6 +39,7 @@ import (
"github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio-go/v7/pkg/tags" "github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/bpool" "github.com/minio/minio/internal/bpool"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/cachevalue" "github.com/minio/minio/internal/cachevalue"
"github.com/minio/minio/internal/config/storageclass" "github.com/minio/minio/internal/config/storageclass"
xioutil "github.com/minio/minio/internal/ioutil" xioutil "github.com/minio/minio/internal/ioutil"
@@ -1183,6 +1184,14 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
opts.CheckPrecondFn = nil opts.CheckPrecondFn = nil
} }
if opts.DataMovement {
resolve, err := z.checkDataMovementCopies(ctx, bucket, object, opts)
if err != nil {
return ObjectInfo{}, err
}
opts.replicaObjectInfo = resolve
}
idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true) idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true)
if err != nil { if err != nil {
return ObjectInfo{}, err return ObjectInfo{}, err
@@ -1202,7 +1211,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
} }
// Public overwrites need all pools for Object Lock. Unaddressed replica // Public overwrites need all pools for Object Lock. Unaddressed replica
// callbacks retain their set-local reads unless reconciliation is needed. // callbacks retain their set-local reads unless reconciliation is needed.
if opts.ReplicaLockReconcile || !opts.ReplicationRequest { if opts.replicaObjectInfo == nil && (opts.ReplicaLockReconcile || !opts.ReplicationRequest) {
opts.replicaObjectInfo = z.replicaObjectInfo opts.replicaObjectInfo = z.replicaObjectInfo
} }
oi, err := z.serverPools[idx].PutObject(ctx, bucket, object, data, opts) oi, err := z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
@@ -1243,6 +1252,25 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
return ObjectInfo{}, z.deletePrefix(ctx, bucket, object) return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
} }
if opts.DataMovement {
opts.NoLock = true
if _, err := z.checkDataMovementCopies(ctx, bucket, object, opts); err != nil {
return objInfo, err
}
// A marker still needs a destination when it is the only version of
// this key outside the draining pool.
idx, err := z.getPoolIdxNoLock(ctx, bucket, object, 0)
if err != nil {
return objInfo, err
}
if idx == opts.SrcPoolIdx {
return objInfo, DataMovementOverwriteErr{Bucket: bucket, Object: object, VersionID: opts.VersionID, Err: errDataMovementSrcDstPoolSame}
}
objInfo, err = z.serverPools[idx].DeleteObject(ctx, bucket, object, opts)
objInfo.Name = decodeDirObject(object)
return objInfo, err
}
// Resolve a physical purge by its addressed version even on a replica // Resolve a physical purge by its addressed version even on a replica
// receiver: latest-key routing can select a different pool or miss copies. // receiver: latest-key routing can select a different pool or miss copies.
// Marker creation and specialized movement/scanner operations retain their // Marker creation and specialized movement/scanner operations retain their
@@ -1328,24 +1356,13 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
return pinfo.ObjInfo, nil return pinfo.ObjInfo, nil
} }
// Datamovement must never be allowed on the same pool. // Retire older copies before the newest copy for unversioned objects.
if opts.DataMovement && opts.SrcPoolIdx == pinfo.Index {
return pinfo.ObjInfo, DataMovementOverwriteErr{
Bucket: bucket,
Object: object,
VersionID: opts.VersionID,
Err: errDataMovementSrcDstPoolSame,
}
}
if opts.DataMovement {
objInfo, err = z.serverPools[pinfo.Index].DeleteObject(ctx, bucket, object, opts)
objInfo.Name = decodeDirObject(object)
return objInfo, err
}
// Delete concurrently in all server pools with read quorum error for unversioned objects.
if len(noReadQuorumPools) > 0 && !opts.Versioned && !opts.VersionSuspended { if len(noReadQuorumPools) > 0 && !opts.Versioned && !opts.VersionSuspended {
if pinfo.ObjInfo.TransitionedObject.Status == lifecycle.TransitionComplete {
// Restored and remote copies can share a tier object. Reuse the
// reconciliation which keeps that reference out of premature GC.
return z.deleteObjectReconciled(ctx, bucket, object, opts)
}
return z.deleteObjectFromAllPools(ctx, bucket, object, opts, noReadQuorumPools) return z.deleteObjectFromAllPools(ctx, bucket, object, opts, noReadQuorumPools)
} }
@@ -1356,6 +1373,21 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
return objInfo, err return objInfo, err
} }
// A delete marker hides the object in every pool. When more than one pool
// holds the object, put the marker into the first one with a readable
// version whose set can take the write, so that a set without quorum does
// not fail the delete while another pool could take the marker.
if opts.VersionID == "" && len(noReadQuorumPools) > 1 {
for idx, pool := range z.serverPools {
if !poolReadable(noReadQuorumPools, idx) || deleteQuorumMissing(pool, object) {
continue
}
objInfo, err = pool.DeleteObject(ctx, bucket, object, opts)
objInfo.Name = decodeDirObject(object)
return objInfo, err
}
}
for _, pool := range z.serverPools { for _, pool := range z.serverPools {
objInfo, err := pool.DeleteObject(ctx, bucket, object, opts) objInfo, err := pool.DeleteObject(ctx, bucket, object, opts)
if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
@@ -1376,36 +1408,76 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
} }
func (z *erasureServerPools) deleteObjectFromAllPools(ctx context.Context, bucket string, object string, opts ObjectOptions, poolIndices []poolErrs) (objInfo ObjectInfo, err error) { func (z *erasureServerPools) deleteObjectFromAllPools(ctx context.Context, bucket string, object string, opts ObjectOptions, poolIndices []poolErrs) (objInfo ObjectInfo, err error) {
derrs := make([]error, len(poolIndices))
dobjects := make([]ObjectInfo, len(poolIndices))
// Delete concurrently in all server pools that reported no error or read quorum error
// where the read quorum issue is from metadata inconsistency.
var wg sync.WaitGroup
for idx, pe := range poolIndices {
if v, ok := pe.Err.(InsufficientReadQuorum); ok && v.Type != RQInconsistentMeta {
derrs[idx] = InsufficientWriteQuorum{}
continue
}
wg.Add(1)
pool := z.serverPools[pe.Index]
go func(idx int, pool *erasureSets) {
defer wg.Done()
dobjects[idx], derrs[idx] = pool.DeleteObject(ctx, bucket, object, opts)
}(idx, pool)
}
wg.Wait()
// the poolIndices array is pre-sorted in order of latest ModTime, we care only about pool with latest object though
// the delete call tries to clean up other pools during DeleteObject call.
objInfo = dobjects[0]
objInfo.Name = decodeDirObject(object) objInfo.Name = decodeDirObject(object)
for _, derr := range derrs {
// Delete in all server pools that reported no error or read quorum error
// where the read quorum issue is from metadata inconsistency. The
// poolIndices array is pre-sorted in order of latest ModTime. Retire the
// older copies first and stop at the first failure, as
// deleteObjectReconciled does: a failed delete then keeps the latest copy
// in place, instead of serving an older write of the object.
for idx := len(poolIndices) - 1; idx >= 0; idx-- {
pe := poolIndices[idx]
if v, ok := pe.Err.(InsufficientReadQuorum); ok && v.Type != RQInconsistentMeta {
return objInfo, InsufficientWriteQuorum{}
}
dobj, derr := z.serverPools[pe.Index].DeleteObject(ctx, bucket, object, opts)
if idx == 0 {
objInfo, err = dobj, derr
break
}
if derr != nil && !isErrObjectNotFound(derr) && !isErrVersionNotFound(derr) { if derr != nil && !isErrObjectNotFound(derr) && !isErrVersionNotFound(derr) {
return objInfo, derr dobj.Name = decodeDirObject(object)
return dobj, derr
} }
} }
return objInfo, derrs[0] objInfo.Name = decodeDirObject(object)
return objInfo, err
}
// deleteQuorumMissing reports whether the set of object in pool has fewer
// online drives than a delete needs (N/2+1, as in erasureObjects.DeleteObjects).
func deleteQuorumMissing(pool *erasureSets, object string) bool {
disks := pool.getHashedSet(object).getDisks()
return countOnlineDisks(disks) < len(disks)/2+1
}
// poolReadable reports whether pools lists pool idx with a readable version.
func poolReadable(pools []poolErrs, idx int) bool {
for _, pe := range pools {
if pe.Index == idx {
return pe.Err == nil
}
}
return false
}
// copyWithoutDeleteQuorum avoids starting a known unwritable cleanup. This
// reduces partial deletion, but successful cleanup must still precede removing
// the authoritative copy: drives can fail after this preflight.
func (z *erasureServerPools) copyWithoutDeleteQuorum(ctx context.Context, bucket, object, versionID string) bool {
for _, pool := range z.serverPools {
if deleteQuorumMissing(pool, object) {
_, err := pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: versionID, NoLock: true})
if err == nil || isErrReadQuorum(err) || isErrMethodNotAllowed(err) {
return true
}
}
}
return false
}
// deletesData reports whether deleting obj removes an object or a version, as
// opposed to adding a delete marker (see erasureObjects.DeleteObjects).
func deletesData(obj ObjectToDelete, opts ObjectOptions) bool {
if obj.VersionID != "" {
return true
}
versioned := opts.Versioned
if opts.PrefixEnabledFn != nil {
versioned = opts.PrefixEnabledFn(obj.ObjectName)
}
return !versioned && !opts.VersionSuspended
} }
func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) { func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
@@ -1414,6 +1486,11 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
objSets := set.NewStringSet() objSets := set.NewStringSet()
for i := range derrs { for i := range derrs {
objects[i].ObjectName = encodeDirObject(objects[i].ObjectName) objects[i].ObjectName = encodeDirObject(objects[i].ObjectName)
dobjects[i] = DeletedObject{
ObjectName: decodeDirObject(objects[i].ObjectName),
VersionID: objects[i].VersionID,
ReplicationState: objects[i].ReplicationState(),
}
derrs[i] = checkDelObjArgs(ctx, bucket, objects[i].ObjectName) derrs[i] = checkDelObjArgs(ctx, bucket, objects[i].ObjectName)
objSets.Add(objects[i].ObjectName) objSets.Add(objects[i].ObjectName)
@@ -1431,25 +1508,64 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
ctx = lkctx.Context() ctx = lkctx.Context()
defer multiDeleteLock.Unlock(lkctx) defer multiDeleteLock.Unlock(lkctx)
// Removing data needs the same authoritative-copy protection as a single
// DELETE. An online-drive preflight cannot protect against later I/O errors.
// Keep marker creation after data removal, including when the same request
// explicitly removes a null version and creates its replacement marker.
batch, batchIdx := objects, []int(nil)
if !z.SinglePool() {
batch = make([]ObjectToDelete, 0, len(objects))
batchIdx = make([]int, 0, len(objects))
data, dataIdx := make([]ObjectToDelete, 0, len(objects)), make([]int, 0, len(objects))
for i, obj := range objects {
if derrs[i] != nil {
continue
}
if deletesData(obj, opts) {
if z.copyWithoutDeleteQuorum(ctx, bucket, obj.ObjectName, obj.VersionID) {
derrs[i] = InsufficientWriteQuorum{}
continue
}
data = append(data, obj)
dataIdx = append(dataIdx, i)
continue
}
batch = append(batch, obj)
batchIdx = append(batchIdx, i)
}
if len(data) > 0 {
deleted, errs := z.deleteObjectsReconciled(ctx, bucket, data, opts)
for j, i := range dataIdx {
dobjects[i], derrs[i] = deleted[j], errs[j]
}
}
}
dObjectsByPool := make([][]DeletedObject, len(z.serverPools)) dObjectsByPool := make([][]DeletedObject, len(z.serverPools))
dErrsByPool := make([][]error, len(z.serverPools)) dErrsByPool := make([][]error, len(z.serverPools))
eg := errgroup.WithNErrs(len(z.serverPools)).WithConcurrency(len(z.serverPools)) if len(batch) > 0 {
for i, pool := range z.serverPools { eg := errgroup.WithNErrs(len(z.serverPools)).WithConcurrency(len(z.serverPools))
pool := pool for i, pool := range z.serverPools {
eg.Go(func() error { pool := pool
dObjectsByPool[i], dErrsByPool[i] = pool.DeleteObjects(ctx, bucket, objects, opts) eg.Go(func() error {
return nil dObjectsByPool[i], dErrsByPool[i] = pool.DeleteObjects(ctx, bucket, batch, opts)
}, i) return nil
}, i)
}
eg.Wait() // wait to check all the pools.
} }
eg.Wait() // wait to check all the pools.
for i := range dobjects { for j := range batch {
i := j
if batchIdx != nil {
i = batchIdx[j]
}
// Iterate over pools // Iterate over pools
for pool := range z.serverPools { for pool := range z.serverPools {
if dErrsByPool[pool][i] == nil && dObjectsByPool[pool][i].found { if dErrsByPool[pool][j] == nil && dObjectsByPool[pool][j].found {
// A fast exit when the object is found and removed // A fast exit when the object is found and removed
dobjects[i] = dObjectsByPool[pool][i] dobjects[i] = dObjectsByPool[pool][j]
derrs[i] = nil derrs[i] = nil
break break
} }
@@ -1457,8 +1573,8 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
// No error related to this object is found, assign this pool result // No error related to this object is found, assign this pool result
// whether it is nil because there is no object found or because of // whether it is nil because there is no object found or because of
// some other errors such erasure quorum errors. // some other errors such erasure quorum errors.
dobjects[i] = dObjectsByPool[pool][i] dobjects[i] = dObjectsByPool[pool][j]
derrs[i] = dErrsByPool[pool][i] derrs[i] = dErrsByPool[pool][j]
} }
} }
} }
@@ -1916,27 +2032,45 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
return ListMultipartsInfo{}, toObjectErr(err, bucket) return ListMultipartsInfo{}, toObjectErr(err, bucket)
} }
scan, err := startMultipartScan(ctx, false)
if err != nil {
return ListMultipartsInfo{}, err
}
defer scan.close()
var uploads []MultipartInfo poolResult := ListMultipartsInfo{}
poolResult.MaxUploads = maxUploads
poolResult.KeyMarker = keyMarker
poolResult.Prefix = prefix
poolResult.Delimiter = delimiter
// Preserve the legacy cache view, including its incomplete pagination.
// Persistent discovery remains an explicit admin diagnostic, not a
// prerequisite for public listing or known-upload operations.
if prefix == "" {
z.mpCache.Range(func(_ string, mp MultipartInfo) bool {
if mp.Bucket == bucket {
poolResult.Uploads = append(poolResult.Uploads, mp)
}
return true
})
sort.Slice(poolResult.Uploads, func(i int, j int) bool {
return poolResult.Uploads[i].Initiated.Before(poolResult.Uploads[j].Initiated)
})
return poolResult, nil
}
if z.SinglePool() {
return z.serverPools[0].getHashedSet(prefix).listMultipartUploadsExact(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
}
for idx, pool := range z.serverPools { for idx, pool := range z.serverPools {
if z.IsSuspended(idx) { if z.IsSuspended(idx) {
continue continue
} }
poolUploads, err := pool.scanMultipartUploads(scan, bucket, idx) result, err := pool.getHashedSet(prefix).listMultipartUploadsExact(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
if err != nil { if err != nil {
return ListMultipartsInfo{}, err return result, err
} }
uploads = append(uploads, poolUploads...) poolResult.Uploads = append(poolResult.Uploads, result.Uploads...)
} }
// Confirmed historical records lack a recoverable bucket/key and are return poolResult, nil
// omitted by the scan. They do not block listing identifiable uploads.
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
} }
// Initiate a new multipart upload on a hashedSet based on object name. // Initiate a new multipart upload on a hashedSet based on object name.
@@ -2230,6 +2364,13 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
} }
opts.CheckPrecondFn = nil opts.CheckPrecondFn = nil
} }
if opts.DataMovement {
resolve, err := z.checkDataMovementCopies(ctx, bucket, encodeDirObject(object), opts)
if err != nil {
return ObjectInfo{}, err
}
opts.replicaObjectInfo = resolve
}
} }
// Hold write locks to verify uploaded parts, also disallows any // Hold write locks to verify uploaded parts, also disallows any
@@ -2249,7 +2390,7 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
if opts.ReplicaLockReconcile || opts.DataMovement { if opts.ReplicaLockReconcile || opts.DataMovement {
opts.ReplicaLockReconcile = true opts.ReplicaLockReconcile = true
} }
if opts.ReplicaLockReconcile || !opts.ReplicationRequest { if opts.replicaObjectInfo == nil && (opts.ReplicaLockReconcile || !opts.ReplicationRequest) {
opts.replicaObjectInfo = z.replicaObjectInfo opts.replicaObjectInfo = z.replicaObjectInfo
} }
@@ -2727,7 +2868,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
return nil return nil
} }
} }
fivs, err := entry.fileInfoVersions(bucket) fivs, err := entry.allFileInfoVersions(bucket)
if err != nil { if err != nil {
return healObjectFn(bucket, entry.name, "", scanMode) return healObjectFn(bucket, entry.name, "", scanMode)
} }
@@ -3240,6 +3381,11 @@ func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, obje
defer ns.Unlock(lkctx) defer ns.Unlock(lkctx)
opts.NoLock = true opts.NoLock = true
} }
if opts.DataMovement {
if _, err := z.checkDataMovementCopies(ctx, bucket, object, opts); err != nil {
return err
}
}
idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size) idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size)
if err != nil { if err != nil {
return err return err
+1 -25
View File
@@ -886,31 +886,7 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
} }
func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) { func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil { return s.getHashedSet(prefix).ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
return ListMultipartsInfo{}, err
}
scan, err := startMultipartScan(ctx, false)
if err != nil {
return ListMultipartsInfo{}, err
}
defer scan.close()
uploads, err := s.scanMultipartUploads(scan, bucket, 0)
if err != nil {
return ListMultipartsInfo{}, err
}
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
}
func (s *erasureSets) scanMultipartUploads(scan *multipartScan, bucket string, poolIdx int) ([]MultipartInfo, error) {
var uploads []MultipartInfo
for i, set := range s.sets {
setUploads, err := set.scanMultipartUploads(scan, bucket, poolIdx, i)
if err != nil {
return nil, err
}
uploads = append(uploads, setUploads...)
}
return uploads, nil
} }
func (s *erasureSets) listMultipartUploadsExact(ctx context.Context, bucket, object string) (ListMultipartsInfo, error) { func (s *erasureSets) listMultipartUploadsExact(ctx context.Context, bucket, object string) (ListMultipartsInfo, error) {
+104
View File
@@ -0,0 +1,104 @@
//go:build consistency_strict
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"strings"
"testing"
"time"
)
func TestPutObjectGenerationFollowsCommitLock(t *testing.T) {
z, bucket := consistencyPools(t)
for _, preserve := range []bool{false, true} {
t.Run(map[bool]string{false: "generated", true: "preserved"}[preserve], func(t *testing.T) {
const object = "ordered-generation"
set := z.serverPools[0].getHashedSet(object)
lock := set.NewNSLock(bucket, object)
lc, err := lock.GetLock(t.Context(), globalOperationTimeout)
if err != nil {
t.Fatal(err)
}
locked := true
defer func() {
if locked {
lock.Unlock(lc)
}
}()
opts := ObjectOptions{}
if preserve {
opts.MTime = time.Unix(1234567890, 0)
}
reader := mustGetPutObjReader(t, strings.NewReader("payload"), 7, "", "")
type result struct {
info ObjectInfo
err error
}
done := make(chan result, 1)
go func() {
info, err := set.PutObject(t.Context(), bucket, object, reader, opts)
done <- result{info, err}
}()
// Wait for the real writer to queue behind the held namespace
// lock. No elapsed delay is used to infer its progress.
deadline := time.Now().Add(5 * time.Second)
for {
set.nsMutex.lockMapMutex.Lock()
entry := set.nsMutex.lockMap[pathJoin(bucket, object)]
waiting := entry != nil && entry.ref == 2
set.nsMutex.lockMapMutex.Unlock()
if waiting {
break
}
select {
case got := <-done:
t.Fatalf("PUT bypassed the held commit lock: %v", got.err)
default:
}
if time.Now().After(deadline) {
t.Fatal("PUT did not queue for the commit lock")
}
time.Sleep(time.Millisecond)
}
beforeUnlock := UTCNow()
lock.Unlock(lc)
locked = false
got := <-done
if got.err != nil {
t.Fatal(got.err)
}
if preserve {
if !got.info.ModTime.Equal(opts.MTime) {
t.Fatalf("explicit modification time changed: %s", got.info.ModTime)
}
} else if got.info.ModTime.Before(beforeUnlock) {
t.Fatalf("generation %s predates commit lock release %s", got.info.ModTime, beforeUnlock)
}
for i, disk := range set.getDisks() {
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, nullVersionID, ReadOptions{})
if err != nil || !fi.ModTime.Equal(got.info.ModTime) {
t.Errorf("disk %d stored a different generation: %s, %v", i, fi.ModTime, err)
}
}
})
}
}
+1 -1
View File
@@ -423,7 +423,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
encodedEntryName := encodeDirObject(entry.name) encodedEntryName := encodeDirObject(entry.name)
var result healEntryResult var result healEntryResult
fivs, err := entry.fileInfoVersions(bucket) fivs, err := entry.allFileInfoVersions(bucket)
if err != nil { if err != nil {
res, err := healInPool(ctx, bucket, encodedEntryName, "", res, err := healInPool(ctx, bucket, encodedEntryName, "",
madmin.HealOpts{ madmin.HealOpts{
+8 -2
View File
@@ -85,15 +85,21 @@ func TestIAMPolicyDeletionRemainsExplicit(t *testing.T) {
} }
p, err := sys.store.GetPolicy("readwrite") p, err := sys.store.GetPolicy("readwrite")
mustIAM(t, err) mustIAM(t, err)
native, err := sys.InfoPolicy("readwrite")
mustIAM(t, err)
if err := sys.DeletePolicy(ctx, "readwrite", true); err == nil { if err := sys.DeletePolicy(ctx, "readwrite", true); err == nil {
t.Fatal("local pristine builtin policy became deletable") t.Fatal("local pristine builtin policy became deletable")
} }
_, err = sys.SetPolicy(ctx, "readwrite", p) _, err = sys.SetPolicy(ctx, "readwrite", p)
mustIAM(t, err) mustIAM(t, err)
mustIAM(t, sys.DeletePolicy(ctx, "readwrite", true)) mustIAM(t, sys.DeletePolicy(ctx, "readwrite", true))
assertIAMPolicyResetInfo(t, sys, "readwrite", native)
mustIAM(t, sys.store.LoadIAMCache(ctx, false)) mustIAM(t, sys.store.LoadIAMCache(ctx, false))
if _, err := sys.store.GetPolicy("readwrite"); !errors.Is(err, errNoSuchPolicy) { assertIAMPolicyResetInfo(t, sys, "readwrite", native)
t.Fatalf("reload restored an explicitly deleted override: %v", err) r, err = loadIAMRevision(ctx, sys.store, getPolicyDocPath("readwrite"))
mustIAM(t, err)
if !r.Deleted {
t.Fatal("native policy restoration removed the override tombstone")
} }
_, err = sys.SetPolicy(ctx, "readwrite", p) _, err = sys.SetPolicy(ctx, "readwrite", p)
mustIAM(t, err) mustIAM(t, err)
+15 -11
View File
@@ -123,8 +123,8 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath s
return nil return nil
} }
func (ies *IAMEtcdStore) decodeIAMConfig(item any, data []byte, path string) error { func (ies *IAMEtcdStore) decodeIAMConfig(ctx context.Context, item any, data []byte, path string) error {
data, err := decryptData(data, path) data, err := decryptData(ctx, data, path)
if err != nil { if err != nil {
return err return err
} }
@@ -145,7 +145,7 @@ func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([
if err != nil { if err != nil {
return nil, err return nil, err
} }
data, err = decryptData(data, path) data, err = decryptData(ctx, data, path)
if err == nil { if err == nil {
ies.index.observe(path, data) ies.index.observe(path, data)
} }
@@ -187,7 +187,7 @@ func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map
} }
func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValue, m map[string]PolicyDoc) error { func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValue, m map[string]PolicyDoc) error {
data, err := decryptData(kvs.Value, string(kvs.Key)) data, err := decryptData(ctx, kvs.Value, string(kvs.Key))
if err != nil { if err != nil {
if err == errConfigNotFound { if err == errConfigNotFound {
return errNoSuchPolicy return errNoSuchPolicy
@@ -211,27 +211,31 @@ func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValu
} }
func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error { func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout) // Bound the etcd read without shortening the caller's decryption budget.
cctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
defer cancel() defer cancel()
// Retrieve all keys and values to avoid too many calls to etcd in case of // Retrieve all keys and values to avoid too many calls to etcd in case of
// a large number of policies // a large number of policies
r, err := ies.client.Get(ctx, iamConfigPoliciesPrefix, etcd.WithPrefix()) r, err := ies.client.Get(cctx, iamConfigPoliciesPrefix, etcd.WithPrefix())
if err != nil { if err != nil {
return err return err
} }
// Parse all values to construct the policies data model. // Parse all values to construct the policies data model.
for _, kvs := range r.Kvs { for _, kvs := range r.Kvs {
if err := ctx.Err(); err != nil {
return err
}
if err = ies.getPolicyDocKV(ctx, kvs, m); err != nil && !errors.Is(err, errNoSuchPolicy) { if err = ies.getPolicyDocKV(ctx, kvs, m); err != nil && !errors.Is(err, errNoSuchPolicy) {
return err return err
} }
} }
return nil return ctx.Err()
} }
func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]UserIdentity, basePrefix string) error { func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]UserIdentity, basePrefix string) error {
var u UserIdentity var u UserIdentity
err := ies.decodeIAMConfig(&u, userkv.Value, string(userkv.Key)) err := ies.decodeIAMConfig(ctx, &u, userkv.Value, string(userkv.Key))
if err != nil { if err != nil {
if err == errConfigNotFound { if err == errConfigNotFound {
return errNoSuchUser return errNoSuchUser
@@ -392,9 +396,9 @@ func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, user
return nil return nil
} }
func (ies *IAMEtcdStore) getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error { func (ies *IAMEtcdStore) getMappedPolicy(ctx context.Context, kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
var p MappedPolicy var p MappedPolicy
err := ies.decodeIAMConfig(&p, kv.Value, string(kv.Key)) err := ies.decodeIAMConfig(ctx, &p, kv.Value, string(kv.Key))
if err != nil { if err != nil {
if err == errConfigNotFound { if err == errConfigNotFound {
return errNoSuchPolicy return errNoSuchPolicy
@@ -434,7 +438,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
// Parse all policies mapping to create the proper data model // Parse all policies mapping to create the proper data model
for _, kv := range r.Kvs { for _, kv := range r.Kvs {
if err = ies.getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) { if err = ies.getMappedPolicy(ctx, kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
return err return err
} }
} }
+40 -5
View File
@@ -108,7 +108,10 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPat
return nil return nil
} }
func decryptData(data []byte, objPath string) ([]byte, error) { func decryptData(ctx context.Context, data []byte, objPath string) ([]byte, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
if utf8.Valid(data) { if utf8.Valid(data) {
return data, nil return data, nil
} }
@@ -118,13 +121,17 @@ func decryptData(data []byte, objPath string) ([]byte, error) {
return pdata, nil return pdata, nil
} }
if GlobalKMS != nil { if GlobalKMS != nil {
pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ pdata, err = config.DecryptBytes(ctx, GlobalKMS, data, kms.Context{
minioMetaBucket: path.Join(minioMetaBucket, objPath), minioMetaBucket: path.Join(minioMetaBucket, objPath),
}) })
if err == nil { if err == nil {
return pdata, nil return pdata, nil
} }
pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ // Cancellation must not trigger the legacy associated-data fallback.
if err := ctx.Err(); err != nil {
return nil, err
}
pdata, err = config.DecryptBytes(ctx, GlobalKMS, data, kms.Context{
minioMetaBucket: objPath, minioMetaBucket: objPath,
}) })
if err == nil { if err == nil {
@@ -134,12 +141,37 @@ func decryptData(data []byte, objPath string) ([]byte, error) {
return nil, err return nil, err
} }
// iamConfigNameTooLong identifies length errors for valid keys with an ASCII
// component that cannot be stored. Normalized LDAP DNs escape non-ASCII bytes.
// Byte length alone is insufficient for Unicode filenames on macOS/Windows,
// and a total-path limit can make an existing record unreadable after a move.
func iamConfigNameTooLong(objPath string, err error) bool {
var invalid ObjectNameInvalid
if !errors.Is(err, errFileNameTooLong) && !errors.As(err, &invalid) {
return false
}
if checkBucketAndObjectNames(context.Background(), minioMetaBucket, objPath) != nil {
return false
}
for component := range strings.SplitSeq(objPath, SlashSeparator) {
if len(component) > 255 && utf8.RuneCountInString(component) == len(component) {
return true
}
}
return false
}
func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context, objPath string) ([]byte, ObjectInfo, error) { func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context, objPath string) ([]byte, ObjectInfo, error) {
data, meta, err := readConfigWithMetadata(ctx, iamOS.objAPI, objPath, ObjectOptions{}) data, meta, err := readConfigWithMetadata(ctx, iamOS.objAPI, objPath, ObjectOptions{})
if err != nil { if err != nil {
if iamConfigNameTooLong(objPath, err) {
// A missing record, not a storage failure: lookups of the
// identity proceed, while saving its record still fails.
err = errConfigNotFound
}
return nil, meta, err return nil, meta, err
} }
data, err = decryptData(data, objPath) data, err = decryptData(ctx, data, objPath)
if err != nil { if err != nil {
return nil, meta, err return nil, meta, err
} }
@@ -251,12 +283,15 @@ func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]Po
return item.Err return item.Err
} }
if err := ctx.Err(); err != nil {
return err
}
policyName := path.Dir(item.Item) policyName := path.Dir(item.Item)
if err := iamOS.loadPolicyDoc(ctx, policyName, m); err != nil && !errors.Is(err, errNoSuchPolicy) { if err := iamOS.loadPolicyDoc(ctx, policyName, m); err != nil && !errors.Is(err, errNoSuchPolicy) {
return err return err
} }
} }
return nil return ctx.Err()
} }
func (iamOS *IAMObjectStore) loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error) { func (iamOS *IAMObjectStore) loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error) {
+181
View File
@@ -0,0 +1,181 @@
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"sync/atomic"
"testing"
"time"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/kms"
"github.com/puzpuzpuz/xsync/v3"
etcd "go.etcd.io/etcd/client/v3"
)
type iamPolicyBudgetGetObserver struct {
etcd.KV
elapsed time.Duration
budget time.Duration
block bool
}
func (k *iamPolicyBudgetGetObserver) Get(ctx context.Context, key string, opts ...etcd.OpOption) (*etcd.GetResponse, error) {
start := time.Now()
if d, ok := ctx.Deadline(); ok {
k.budget = time.Until(d)
}
defer func() { k.elapsed = time.Since(start) }()
if k.block {
<-ctx.Done()
return nil, ctx.Err()
}
return k.KV.Get(ctx, key, opts...)
}
func iamPolicyBudgetCall(ctx context.Context, store *IAMStoreSys, operation string) (int, error) {
switch operation {
case "policies":
m, err := store.ListPolicies(ctx, "")
return len(m), err
case "docs":
m, err := store.ListPolicyDocs(ctx, "")
return len(m), err
default:
m := xsync.NewMapOf[string, MappedPolicy]()
err := store.IAMStorageAPI.(*IAMEtcdStore).loadMappedPolicies(ctx, regUser, false, m)
return m.Size(), err
}
}
func prepareIAMPolicyBudgetFixture(t *testing.T, operation string) (*IAMStoreSys, *iamPolicyBudgetGetObserver, *atomic.Int32) {
t.Helper()
ctx, sys, _ := prepareIAMRevisionFixture(t, "etcd")
backend := sys.store.IAMStorageAPI.(*IAMEtcdStore)
builtin, err := kms.NewBuiltin("cancellation-key", bytes.Repeat([]byte{43}, 32))
mustIAM(t, err)
var doc any
path := getPolicyDocPath("budget-slow")
if operation == "mappings" {
path = getMappedPolicyPath("budget-slow", regUser, false)
doc = newMappedPolicy("readonly")
} else {
p, err := sys.store.GetPolicy("readonly")
mustIAM(t, err)
doc = newPolicyDoc(p)
}
plain, err := json.Marshal(doc)
mustIAM(t, err)
encrypted, err := config.EncryptBytes(builtin, plain, kms.Context{minioMetaBucket: minioMetaBucket + "/" + path})
mustIAM(t, err)
_, err = backend.client.Put(ctx, path, string(encrypted))
mustIAM(t, err)
calls := &atomic.Int32{}
priorKMS := GlobalKMS
GlobalKMS = connectCancellationKES(t, cancellationDecryptHandler(t, builtin, func(r *http.Request) {
calls.Add(1)
select {
case <-time.After(450 * time.Millisecond):
case <-r.Context().Done():
}
}))
t.Cleanup(func() { GlobalKMS = priorKMS })
priorTimeout := defaultContextTimeout
defaultContextTimeout = 200 * time.Millisecond
t.Cleanup(func() { defaultContextTimeout = priorTimeout })
observer := &iamPolicyBudgetGetObserver{KV: backend.client.KV}
backend.client.KV = observer
t.Cleanup(func() { backend.client.KV = observer.KV })
return sys.store, observer, calls
}
// A fast etcd Get followed by slow KES decryption must retain the caller's
// budget, even when decryption lasts longer than the etcd read timeout.
func TestIAMPolicyEtcdDecodeBudget(t *testing.T) {
for _, operation := range []string{"policies", "docs", "mappings"} {
t.Run(operation, func(t *testing.T) {
store, observer, calls := prepareIAMPolicyBudgetFixture(t, operation)
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
start := time.Now()
n, err := iamPolicyBudgetCall(ctx, store, operation)
t.Logf("operation=%s total_ms=%.3f get_ms=%.3f get_budget_ms=%.3f decrypts=%d count=%d error=%v caller_error=%v", operation, float64(time.Since(start).Microseconds())/1000, float64(observer.elapsed.Microseconds())/1000, float64(observer.budget.Microseconds())/1000, calls.Load(), n, err, ctx.Err())
if err != nil {
t.Fatalf("valid caller budget was shortened after etcd Get completed: %v", err)
}
if observer.elapsed >= 200*time.Millisecond || observer.budget <= 0 || observer.budget > 210*time.Millisecond {
t.Fatalf("etcd read budget invalid: %+v", observer)
}
if n == 0 || calls.Load() != 1 || ctx.Err() != nil {
t.Fatal("successful result was incomplete")
}
})
}
}
// The correction must retain bounded etcd RPCs and caller cancellation. A
// failed list cannot publish its private map or advance the cache timestamp.
func TestIAMPolicyEtcdBudgetGuards(t *testing.T) {
for _, operation := range []string{"policies", "docs", "mappings"} {
for _, mode := range []string{"read_timeout", "caller_deadline", "caller_cancel"} {
t.Run(fmt.Sprintf("%s/%s", operation, mode), func(t *testing.T) {
store, observer, calls := prepareIAMPolicyBudgetFixture(t, operation)
cache := store.rlock()
before, count := cache.updatedAt, len(cache.iamPolicyDocsMap)
store.runlock()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
want := context.DeadlineExceeded
if mode == "read_timeout" {
observer.block = true
}
if mode == "caller_deadline" {
cancel()
ctx, cancel = context.WithTimeout(context.Background(), 80*time.Millisecond)
defer cancel()
}
var timer *time.Timer
if mode == "caller_cancel" {
timer = time.AfterFunc(80*time.Millisecond, cancel)
defer timer.Stop()
want = context.Canceled
}
start := time.Now()
_, err := iamPolicyBudgetCall(ctx, store, operation)
t.Logf("operation=%s mode=%s elapsed_ms=%.3f decrypts=%d error=%v", operation, mode, float64(time.Since(start).Microseconds())/1000, calls.Load(), err)
if !errors.Is(err, want) {
t.Fatalf("expected %v, got %v", want, err)
}
if mode == "read_timeout" && (calls.Load() != 0 || ctx.Err() != nil) {
t.Fatal("read timeout did not remain local to the etcd RPC")
}
cache = store.rlock()
unchanged := cache.updatedAt.Equal(before) && len(cache.iamPolicyDocsMap) == count
store.runlock()
if !unchanged {
t.Fatal("failed list published a cache replacement")
}
})
}
}
}
+182
View File
@@ -0,0 +1,182 @@
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
"sync"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
"github.com/pgsty/silo-pkg/v3/policy"
)
type iamPolicyRefreshContext struct{}
// Keep IAMStorageAPI as *IAMObjectStore so LoadIAMCache exercises the actual
// 32-worker object loader. Capture bytes and release the physical read lock
// before pausing; concurrent writes can commit against real object storage.
type iamPolicyRefreshReadGate struct {
ObjectLayer
policyPath, userPath string
listEntered, listRelease, refreshEntered, refreshRelease chan struct{}
listOnce, refreshOnce sync.Once
}
func (g *iamPolicyRefreshReadGate) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
r, err := g.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
if err != nil {
return nil, err
}
mode, _ := ctx.Value(iamPolicyRefreshContext{}).(string)
var release <-chan struct{}
if bucket == minioMetaBucket && mode == "list" && object == g.policyPath {
release = g.listRelease
}
if bucket == minioMetaBucket && mode == "refresh" && object == g.userPath {
release = g.refreshRelease
}
if release == nil {
return r, nil
}
data, err := io.ReadAll(r)
info := r.ObjInfo
r.Close()
if err != nil {
return nil, err
}
if mode == "list" {
g.listOnce.Do(func() { close(g.listEntered) })
} else {
g.refreshOnce.Do(func() { close(g.refreshEntered) })
}
select {
case <-release:
return NewGetObjectReaderFromReader(bytes.NewReader(data), info, opts)
case <-ctx.Done():
return nil, ctx.Err()
}
}
func TestIAMPolicyReloadActualObjectRefresh(t *testing.T) {
for _, docs := range []bool{false, true} {
for _, canceled := range []bool{false, true} {
t.Run(fmt.Sprintf("docs=%t/cancel=%t", docs, canceled), func(t *testing.T) {
ctx, sys, obj := prepareIAMRevisionFixture(t)
store := sys.store
rw, err := store.GetPolicy("readwrite")
mustIAM(t, err)
ro, err := store.GetPolicy("readonly")
mustIAM(t, err)
for _, name := range []string{"review-edit", "review-delete"} {
_, err = store.SetPolicy(ctx, name, rw)
mustIAM(t, err)
}
const parent = "review-parent"
_, err = sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "synthetic-review-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
secret, err := getTokenSigningKey()
mustIAM(t, err)
issue := func(typ string) auth.Credentials {
c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent, tokenRevokeTypeClaim: typ}, secret)
mustIAM(t, err)
c.ParentUser = parent
_, err = store.SetTempUser(ctx, c.AccessKey, c, "")
mustIAM(t, err)
return c
}
revoked, retained := issue("selected"), issue("other")
gate := &iamPolicyRefreshReadGate{ObjectLayer: obj, policyPath: getPolicyDocPath("review-edit"), userPath: getUserIdentityPath(parent, regUser), listEntered: make(chan struct{}), listRelease: make(chan struct{}), refreshEntered: make(chan struct{}), refreshRelease: make(chan struct{})}
store.IAMStorageAPI.(*IAMObjectStore).objAPI = gate
freeList := sync.OnceFunc(func() { close(gate.listRelease) })
freeRefresh := sync.OnceFunc(func() { close(gate.refreshRelease) })
var wg sync.WaitGroup
t.Cleanup(func() { freeList(); freeRefresh(); wg.Wait() })
start := func(fn func() error) <-chan error {
done := make(chan error, 1)
wg.Go(func() { done <- fn() })
return done
}
refreshDone := start(func() error {
return store.LoadIAMCache(context.WithValue(ctx, iamPolicyRefreshContext{}, "refresh"), false)
})
awaitIAMPolicyReload(t, gate.refreshEntered)
listCtx, cancel := context.WithCancel(context.WithValue(ctx, iamPolicyRefreshContext{}, "list"))
defer cancel()
listDone := start(func() error { return iamPolicyReloadCall(listCtx, store, docs) })
awaitIAMPolicyReload(t, gate.listEntered)
read := start(func() error {
if _, ok := store.GetUser(parent); !ok {
return errors.New("cached user lost")
}
_, err := store.GetPolicy("review-edit")
return err
})
select {
case err := <-read:
mustIAM(t, err)
case <-time.After(time.Second):
t.Fatal("actual object reload blocked cached reads")
}
writes := []<-chan error{
start(func() error { _, err := store.SetPolicy(ctx, "review-edit", ro); return err }),
start(func() error { return store.DeletePolicy(ctx, "review-delete", false) }),
start(func() error { _, err := store.RevokeTokens(ctx, parent, "selected"); return err }),
}
if canceled {
cancel()
} else {
freeList()
}
err = <-listDone
if canceled {
if !errors.Is(err, context.Canceled) {
t.Fatalf("cancel error: %v", err)
}
} else {
mustIAM(t, err)
}
for _, done := range writes {
mustIAM(t, <-done)
}
freeRefresh()
mustIAM(t, <-refreshDone)
p, err := store.GetPolicy("review-edit")
mustIAM(t, err)
if p.IsAllowed(policy.Args{Action: policy.PutObjectAction, BucketName: "bucket", ObjectName: "object"}) || !p.IsAllowed(policy.Args{Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}) {
t.Fatal("actual object refresh restored stale policy")
}
if _, err := store.GetPolicy("review-delete"); !errors.Is(err, errNoSuchPolicy) {
t.Fatal("deleted policy restored")
}
if _, ok := store.GetUser(revoked.AccessKey); ok {
t.Fatal("revoked STS token restored")
}
if _, ok := store.GetUser(retained.AccessKey); !ok {
t.Fatal("unrelated token lost")
}
})
}
}
}
+373
View File
@@ -0,0 +1,373 @@
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
"github.com/pgsty/silo-pkg/v3/policy"
)
type iamPolicyReloadStore struct {
IAMStorageAPI
policies func(context.Context, map[string]PolicyDoc) error
users func(context.Context, IAMUserType, map[string]UserIdentity) error
}
func (s *iamPolicyReloadStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
return s.policies(ctx, m)
}
func (s *iamPolicyReloadStore) loadUsers(ctx context.Context, typ IAMUserType, m map[string]UserIdentity) error {
if s.users != nil {
return s.users(ctx, typ, m)
}
return s.IAMStorageAPI.loadUsers(ctx, typ, m)
}
func iamPolicyReloadCall(ctx context.Context, store *IAMStoreSys, docs bool) error {
if docs {
_, err := store.ListPolicyDocs(ctx, "")
return err
}
_, err := store.ListPolicies(ctx, "")
return err
}
func awaitIAMPolicyReload(t *testing.T, ch <-chan struct{}) {
t.Helper()
select {
case <-ch:
case <-time.After(5 * time.Second):
t.Fatal("IAM operation did not reach the controlled barrier")
}
}
// Both list entry points must allow cached readers while preserving the writer
// ordering used by policy edits, peer notifications and credential revocation.
// A refresh that captured the old state must not publish after those writes.
func TestIAMPolicyReloadConcurrentChanges(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
for _, docs := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/docs=%t", backend, docs), func(t *testing.T) {
ctx, sys, obj := prepareIAMRevisionFixture(t, backend)
store := sys.store
original := store.IAMStorageAPI
rw, err := store.GetPolicy("readwrite")
mustIAM(t, err)
ro, err := store.GetPolicy("readonly")
mustIAM(t, err)
for _, name := range []string{"reload-edit", "reload-delete", "reload-peer"} {
_, err := store.SetPolicy(ctx, name, rw)
mustIAM(t, err)
}
const parent = "reload-parent"
_, err = sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "synthetic-reload-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
secret, err := getTokenSigningKey()
mustIAM(t, err)
issue := func(typ string) auth.Credentials {
c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent, tokenRevokeTypeClaim: typ}, secret)
mustIAM(t, err)
c.ParentUser = parent
_, err = store.SetTempUser(ctx, c.AccessKey, c, "")
mustIAM(t, err)
return c
}
revoked, retained := issue("selected"), issue("other")
peer := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
if backend == "etcd" {
peer.IAMStorageAPI = newIAMEtcdStore(original.(*IAMEtcdStore).client, MinIOUsersSysType)
}
mustIAM(t, peer.LoadIAMCache(ctx, true))
type refreshKey struct{}
loading, releaseLoad := make(chan struct{}), make(chan struct{})
refreshing, releaseRefresh := make(chan struct{}), make(chan struct{})
unblockLoad := sync.OnceFunc(func() { close(releaseLoad) })
unblockRefresh := sync.OnceFunc(func() { close(releaseRefresh) })
var wg sync.WaitGroup
t.Cleanup(func() { unblockLoad(); unblockRefresh(); wg.Wait() })
store.IAMStorageAPI = &iamPolicyReloadStore{
IAMStorageAPI: original,
policies: func(ctx context.Context, m map[string]PolicyDoc) error {
if err := original.loadPolicyDocs(ctx, m); err != nil {
return err
}
if ctx.Value(refreshKey{}) == nil {
close(loading)
select {
case <-releaseLoad:
case <-ctx.Done():
return ctx.Err()
}
}
return nil
},
users: func(ctx context.Context, typ IAMUserType, m map[string]UserIdentity) error {
if err := original.loadUsers(ctx, typ, m); err != nil {
return err
}
if typ == stsUser && ctx.Value(refreshKey{}) != nil {
close(refreshing)
<-releaseRefresh
}
return nil
},
}
start := func(fn func() error) <-chan error {
out := make(chan error, 1)
wg.Go(func() { out <- fn() })
return out
}
refreshDone := start(func() error { return store.LoadIAMCache(context.WithValue(ctx, refreshKey{}, true), false) })
awaitIAMPolicyReload(t, refreshing)
listDone := start(func() error { return iamPolicyReloadCall(ctx, store, docs) })
awaitIAMPolicyReload(t, loading)
reader := start(func() error {
if _, ok := store.GetUser(parent); !ok {
return errors.New("committed user disappeared during policy loading")
}
_, err := store.GetPolicy("reload-edit")
return err
})
select {
case err := <-reader:
mustIAM(t, err)
case <-time.After(time.Second):
t.Fatal("policy loading blocked cached authentication")
}
// The peer commits independently; its notification waits alongside
// local changes while the older list snapshot remains in flight.
_, err = peer.SetPolicy(ctx, "reload-peer", ro)
mustIAM(t, err)
writersStarted := make(chan struct{}, 4)
startWrite := func(fn func() error) <-chan error {
return start(func() error { writersStarted <- struct{}{}; return fn() })
}
writes := []<-chan error{
startWrite(func() error { _, err := store.SetPolicy(ctx, "reload-edit", ro); return err }),
startWrite(func() error { return store.DeletePolicy(ctx, "reload-delete", false) }),
startWrite(func() error { _, err := store.RevokeTokens(ctx, parent, "selected"); return err }),
startWrite(func() error { return store.PolicyNotificationHandler(ctx, "reload-peer") }),
}
for range writes {
<-writersStarted
}
select {
case err := <-writes[0]:
t.Fatalf("policy write overtook the pending reload: %v", err)
case <-time.After(25 * time.Millisecond):
}
for _, done := range writes[1:] {
select {
case err := <-done:
t.Fatalf("IAM mutation overtook the pending reload: %v", err)
default:
}
}
unblockLoad()
mustIAM(t, <-listDone)
for _, done := range writes {
mustIAM(t, <-done)
}
unblockRefresh()
mustIAM(t, <-refreshDone)
for _, name := range []string{"reload-edit", "reload-peer"} {
p, err := store.GetPolicy(name)
mustIAM(t, err)
if p.IsAllowed(policy.Args{Action: policy.PutObjectAction, BucketName: "bucket", ObjectName: "object"}) {
t.Fatalf("stale list/refresh restored the old grant: %s", name)
}
if !p.IsAllowed(policy.Args{Action: policy.GetObjectAction, BucketName: "bucket", ObjectName: "object"}) {
t.Fatalf("reload lost the committed read grant: %s", name)
}
}
if _, err := store.GetPolicy("reload-delete"); !errors.Is(err, errNoSuchPolicy) {
t.Fatalf("deleted policy returned: %v", err)
}
if _, ok := store.GetUser(revoked.AccessKey); ok {
t.Fatal("stale list/refresh restored a revoked token")
}
for _, key := range []string{retained.AccessKey, parent} {
if _, ok := store.GetUser(key); !ok {
t.Fatal("reload removed an unrelated credential")
}
}
})
}
}
}
func TestIAMPolicyReloadFailurePreservesCache(t *testing.T) {
for _, docs := range []bool{false, true} {
for _, failure := range []string{"storage", "cancel_partial", "caller_deadline_partial"} {
t.Run(fmt.Sprintf("docs=%t/%s", docs, failure), func(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t)
var cancel context.CancelFunc
if failure == "caller_deadline_partial" {
ctx, cancel = context.WithTimeout(ctx, 100*time.Millisecond)
} else {
ctx, cancel = context.WithCancel(ctx)
}
defer cancel()
store := sys.store
cache := store.rlock()
updated := cache.updatedAt
store.runlock()
injected := errors.New("policy read failed")
entered := make(chan struct{})
original := store.IAMStorageAPI
store.IAMStorageAPI = &iamPolicyReloadStore{IAMStorageAPI: original, policies: func(ctx context.Context, m map[string]PolicyDoc) error {
m["partial-result"] = PolicyDoc{}
close(entered)
if failure == "storage" {
return injected
}
<-ctx.Done()
// A canceled enumeration can close without yielding an error.
return nil
}}
done := make(chan error, 1)
go func() { done <- iamPolicyReloadCall(ctx, store, docs) }()
awaitIAMPolicyReload(t, entered)
want := injected
switch failure {
case "cancel_partial":
cancel()
want = context.Canceled
case "caller_deadline_partial":
want = context.DeadlineExceeded
}
select {
case err := <-done:
if !errors.Is(err, want) {
t.Fatalf("got %v, want %v", err, want)
}
case <-time.After(time.Second):
cancel()
<-done
t.Fatal("reload did not honor caller cancellation")
}
cache = store.rlock()
_, partial := cache.iamPolicyDocsMap["partial-result"]
unchanged := cache.updatedAt.Equal(updated)
store.runlock()
if partial || !unchanged {
t.Fatal("failed/canceled reload published a partial policy map")
}
store.IAMStorageAPI = original
// Cancellation must release both locks so the next operation works.
mustIAM(t, iamPolicyReloadCall(context.Background(), store, docs))
if _, err := store.GetPolicy("readwrite"); err != nil {
t.Fatal("reload lost default policies")
}
})
}
}
}
type iamPolicyRelockObserver struct {
IAMStorageAPI
calls atomic.Int32
reacquiring chan struct{}
}
func (s *iamPolicyRelockObserver) lock() *iamCache {
if s.calls.Add(1) == 2 {
close(s.reacquiring)
}
return s.IAMStorageAPI.lock()
}
func TestIAMPolicyReloadCallerBudgetPreserved(t *testing.T) {
for _, docs := range []bool{false, true} {
t.Run(fmt.Sprintf("docs=%t", docs), func(t *testing.T) {
_, sys, _ := prepareIAMRevisionFixture(t)
original := sys.store.IAMStorageAPI
prior := defaultContextTimeout
defaultContextTimeout = 50 * time.Millisecond
t.Cleanup(func() { defaultContextTimeout = prior })
sys.store.IAMStorageAPI = &iamPolicyReloadStore{IAMStorageAPI: original, policies: func(ctx context.Context, m map[string]PolicyDoc) error {
select {
case <-time.After(120 * time.Millisecond):
return original.loadPolicyDocs(ctx, m)
case <-ctx.Done():
return ctx.Err()
}
}}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
err := iamPolicyReloadCall(ctx, sys.store, docs)
if err != nil {
t.Fatalf("reload cut off a still-valid caller budget: err=%v callerErr=%v", err, ctx.Err())
}
})
}
}
func TestIAMPolicyReloadCancelWhileReacquiringCache(t *testing.T) {
for _, docs := range []bool{false, true} {
t.Run(fmt.Sprintf("docs=%t", docs), func(t *testing.T) {
_, sys, _ := prepareIAMRevisionFixture(t)
store := sys.store
original := store.IAMStorageAPI
observer := &iamPolicyRelockObserver{IAMStorageAPI: original, reacquiring: make(chan struct{})}
entered, release := make(chan struct{}), make(chan struct{})
store.IAMStorageAPI = &iamPolicyReloadStore{IAMStorageAPI: observer, policies: func(ctx context.Context, m map[string]PolicyDoc) error {
m["partial-reload"] = PolicyDoc{}
close(entered)
<-release
return nil
}}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() { done <- iamPolicyReloadCall(ctx, store, docs) }()
awaitIAMPolicyReload(t, entered)
cache := store.rlock()
before := cache.updatedAt
count := len(cache.iamPolicyDocsMap)
close(release)
awaitIAMPolicyReload(t, observer.reacquiring)
// This reader forces the reload to wait before its commit step.
// The request is canceled after I/O completed, before publishing.
cancel()
store.runlock()
err := <-done
if !errors.Is(err, context.Canceled) {
t.Fatalf("cancel before publish lost: %v", err)
}
cache = store.rlock()
_, partial := cache.iamPolicyDocsMap["partial-reload"]
unchanged := cache.updatedAt.Equal(before) && len(cache.iamPolicyDocsMap) == count
store.runlock()
if partial || !unchanged {
t.Fatal("canceled complete load still published")
}
})
}
}
+305
View File
@@ -0,0 +1,305 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"encoding/json"
"errors"
"fmt"
"reflect"
"sync"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/grid"
"github.com/pgsty/silo-pkg/v3/policy"
)
func newIAMPolicyResetSibling(ctx context.Context, t *testing.T, source *IAMSys, obj ObjectLayer) *IAMSys {
t.Helper()
store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
if original, ok := source.store.IAMStorageAPI.(*IAMEtcdStore); ok {
store.IAMStorageAPI = newIAMEtcdStore(original.client, MinIOUsersSysType)
}
mustIAM(t, store.LoadIAMCache(ctx, true))
return &IAMSys{store: store, usersSysType: MinIOUsersSysType}
}
func assertIAMPolicyResetInfo(t *testing.T, sys *IAMSys, name string, want *madmin.PolicyInfo) {
t.Helper()
got, err := sys.InfoPolicy(name)
mustIAM(t, err)
var gotPolicy, wantPolicy policy.Policy
mustIAM(t, json.Unmarshal(got.Policy, &gotPolicy))
mustIAM(t, json.Unmarshal(want.Policy, &wantPolicy))
// Actions and resources are sets, whose JSON array order is not stable.
// Compare their exact contents while retaining all native metadata checks.
gotMetadata, wantMetadata := *got, *want
gotMetadata.Policy, wantMetadata.Policy = nil, nil
if !gotPolicy.Equals(wantPolicy) || !reflect.DeepEqual(gotMetadata, wantMetadata) {
t.Fatalf("policy definition or native metadata changed: got %+v, want %+v", got, want)
}
}
// The first InfoPolicy call after Reset must work without any inventory read,
// refresh, or retry. The durable tombstone belongs to the removed override;
// it must still reject stale overrides without suppressing the native policy.
func TestIAMBuiltinPolicyResetRestoresNativePolicy(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
t.Run(backend, func(t *testing.T) {
ctx, sys, obj := prepareIAMRevisionFixture(t, backend)
for _, builtin := range policy.DefaultPolicies {
t.Run(builtin.Name, func(t *testing.T) {
name := builtin.Name
native, err := sys.InfoPolicy(name)
mustIAM(t, err)
override := builtin.Definition
override.ID = "reset-override"
written, err := sys.SetPolicy(ctx, name, override)
mustIAM(t, err)
mustIAM(t, sys.DeletePolicy(ctx, name, true))
assertIAMPolicyResetInfo(t, sys, name, native)
path := getPolicyDocPath(name)
deleted, err := loadIAMRevision(ctx, sys.store, path)
mustIAM(t, err)
if !deleted.Deleted || !deleted.timestamp().After(written) {
t.Fatalf("Reset lost the override deletion revision: %+v", deleted)
}
for _, at := range []time.Time{written, deleted.timestamp()} {
_, err = sys.SetPolicy(withIAMReplicationTime(ctx, at), name, override)
if !errors.Is(err, errIAMStaleUpdate) {
t.Fatalf("Reset accepted a stale override at %s: %v", at, err)
}
assertIAMPolicyResetInfo(t, sys, name, native)
}
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
assertIAMPolicyResetInfo(t, sys, name, native)
fresh := newIAMPolicyResetSibling(ctx, t, sys, obj)
assertIAMPolicyResetInfo(t, fresh, name, native)
for _, docs := range []bool{false, true} {
mustIAM(t, iamPolicyReloadCall(ctx, fresh.store, docs))
assertIAMPolicyResetInfo(t, fresh, name, native)
}
after, err := loadIAMRevision(ctx, fresh.store, path)
mustIAM(t, err)
if !reflect.DeepEqual(after, deleted) || !fresh.store.revisionIndex().get(path).Deleted {
t.Fatal("default restoration changed the durable deletion or revision index")
}
// A deliberate new edit remains possible after Reset.
_, err = sys.SetPolicy(ctx, name, override)
mustIAM(t, err)
current, err := sys.store.GetPolicyDoc(name)
mustIAM(t, err)
if current.Policy.ID != override.ID || current.UpdateDate.IsZero() {
t.Fatal("Reset prevented a subsequent explicit override")
}
mustIAM(t, sys.DeletePolicy(ctx, name, true))
})
}
})
}
}
// Peer and watch notifications carry a name, not a version. Reload committed
// storage so delayed deletion events cannot erase a recreated policy. Built-in
// defaults retain mappings; a genuinely deleted custom policy does not.
func TestIAMPolicyResetNotificationLoadsCommittedState(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
for _, delivery := range []string{"peer", "watch"} {
for _, recreate := range []bool{false, true} {
for _, name := range []string{"writeonly", "reset-custom"} {
t.Run(fmt.Sprintf("%s/%s/recreate=%t/%s", backend, delivery, recreate, name), func(t *testing.T) {
ctx, source, obj := prepareIAMRevisionFixture(t, backend)
globalObjLayerMutex.Lock()
globalObjectAPI = obj
globalObjLayerMutex.Unlock()
builtin := name == "writeonly"
var want *madmin.PolicyInfo
if builtin {
var err error
want, err = source.InfoPolicy(name)
mustIAM(t, err)
}
p, err := source.store.GetPolicy("readwrite")
mustIAM(t, err)
p.ID = "reset-original-override"
_, err = source.SetPolicy(ctx, name, p)
mustIAM(t, err)
_, err = source.CreateUser(ctx, "reset-user", madmin.AddOrUpdateUserReq{SecretKey: "synthetic-reset-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
_, err = source.AddUsersToGroup(ctx, "reset-group", []string{"reset-user"})
mustIAM(t, err)
for _, group := range []bool{false, true} {
entity := "reset-user"
if group {
entity = "reset-group"
}
_, err = source.PolicyDBSet(ctx, entity, "readonly,"+name, regUser, group)
mustIAM(t, err)
}
if err := source.DeletePolicy(ctx, name, true); !errors.Is(err, errPolicyInUse) {
t.Fatalf("Reset bypassed policy-in-use protection: %v", err)
}
sibling := newIAMPolicyResetSibling(ctx, t, source, obj)
mappings := make(map[bool]MappedPolicy)
mappings[false], _ = sibling.store.GetMappedPolicy("reset-user", false)
mappings[true], _ = sibling.store.GetMappedPolicy("reset-group", true)
// A remote site may delete before this node has received the
// corresponding mapping changes. Local deletion stays guarded.
mustIAM(t, source.DeletePolicy(withIAMReplicationTime(ctx, UTCNow()), name, true))
if recreate {
p.ID = "reset-recreated-override"
_, err = source.SetPolicy(ctx, name, p)
mustIAM(t, err)
want, err = source.InfoPolicy(name)
mustIAM(t, err)
}
before, err := loadIAMRevision(ctx, source.store, getPolicyDocPath(name))
mustIAM(t, err)
globalIAMSys = sibling
defer func() { globalIAMSys = source }()
for range 2 {
if delivery == "peer" {
_, remoteErr := (&peerRESTServer{}).DeletePolicyHandler(grid.NewMSSWith(map[string]string{peerRESTPolicy: name}))
if remoteErr != nil {
t.Fatal(remoteErr)
}
} else {
mustIAM(t, sibling.loadWatchedEvent(ctx, iamWatchEvent{keyPath: getPolicyDocPath(name)}))
}
if builtin || recreate {
assertIAMPolicyResetInfo(t, sibling, name, want)
} else if _, err := sibling.InfoPolicy(name); !errors.Is(err, errNoSuchPolicy) {
t.Fatalf("deleted custom policy returned: %v", err)
}
for group, original := range mappings {
entity := "reset-user"
if group {
entity = "reset-group"
}
got, _ := sibling.store.GetMappedPolicy(entity, group)
if builtin || recreate {
if !reflect.DeepEqual(got, original) {
t.Fatalf("notification changed surviving mappings: %+v", got)
}
} else if got.Policies != "readonly" {
t.Fatalf("custom deletion did not retain only the unrelated mapping: %+v", got)
}
}
}
after, err := loadIAMRevision(ctx, source.store, getPolicyDocPath(name))
mustIAM(t, err)
if !reflect.DeepEqual(after, before) {
t.Fatal("notification changed durable policy storage")
}
})
}
}
}
}
}
func TestIAMPolicyResetNotificationSurvivesOlderRefresh(t *testing.T) {
ctx, source, obj := prepareIAMRevisionFixture(t)
p, err := source.store.GetPolicy("readwrite")
mustIAM(t, err)
p.ID = "old-override"
_, err = source.SetPolicy(ctx, "writeonly", p)
mustIAM(t, err)
sibling := newIAMPolicyResetSibling(ctx, t, source, obj)
original := sibling.store.IAMStorageAPI
loading, release := make(chan struct{}), make(chan struct{})
unblock := sync.OnceFunc(func() { close(release) })
var wg sync.WaitGroup
t.Cleanup(func() { unblock(); wg.Wait() })
sibling.store.IAMStorageAPI = &iamPolicyReloadStore{
IAMStorageAPI: original,
policies: original.loadPolicyDocs,
users: func(ctx context.Context, typ IAMUserType, m map[string]UserIdentity) error {
if typ == regUser {
close(loading)
<-release
}
return original.loadUsers(ctx, typ, m)
},
}
done := make(chan error, 1)
wg.Go(func() { done <- sibling.store.LoadIAMCache(ctx, false) })
awaitIAMPolicyReload(t, loading)
mustIAM(t, source.DeletePolicy(ctx, "writeonly", true))
p.ID = "newer-override"
_, err = source.SetPolicy(ctx, "writeonly", p)
mustIAM(t, err)
want, err := source.InfoPolicy("writeonly")
mustIAM(t, err)
// The delayed deletion notification must observe the newer edit, and
// prevent a refresh that captured the old override from replacing it.
mustIAM(t, sibling.DeletePolicy(ctx, "writeonly", false))
unblock()
mustIAM(t, <-done)
assertIAMPolicyResetInfo(t, sibling, "writeonly", want)
}
type iamPolicyResetFailureStore struct {
IAMStorageAPI
err error
}
func (s *iamPolicyResetFailureStore) saveIAMConfig(context.Context, any, string, ...options) error {
return s.err
}
func (s *iamPolicyResetFailureStore) loadPolicyDoc(context.Context, string, map[string]PolicyDoc) error {
return s.err
}
func TestIAMBuiltinPolicyResetStorageFailurePreservesOverride(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
t.Run(backend, func(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
p, err := sys.store.GetPolicy("readonly")
mustIAM(t, err)
_, err = sys.SetPolicy(ctx, "writeonly", p)
mustIAM(t, err)
before, err := sys.InfoPolicy("writeonly")
mustIAM(t, err)
stored, err := loadIAMRevision(ctx, sys.store, getPolicyDocPath("writeonly"))
mustIAM(t, err)
original := sys.store.IAMStorageAPI
injected := errors.New("injected policy storage failure")
sys.store.IAMStorageAPI = &iamPolicyResetFailureStore{IAMStorageAPI: original, err: injected}
for _, notify := range []bool{true, false} {
if err := sys.DeletePolicy(ctx, "writeonly", notify); !errors.Is(err, injected) {
t.Fatalf("storage failure became a successful Reset, notify=%t: %v", notify, err)
}
assertIAMPolicyResetInfo(t, sys, "writeonly", before)
}
if err := sys.loadWatchedEvent(ctx, iamWatchEvent{keyPath: getPolicyDocPath("writeonly")}); !errors.Is(err, injected) {
t.Fatalf("watch storage failure became a default policy: %v", err)
}
assertIAMPolicyResetInfo(t, sys, "writeonly", before)
after, err := loadIAMRevision(ctx, original, getPolicyDocPath("writeonly"))
mustIAM(t, err)
if !reflect.DeepEqual(after, stored) {
t.Fatal("failed Reset changed the durable override")
}
})
}
}
+18
View File
@@ -29,6 +29,7 @@ const (
iamRevisionPeerPath = "/v3/site-replication/peer/iam-revisions" iamRevisionPeerPath = "/v3/site-replication/peer/iam-revisions"
iamUserBoundaryType = "silo-user-revocation" iamUserBoundaryType = "silo-user-revocation"
iamGroupBoundaryType = "silo-group-revocation" iamGroupBoundaryType = "silo-group-revocation"
iamSTSDeleteType = "silo-sts-revocation"
maxIAMRevisionBatch = 128 maxIAMRevisionBatch = 128
) )
@@ -49,6 +50,7 @@ type iamGroupBoundary struct {
// before applying any change that would lose revocation or member metadata. // before applying any change that would lose revocation or member metadata.
type iamReplicationItem struct { type iamReplicationItem struct {
madmin.SRIAMItem madmin.SRIAMItem
STSExpiresAt time.Time `json:"stsExpiresAt,omitempty"`
GroupGrants map[string]time.Time `json:"groupGrants,omitempty"` GroupGrants map[string]time.Time `json:"groupGrants,omitempty"`
GroupSnapshot bool `json:"groupSnapshot,omitempty"` GroupSnapshot bool `json:"groupSnapshot,omitempty"`
UserRevocation *iamUserBoundary `json:"userRevocation,omitempty"` UserRevocation *iamUserBoundary `json:"userRevocation,omitempty"`
@@ -155,6 +157,13 @@ type (
func (c *SiteReplicationSys) replicationItem(ctx context.Context, item madmin.SRIAMItem) (iamReplicationItem, error) { func (c *SiteReplicationSys) replicationItem(ctx context.Context, item madmin.SRIAMItem) (iamReplicationItem, error) {
out := iamReplicationItem{SRIAMItem: item} out := iamReplicationItem{SRIAMItem: item}
if item.Type == iamSTSDeleteType {
r, err := loadIAMRevision(ctx, globalIAMSys.store, getUserIdentityPath(item.Name, stsUser))
if err != nil {
return out, err
}
out.STSExpiresAt = r.ExpiresAt
}
if item.Type == madmin.SRIAMItemSvcAcc && item.SvcAccChange != nil { if item.Type == madmin.SRIAMItemSvcAcc && item.SvcAccChange != nil {
var key string var key string
if item.SvcAccChange.Create != nil { if item.SvcAccChange.Create != nil {
@@ -242,6 +251,15 @@ func applyIAMReplicationItem(ctx context.Context, item iamReplicationItem) error
ctx = context.WithValue(ctx, iamRecordBoundaryKey{}, item.RevokedBefore) ctx = context.WithValue(ctx, iamRecordBoundaryKey{}, item.RevokedBefore)
} }
switch item.Type { switch item.Type {
case iamSTSDeleteType:
if item.Name == "" || strings.ContainsAny(item.Name, "/\\") || item.UpdatedAt.IsZero() {
return errSRInvalidRequest(errInvalidArgument)
}
if err := globalIAMSys.store.revokeTempUser(withIAMReplicationTime(ctx, item.UpdatedAt), item.Name, item.STSExpiresAt); err != nil {
return iamReplicationError(err)
}
globalIAMSys.notifyForUser(ctx, item.Name, true)
return nil
case iamUserBoundaryType: case iamUserBoundaryType:
if item.UserRevocation == nil || item.UserRevocation.User == "" || item.UserRevocation.Before.IsZero() { if item.UserRevocation == nil || item.UserRevocation.User == "" || item.UserRevocation.Before.IsZero() {
return errSRInvalidRequest(errInvalidArgument) return errSRInvalidRequest(errInvalidArgument)
+32 -8
View File
@@ -348,6 +348,13 @@ func saveIAMRevision(ctx context.Context, store IAMStorageAPI, path string, item
// STS access keys identify immutable tokens, not reusable user names. // STS access keys identify immutable tokens, not reusable user names.
return errIAMStaleUpdate return errIAMStaleUpdate
} }
if strings.HasPrefix(path, iamConfigSTSPrefix) && deleted {
// STS keys are immutable. A delayed revocation must also defeat a
// replayed creation that was recorded with a newer event timestamp.
if origin, replicated := iamReplicationTime(ctx); replicated && previous.timestamp().After(origin) {
ctx = withIAMReplicationTime(ctx, previous.timestamp())
}
}
if origin, replicated := iamReplicationTime(ctx); replicated { if origin, replicated := iamReplicationTime(ctx); replicated {
*at = origin *at = origin
if previous.timestamp().After(origin) || (previous.Deleted && !deleted && !origin.After(previous.timestamp())) { if previous.timestamp().After(origin) || (previous.Deleted && !deleted && !origin.After(previous.timestamp())) {
@@ -394,11 +401,18 @@ func saveIAMRevision(ctx context.Context, store IAMStorageAPI, path string, item
// Retain only the parent name for root-account exclusion during heal. // Retain only the parent name for root-account exclusion during heal.
v.Credentials = auth.Credentials{ParentUser: previous.Credentials.ParentUser} v.Credentials = auth.Credentials{ParentUser: previous.Credentials.ParentUser}
v.RevokedBefore = *at v.RevokedBefore = *at
if strings.HasPrefix(path, iamConfigSTSPrefix) && !previous.Credentials.Expiration.IsZero() && !previous.Credentials.Expiration.Equal(timeSentinel) { if strings.HasPrefix(path, iamConfigSTSPrefix) {
// The signed STS token cannot authorize beyond this time, even // Carry expiry when a revocation arrives before the token itself.
// if an offline site replays it with a newer event timestamp. // It is safe to forget this key only after its signed lifetime.
v.ExpiresAt = previous.Credentials.Expiration.Add(globalMaxSkewTime) if expiration := previous.Credentials.Expiration; !expiration.IsZero() && !expiration.Equal(timeSentinel) && expiration.Add(globalMaxSkewTime).After(v.ExpiresAt) {
opts = []options{{ttl: max(1, int64(time.Until(v.ExpiresAt).Seconds())+1)}} v.ExpiresAt = expiration.Add(globalMaxSkewTime)
}
if previous.ExpiresAt.After(v.ExpiresAt) {
v.ExpiresAt = previous.ExpiresAt
}
if !v.ExpiresAt.IsZero() {
opts = []options{{ttl: max(1, int64(time.Until(v.ExpiresAt).Seconds())+1)}}
}
} }
} else { } else {
if v.Credentials.SessionToken != "" && v.Credentials.Claims == nil { if v.Credentials.SessionToken != "" && v.Credentials.Claims == nil {
@@ -489,6 +503,12 @@ func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool
name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile) name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigUsersPrefix), "/"+iamIdentityFile)
item.Type = madmin.SRIAMItemIAMUser item.Type = madmin.SRIAMItemIAMUser
item.IAMUser = &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true} item.IAMUser = &madmin.SRIAMUser{AccessKey: name, IsDeleteReq: true}
case strings.HasPrefix(path, iamConfigSTSPrefix):
if !r.ExpiresAt.IsZero() && UTCNow().After(r.ExpiresAt) {
return item, false
}
item.Type = iamSTSDeleteType
item.Name = strings.TrimSuffix(strings.TrimPrefix(path, iamConfigSTSPrefix), "/"+iamIdentityFile)
case strings.HasPrefix(path, iamConfigServiceAccountsPrefix): case strings.HasPrefix(path, iamConfigServiceAccountsPrefix):
name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigServiceAccountsPrefix), "/"+iamIdentityFile) name := strings.TrimSuffix(strings.TrimPrefix(path, iamConfigServiceAccountsPrefix), "/"+iamIdentityFile)
if name == siteReplicatorSvcAcc || r.Credentials.ParentUser == globalActiveCred.AccessKey { if name == siteReplicatorSvcAcc || r.Credentials.ParentUser == globalActiveCred.AccessKey {
@@ -518,8 +538,7 @@ func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool
item.Type = madmin.SRIAMItemPolicyMapping item.Type = madmin.SRIAMItemPolicyMapping
item.PolicyMapping = &madmin.SRPolicyMapping{UserOrGroup: strings.TrimSuffix(name, ".json"), UserType: int(typ), IsGroup: prefix == "groups"} item.PolicyMapping = &madmin.SRPolicyMapping{UserOrGroup: strings.TrimSuffix(name, ".json"), UserType: int(typ), IsGroup: prefix == "groups"}
default: default:
// Expired STS credentials are not replayed. Parent revocations and // Other config paths do not represent replicated IAM deletions.
// their retained timestamp reject delayed copies of derived tokens.
return item, false return item, false
} }
return item, true return item, true
@@ -527,6 +546,8 @@ func iamDeletionItem(path string, r iamRevision) (item madmin.SRIAMItem, ok bool
func iamDeletionPath(item madmin.SRIAMItem) string { func iamDeletionPath(item madmin.SRIAMItem) string {
switch item.Type { switch item.Type {
case iamSTSDeleteType:
return getUserIdentityPath(item.Name, stsUser)
case madmin.SRIAMItemIAMUser: case madmin.SRIAMItemIAMUser:
if item.IAMUser != nil && item.IAMUser.IsDeleteReq { if item.IAMUser != nil && item.IAMUser.IsDeleteReq {
return getUserIdentityPath(item.IAMUser.AccessKey, regUser) return getUserIdentityPath(item.IAMUser.AccessKey, regUser)
@@ -582,6 +603,9 @@ func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
continue continue
} }
out := iamReplicationItem{SRIAMItem: item} out := iamReplicationItem{SRIAMItem: item}
if item.Type == iamSTSDeleteType {
out.STSExpiresAt = r.ExpiresAt
}
if item.Type == madmin.SRIAMItemIAMUser && !r.Deleted { if item.Type == madmin.SRIAMItemIAMUser && !r.Deleted {
out.Type, out.IAMUser = iamUserBoundaryType, nil out.Type, out.IAMUser = iamUserBoundaryType, nil
out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore} out.UserRevocation = &iamUserBoundary{User: item.IAMUser.AccessKey, Before: r.RevokedBefore}
@@ -593,7 +617,7 @@ func (c *SiteReplicationSys) healIAMDeletions(ctx context.Context) (err error) {
byType[item.Type] = append(byType[item.Type], out) byType[item.Type] = append(byType[item.Type], out)
} }
var items []iamReplicationItem var items []iamReplicationItem
for _, typ := range []string{madmin.SRIAMItemPolicyMapping, madmin.SRIAMItemIAMUser, madmin.SRIAMItemSvcAcc, madmin.SRIAMItemGroupInfo, madmin.SRIAMItemPolicy} { for _, typ := range []string{madmin.SRIAMItemPolicyMapping, madmin.SRIAMItemIAMUser, madmin.SRIAMItemSvcAcc, iamSTSDeleteType, madmin.SRIAMItemGroupInfo, madmin.SRIAMItemPolicy} {
items = append(items, byType[typ]...) items = append(items, byType[typ]...)
} }
if len(items) == 0 { if len(items) == 0 {
+210
View File
@@ -7,14 +7,19 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"errors" "errors"
"fmt"
"os" "os"
"path/filepath"
"runtime"
"strings" "strings"
"sync/atomic" "sync/atomic"
"testing" "testing"
"time" "time"
"github.com/minio/madmin-go/v3" "github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/grid" "github.com/minio/minio/internal/grid"
"github.com/pgsty/silo-pkg/v3/ldap"
xnet "github.com/pgsty/silo-pkg/v3/net" xnet "github.com/pgsty/silo-pkg/v3/net"
) )
@@ -93,6 +98,211 @@ func TestIAMRevocationTombstoneReplayIsIdempotent(t *testing.T) {
} }
} }
// An LDAP parent named by a normalized DN longer than NAME_MAX has no record
// on object storage. Its lookups must find nothing instead of failing login.
func TestLoadIAMRevisionNameTooLong(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
t.Run(backend, func(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
store := sys.store.IAMStorageAPI
// 160 bytes of UTF-8 become 384 characters.
parent, err := ldap.NormalizeDN("uid=Сотрудник,ou=Отдел разработки,ou=Дирекция информационных технологий,ou=people,ou=swengg,dc=min,dc=io")
mustIAM(t, err)
if len(parent) <= 255 {
t.Fatalf("normalized DN has only %d characters", len(parent))
}
parentPath := getUserIdentityPath(parent, regUser)
for _, path := range []string{parentPath, getUserIdentityPath("missing-parent", regUser)} {
r, err := loadIAMRevision(ctx, store, path)
if err != nil || r.Deleted || !r.timestamp().IsZero() || !r.RevokedBefore.IsZero() {
t.Fatalf("absent revision %.40s...: %+v, %v", path, r, err)
}
}
if backend == "object" {
var invalid ObjectNameInvalid
if _, err := loadIAMRevision(ctx, store, iamConfigUsersPrefix+"a//b/"+iamIdentityFile); !errors.As(err, &invalid) {
t.Fatalf("invalid name not caused by length was masked: %v", err)
}
}
claims := map[string]any{}
mustIAM(t, setIAMParentRevocationClaim(ctx, store, parent, claims))
if _, ok := claims[iamParentRevocationClaim]; ok {
t.Fatalf("unexpected revocation claim: %v", claims)
}
if _, err := sys.PolicyDBGet(parent); err != nil {
t.Fatalf("policy lookup failed: %v", err)
}
// The call that failed in AssumeRoleWithLDAPIdentity.
secret, err := getTokenSigningKey()
mustIAM(t, err)
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
mustIAM(t, err)
cred.ParentUser = parent
_, err = sys.SetTempUser(ctx, cred.AccessKey, cred, "")
mustIAM(t, err)
if _, ok := sys.store.GetUser(cred.AccessKey); !ok {
t.Fatal("temporary credential not cached")
}
// Object storage still cannot record a revocation for this parent;
// the failure must be reported. Etcd has no such limit.
err = saveIAMRevision(ctx, store, parentPath, &iamUserRevocation{})
if backend == "object" {
if !iamConfigNameTooLong(parentPath, err) {
t.Fatalf("expected name-too-long error, got %v", err)
}
return
}
mustIAM(t, err)
if err := checkIAMParentRevision(ctx, store, cred); !errors.Is(err, errIAMStaleUpdate) {
t.Fatalf("revoked parent accepted: %v", err)
}
})
}
}
// checkPathLength does not count backslashes outside Windows, so this escaped
// DN passes it. Once config/iam/users exists, as after creating any built-in
// user, the filesystem rejects the element with ENAMETOOLONG instead.
func TestLoadIAMRevisionFilesystemNameTooLong(t *testing.T) {
ctx, sys, obj := prepareIAMRevisionFixture(t)
parent, err := ldap.NormalizeDN("cn=Сидоров П.П.,ou=Отдел ИТ-поддержки,ou=Пользователи,ou=people,ou=swengg,dc=min,dc=io")
mustIAM(t, err)
parentPath := getUserIdentityPath(parent, regUser)
if len(parent) <= 255 || checkPathLength(parentPath) != nil {
t.Fatalf("normalized DN must exceed NAME_MAX yet pass checkPathLength: %d characters, %v", len(parent), checkPathLength(parentPath))
}
_, err = sys.CreateUser(ctx, "existing-user", madmin.AddOrUpdateUserReq{SecretKey: "synthetic-user-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
mustIAM(t, saveIAMRevision(ctx, sys.store.IAMStorageAPI, getMappedPolicyPath("existing-parent", stsUser, false), &MappedPolicy{Version: 1, Policies: "readwrite"}))
store := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, LDAPUsersSysType)}
mustIAM(t, store.LoadIAMCache(ctx, false))
r, err := loadIAMRevision(ctx, store.IAMStorageAPI, parentPath)
if err != nil || r.Deleted || !r.timestamp().IsZero() {
t.Fatalf("absent revision: %+v, %v", r, err)
}
// LDAP mode reads policydb/sts-users/<DN>.json.
if _, err := store.PolicyDBGet(parent); err != nil {
t.Fatalf("policy lookup failed: %v", err)
}
secret, err := getTokenSigningKey()
mustIAM(t, err)
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
mustIAM(t, err)
cred.ParentUser = parent
if _, err := store.SetTempUser(ctx, cred.AccessKey, cred, ""); err != nil {
t.Fatalf("temporary credential rejected: %v", err)
}
err = saveIAMRevision(ctx, store.IAMStorageAPI, parentPath, &iamUserRevocation{})
if !iamConfigNameTooLong(parentPath, err) {
t.Fatalf("expected name-too-long error, got %v", err)
}
}
func TestIAMConfigNameTooLong(t *testing.T) {
long := iamConfigUsersPrefix + strings.Repeat("x", 256) + "/" + iamIdentityFile
short := iamConfigUsersPrefix + strings.Repeat("x", 200) + "/" + iamIdentityFile
invalid := iamConfigUsersPrefix + "a//b/" + iamIdentityFile
escaped := iamConfigUsersPrefix + strings.Repeat("\\d0", 86) + "/" + iamIdentityFile
limit := iamConfigUsersPrefix + strings.Repeat("x", 255) + "/" + iamIdentityFile
unicode := iamConfigUsersPrefix + strings.Repeat("界", 100) + "/" + iamIdentityFile
deep := iamConfigUsersPrefix + strings.Repeat("nested/", 160) + iamIdentityFile
for _, tc := range []struct {
name string
path string
err error
want bool
}{
{"NAME_MAX", long, ObjectNameInvalid{Bucket: minioMetaBucket, Object: long}, true},
{"escaped DN", escaped, ObjectNameInvalid{Bucket: minioMetaBucket, Object: escaped}, runtime.GOOS != "windows"},
{"short component", short, ObjectNameInvalid{Bucket: minioMetaBucket, Object: short}, false},
{"exact component limit", limit, ObjectNameInvalid{Bucket: minioMetaBucket, Object: limit}, false},
{"unicode byte length", unicode, ObjectNameInvalid{Bucket: minioMetaBucket, Object: unicode}, false},
{"total path length", deep, ObjectNameInvalid{Bucket: minioMetaBucket, Object: deep}, false},
{"short drive path", short, errFileNameTooLong, false},
{"policy suffix at limit", getMappedPolicyPath(strings.Repeat("x", 250), stsUser, false), errFileNameTooLong, false},
{"policy suffix exceeds limit", getMappedPolicyPath(strings.Repeat("x", 251), stsUser, false), errFileNameTooLong, true},
{"drive", long, errFileNameTooLong, true},
{"wrapped drive", long, fmt.Errorf("read: %w", errFileNameTooLong), true},
{"wrapped object", long, fmt.Errorf("read: %w", ObjectNameInvalid{Bucket: minioMetaBucket, Object: long}), true},
{"no error", long, nil, false},
{"not found", long, errConfigNotFound, false},
{"offline drive", long, errDiskNotFound, false},
{"read quorum", long, InsufficientReadQuorum{}, false},
{"access denied", long, PrefixAccessDenied{}, false},
{"deadline", long, context.DeadlineExceeded, false},
{"invalid name", invalid, ObjectNameInvalid{Bucket: minioMetaBucket, Object: invalid}, false},
{"invalid long name", invalid + strings.Repeat("x", 256), errFileNameTooLong, false},
} {
t.Run(tc.name, func(t *testing.T) {
if got := iamConfigNameTooLong(tc.path, tc.err); got != tc.want {
t.Errorf("got %t, want %t", got, tc.want)
}
})
}
}
// Moving a drive can make an existing record exceed the total path limit.
// A length error must not turn that parent's revocation into a missing record.
func TestLoadIAMRevisionDeepPath(t *testing.T) {
if runtime.GOOS != "darwin" {
t.Skip("exercises the Darwin total-path limit")
}
for _, tc := range []struct {
name string
parent string
}{
{"ASCII", strings.Repeat("p", 200)},
{"Unicode", strings.Repeat("界", 100)},
} {
t.Run(tc.name, func(t *testing.T) {
resetTestGlobals()
defer resetTestGlobals()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
base := t.TempDir()
oldRoot := filepath.Join(base, "drive")
mustIAM(t, os.Mkdir(oldRoot, 0o700))
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, oldRoot))
mustIAM(t, err)
defer func() { obj.Shutdown(ctx) }()
store := newIAMObjectStore(obj, LDAPUsersSysType)
cred := auth.Credentials{ParentUser: tc.parent}
parentPath := getUserIdentityPath(tc.parent, regUser)
mustIAM(t, store.saveIAMConfig(ctx, &UserIdentity{Version: 1, Deleted: true, RevokedBefore: UTCNow()}, parentPath))
if err := checkIAMParentRevision(ctx, store, cred); !errors.Is(err, errIAMStaleUpdate) {
t.Fatalf("original parent revocation ineffective: %v", err)
}
mustIAM(t, obj.Shutdown(ctx))
deep := base
for len(deep) < 850 {
deep = filepath.Join(deep, strings.Repeat("d", 80))
}
mustIAM(t, os.MkdirAll(deep, 0o700))
newRoot := filepath.Join(deep, "drive")
mustIAM(t, checkPathLength(newRoot))
mustIAM(t, os.Rename(oldRoot, newRoot))
// The record still exists; a relative read avoids the total-path limit.
root, err := os.OpenRoot(newRoot)
mustIAM(t, err)
defer root.Close()
data, err := root.ReadFile(filepath.Join(minioMetaBucket, parentPath, xlStorageFormatFile))
mustIAM(t, err)
if len(data) == 0 {
t.Fatal("missing physical revocation record")
}
obj, _, err = initObjectLayer(ctx, mustGetPoolEndpoints(0, newRoot))
mustIAM(t, err)
store = newIAMObjectStore(obj, LDAPUsersSysType)
var invalid ObjectNameInvalid
if err := checkIAMParentRevision(ctx, store, cred); !errors.As(err, &invalid) {
t.Fatalf("expected path error for an existing parent revocation, got %v", err)
}
})
}
}
// Run with both object storage and etcd through TestIAMRevocation*Lifecycle. // Run with both object storage and etcd through TestIAMRevocation*Lifecycle.
// The object-store case uses the real peer RPC and deletion handler, so a // The object-store case uses the real peer RPC and deletion handler, so a
// spurious notification actually destroys the parent instead of only counting it. // spurious notification actually destroys the parent instead of only counting it.
+153 -76
View File
@@ -647,6 +647,20 @@ func setDefaultCannedPolicies(policies map[string]PolicyDoc) {
} }
} }
// Remove a stored policy from cache. A built-in policy's deletion removes its
// override, so its native definition remains available with native metadata.
// The durable tombstone is retained separately to reject stale overrides.
func removePolicyFromCache(name string, policies map[string]PolicyDoc) (restoredDefault bool) {
delete(policies, name)
for _, p := range policy.DefaultPolicies {
if p.Name == name {
policies[name] = defaultPolicyDoc(p.Definition)
return true
}
}
return false
}
// LoadIAMCache reads all IAM items and populates a new iamCache object and // LoadIAMCache reads all IAM items and populates a new iamCache object and
// replaces the in-memory cache object. // replaces the in-memory cache object.
func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) error { func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) error {
@@ -711,13 +725,18 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil { if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil {
return err return err
} }
// Rebuild STS revisions as well: a missed etcd watch event must not
// leave a cached token valid or hide its tombstone after a restart.
if err := store.loadUsers(ctx, stsUser, newCache.iamSTSAccountsMap); err != nil {
return err
}
newCache.buildUserGroupMemberships() newCache.buildUserGroupMemberships()
} }
for name := range newCache.iamPolicyDocsMap { for name := range newCache.iamPolicyDocsMap {
if store.revisionIndex().get(getPolicyDocPath(name)).Deleted { if store.revisionIndex().get(getPolicyDocPath(name)).Deleted {
delete(newCache.iamPolicyDocsMap, name) removePolicyFromCache(name, newCache.iamPolicyDocsMap)
} }
} }
@@ -754,6 +773,14 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
cache.updatedAt = time.Now() cache.updatedAt = time.Now()
} }
// Revision loading is independent of cache replacement. Even if ongoing
// IAM writes defer the full swap, a missed STS notification must converge.
for key := range cache.iamSTSAccountsMap {
if store.revisionIndex().get(getUserIdentityPath(key, stsUser)).Deleted {
delete(cache.iamSTSAccountsMap, key)
}
}
return nil return nil
} }
@@ -1363,9 +1390,9 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
} }
// PolicyNotificationHandler - loads given policy from storage. If not present, // PolicyNotificationHandler - loads given policy from storage. If not present,
// deletes from cache. This notification only reads from storage, and updates // removes its override from cache. This notification only reads from storage,
// cache. When the notification is for a policy deletion, it updates the // and updates cache. A deleted built-in override restores its native policy;
// user-policy and group-policy maps as well. // a deleted custom policy is removed from user-policy and group-policy maps.
func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy string) error { func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy string) error {
if policy == "" { if policy == "" {
return errInvalidArgument return errInvalidArgument
@@ -1376,8 +1403,10 @@ func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy
err := store.loadPolicyDoc(ctx, policy, cache.iamPolicyDocsMap) err := store.loadPolicyDoc(ctx, policy, cache.iamPolicyDocsMap)
if errors.Is(err, errNoSuchPolicy) { if errors.Is(err, errNoSuchPolicy) {
// policy was deleted, update cache. if removePolicyFromCache(policy, cache.iamPolicyDocsMap) {
delete(cache.iamPolicyDocsMap, policy) cache.updatedAt = time.Now()
return nil
}
// update user policy map // update user policy map
cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool { cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool {
@@ -1413,74 +1442,77 @@ func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy
cache.updatedAt = time.Now() cache.updatedAt = time.Now()
return nil return nil
} }
if err == nil {
cache.updatedAt = time.Now()
}
return err return err
} }
// DeletePolicy - deletes policy from storage and cache. When this called in // DeletePolicy - deletes a stored policy and restores the native definition
// response to a notification (i.e. isFromNotification = true), it skips the // when it was a built-in override. Notifications reload committed storage, so
// validation of policy usage and the attempt to delete in the backend as well // a delayed deletion event cannot remove a subsequently recreated policy.
// (as this is already done by the notifying node).
func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string, isFromNotification bool) error { func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string, isFromNotification bool) error {
if policy == "" { if policy == "" {
return errInvalidArgument return errInvalidArgument
} }
if isFromNotification {
return store.PolicyNotificationHandler(ctx, policy)
}
cache := store.lock() cache := store.lock()
defer store.unlock() defer store.unlock()
if !isFromNotification { if err := store.checkIAMRevision(ctx, getPolicyDocPath(policy), true); err != nil {
if err := store.checkIAMRevision(ctx, getPolicyDocPath(policy), true); err != nil { return err
return err }
if _, replicated := iamReplicationTime(ctx); !replicated {
if _, exists := cache.iamPolicyDocsMap[policy]; !exists {
// Preserve idempotent local deletion without creating a durable
// revocation for a misspelled/nonexistent policy name.
return nil
} }
if _, replicated := iamReplicationTime(ctx); !replicated { // Check if policy is mapped to any existing user or group. If so, we do not
if _, exists := cache.iamPolicyDocsMap[policy]; !exists { // allow deletion of the policy. If the policy is mapped to an STS account,
// Preserve idempotent local deletion without creating a durable // we do allow deletion.
// revocation for a misspelled/nonexistent policy name. users := []string{}
return nil groups := []string{}
cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool {
pset := mp.policySet()
if store.getUsersSysType() == MinIOUsersSysType {
if _, ok := cache.iamUsersMap[u]; !ok {
// This case can happen when a temporary account is
// deleted or expired - remove it from userPolicyMap.
cache.iamUserPolicyMap.Delete(u)
return true
}
} }
// Check if policy is mapped to any existing user or group. If so, we do not if pset.Contains(policy) {
// allow deletion of the policy. If the policy is mapped to an STS account, users = append(users, u)
// we do allow deletion.
users := []string{}
groups := []string{}
cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool {
pset := mp.policySet()
if store.getUsersSysType() == MinIOUsersSysType {
if _, ok := cache.iamUsersMap[u]; !ok {
// This case can happen when a temporary account is
// deleted or expired - remove it from userPolicyMap.
cache.iamUserPolicyMap.Delete(u)
return true
}
}
if pset.Contains(policy) {
users = append(users, u)
}
return true
})
cache.iamGroupPolicyMap.Range(func(g string, mp MappedPolicy) bool {
pset := mp.policySet()
if pset.Contains(policy) {
groups = append(groups, g)
}
return true
})
if len(users) != 0 || len(groups) != 0 {
return errPolicyInUse
} }
} return true
})
err := store.deletePolicyDoc(ctx, policy) cache.iamGroupPolicyMap.Range(func(g string, mp MappedPolicy) bool {
if errors.Is(err, errNoSuchPolicy) { pset := mp.policySet()
// Ignore error if policy is already deleted. if pset.Contains(policy) {
err = nil groups = append(groups, g)
} }
if err != nil { return true
return err })
if len(users) != 0 || len(groups) != 0 {
return errPolicyInUse
} }
} }
delete(cache.iamPolicyDocsMap, policy) err := store.deletePolicyDoc(ctx, policy)
if errors.Is(err, errNoSuchPolicy) {
// Ignore error if policy is already deleted.
err = nil
}
if err != nil {
return err
}
removePolicyFromCache(policy, cache.iamPolicyDocsMap)
cache.updatedAt = time.Now() cache.updatedAt = time.Now()
return nil return nil
@@ -1568,10 +1600,22 @@ func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (
defer store.unlock() defer store.unlock()
m := map[string]PolicyDoc{} m := map[string]PolicyDoc{}
err := store.loadPolicyDocs(ctx, m) // Keep cache writers serialized, but let authentication use the committed
// cache while the replacement is loaded into this private map.
err := func() error {
store.IAMStorageAPI.unlock()
defer store.IAMStorageAPI.lock()
if err := store.loadPolicyDocs(ctx, m); err != nil {
return err
}
return ctx.Err()
}()
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err := ctx.Err(); err != nil {
return nil, err
}
// Sets default canned policies // Sets default canned policies
setDefaultCannedPolicies(m) setDefaultCannedPolicies(m)
@@ -1596,10 +1640,20 @@ func (store *IAMStoreSys) ListPolicyDocs(ctx context.Context, bucketName string)
defer store.unlock() defer store.unlock()
m := map[string]PolicyDoc{} m := map[string]PolicyDoc{}
err := store.loadPolicyDocs(ctx, m) err := func() error {
store.IAMStorageAPI.unlock()
defer store.IAMStorageAPI.lock()
if err := store.loadPolicyDocs(ctx, m); err != nil {
return err
}
return ctx.Err()
}()
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err := ctx.Err(); err != nil {
return nil, err
}
// Sets default canned policies // Sets default canned policies
setDefaultCannedPolicies(m) setDefaultCannedPolicies(m)
@@ -2184,45 +2238,68 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
// RevokeTokens - revokes all temporary credentials, or those with matching type, // RevokeTokens - revokes all temporary credentials, or those with matching type,
// associated with the parent user. // associated with the parent user.
func (store *IAMStoreSys) RevokeTokens(ctx context.Context, parentUser string, tokenRevokeType string) error { func (store *IAMStoreSys) RevokeTokens(ctx context.Context, parentUser string, tokenRevokeType string) ([]string, error) {
if parentUser == "" { if parentUser == "" {
return errInvalidArgument return nil, errInvalidArgument
} }
before := UTCNow()
cache := store.lock() cache := store.lock()
defer store.unlock() defer store.unlock()
secret, err := getTokenSigningKey() secret, err := getTokenSigningKey()
if err != nil { if err != nil {
return err return nil, err
} }
// Another node may have issued a token whose notification was lost.
var revoked bool // Enumerate durable identities, retaining cached entries for retrying a
for _, ui := range cache.iamSTSAccountsMap { // partially propagated revocation. Do not hold up cached authentication.
if ui.Credentials.ParentUser != parentUser { accounts := maps.Clone(cache.iamSTSAccountsMap)
if err := store.withIAMStorage(ctx, func(ctx context.Context) error {
return store.loadUsers(ctx, stsUser, accounts)
}); err != nil {
return nil, err
}
var revoked []string
var failures []error
for key, ui := range accounts {
if ui.Credentials.ParentUser != parentUser || ui.UpdatedAt.After(before) || ui.Credentials.IsExpired() {
continue continue
} }
if tokenRevokeType != "" { if tokenRevokeType != "" {
claims, err := getClaimsFromTokenWithSecret(ui.Credentials.SessionToken, secret) claims, err := getClaimsFromTokenWithSecret(ui.Credentials.SessionToken, secret)
if err != nil { if err != nil {
continue // skip if token is invalid continue
} }
// skip if token type is given and does not match
if v, _ := claims.Lookup(tokenRevokeTypeClaim); v != tokenRevokeType { if v, _ := claims.Lookup(tokenRevokeTypeClaim); v != tokenRevokeType {
continue continue
} }
} }
if err := store.deleteUserIdentity(ctx, ui.Credentials.AccessKey, stsUser); err != nil { if err := store.deleteUserIdentity(ctx, key, stsUser); err != nil {
return err failures = append(failures, err)
continue
} }
delete(cache.iamSTSAccountsMap, ui.Credentials.AccessKey) delete(cache.iamSTSAccountsMap, key)
revoked = true revoked = append(revoked, key)
} }
if len(revoked) != 0 {
if revoked {
cache.updatedAt = time.Now() cache.updatedAt = time.Now()
} }
return revoked, errors.Join(failures...)
}
// revokeTempUser applies an immutable token's replicated tombstone without
// touching the parent, its policies, or other (including newer) tokens.
func (store *IAMStoreSys) revokeTempUser(ctx context.Context, key string, expiresAt time.Time) error {
cache := store.lock()
defer store.unlock()
u := UserIdentity{Version: 1, Deleted: true, ExpiresAt: expiresAt}
if err := store.saveUserIdentity(ctx, key, stsUser, &u); err != nil {
return err
}
delete(cache.iamSTSAccountsMap, key)
cache.updatedAt = time.Now()
if store.group != nil {
store.group.Forget(key)
}
return nil return nil
} }
+254
View File
@@ -0,0 +1,254 @@
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/grid"
xnet "github.com/pgsty/silo-pkg/v3/net"
)
func TestIAMSTSRevocationDurableScope(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
for _, notify := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/notify=%t", backend, notify), func(t *testing.T) {
ctx, sys, obj := prepareIAMRevisionFixture(t, backend)
for _, parent := range []string{"sts-parent", "other-parent"} {
_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "synthetic-parent-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
}
sibling := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
if backend == "etcd" {
sibling.IAMStorageAPI = newIAMEtcdStore(sys.store.IAMStorageAPI.(*IAMEtcdStore).client, MinIOUsersSysType)
}
mustIAM(t, sibling.LoadIAMCache(ctx, false))
secret, err := getTokenSigningKey()
mustIAM(t, err)
issue := func(store *IAMStoreSys, parent, typ string) auth.Credentials {
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent, tokenRevokeTypeClaim: typ}, secret)
mustIAM(t, err)
cred.ParentUser = parent
_, err = store.SetTempUser(ctx, cred.AccessKey, cred, "")
mustIAM(t, err)
return cred
}
first := issue(sys.store, "sts-parent", "selected")
uncached := issue(sibling, "sts-parent", "selected")
otherType := issue(sys.store, "sts-parent", "other-type")
otherUser := issue(sys.store, "other-parent", "selected")
if _, ok := sys.store.GetUser(uncached.AccessKey); ok {
t.Fatal("fixture cached the remote token")
}
for _, c := range []auth.Credentials{first, otherType, otherUser} {
mustIAM(t, sibling.UserNotificationHandler(ctx, c.AccessKey, stsUser))
}
prior := globalNotificationSys
globalNotificationSys = &NotificationSys{}
t.Cleanup(func() { globalNotificationSys = prior })
if notify {
tg, err := grid.SetupTestGrid(2)
mustIAM(t, err)
t.Cleanup(tg.Cleanup)
mustIAM(t, loadUserRPC.Register(tg.Managers[1], func(r *grid.MSS) (grid.NoPayload, *grid.RemoteErr) {
if err := sibling.UserNotificationHandler(ctx, r.Get(peerRESTUser), stsUser); err != nil {
return grid.NoPayload{}, grid.NewRemoteErr(err)
}
return grid.NoPayload{}, nil
}))
host, err := xnet.ParseHost(strings.TrimPrefix(tg.Hosts[1], "http://"))
mustIAM(t, err)
globalNotificationSys = &NotificationSys{peerClients: []*peerRESTClient{{host: host, gridConn: func() *grid.Connection { return tg.Managers[0].Connection(tg.Hosts[1]) }}}}
}
mustIAM(t, sys.RevokeTokens(ctx, "sts-parent", "selected"))
for _, c := range []auth.Credentials{first, uncached} {
r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(c.AccessKey, stsUser))
mustIAM(t, err)
if !r.Deleted {
t.Errorf("durable token was omitted: %s", c.AccessKey)
}
if _, ok := iamDeletionItem(getUserIdentityPath(c.AccessKey, stsUser), r); !ok {
t.Error("STS revocation cannot be replayed")
}
if notify && backend == "object" {
if _, ok := sibling.GetUser(c.AccessKey); ok {
t.Error("notified sibling accepted revoked token")
}
}
}
fresh := issue(sibling, "sts-parent", "selected")
// A missed notification must converge through the normal cache refresh.
mustIAM(t, sibling.LoadIAMCache(ctx, false))
for _, c := range []auth.Credentials{first, uncached} {
if _, ok := sibling.GetUser(c.AccessKey); ok {
t.Error("refresh retained revoked token")
}
}
for _, c := range []auth.Credentials{otherType, otherUser, fresh} {
mustIAM(t, sibling.UserNotificationHandler(ctx, c.AccessKey, stsUser))
if _, ok := sibling.GetUser(c.AccessKey); !ok {
t.Error("revocation removed an out-of-scope or new token")
}
}
if _, ok := sibling.GetUser("sts-parent"); !ok {
t.Error("revocation removed parent")
}
})
}
}
}
// Deletion is keyed by the immutable STS access key, so delayed delivery cannot
// affect a newly issued token and a delayed creation cannot undo revocation.
func TestIAMSTSRevocationReorderedReplay(t *testing.T) {
for _, backend := range []string{"object", "etcd"} {
for _, missing := range []bool{false, true} {
t.Run(fmt.Sprintf("%s/missing=%t", backend, missing), func(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t, backend)
const parent = "sts-replay-parent"
_, err := sys.CreateUser(ctx, parent, madmin.AddOrUpdateUserReq{SecretKey: "synthetic-parent-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
secret, err := getTokenSigningKey()
mustIAM(t, err)
cred, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
mustIAM(t, err)
cred.ParentUser = parent
created := UTCNow().Add(-time.Minute)
peer := madmin.SRSTSCredential{AccessKey: cred.AccessKey, SecretKey: cred.SecretKey, SessionToken: cred.SessionToken, ParentUser: parent}
if !missing {
mustIAM(t, globalSiteReplicationSys.PeerSTSAccHandler(ctx, &peer, created))
}
fresh, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: parent}, secret)
mustIAM(t, err)
fresh.ParentUser = parent
_, err = sys.SetTempUser(ctx, fresh.AccessKey, fresh, "")
mustIAM(t, err)
// Older than the live record on this peer: deletion must still win.
item := iamReplicationItem{SRIAMItem: madmin.SRIAMItem{Type: iamSTSDeleteType, Name: cred.AccessKey, UpdatedAt: created.Add(-time.Second)}, STSExpiresAt: cred.Expiration.Add(globalMaxSkewTime)}
for range 2 {
mustIAM(t, applyIAMReplicationItem(ctx, item))
mustIAM(t, globalSiteReplicationSys.PeerSTSAccHandler(ctx, &peer, UTCNow()))
mustIAM(t, sys.store.LoadIAMCache(ctx, false))
if _, ok := sys.GetUser(ctx, cred.AccessKey); ok {
t.Fatal("delayed STS creation undid revocation")
}
if _, ok := sys.GetUser(ctx, fresh.AccessKey); !ok {
t.Fatal("delayed revocation removed new token")
}
}
r, err := loadIAMRevision(ctx, sys.store, getUserIdentityPath(cred.AccessKey, stsUser))
mustIAM(t, err)
if !r.Deleted || !r.ExpiresAt.Equal(item.STSExpiresAt) || r.Credentials.SessionToken != "" || r.Credentials.SecretKey != "" {
t.Fatal("revocation lost expiry or retained secret material")
}
for _, status := range []madmin.AccountStatus{madmin.AccountDisabled, madmin.AccountEnabled} {
_, err = sys.SetUserStatus(ctx, parent, status)
mustIAM(t, err)
}
if _, ok := sys.GetUser(ctx, cred.AccessKey); ok {
t.Fatal("parent re-enable revived revoked token")
}
})
}
}
}
// Pause the durable scan while another node persists a new credential. The
// revocation's original cutoff must survive the scan and leave that token live.
type iamSTSScanGateStore struct {
IAMStorageAPI
entered chan struct{}
resume chan struct{}
}
func (s *iamSTSScanGateStore) loadUsers(ctx context.Context, typ IAMUserType, users map[string]UserIdentity) error {
close(s.entered)
select {
case <-s.resume:
case <-ctx.Done():
return ctx.Err()
}
return s.IAMStorageAPI.loadUsers(ctx, typ, users)
}
func TestIAMSTSRevocationConcurrentIssuance(t *testing.T) {
ctx, sys, obj := prepareIAMRevisionFixture(t)
_, err := sys.CreateUser(ctx, "sts-race-parent", madmin.AddOrUpdateUserReq{SecretKey: "synthetic-parent-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
sibling := &IAMStoreSys{IAMStorageAPI: newIAMObjectStore(obj, MinIOUsersSysType)}
secret, err := getTokenSigningKey()
mustIAM(t, err)
issue := func() auth.Credentials {
c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: "sts-race-parent"}, secret)
mustIAM(t, err)
c.ParentUser = "sts-race-parent"
_, err = sibling.SetTempUser(ctx, c.AccessKey, c, "")
mustIAM(t, err)
return c
}
old := issue()
gate := &iamSTSScanGateStore{IAMStorageAPI: sys.store.IAMStorageAPI, entered: make(chan struct{}), resume: make(chan struct{})}
sys.store.IAMStorageAPI = gate
done := make(chan error, 1)
go func() { done <- sys.RevokeTokens(ctx, "sts-race-parent", "") }()
<-gate.entered
fresh := issue()
close(gate.resume)
mustIAM(t, <-done)
mustIAM(t, sibling.UserNotificationHandler(ctx, old.AccessKey, stsUser))
if _, ok := sibling.GetUser(old.AccessKey); ok {
t.Fatal("old uncached token survived")
}
mustIAM(t, sibling.UserNotificationHandler(ctx, fresh.AccessKey, stsUser))
if _, ok := sibling.GetUser(fresh.AccessKey); !ok {
t.Fatal("new token was included in earlier revocation")
}
}
type iamSTSFailureStore struct {
IAMStorageAPI
failedPath string
}
func (s *iamSTSFailureStore) saveIAMConfig(ctx context.Context, item any, path string, opts ...options) error {
if path == s.failedPath {
return errIAMInjectedWrite
}
return s.IAMStorageAPI.saveIAMConfig(ctx, item, path, opts...)
}
func TestIAMSTSRevocationPartialFailure(t *testing.T) {
ctx, sys, _ := prepareIAMRevisionFixture(t)
_, err := sys.CreateUser(ctx, "sts-partial-parent", madmin.AddOrUpdateUserReq{SecretKey: "synthetic-parent-password", Status: madmin.AccountEnabled})
mustIAM(t, err)
secret, err := getTokenSigningKey()
mustIAM(t, err)
var keys []string
for range 2 {
c, err := auth.GetNewCredentialsWithMetadata(map[string]any{"exp": UTCNow().Add(time.Hour).Unix(), parentClaim: "sts-partial-parent"}, secret)
mustIAM(t, err)
c.ParentUser = "sts-partial-parent"
_, err = sys.SetTempUser(ctx, c.AccessKey, c, "")
mustIAM(t, err)
keys = append(keys, c.AccessKey)
}
sys.store.IAMStorageAPI = &iamSTSFailureStore{IAMStorageAPI: sys.store.IAMStorageAPI, failedPath: getUserIdentityPath(keys[0], stsUser)}
revoked, err := sys.store.RevokeTokens(ctx, "sts-partial-parent", "")
if !errors.Is(err, errIAMInjectedWrite) || len(revoked) != 1 || revoked[0] != keys[1] {
t.Fatalf("committed keys or failure lost: %v %v", revoked, err)
}
if _, ok := sys.store.GetUser(keys[1]); ok {
t.Fatal("committed revocation retained cache entry")
}
if _, ok := sys.store.GetUser(keys[0]); !ok {
t.Fatal("failed revocation claimed cache success")
}
}
+8 -1
View File
@@ -717,7 +717,14 @@ func (sys *IAMSys) RevokeTokens(ctx context.Context, accessKey, tokenRevokeType
return errServerNotInitialized return errServerNotInitialized
} }
return sys.store.RevokeTokens(ctx, accessKey, tokenRevokeType) keys, err := sys.store.RevokeTokens(ctx, accessKey, tokenRevokeType)
// Publish every committed tombstone even if another token failed to save.
for _, key := range keys {
sys.notifyForUser(ctx, key, true)
// Offline peers recover this same record through IAM deletion healing.
iamLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{Type: iamSTSDeleteType, Name: key}))
}
return err
} }
// DeleteUser - delete user (only for long-term users not STS users). // DeleteUser - delete user (only for long-term users not STS users).
+256
View File
@@ -0,0 +1,256 @@
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/minio/kms-go/kes"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/kms"
)
// Use the configured KES client over TLS, rather than a mocked KMS interface:
// cancellation must survive every layer, including the HTTP request context.
func connectCancellationKES(t *testing.T, handler http.Handler) *kms.KMS {
t.Helper()
for _, key := range []string{kms.EnvKMSEndpoint, kms.EnvKMSSecretKey, kms.EnvKMSSecretKeyFile} {
t.Setenv(key, "")
mustIAM(t, os.Unsetenv(key))
}
server := httptest.NewTLSServer(handler)
t.Cleanup(server.Close)
ca := filepath.Join(t.TempDir(), "ca.pem")
mustIAM(t, os.WriteFile(ca, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}), 0o600))
key, err := kes.GenerateAPIKey(nil)
mustIAM(t, err)
t.Setenv(kms.EnvKESEndpoint, server.URL)
t.Setenv(kms.EnvKESAPIKey, key.String())
t.Setenv(kms.EnvKESDefaultKey, "cancellation-key")
t.Setenv(kms.EnvKESServerCA, ca)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
k, err := kms.Connect(ctx, nil)
mustIAM(t, err)
return k
}
func cancellationDecryptHandler(t *testing.T, k *kms.KMS, before func(*http.Request)) http.Handler {
t.Helper()
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// KMS Connect periodically describes the default key to keep it cached.
// This maintenance request is independent of the decrypt under test.
if r.Method == http.MethodGet && r.URL.Path == "/v1/key/describe/cancellation-key" {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{"name": "cancellation-key"})
return
}
if r.URL.Path != "/v1/key/decrypt/cancellation-key" {
t.Errorf("unexpected KES path: %s", r.URL.Path)
w.WriteHeader(http.StatusNotFound)
return
}
var request struct {
Ciphertext []byte `json:"ciphertext"`
Context []byte `json:"context"`
}
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Error(err)
return
}
// Drain the HTTP/1 body so net/http can observe a peer disconnect while
// the handler is blocked in the simulated KES operation.
_, _ = io.Copy(io.Discard, r.Body)
_ = r.Body.Close()
before(r)
if r.Context().Err() != nil {
return
}
var aad kms.Context
if err := json.Unmarshal(request.Context, &aad); err != nil {
t.Error(err)
return
}
plain, err := k.Decrypt(r.Context(), &kms.DecryptRequest{Name: "cancellation-key", Ciphertext: request.Ciphertext, AssociatedData: aad})
w.Header().Set("Content-Type", "application/json")
if err != nil {
w.WriteHeader(http.StatusBadRequest)
_ = json.NewEncoder(w).Encode(map[string]string{"message": kes.ErrDecrypt.Error()})
return
}
_ = json.NewEncoder(w).Encode(map[string][]byte{"plaintext": plain})
})
}
func TestKMSRequestCancellation(t *testing.T) {
for _, operation := range []string{"iam", "sse-list"} {
for _, expiry := range []string{"cancel", "deadline"} {
t.Run(operation+"/"+expiry, func(t *testing.T) {
builtin, err := kms.NewBuiltin("cancellation-key", bytes.Repeat([]byte{42}, 32))
mustIAM(t, err)
entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{})
var calls atomic.Int32
var enteredOnce, canceledOnce sync.Once
unblock := sync.OnceFunc(func() { close(release) })
client := connectCancellationKES(t, cancellationDecryptHandler(t, builtin, func(r *http.Request) {
calls.Add(1)
enteredOnce.Do(func() { close(entered) })
select {
case <-r.Context().Done():
canceledOnce.Do(func() { close(canceled) })
case <-release:
}
}))
t.Cleanup(unblock)
prior := GlobalKMS
GlobalKMS = client
t.Cleanup(func() { GlobalKMS = prior })
var run func(context.Context) error
if operation == "iam" {
const name = "config/iam/policies/cancel/policy.json"
plaintext := []byte(`{"Version":1}`)
ciphertext, err := config.EncryptBytes(builtin, plaintext, kms.Context{minioMetaBucket: minioMetaBucket + "/" + name})
mustIAM(t, err)
run = func(ctx context.Context) error {
got, err := decryptData(ctx, ciphertext, name)
if err == nil && !bytes.Equal(got, plaintext) {
return errors.New("config plaintext changed")
}
return err
}
} else {
var metadata []map[string]string
var buckets, names []string
var expected []crypto.ObjectKey
for i := range 3 {
name := fmt.Sprintf("object-%d", i)
dek, err := builtin.GenerateKey(context.Background(), &kms.GenerateKeyRequest{AssociatedData: kms.Context{"bucket": "bucket/" + name}})
mustIAM(t, err)
key := crypto.GenerateKey(dek.Plaintext, nil)
sealed := key.Seal(dek.Plaintext, crypto.GenerateIV(nil), crypto.S3.String(), "bucket", name)
metadata = append(metadata, crypto.S3.CreateMetadata(nil, dek.KeyID, dek.Ciphertext, sealed))
buckets, names = append(buckets, "bucket"), append(names, name)
expected = append(expected, key)
}
run = func(ctx context.Context) error {
keys, err := crypto.S3.UnsealObjectKeys(ctx, client, metadata, buckets, names)
if err != nil {
return err
}
if len(keys) != len(expected) {
return errors.New("partial key list returned as success")
}
for i := range keys {
if keys[i] != expected[i] {
return errors.New("object key or ordering changed")
}
}
return nil
}
}
ctx, cancel := context.WithCancel(context.Background())
if expiry == "deadline" {
cancel()
ctx, cancel = context.WithTimeout(context.Background(), 300*time.Millisecond)
}
defer cancel()
done := make(chan error, 1)
go func() { done <- run(ctx) }()
select {
case <-entered:
case err := <-done:
t.Fatalf("decrypt returned before reaching KES: %v (type=%v)", err, client.Type)
case <-time.After(5 * time.Second):
t.Fatal("decrypt did not reach KES")
}
want := context.DeadlineExceeded
if expiry == "cancel" {
cancel()
want = context.Canceled
}
select {
case err := <-done:
if !errors.Is(err, want) {
t.Fatalf("got %v, want %v", err, want)
}
// The current KES SDK may sleep twice for up to 999ms after a
// deadline error. Include that existing backoff plus the 300ms
// request deadline; cancellation must still stop new decrypts.
case <-time.After(3 * time.Second):
unblock()
<-done
t.Fatal("request exceeded the cancellation and KES retry budget")
}
awaitIAMPolicyReload(t, canceled)
if n := calls.Load(); n != 1 {
t.Fatalf("cancellation launched a fallback or more object decryptions: %d calls", n)
}
unblock()
mustIAM(t, run(context.Background()))
before := calls.Load()
cctx, ccancel := context.WithCancel(context.Background())
ccancel()
if err := run(cctx); !errors.Is(err, context.Canceled) || calls.Load() != before {
t.Fatal("already-canceled request still issued decryption work")
}
})
}
}
}
func TestIAMDecryptAssociatedDataCompatibility(t *testing.T) {
builtin, err := kms.NewBuiltin("cancellation-key", bytes.Repeat([]byte{42}, 32))
mustIAM(t, err)
var calls atomic.Int32
client := connectCancellationKES(t, cancellationDecryptHandler(t, builtin, func(*http.Request) { calls.Add(1) }))
prior := GlobalKMS
GlobalKMS = client
t.Cleanup(func() { GlobalKMS = prior })
const name = "config/iam/policies/compat/policy.json"
plaintext := []byte(`{"Version":1}`)
for _, legacy := range []bool{false, true} {
t.Run(fmt.Sprint(legacy), func(t *testing.T) {
aad, wantCalls := minioMetaBucket+"/"+name, int32(1)
if legacy {
aad, wantCalls = name, 2
}
ciphertext, err := config.EncryptBytes(builtin, plaintext, kms.Context{minioMetaBucket: aad})
mustIAM(t, err)
before := calls.Load()
got, err := decryptData(context.Background(), ciphertext, name)
mustIAM(t, err)
if !bytes.Equal(got, plaintext) || calls.Load()-before != wantCalls {
t.Fatal("current/legacy encrypted IAM records no longer decrypt correctly")
}
})
}
}
+16 -16
View File
@@ -40,7 +40,7 @@ func requireMultipartUploadKeys(t *testing.T, got ListMultipartsInfo, want ...st
} }
} }
func TestListMultipartUploadsS3Compatibility(t *testing.T) { func TestMultipartDiagnosticScanPagination(t *testing.T) {
obj, dirs, err := prepareErasureSets32(t.Context()) obj, dirs, err := prepareErasureSets32(t.Context())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -81,14 +81,14 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
uploadIDs[object] = mp.UploadID uploadIDs[object] = mp.UploadID
} }
// Durable multipart metadata, rather than this node-local cache, must be // The diagnostic scanner must discover metadata independently of the cache.
// authoritative after a restart or when another node handles the request. // These checks do not claim that public listing offers durable pagination.
z.mpCache.Range(func(uploadID string, _ MultipartInfo) bool { z.mpCache.Range(func(uploadID string, _ MultipartInfo) bool {
z.mpCache.Delete(uploadID) z.mpCache.Delete(uploadID)
return true return true
}) })
all, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100) all, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 100)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -97,7 +97,7 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
t.Fatalf("complete listing has truncation state: %+v", all) t.Fatalf("complete listing has truncation state: %+v", all)
} }
first, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 1) first, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 1)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -108,31 +108,31 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
objects[0], uploadIDs[objects[0]]) objects[0], uploadIDs[objects[0]])
} }
rest, err := z.ListMultipartUploads(t.Context(), bucket, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 100) rest, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 100)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
requireMultipartUploadKeys(t, rest, objects[1:]...) requireMultipartUploadKeys(t, rest, objects[1:]...)
afterKey, err := z.ListMultipartUploads(t.Context(), bucket, "", objects[1], "", "", 100) afterKey, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", objects[1], "", "", 100)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
requireMultipartUploadKeys(t, afterKey, objects[2:]...) requireMultipartUploadKeys(t, afterKey, objects[2:]...)
prefixed, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", "", 100) prefixed, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "t/", "", "", "", 100)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
requireMultipartUploadKeys(t, prefixed, objects[:3]...) requireMultipartUploadKeys(t, prefixed, objects[:3]...)
nested, err := z.ListMultipartUploads(t.Context(), bucket, "t/a_b/", "", "", "", 100) nested, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "t/a_b/", "", "", "", 100)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
requireMultipartUploadKeys(t, nested, objects[:2]...) requireMultipartUploadKeys(t, nested, objects[:2]...)
grouped, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", SlashSeparator, 100) grouped, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "t/", "", "", SlashSeparator, 100)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -141,7 +141,7 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
t.Fatalf("common prefixes = %v, want %v", grouped.CommonPrefixes, want) t.Fatalf("common prefixes = %v, want %v", grouped.CommonPrefixes, want)
} }
groupPage, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", SlashSeparator, 1) groupPage, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "t/", "", "", SlashSeparator, 1)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -153,7 +153,7 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
groupPage.NextKeyMarker, groupPage.NextUploadIDMarker, groupPage.IsTruncated) groupPage.NextKeyMarker, groupPage.NextUploadIDMarker, groupPage.IsTruncated)
} }
groupRest, err := z.ListMultipartUploads(t.Context(), bucket, "t/", groupPage.NextKeyMarker, "", SlashSeparator, 1) groupRest, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "t/", groupPage.NextKeyMarker, "", SlashSeparator, 1)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -197,12 +197,12 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil { er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil {
t.Fatal(err) t.Fatal(err)
} }
legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100) legacy, err := z.scanMultipartUploadsForTest(t.Context(), bucket, legacyObject, "", "", "", 100)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
requireMultipartUploadKeys(t, legacy) requireMultipartUploadKeys(t, legacy)
remaining, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100) remaining, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 100)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -280,7 +280,7 @@ func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) {
return true return true
}) })
page, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 2) page, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", "", "", "", 2)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -289,7 +289,7 @@ func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) {
t.Fatalf("first global page = %+v", page) t.Fatalf("first global page = %+v", page)
} }
rest, err := z.ListMultipartUploads(t.Context(), bucket, "", page.NextKeyMarker, page.NextUploadIDMarker, "", 2) rest, err := z.scanMultipartUploadsForTest(t.Context(), bucket, "", page.NextKeyMarker, page.NextUploadIDMarker, "", 2)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -0,0 +1,94 @@
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
"context"
"io"
"testing"
"github.com/minio/minio/internal/config/storageclass"
)
type compatibilityFailedOverwrite struct {
StorageAPI
commit bool
}
func (d compatibilityFailedOverwrite) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
if d.commit {
if _, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts); err != nil {
return RenameDataResp{}, err
}
}
return RenameDataResp{}, errDiskNotFound
}
// Maintenance compatibility requirement: after a failed overwrite with one residue, keep the previously acknowledged quorum readable.
// This tests a known history; it does not claim the resulting snapshot proves it.
func TestMaintenancePartialFailedOverwriteAvailability(t *testing.T) {
// Lock RPC tests configure a single-drive backend (EC:0). Pin this
// fixture to 12+4 so its fifteen old shards really have read quorum.
previousStorageClass := globalStorageClass
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
t.Cleanup(func() { globalStorageClass.Update(previousStorageClass) })
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dirs, err := getRandomDisks(16)
if err != nil {
t.Fatal(err)
}
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, dirs...))
if err != nil {
removeRoots(dirs)
t.Fatal(err)
}
z := obj.(*erasureServerPools)
previous := newObjectLayerFn()
setObjectLayer(z)
defer func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) }()
bucket := "maintenance-compatibility"
key := "partial-overwrite"
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
oldBody := bytes.Repeat([]byte("old-A"), 65536)
newBody := bytes.Repeat([]byte("new-B"), 65536)
put := func(body []byte) error {
_, err := z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
return err
}
if err := put(oldBody); err != nil {
t.Fatal(err)
}
set := z.serverPools[0].getHashedSet(key)
getDisks := set.getDisks
disks := getDisks()
fi, err := disks[0].ReadVersion(ctx, "", bucket, key, "", ReadOptions{})
if err != nil || fi.Erasure.DataBlocks != 12 || fi.Erasure.ParityBlocks != 4 {
t.Fatalf("compatibility fixture must have 12+4 erasure coding: %+v, %v", fi.Erasure, err)
}
wrapped := append([]StorageAPI(nil), disks...)
for i, d := range disks {
wrapped[i] = compatibilityFailedOverwrite{d, i == 0}
}
set.getDisks = func() []StorageAPI { return wrapped }
err = put(newBody)
set.getDisks = getDisks
if err == nil {
t.Fatal("partial overwrite must fail")
}
t.Logf("Overwrite completed with failure; one new residue, fifteen original shards: %v", err)
r, err := z.GetObjectNInfo(ctx, bucket, key, nil, nil, ObjectOptions{})
if err != nil {
t.Fatalf("MAINTENANCE COMPATIBILITY: previously readable acknowledged value now refused: %v", err)
}
data, err := io.ReadAll(r)
r.Close()
if err != nil || !bytes.Equal(data, oldBody) {
t.Fatalf("old acknowledged payload mismatch: bytes=%d err=%v", len(data), err)
}
}
+40 -12
View File
@@ -177,14 +177,12 @@ func (e metaCacheEntry) isInDir(dir, separator string) bool {
} }
// isLatestDeletemarker returns whether the latest version is a delete marker. // isLatestDeletemarker returns whether the latest version is a delete marker.
// Free versions are not versions of the object and are skipped.
// If metadata is NOT versioned false will always be returned. // If metadata is NOT versioned false will always be returned.
// If v2 and UNABLE to load metadata true will be returned. // If v2 and UNABLE to load metadata true will be returned.
func (e *metaCacheEntry) isLatestDeletemarker() bool { func (e *metaCacheEntry) isLatestDeletemarker() bool {
if e.cached != nil { if e.cached != nil {
if len(e.cached.versions) == 0 { return e.cached.isLatestDeleteMarker()
return true
}
return e.cached.versions[0].header.Type == DeleteType
} }
if !isXL2V1Format(e.metadata) { if !isXL2V1Format(e.metadata) {
return false return false
@@ -196,10 +194,10 @@ func (e *metaCacheEntry) isLatestDeletemarker() bool {
} }
// Fall back... // Fall back...
xlMeta, err := e.xlmeta() xlMeta, err := e.xlmeta()
if err != nil || len(xlMeta.versions) == 0 { if err != nil {
return true return true
} }
return xlMeta.versions[0].header.Type == DeleteType return xlMeta.isLatestDeleteMarker()
} }
// isAllFreeVersions returns if all objects are free versions. // isAllFreeVersions returns if all objects are free versions.
@@ -287,9 +285,21 @@ func (e *metaCacheEntry) xlmeta() (*xlMetaV2, error) {
return e.cached, nil return e.cached, nil
} }
// fileInfoVersions returns the metadata as FileInfoVersions. // fileInfoVersions returns the metadata as FileInfoVersions. The free
// versions of deleted tiered versions are not versions of the object; they
// are in FreeVersions.
// If entry is a directory it is returned as that. // If entry is a directory it is returned as that.
func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, error) { func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, error) {
return e.getFileInfoVersions(bucket, false)
}
// allFileInfoVersions returns the metadata as FileInfoVersions with the free
// versions among the versions, for healing.
func (e *metaCacheEntry) allFileInfoVersions(bucket string) (FileInfoVersions, error) {
return e.getFileInfoVersions(bucket, true)
}
func (e *metaCacheEntry) getFileInfoVersions(bucket string, inclFreeVersions bool) (FileInfoVersions, error) {
if e.isDir() { if e.isDir() {
return FileInfoVersions{ return FileInfoVersions{
Volume: bucket, Volume: bucket,
@@ -304,7 +314,7 @@ func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, erro
}, nil }, nil
} }
// Too small gains to reuse cache here. // Too small gains to reuse cache here.
return getFileInfoVersions(e.metadata, bucket, e.name, true) return getFileInfoVersions(e.metadata, bucket, e.name, inclFreeVersions)
} }
// metaCacheEntries is a slice of metacache entries. // metaCacheEntries is a slice of metacache entries.
@@ -781,6 +791,15 @@ func mergeEntryChannelsWithResolver(ctx context.Context, in []chan metaCacheEntr
} }
last := "" last := ""
var toMerge []int var toMerge []int
var directoryCollisions []int
discardDirectories := func() error {
for _, idx := range directoryCollisions {
if err := selectFrom(idx); err != nil {
return err
}
}
return nil
}
// Choose the best to return. // Choose the best to return.
for { for {
@@ -790,6 +809,7 @@ func mergeEntryChannelsWithResolver(ctx context.Context, in []chan metaCacheEntr
best := top[0] best := top[0]
bestIdx := 0 bestIdx := 0
toMerge = toMerge[:0] toMerge = toMerge[:0]
directoryCollisions = directoryCollisions[:0]
for i, other := range top[1:] { for i, other := range top[1:] {
otherIdx := i + 1 otherIdx := i + 1
if other == nil { if other == nil {
@@ -819,10 +839,9 @@ func mergeEntryChannelsWithResolver(ctx context.Context, in []chan metaCacheEntr
if serverDebugLog { if serverDebugLog {
console.Debugln("mergeEntryChannels: discarding directory", other.name, "for object", best.name) console.Debugln("mergeEntryChannels: discarding directory", other.name, "for object", best.name)
} }
// Discard the directory. // A pending object can resolve to absence. Keep the
if err := selectFrom(otherIdx); err != nil { // directory until the whole object group is confirmed.
return err directoryCollisions = append(directoryCollisions, otherIdx)
}
continue continue
} }
// Replace directory with object. // Replace directory with object.
@@ -838,6 +857,7 @@ func mergeEntryChannelsWithResolver(ctx context.Context, in []chan metaCacheEntr
} }
if best.name > other.name { if best.name > other.name {
toMerge = toMerge[:0] toMerge = toMerge[:0]
directoryCollisions = directoryCollisions[:0]
best = other best = other
bestIdx = otherIdx bestIdx = otherIdx
} }
@@ -866,6 +886,11 @@ func mergeEntryChannelsWithResolver(ctx context.Context, in []chan metaCacheEntr
if err != nil { if err != nil {
return err return err
} }
if resolved != nil {
if err := discardDirectories(); err != nil {
return err
}
}
if resolved != nil && resolved.name > last { if resolved != nil && resolved.name > last {
select { select {
case <-ctxDone: case <-ctxDone:
@@ -876,6 +901,9 @@ func mergeEntryChannelsWithResolver(ctx context.Context, in []chan metaCacheEntr
} }
continue continue
} }
if err := discardDirectories(); err != nil {
return err
}
// Merge any unmerged // Merge any unmerged
if len(toMerge) > 0 { if len(toMerge) > 0 {
@@ -0,0 +1,61 @@
//go:build consistency_strict
// Preserved diagnostic for the deferred strict-read policy. This is not a
// maintenance release gate; failures record intentionally unresolved risks.
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
"testing"
"time"
)
func TestStrictListFallbackRejectsSplitGenerations(t *testing.T) {
z, bucket, _ := nullQuorumBackend(t)
set := z.serverPools[0].sets[0]
disks := set.getDisks()
const object = "object"
mtime := time.Now().UTC().Add(-time.Hour)
var generations [3][][]byte
for generation := range generations {
body := bytes.Repeat([]byte{byte('a' + generation)}, 8192)
nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime.Add(time.Duration(generation) * time.Minute)})
generations[generation] = make([][]byte, len(disks))
for i, disk := range disks {
generations[generation][i] = mustReadNullQuorumMeta(t, disk, bucket, object)
}
}
// All sixteen drives are online, but the 7/5/4 split leaves no generation
// with the eight matching snapshots required by the listing resolver.
for i, disk := range disks {
generation := 0
if i >= 7 {
generation = 1
}
if i >= 12 {
generation = 2
}
if err := disk.WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, generations[generation][i]); err != nil {
t.Fatal(err)
}
}
if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrReadQuorum(err) {
t.Fatalf("split metadata must remain a read-quorum failure: %v", err)
}
entries := make(metaCacheEntries, len(disks))
for i, disk := range disks {
entries[i] = metaCacheEntry{name: object, metadata: mustReadNullQuorumMeta(t, disk, bucket, object)}
if _, err := entries[i].xlmeta(); err != nil {
t.Fatalf("drive %d no longer has valid metadata: %v", i, err)
}
}
resolver := metadataResolutionParams{dirQuorum: 8, objQuorum: 8, bucket: bucket, requestedVersions: 1}
entry, err := z.resolveListingGroup(t.Context(), bucket, []*metaCacheEntry{set.resolveListEntry(entries, nil, &resolver)})
if entry != nil || !isErrReadQuorum(err) {
t.Fatalf("split live metadata became absence: entry=%v err=%v", entry, err)
}
}
+243
View File
@@ -0,0 +1,243 @@
// Copyright (c) 2026 Ruohang Feng
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
"context"
"encoding/xml"
"fmt"
"io"
"net/http"
"net/http/httptest"
"slices"
"testing"
"time"
)
type listingDirectoryWalkSnapshot struct {
StorageAPI
bucket string
snapshot []byte
}
func (d listingDirectoryWalkSnapshot) WalkDir(ctx context.Context, opts WalkDirOptions, out io.Writer) error {
if opts.Bucket != d.bucket {
return d.StorageAPI.WalkDir(ctx, opts, out)
}
_, err := out.Write(d.snapshot)
return err
}
// Capture two actual directory-walk generations for a directory object, then
// acknowledge its deletion. A different pool still contains dir/child. The
// stale source walkers disagree, while every current source drive is absent.
func TestListingRecoveryDeletedDirectoryKeepsCommonPrefix(t *testing.T) {
for _, marker := range []bool{false, true} {
for _, sourcePool := range []int{0, 1} {
t.Run(fmt.Sprintf("pool=%d/marker=%t", sourcePool, marker), func(t *testing.T) {
z, bucket, router := listingPoolsBackend(t)
ctx := t.Context()
source := z.serverPools[sourcePool].sets[0]
target := z.serverPools[1-sourcePool].sets[0]
childBody := []byte("surviving child")
put := func(set *erasureObjects, key string, tm time.Time) {
var body []byte
if key == "dir/child" {
body = childBody
}
_, err := set.PutObject(ctx, bucket, encodeDirObject(key), mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: tm, VersionSuspended: marker})
if err != nil {
t.Fatal(err)
}
}
tm := time.Now().UTC().Add(-time.Hour)
put(target, "dir/child", tm)
disks, switchDisks := installListingDiskSwitch(source)
snapshots := make([][]byte, len(disks))
for gen := range 3 {
if gen < 2 {
put(source, "dir/", tm.Add(time.Duration(gen)*time.Minute))
} else {
if _, err := source.DeleteObject(ctx, bucket, encodeDirObject("dir/"), ObjectOptions{VersionSuspended: marker, DeleteMarker: marker, MTime: tm.Add(2 * time.Minute)}); err != nil {
t.Fatal(err)
}
}
for i, disk := range disks {
wanted := 0
if i >= 6 {
wanted = 1
}
if i >= 12 {
wanted = 2
}
if wanted != gen {
continue
}
var b bytes.Buffer
if err := disk.WalkDir(ctx, WalkDirOptions{Bucket: bucket, Recursive: false}, &b); err != nil {
t.Fatal(err)
}
snapshots[i] = append([]byte(nil), b.Bytes()...)
}
}
for _, key := range []string{"dir/child", "dir/"} {
request, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucket, key), 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
response := httptest.NewRecorder()
router.ServeHTTP(response, request.WithContext(ctx))
t.Logf("GET %s HTTP %d", key, response.Code)
want := http.StatusOK
if key == "dir/" {
want = http.StatusNotFound
}
if response.Code != want {
t.Fatalf("GET %s: got %d want %d: %s", key, response.Code, want, response.Body.String())
}
if key == "dir/child" && !bytes.Equal(response.Body.Bytes(), childBody) {
t.Fatal("surviving child content changed")
}
}
wrapped := make([]StorageAPI, len(disks))
for i, disk := range disks {
wrapped[i] = listingDirectoryWalkSnapshot{disk, bucket, snapshots[i]}
}
switchDisks(wrapped)
t.Cleanup(func() { switchDisks(disks) })
req, err := newTestSignedRequestV4(http.MethodGet, getListObjectsV2URL("", bucket, "", "1000", "", "", "/"), 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req.WithContext(ctx))
t.Logf("LIST HTTP %d body=%s", rec.Code, rec.Body.String())
var page ListObjectsV2Response
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &page) != nil {
t.Fatalf("LIST failed: %d %s", rec.Code, rec.Body.String())
}
if len(page.CommonPrefixes) != 1 || page.CommonPrefixes[0].Prefix != "dir/" {
t.Fatalf("acknowledged child lost its CommonPrefix: %+v", page)
}
})
}
}
}
// The directory must remain in the sorted input when an earlier object name
// disappears. Emitting it immediately would skip an intervening name, and
// keeping only one colliding directory would leave input order significant.
func TestListingRecoveryDirectoryCollisionOrdering(t *testing.T) {
for _, object := range []string{"dir", "dir/"} {
for _, present := range []bool{false, true} {
if present && object == "dir" {
continue
}
for source := range 3 {
t.Run(fmt.Sprintf("object=%s/present=%t/source=%d", object, present, source), func(t *testing.T) {
in := make([]chan metaCacheEntry, 6)
for i := range in {
in[i] = make(chan metaCacheEntry, 1)
entry := metaCacheEntry{name: "dir/"}
if i == source {
entry = metaCacheEntry{name: object, listingSet: &erasureObjects{}}
}
if i >= 3 {
entry = metaCacheEntry{name: []string{"dir-", "dir.ext", "dir0"}[i-3], metadata: []byte{1}}
}
in[i] <- entry
close(in[i])
}
out := make(chan metaCacheEntry, len(in))
err := mergeEntryChannelsWithResolver(t.Context(), in, out, 1, nil, func(group []*metaCacheEntry) (*metaCacheEntry, error) {
if len(group) != 1 || group[0].listingSet == nil {
t.Fatalf("directory entered the object group: %+v", group)
}
if !present {
return nil, nil
}
return &metaCacheEntry{name: object, metadata: []byte{1}}, nil
})
if err != nil {
t.Fatal(err)
}
var names []string
for entry := range out {
names = append(names, entry.name)
if entry.name == "dir/" && entry.isObject() != present {
t.Fatalf("directory collision returned the wrong entry type: %+v", entry)
}
}
want := []string{"dir-", "dir.ext", "dir/", "dir0"}
if !slices.Equal(names, want) {
t.Fatalf("names=%v, want %v", names, want)
}
})
}
}
}
}
// A regular walk result may be stale when a pending peer forces current
// recovery. Keep sibling prefixes until the complete object group is resolved.
func TestListingRecoveryMixedDirectoryCollision(t *testing.T) {
orders := [][3]int{{0, 1, 2}, {0, 2, 1}, {1, 0, 2}, {1, 2, 0}, {2, 0, 1}, {2, 1, 0}}
for _, order := range orders {
for _, present := range []bool{false, true} {
t.Run(fmt.Sprintf("order=%v/present=%t", order, present), func(t *testing.T) {
entries := []metaCacheEntry{
{name: "dir/", metadata: []byte{1}},
{name: "dir/"},
{name: "dir/", listingSet: &erasureObjects{}},
}
in := make([]chan metaCacheEntry, len(order))
for i, index := range order {
in[i] = make(chan metaCacheEntry, 2)
in[i] <- entries[index]
in[i] <- metaCacheEntry{name: fmt.Sprintf("tail-%d", i), metadata: []byte{1}}
close(in[i])
}
out := make(chan metaCacheEntry, 6)
calls := 0
err := mergeEntryChannelsWithResolver(t.Context(), in, out, 1, nil, func(group []*metaCacheEntry) (*metaCacheEntry, error) {
calls++
if len(group) != 2 {
t.Fatalf("incomplete object group: %+v", group)
}
pending := 0
for _, entry := range group {
if entry.name != "dir/" || entry.isDir() {
t.Fatalf("unexpected recovery entry: %+v", entry)
}
if entry.listingSet != nil {
pending++
}
}
if pending != 1 {
t.Fatalf("pending entries=%d, want 1", pending)
}
if !present {
return nil, nil
}
return &metaCacheEntry{name: "dir/", metadata: []byte{1}}, nil
})
if err != nil || calls != 1 {
t.Fatalf("err=%v recovery calls=%d", err, calls)
}
var names []string
for entry := range out {
names = append(names, entry.name)
if entry.listingSet != nil || entry.name == "dir/" && entry.isObject() != present {
t.Fatalf("unexpected merged entry: %+v", entry)
}
}
want := []string{"dir/", "tail-0", "tail-1", "tail-2"}
if !slices.Equal(names, want) {
t.Fatalf("names=%v, want %v", names, want)
}
})
}
}
}
+42 -25
View File
@@ -221,33 +221,50 @@ func TestListingRecoveryNormalPathDoesNotReadOrLock(t *testing.T) {
func TestListingPendingDirectoryCollision(t *testing.T) { func TestListingPendingDirectoryCollision(t *testing.T) {
for _, objectFirst := range []bool{false, true} { for _, objectFirst := range []bool{false, true} {
t.Run(fmt.Sprint(objectFirst), func(t *testing.T) { for _, present := range []bool{true, false} {
in := []chan metaCacheEntry{make(chan metaCacheEntry, 1), make(chan metaCacheEntry, 1)} t.Run(fmt.Sprintf("objectFirst=%v/present=%v", objectFirst, present), func(t *testing.T) {
objectIndex := 1 in := []chan metaCacheEntry{make(chan metaCacheEntry, 2), make(chan metaCacheEntry, 2), make(chan metaCacheEntry, 2)}
if objectFirst { objectIndex := 1
objectIndex = 0 if objectFirst {
} objectIndex = 0
in[objectIndex] <- metaCacheEntry{name: "dir/", listingSet: &erasureObjects{}} }
in[1-objectIndex] <- metaCacheEntry{name: "dir/"} in[objectIndex] <- metaCacheEntry{name: "dir/", listingSet: &erasureObjects{}}
for _, c := range in { in[1-objectIndex] <- metaCacheEntry{name: "dir/"}
close(c) in[2] <- metaCacheEntry{name: "a", metadata: []byte{1}}
} for _, c := range in {
out := make(chan metaCacheEntry, 2) close(c)
calls := 0 }
err := mergeEntryChannelsWithResolver(t.Context(), in, out, 1, nil, func(group []*metaCacheEntry) (*metaCacheEntry, error) { out := make(chan metaCacheEntry, 4)
calls++ calls := 0
if len(group) != 1 || group[0].listingSet == nil { err := mergeEntryChannelsWithResolver(t.Context(), in, out, 1, nil, func(group []*metaCacheEntry) (*metaCacheEntry, error) {
t.Fatalf("wrong pending group: %+v", group) calls++
if len(group) != 1 || group[0].listingSet == nil {
t.Fatalf("wrong pending group: %+v", group)
}
if !present {
return nil, nil
}
return &metaCacheEntry{name: "dir/", metadata: []byte{1}}, nil
})
if err != nil || calls != 1 || len(out) != 2 {
t.Fatalf("err=%v recoveries=%d outputs=%d", err, calls, len(out))
}
first := <-out
if first.name != "a" {
t.Fatalf("expected smaller key first, got %+v", first)
}
entry := <-out
if entry.name != "dir/" || entry.listingSet != nil {
t.Fatalf("unexpected recovered entry: %+v", entry)
}
if present && (entry.isDir() || !entry.isObjectDir()) {
t.Fatalf("object became directory: %+v", entry)
}
if !present && (!entry.isDir() || entry.name != "dir/") {
t.Fatalf("absent pending object dropped live directory prefix: %+v", entry)
} }
return &metaCacheEntry{name: "dir/", metadata: []byte{1}}, nil
}) })
if err != nil || calls != 1 || len(out) != 1 { }
t.Fatalf("err=%v recoveries=%d outputs=%d", err, calls, len(out))
}
if entry := <-out; entry.isDir() || !entry.isObjectDir() {
t.Fatalf("object became directory: %+v", entry)
}
})
} }
} }
+167
View File
@@ -0,0 +1,167 @@
//go:build consistency_strict
// Preserved diagnostic for the deferred strict-read policy. This is not a
// maintenance release gate; failures record intentionally unresolved risks.
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
madmin "github.com/minio/madmin-go/v3"
"io"
"net/http"
"os"
"path/filepath"
"slices"
"testing"
"time"
)
func TestStrictSingleNullQuorumReadAndConsumers(t *testing.T) {
z, bucket, router := nullQuorumBackend(t)
set := z.serverPools[0].sets[0]
disks := set.getDisks()
const object = "object"
oldBody, newBody := bytes.Repeat([]byte{'a'}, 8192), bytes.Repeat([]byte{'b'}, 8192)
oldTime := time.Now().UTC().Add(-time.Hour)
put := func(body []byte, modTime time.Time) {
t.Helper()
_, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: modTime})
if err != nil {
t.Fatal(err)
}
}
put(oldBody, oldTime)
oldMeta := make([][]byte, len(disks))
for i, disk := range disks {
var err error
oldMeta[i], err = disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
if err != nil {
t.Fatal(err)
}
}
put(newBody, oldTime.Add(time.Minute))
newMeta := make([][]byte, len(disks))
for i, disk := range disks {
newMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
}
// Model an incomplete overwrite using real inline shards: fifteen old
// copies retain a read quorum; the final disk contains the newer minority.
// The old quorum does not prove that the newer overwrite was uncommitted.
for i := range 15 {
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
t.Fatal(err)
}
}
checkRead := func(body []byte, modTime time.Time) {
t.Helper()
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object))
head := nullQuorumRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, object))
if get.Code != 200 || !bytes.Equal(get.Body.Bytes(), body) || head.Code != 200 || head.Header().Get("Content-Length") != "8192" {
t.Fatalf("GET/HEAD lost readable quorum: GET=%d HEAD=%d body=%q", get.Code, head.Code, get.Body.String())
}
oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
if err != nil || !oi.ModTime.Equal(modTime) {
t.Fatalf("wrong readable generation: %v, %v", oi.ModTime, err)
}
}
for _, method := range []string{http.MethodGet, http.MethodHead} {
rec := nullQuorumRequest(t, router, method, getGetObjectURL("", bucket, object))
if rec.Code != http.StatusServiceUnavailable {
t.Fatalf("%s exposed an unresolved old quorum: %d", method, rec.Code)
}
}
if _, err := z.HealObject(t.Context(), bucket, object, "", madmin.HealOpts{Remove: true}); !isErrReadQuorum(err) {
t.Fatalf("heal accepted an unresolved old quorum: %v", err)
}
// Reading must not reinterpret the minority as absent or delete it.
minority, err := disks[15].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
if err != nil || !minority.ModTime.Equal(oldTime.Add(time.Minute)) {
t.Fatalf("read mutated the minority: %+v, %v", minority, err)
}
// Now model the safe direction: the latest generation has the quorum.
// Keep the migration, Walk and scanner assertions on that readable state.
for i := range 15 {
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMeta[i]); err != nil {
t.Fatal(err)
}
}
if err := disks[15].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[15]); err != nil {
t.Fatal(err)
}
newTime := oldTime.Add(time.Minute)
checkRead(newBody, newTime)
for _, kind := range []string{"rebalance", "decommission"} {
t.Run(kind, func(t *testing.T) {
var names []string
consume := func(entry metaCacheEntry) {
versions, err := entry.fileInfoVersions(bucket)
if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(newTime) {
t.Errorf("invalid migration metadata: %+v, %v", versions, err)
return
}
// Migration reopens the selected name/version on all drives;
// it must not treat the selected listing metadata as disk agreement.
reader, err := set.GetObjectNInfo(t.Context(), bucket, entry.name, nil, nil, ObjectOptions{VersionID: nullVersionID, NoLock: true, NoDecryption: true})
if err != nil {
t.Error(err)
return
}
body, err := io.ReadAll(reader)
reader.Close()
if err != nil || !bytes.Equal(body, newBody) {
t.Errorf("migration could not read selected object data: %v", err)
}
names = append(names, entry.name)
}
var err error
if kind == "rebalance" {
err = set.listObjectsToRebalance(t.Context(), bucket, consume)
} else {
err = set.listObjectsToDecommission(t.Context(), decomBucketInfo{Name: bucket}, consume)
}
if err != nil || !slices.Equal(names, []string{object}) {
t.Fatalf("migration listing dropped the quorum: %v, %v", names, err)
}
})
}
for _, latestOnly := range []bool{false, true} {
results := make(chan itemOrErr[ObjectInfo], 16)
if err := z.Walk(t.Context(), bucket, "", results, WalkOptions{AskDisks: "strict", LatestOnly: latestOnly}); err != nil {
t.Fatal(err)
}
var names []string
for result := range results {
if result.Err != nil || !result.Item.ModTime.Equal(newTime) {
t.Fatalf("Walk returned invalid metadata: %+v", result)
}
names = append(names, result.Item.Name)
}
if !slices.Equal(names, []string{object}) {
t.Fatalf("Walk dropped the quorum: %v", names)
}
}
// Exercise the scanner's actual abandoned-child path. Make the scan drive
// miss the object, while keeping the latest quorum and an older minority on
// the remaining disks. Its queued heal must read all disks and repair both.
if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[14]); err != nil {
t.Fatal(err)
}
if err := os.RemoveAll(filepath.Join(disks[15].Endpoint().Path, bucket, object)); err != nil {
t.Fatal(err)
}
scanNullQuorumAbandoned(t, z, bucket, object, newTime)
for i, disk := range disks {
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{Healing: true})
if err != nil || !fi.ModTime.Equal(newTime) {
t.Fatalf("scanner heal did not reconcile disk %d: %v, %v", i, fi.ModTime, err)
}
}
checkRead(newBody, newTime)
put(newBody, oldTime.Add(2*time.Minute))
checkRead(newBody, oldTime.Add(2*time.Minute))
}
+78 -4
View File
@@ -117,6 +117,10 @@ func (d *nullQuorumWalkDisk) WalkDir(ctx context.Context, opts WalkDirOptions, o
} }
func nullQuorumBackend(t *testing.T) (*erasureServerPools, string, http.Handler) { func nullQuorumBackend(t *testing.T) (*erasureServerPools, string, http.Handler) {
return nullQuorumBackendWithVersioning(t, false)
}
func nullQuorumBackendWithVersioning(t *testing.T, suspended bool) (*erasureServerPools, string, http.Handler) {
t.Helper() t.Helper()
ctx, cancel := context.WithCancel(t.Context()) ctx, cancel := context.WithCancel(t.Context())
obj, dirs, err := prepareErasure16(ctx) obj, dirs, err := prepareErasure16(ctx)
@@ -133,10 +137,16 @@ func nullQuorumBackend(t *testing.T) (*erasureServerPools, string, http.Handler)
removeRoots(dirs) removeRoots(dirs)
setObjectLayer(previous) setObjectLayer(previous)
}) })
bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{}) bucket, router, err := initAPIHandlerTest(ctx, z, nil, MakeBucketOptions{VersioningEnabled: suspended})
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if suspended {
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig,
[]byte(`<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>`)); err != nil {
t.Fatal(err)
}
}
return z, bucket, router return z, bucket, router
} }
@@ -158,6 +168,36 @@ func nullQuorumPut(t *testing.T, z *erasureServerPools, bucket, object string, b
} }
} }
func nullQuorumCrossParityOverwrite(t *testing.T, z *erasureServerPools, pool int, bucket, object string, oldBody, newBody []byte, oldTime time.Time, opts ObjectOptions) time.Time {
t.Helper()
set := z.serverPools[pool].getHashedSet(object)
disks := set.getDisks()
put := func(body []byte, opts ObjectOptions) {
if _, err := z.serverPools[pool].PutObject(t.Context(), bucket, object,
mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), opts); err != nil {
t.Fatal(err)
}
}
oldOpts := opts
oldOpts.MTime = oldTime
oldOpts.MaxParity = false
put(oldBody, oldOpts)
old := make([][]byte, len(disks))
for i, disk := range disks {
old[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
}
newTime := oldTime.Add(time.Minute)
opts.MTime = newTime
opts.MaxParity = true
put(newBody, opts)
for i := 12; i < len(disks); i++ {
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, old[i]); err != nil {
t.Fatal(err)
}
}
return newTime
}
func nullQuorumListV2Keys(t *testing.T, router http.Handler, bucket string) (int, []string) { func nullQuorumListV2Keys(t *testing.T, router http.Handler, bucket string) (int, []string) {
t.Helper() t.Helper()
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1000", "", "", "")) rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectsV2URL("", bucket, "", "1000", "", "", ""))
@@ -175,6 +215,29 @@ func nullQuorumListV2Keys(t *testing.T, router http.Handler, bucket string) (int
return rec.Code, keys return rec.Code, keys
} }
type nullQuorumListedVersion struct {
Key string
VersionID string `xml:"VersionId"`
LastModified string
Size int64
IsLatest bool
}
func nullQuorumListVersions(t *testing.T, router http.Handler, bucket string) (int, []nullQuorumListedVersion) {
t.Helper()
rec := nullQuorumRequest(t, router, http.MethodGet, getListObjectVersionsURL("", bucket, "", "1000", ""))
if rec.Code != http.StatusOK {
return rec.Code, nil
}
var list struct {
Versions []nullQuorumListedVersion `xml:"Version"`
}
if err := xml.Unmarshal(rec.Body.Bytes(), &list); err != nil {
t.Fatal(err)
}
return rec.Code, list.Versions
}
func TestListObjectsSingleNullQuorumHTTP(t *testing.T) { func TestListObjectsSingleNullQuorumHTTP(t *testing.T) {
z, bucket, router := nullQuorumBackend(t) z, bucket, router := nullQuorumBackend(t)
oldBody := bytes.Repeat([]byte{'a'}, 8192) oldBody := bytes.Repeat([]byte{'a'}, 8192)
@@ -815,11 +878,22 @@ func scanNullQuorumAbandoned(t *testing.T, z *erasureServerPools, bucket, object
disks := z.serverPools[0].sets[0].getDisks() disks := z.serverPools[0].sets[0].getDisks()
seq := newBgHealSequence() seq := newBgHealSequence()
defer seq.cancelCtx() defer seq.cancelCtx()
previousState, previousRoutine := globalBackgroundHealState, globalBackgroundHealRoutine // Disk monitors may still read the shared state between object-layer tests.
globalBackgroundHealState = &allHealState{healSeqMap: map[string]*healSequence{"test": seq}} // Install the test sequence under its lock without replacing the state.
state := globalBackgroundHealState
state.Lock()
previousSequences := state.healSeqMap
state.healSeqMap = map[string]*healSequence{"test": seq}
state.Unlock()
previousRoutine := globalBackgroundHealRoutine
routine := &healRoutine{tasks: make(chan healTask)} routine := &healRoutine{tasks: make(chan healTask)}
globalBackgroundHealRoutine = routine globalBackgroundHealRoutine = routine
defer func() { globalBackgroundHealState, globalBackgroundHealRoutine = previousState, previousRoutine }() defer func() {
globalBackgroundHealRoutine = previousRoutine
state.Lock()
state.healSeqMap = previousSequences
state.Unlock()
}()
workerDone := make(chan struct{}) workerDone := make(chan struct{})
var healed []string var healed []string
go func() { go func() {
+5
View File
@@ -193,6 +193,11 @@ func (o *listPathOptions) shouldSkip(ctx context.Context, entry metaCacheEntry)
if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() && !entry.isObjectDir() { if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() && !entry.isObjectDir() {
return true return true
} }
// Free versions are not versions of the object. Without other versions
// there is nothing to list, and the entry must not count toward the limit.
if o.InclDeleted && entry.isObject() && entry.isAllFreeVersions() {
return true
}
if o.Lifecycle != nil || o.Replication.Config != nil { if o.Lifecycle != nil || o.Replication.Config != nil {
return triggerExpiryAndRepl(ctx, *o, entry) return triggerExpiryAndRepl(ctx, *o, entry)
} }
+1
View File
@@ -186,6 +186,7 @@ func TestGetAPIEndpoints(t *testing.T) {
} }
func TestCheckLocalServerAddr(t *testing.T) { func TestCheckLocalServerAddr(t *testing.T) {
useTestEndpointDNS(t)
testCases := []struct { testCases := []struct {
serverAddr string serverAddr string
expectedErr error expectedErr error
+11
View File
@@ -834,6 +834,9 @@ func (de DataMovementOverwriteErr) Error() string {
if de.VersionID != "" { if de.VersionID != "" {
objInfoStr = fmt.Sprintf("%s version-id=%s", objInfoStr, de.VersionID) objInfoStr = fmt.Sprintf("%s version-id=%s", objInfoStr, de.VersionID)
} }
if de.Err == errDataMovementSuperseded {
return fmt.Sprintf("data movement skipped, another pool holds a newer copy of %s", objInfoStr)
}
return fmt.Sprintf("invalid data movement operation, source and destination pool are the same for %s", objInfoStr) return fmt.Sprintf("invalid data movement operation, source and destination pool are the same for %s", objInfoStr)
} }
@@ -841,3 +844,11 @@ func isDataMovementOverWriteErr(err error) bool {
var de DataMovementOverwriteErr var de DataMovementOverwriteErr
return errors.As(err, &de) return errors.As(err, &de)
} }
// isDataMovementSuperseded reports a version that was not moved because
// another pool already holds a newer copy, which reads return. The source copy
// is a stale duplicate: the move is complete, and the source may be removed.
func isDataMovementSuperseded(err error) bool {
var de DataMovementOverwriteErr
return errors.As(err, &de) && de.Err == errDataMovementSuperseded
}
+5 -2
View File
@@ -399,12 +399,15 @@ func getCompleteMultipartMD5(parts []CompletePart) string {
return s3MD5 return s3MD5
} }
// Clean unwanted fields from metadata // Keep metadata written by pre-release builds out of public object metadata.
const rollbackIDKey = ReservedMetadataPrefixLower + "rollback-id"
// Clean unwanted fields from metadata.
func cleanMetadata(metadata map[string]string) map[string]string { func cleanMetadata(metadata map[string]string) map[string]string {
// Remove STANDARD StorageClass // Remove STANDARD StorageClass
metadata = removeStandardStorageClass(metadata) metadata = removeStandardStorageClass(metadata)
// Clean meta etag keys 'md5Sum', 'etag', "expires", "x-amz-tagging". // Clean meta etag keys 'md5Sum', 'etag', "expires", "x-amz-tagging".
return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging, "last-modified", VersionPurgeStatusKey) return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging, "last-modified", VersionPurgeStatusKey, rollbackIDKey)
} }
// Filter X-Amz-Storage-Class field only if it is set to STANDARD. // Filter X-Amz-Storage-Class field only if it is set to STANDARD.
+15
View File
@@ -28,6 +28,7 @@ import (
"github.com/minio/minio/internal/amztime" "github.com/minio/minio/internal/amztime"
"github.com/minio/minio/internal/bucket/lifecycle" "github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/bucket/replication"
"github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/event" "github.com/minio/minio/internal/event"
"github.com/minio/minio/internal/hash" "github.com/minio/minio/internal/hash"
@@ -499,6 +500,20 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
} }
} }
sendEvent(evArgs) sendEvent(evArgs)
// ILM purges carry a delete replication decision. Queue the purge
// after the local delete, just as the multi-object DELETE handler
// does for API-originated deletes.
if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) {
if dobj.VersionPurgeStatus() == replication.VersionPurgePending ||
dobj.DeleteMarkerReplicationStatus() == replication.Pending {
scheduleReplicationDelete(ctx, DeletedObjectReplicationInfo{
DeletedObject: dobj,
Bucket: bucket,
EventType: ReplicateILMDelete,
}, o)
}
}
} }
} }
} }
+652
View File
@@ -0,0 +1,652 @@
//go:build consistency_strict
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"testing"
"time"
"github.com/minio/minio/internal/cachevalue"
"github.com/minio/minio/internal/config/drive"
"github.com/minio/minio/internal/config/storageclass"
xhttp "github.com/minio/minio/internal/http"
xioutil "github.com/minio/minio/internal/ioutil"
)
func mutationPools(t testing.TB) (*erasureServerPools, string) {
t.Helper()
// These scenarios require a 4+4 set: five physical commits form a
// readable generation, while four successful replies fail write quorum.
// Other server tests can leave EC:0 configured before this fixture runs.
previousStorageClass := globalStorageClass
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
t.Cleanup(func() { globalStorageClass.Update(previousStorageClass) })
ctx, cancel := context.WithCancel(context.Background())
dirs, err := getRandomDisks(8)
if err != nil {
t.Fatal(err)
}
endpoints := mustGetPoolEndpoints(0, dirs[:8]...)
obj, _, err := initObjectLayer(ctx, endpoints)
if err != nil {
cancel()
removeRoots(dirs)
t.Fatal(err)
}
z := obj.(*erasureServerPools)
old := newObjectLayerFn()
setObjectLayer(z)
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(old) })
bucket := "scope-probe"
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
return z, bucket
}
func mutationPut(t testing.TB, z *erasureServerPools, bucket, obj string, pool int, body string, when time.Time) {
t.Helper()
_, err := z.serverPools[pool].PutObject(context.Background(), bucket, obj, mustGetPutObjReader(t, bytes.NewBufferString(body), int64(len(body)), "", ""), ObjectOptions{MTime: when})
if err != nil {
t.Fatal(err)
}
}
type mutationAmbiguousRename struct {
StorageAPI
index int
entered chan<- struct{}
release <-chan struct{}
}
func (d mutationAmbiguousRename) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
if d.index >= 5 {
return RenameDataResp{}, errDiskNotFound
}
r, e := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, o)
d.entered <- struct{}{}
<-d.release
if d.index == 4 && e == nil {
return r, context.DeadlineExceeded
}
return r, e
}
// A physical fifth commit loses its reply. B successfully updates tags on all
// five copies of A. A then undoes four known successes. B's tag update retains
// the DataDir and ModTime, so the proposed ownership predicate cannot see it.
func TestMutationRollbackPreservesAcknowledgedTags(t *testing.T) {
z, bucket := mutationPools(t)
const object = "tag-undo"
base := time.Now().Add(-time.Hour)
mutationPut(t, z, bucket, object, 0, "initial", base)
set := z.serverPools[0].getHashedSet(object)
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
entered, release := make(chan struct{}, 5), make(chan struct{})
var once sync.Once
unblock := func() { once.Do(func() { close(release) }) }
defer unblock()
wrapped := append([]StorageAPI(nil), disks...)
for i, d := range disks {
wrapped[i] = mutationAmbiguousRename{d, i, entered, release}
}
swap(wrapped)
aDone := make(chan error, 1)
go func() {
body := "pending-A"
_, e := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Minute)})
aDone <- e
}()
for range 5 {
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("A did not finish physical commits")
}
}
swap(disks)
const tags = "owner=acknowledged-B"
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{NoLock: true}); err != nil {
t.Fatalf("B tags failed: %v", err)
}
count := func() int {
n := 0
for _, d := range disks {
fi, e := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
if e == nil && fi.Metadata[xhttp.AmzObjectTagging] == tags {
n++
}
}
return n
}
if n := count(); n != 5 {
t.Fatalf("B did not update all five A copies: %d", n)
}
t.Log("B tag update acknowledged on five physical copies; fifth A reply will report timeout")
unblock()
select {
case e := <-aDone:
if e == nil {
t.Fatal("A unexpectedly succeeded")
}
t.Logf("A result=%v", e)
case <-time.After(5 * time.Second):
t.Fatal("A did not finish")
}
if n := count(); n != 5 {
t.Errorf("acknowledged tags lost physical copies: got %d, want 5", n)
}
oi, e := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
if e != nil {
t.Fatal(e)
}
if oi.UserTags != tags {
t.Fatalf("version-scoped rollback lost acknowledged tag update: tags=%q", oi.UserTags)
}
}
type delayedRenameResult struct {
StorageAPI
fail bool
entered chan<- struct{}
release <-chan struct{}
}
func (d delayedRenameResult) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
if d.fail {
return RenameDataResp{}, errDiskNotFound
}
r, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
d.entered <- struct{}{}
<-d.release // Delay the reply, after the physical mutation and local unlock.
return r, err
}
// One pool, eight real disk directories, no edited object metadata. A commits
// on four disks (below write quorum five); its replies are delayed. Model a lost
// namespace lease by letting B execute with NoLock, like the previous lease
// fixture. B adds an explicit version and succeeds on seven disks. A then
// fails quorum and rolls back. A must not erase the separately acknowledged B.
func TestMutationRollbackPreservesConcurrentVersion(t *testing.T) {
z, bucket := mutationPools(t)
const object = "rollback-and-version"
size, _ := strconv.Atoi(os.Getenv("SILO_TEST_MUTATION_BYTES"))
payload := func(v string) string {
if size > len(v) {
return v + strings.Repeat(".", size-len(v))
}
return v
}
base := time.Now().Add(-time.Hour)
mutationPut(t, z, bucket, object, 0, payload("initial-null"), base)
set := z.serverPools[0].getHashedSet(object)
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
entered, release := make(chan struct{}, 4), make(chan struct{})
var once sync.Once
unblock := func() { once.Do(func() { close(release) }) }
defer unblock()
aDisks := append([]StorageAPI(nil), disks...)
for i, d := range disks {
aDisks[i] = delayedRenameResult{d, i >= 4, entered, release}
}
swap(aDisks)
aDone := make(chan error, 1)
go func() {
body := payload("pending-null-A")
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object,
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
ObjectOptions{NoLock: true, MTime: base.Add(time.Minute)})
aDone <- err
}()
for range 4 {
select {
case <-entered:
case err := <-aDone:
t.Fatalf("A returned early: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("A did not reach the reply barrier")
}
}
t.Log("A physically committed on four disks; replies held; other four RenameData calls failed")
bDisks := append([]StorageAPI(nil), disks...)
bDisks[7] = nil
swap(bDisks)
bVersion := mustGetUUID()
body := payload("acknowledged-version-B")
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object,
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
ObjectOptions{NoLock: true, Versioned: true, VersionID: bVersion, MTime: base.Add(2 * time.Minute)})
if err != nil {
t.Fatalf("B failed: %v", err)
}
countB := func() int {
n := 0
for _, d := range disks {
if _, e := d.ReadVersion(t.Context(), "", bucket, object, bVersion, ReadOptions{}); e == nil {
n++
}
}
return n
}
if n := countB(); n != 7 {
t.Fatalf("B was not committed on the expected seven drives: %d", n)
}
t.Log("B succeeded on seven drives and its version is present on all seven")
unblock()
select {
case err := <-aDone:
if err == nil {
t.Fatal("A unexpectedly succeeded")
}
t.Logf("A failed and rolled back: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("A did not finish")
}
swap(disks)
t.Logf("B metadata survives on %d of eight drives after A's rollback", countB())
if n := countB(); n != 7 {
t.Fatalf("unrelated rollback removed B's acknowledged copies: got %d, want 7", n)
}
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: bVersion})
if err != nil {
t.Fatalf("acknowledged B became unreadable after unrelated rollback: %v", err)
}
defer r.Close()
got, err := io.ReadAll(r)
if err != nil || !bytes.Equal(got, []byte(body)) {
t.Fatalf("B payload changed: readErr=%v length=%d", err, len(got))
}
}
func TestMutationDeleteMarkerDoesNotReenter(t *testing.T) {
disk, _, err := newXLStorageTestSetup(t)
if err != nil {
t.Fatal(err)
}
defer disk.Close()
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
t.Fatal(err)
}
fi := FileInfo{Volume: "bucket", Name: "absent", VersionID: mustGetUUID(), Deleted: true, ModTime: UTCNow()}
done := make(chan error, 1)
go func() { done <- disk.storage.DeleteVersion(t.Context(), "bucket", "absent", fi, true, DeleteOptions{}) }()
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(2 * time.Second):
t.Fatal("DeleteVersion(forceDelMarker) blocked acquiring the same guard again through WriteMetadata")
}
}
// Cleanup can arrive after an RPC reply is lost while the real disk commit is
// still running. Both single and bulk cleanup must retain its staged metadata.
func TestMutationStagingCleanupWaitsForCommit(t *testing.T) {
for _, bulk := range []bool{false, true} {
t.Run(strconv.FormatBool(bulk), func(t *testing.T) {
disk, root, err := newXLStorageTestSetup(t)
if err != nil {
t.Fatal(err)
}
defer disk.Close()
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
t.Fatal(err)
}
var lastSuccess int64
gate := &stagedRenameContext{
Context: context.Background(), meta: filepath.Join(root, minioMetaTmpBucket, "staged", xlStorageFormatFile),
entered: make(chan struct{}), release: make(chan struct{}),
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
}
unblock := sync.OnceFunc(func() { close(gate.release) })
defer unblock()
fi := newFileInfo("object", 2, 2)
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
fi.Fresh = true
committed := make(chan error, 1)
go func() {
_, err := disk.RenameData(gate, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
committed <- err
}()
select {
case <-gate.entered:
case <-time.After(5 * time.Second):
t.Fatal("commit did not reach staging barrier")
}
cleaned := make(chan error, 1)
go func() {
if bulk {
cleaned <- disk.DeleteBulk(t.Context(), minioMetaTmpBucket, pathJoin("staged", xlStorageFormatFile))
} else {
cleaned <- disk.Delete(t.Context(), minioMetaTmpBucket, "staged", DeleteOptions{Recursive: true})
}
}()
early := false
select {
case err := <-cleaned:
early = true
t.Errorf("cleanup overtook admitted disk commit: %v", err)
case <-time.After(100 * time.Millisecond):
}
unblock()
select {
case err := <-committed:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("commit did not finish")
}
if !early {
select {
case <-cleaned:
case <-time.After(5 * time.Second):
t.Fatal("cleanup did not finish")
}
}
got, err := disk.ReadVersion(t.Context(), "", "bucket", "object", "", ReadOptions{})
if err != nil || got.DataDir != fi.DataDir || !got.ModTime.Equal(fi.ModTime) {
t.Fatalf("cleanup lost committed metadata: got=%+v err=%v", got, err)
}
})
}
}
type reviewReadGate struct {
StorageAPI
ready chan<- struct{}
release <-chan struct{}
}
func (d reviewReadGate) ReadXL(ctx context.Context, volume, path string, data bool) (RawFileInfo, error) {
r, err := d.StorageAPI.ReadXL(ctx, volume, path, data)
d.ready <- struct{}{}
if d.release != nil {
<-d.release
}
return r, err
}
func TestReviewFastReadAlreadyWaitsForSlowDisk(t *testing.T) {
z, bucket := mutationPools(t)
const object = "healthy-slow-read"
mutationPut(t, z, bucket, object, 0, "healthy-inline", time.Now())
set := z.serverPools[0].getHashedSet(object)
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
ready, release := make(chan struct{}, len(disks)), make(chan struct{})
var once sync.Once
unblock := func() { once.Do(func() { close(release) }) }
defer unblock()
wrapped := append([]StorageAPI(nil), disks...)
for i, d := range disks {
g := reviewReadGate{StorageAPI: d, ready: ready}
if i == len(disks)-1 {
g.release = release
}
wrapped[i] = g
}
swap(wrapped)
done := make(chan error, 1)
go func() {
_, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: true}, true)
done <- err
}()
for range disks {
select {
case <-ready:
case <-time.After(5 * time.Second):
t.Fatal("reads did not reach barrier")
}
}
select {
case err := <-done:
t.Fatalf("read returned before the final disk: %v", err)
case <-time.After(100 * time.Millisecond):
t.Log("seven healthy replies are insufficient to return; final unbuffered mrfCheck send is still waiting")
}
unblock()
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("read did not finish after slow disk release")
}
}
// Pause the real trash worker at its disk-space callback, after moving the
// external shards but before DeleteVersion can replace xl.meta. An outer join
// must also contain this nested worker, and later mutations must remain excluded.
func TestMutationDeleteJoinsNestedWorker(t *testing.T) {
z, bucket := mutationPools(t)
const object = "nested-timeout"
mutationPut(t, z, bucket, object, 0, strings.Repeat("x", 2<<20), time.Now())
disk := z.serverPools[0].getHashedSet(object).getDisks()[0].(*xlStorageDiskIDCheck)
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
if err != nil {
t.Fatal(err)
}
entered, release := make(chan struct{}), make(chan struct{})
var once sync.Once
unblock := func() { once.Do(func() { close(release) }) }
defer unblock()
oldCache := disk.storage.diskInfoCache
disk.storage.diskInfoCache = cachevalue.NewFromFunc(time.Hour, cachevalue.Opts{}, func(context.Context) (DiskInfo, error) {
close(entered)
<-release
return DiskInfo{Used: 100, Free: 100, UsedInodes: 100, FreeInodes: 100}, nil
})
defer func() { disk.storage.diskInfoCache = oldCache }()
oldTimeout := globalDriveConfig.GetMaxTimeout()
globalDriveConfig.Update(drive.Config{MaxTimeout: 1200 * time.Millisecond})
defer globalDriveConfig.Update(drive.Config{MaxTimeout: oldTimeout})
done := make(chan error, 1)
go func() {
_, e := xioutil.WithDeadline(context.Background(), globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
return struct{}{}, disk.storage.DeleteVersion(ctx, bucket, object, fi, false, DeleteOptions{UndoWrite: true})
})
done <- e
}()
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("trash callback not reached")
}
early := false
select {
case e := <-done:
early = true
t.Errorf("outer join returned before nested disk worker completed: %v", e)
case <-time.After(1500 * time.Millisecond):
}
updated := make(chan error, 1)
go func() {
updated <- disk.storage.UpdateMetadata(context.Background(), bucket, object, FileInfo{VersionID: fi.VersionID, Metadata: map[string]string{"after-timeout": "yes"}}, UpdateMetadataOpts{})
}()
writerEarly := false
select {
case e := <-updated:
writerEarly = true
t.Errorf("later metadata writer passed unfinished deletion: %v", e)
case <-time.After(100 * time.Millisecond):
}
unblock()
if !early {
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("delete did not finish")
}
}
if !writerEarly {
select {
case <-updated:
case <-time.After(5 * time.Second):
t.Fatal("later writer did not finish")
}
}
}
// Even an update that leaves all public fields unchanged is a later successful
// operation. An old writer must not regain undo ownership through an ABA cycle.
func TestMutationNoopUpdateRevokesUndo(t *testing.T) {
for _, method := range []string{"update", "write", "aba"} {
t.Run(method, func(t *testing.T) {
z, bucket := mutationPools(t)
const object = "no-op-update"
mutationPut(t, z, bucket, object, 0, "acknowledged", time.Now())
disks := z.serverPools[0].getHashedSet(object).getDisks()
for _, disk := range disks {
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: true})
if err != nil {
t.Fatal(err)
}
switch method {
case "update":
err = disk.UpdateMetadata(t.Context(), bucket, object, fi, UpdateMetadataOpts{})
case "write":
err = disk.WriteMetadata(t.Context(), "", bucket, object, fi)
case "aba":
for _, etag := range []string{"temporary", fi.Metadata["etag"]} {
err = disk.UpdateMetadata(t.Context(), bucket, object, FileInfo{Metadata: map[string]string{"etag": etag}}, UpdateMetadataOpts{})
if err != nil {
t.Fatal(err)
}
}
}
if err != nil {
t.Fatal(err)
}
if err := disk.DeleteVersion(t.Context(), bucket, object, fi, false, DeleteOptions{UndoWrite: true}); err != errFileVersionNotFound {
t.Fatalf("old writer retained undo after %s: %v", method, err)
}
}
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
defer r.Close()
body, err := io.ReadAll(r)
if err != nil || string(body) != "acknowledged" {
t.Fatalf("later acknowledged state lost: %q, %v", body, err)
}
})
}
}
func TestMutationCanceledSuccessfulCommitIsNotUndone(t *testing.T) {
z, bucket := mutationPools(t)
const object = "cancel-after-commit"
mutationPut(t, z, bucket, object, 0, "initial", time.Now().Add(-time.Hour))
set := z.serverPools[0].getHashedSet(object)
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
entered, release := make(chan struct{}, len(disks)), make(chan struct{})
var once sync.Once
unblock := func() { once.Do(func() { close(release) }) }
defer unblock()
wrapped := append([]StorageAPI(nil), disks...)
for i, disk := range disks {
wrapped[i] = delayedRenameResult{disk, false, entered, release}
}
swap(wrapped)
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
done := make(chan error, 1)
go func() {
_, err := set.PutObject(ctx, bucket, object, mustGetPutObjReader(t, strings.NewReader("committed"), 9, "", ""), ObjectOptions{})
done <- err
}()
for range disks {
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("physical commit barrier not reached")
}
}
cancel()
unblock()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("commit did not finish")
}
swap(disks)
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
defer r.Close()
body, err := io.ReadAll(r)
if err != nil || string(body) != "committed" {
t.Fatalf("cancellation rolled back an actual successful quorum: %q, %v", body, err)
}
}
// Partial no-op updates may revoke undo on only some disks. That internal
// capability must not split the quorum of otherwise identical object contents.
func TestMutationPartialNoopPreservesReadability(t *testing.T) {
z, bucket := mutationPools(t)
const object = "partial-noop"
set := z.serverPools[0].getHashedSet(object)
const body = "acknowledged-data"
const tags = "owner=unchanged"
_, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{UserDefined: map[string]string{xhttp.AmzObjectTagging: tags}})
if err != nil {
t.Fatal(err)
}
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
partial := append([]StorageAPI(nil), disks...)
for i := 4; i < len(partial); i++ {
partial[i] = nil
}
swap(partial)
if _, err = set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err == nil {
t.Fatal("four drives unexpectedly gave write quorum")
}
swap(disks)
for _, version := range []string{"", nullVersionID} {
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: version})
if err != nil {
t.Fatalf("unchanged acknowledged object became unreadable after no-op partial update: %v", err)
}
got, err := io.ReadAll(r)
r.Close()
if err != nil || string(got) != body || r.ObjInfo.UserTags != tags {
t.Fatalf("body/tags changed: %q %v %+v", got, err, r.ObjInfo)
}
}
}
@@ -0,0 +1,57 @@
//go:build consistency_strict
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
"maps"
)
// Compare logical version state, excluding per-shard checksums/index, physical
// DataDir placement and properties of other versions in the same xl.meta.
// Different DataDir values remain readable for the existing rebalance recovery
// contract; rollback separately requires an exact DataDir match.
func sameFileInfoVersion(a, b FileInfo) bool {
if a.VersionID != b.VersionID || a.Deleted != b.Deleted || a.XLV1 != b.XLV1 || a.Size != b.Size ||
a.TransitionStatus != b.TransitionStatus || a.TransitionTier != b.TransitionTier ||
a.TransitionedObjName != b.TransitionedObjName || a.TransitionVersionID != b.TransitionVersionID ||
!sameVersionMetadata(a.Metadata, b.Metadata) || !bytes.Equal(a.Checksum, b.Checksum) || len(a.Parts) != len(b.Parts) {
return false
}
if !a.Deleted && a.Size != 0 && !a.Erasure.Equal(b.Erasure) {
return false
}
for i, p := range a.Parts {
q := b.Parts[i]
if p.Number != q.Number || p.Size != q.Size || p.ActualSize != q.ActualSize || p.ETag != q.ETag || !bytes.Equal(p.Index, q.Index) {
return false
}
}
return true
}
// The undo capability controls rollback ownership, not the logical contents.
// A partial no-op update can revoke it without changing a single public field.
func sameVersionMetadata(a, b map[string]string) bool {
if a[rollbackIDKey] == b[rollbackIDKey] {
return maps.Equal(a, b)
}
for k, v := range a {
if k != rollbackIDKey {
if other, ok := b[k]; !ok || other != v {
return false
}
}
}
for k := range b {
if k != rollbackIDKey {
if _, ok := a[k]; !ok {
return false
}
}
}
return true
}
+269
View File
@@ -0,0 +1,269 @@
//go:build consistency_strict
// Preserved diagnostic for the deferred strict-read policy. This is not a
// maintenance release gate; failures record intentionally unresolved risks.
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"io"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/minio/minio/internal/dsync"
)
func lateCommitPools(t *testing.T) (*erasureServerPools, string) {
t.Helper()
ctx, cancel := context.WithCancel(context.Background())
dirs, err := getRandomDisks(16)
if err != nil {
t.Fatal(err)
}
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, dirs...))
if err != nil {
cancel()
removeRoots(dirs)
t.Fatal(err)
}
z := obj.(*erasureServerPools)
previous := newObjectLayerFn()
setObjectLayer(z)
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) })
bucket := "late-commit-regression"
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
return z, bucket
}
func lateCommitSwitch(set *erasureObjects) ([]StorageAPI, func([]StorageAPI)) {
original := set.getDisks()
var active atomic.Value
active.Store(original)
set.getDisks = func() []StorageAPI { return active.Load().([]StorageAPI) }
return original, func(disks []StorageAPI) { active.Store(disks) }
}
type lateCommitDisk struct {
StorageAPI
index int
earlyDone chan<- struct{}
releaseRename <-chan struct{}
renameReady chan<- struct{}
releaseUndo <-chan struct{}
}
func (d lateCommitDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
defer func() { d.renameReady <- struct{}{} }()
if d.index >= 1 {
<-d.releaseRename
}
result, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
if err != nil {
return result, err
}
if d.index < 1 {
d.earlyDone <- struct{}{}
<-d.releaseRename
}
// Model a lost successful response. Real disk commit has already happened.
if d.index < 1 || d.index >= 12 {
return RenameDataResp{}, errDiskNotFound
}
return result, nil
}
func (d lateCommitDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, force bool, opts DeleteOptions) error {
if opts.UndoWrite {
<-d.releaseUndo
}
return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, force, opts)
}
// All metadata changes use production PutObject/RenameData/UndoWrite on real disks.
// Fault injection only controls timing, namespace lease age, and transport results.
func TestMutationReadDuringLateCommitAndUndo(t *testing.T) {
z, bucket := lateCommitPools(t)
object := "late-undo-16"
bodyA, bodyB := strings.Repeat("writer-A", 262144), strings.Repeat("writer-B", 262144)
label := func(s string) string {
if s == bodyA {
return "A (2 MiB, exact payload match)"
}
if s == bodyB {
return "B (2 MiB, exact payload match)"
}
if s == "" {
return "empty"
}
return "UNEXPECTED PAYLOAD"
}
set := z.serverPools[0].getHashedSet(object)
put := func(ctx context.Context, body string) (ObjectInfo, error) {
return set.PutObject(ctx, bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true})
}
if _, err := put(t.Context(), "seed"); err != nil {
t.Fatal(err)
}
lockers := make([]*localLocker, 4)
clients := make([]dsync.NetLocker, 4)
for i := range lockers {
lockers[i] = newLocker()
clients[i] = lockers[i]
}
newLock := func() *distLockInstance {
return &distLockInstance{rwMutex: dsync.NewDRWMutex(&dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, "late-commit" }, Timeouts: dsync.DefaultTimeouts}, bucket+"/"+object), opsID: mustGetUUID()}
}
oldLock := newLock()
oldCtx, err := oldLock.GetLock(t.Context(), newDynamicTimeout(time.Second, time.Second))
if err != nil {
t.Fatal(err)
}
defer oldLock.Unlock(oldCtx)
disks, swap := lateCommitSwitch(set)
defer swap(disks)
earlyDone, renameReady := make(chan struct{}, 16), make(chan struct{}, 16)
releaseRename, releaseUndo := make(chan struct{}), make(chan struct{})
var renameOnce, undoOnce sync.Once
unblockRename := func() { renameOnce.Do(func() { close(releaseRename) }) }
unblockUndo := func() { undoOnce.Do(func() { close(releaseUndo) }) }
defer unblockUndo()
defer unblockRename()
wrapped := append([]StorageAPI(nil), disks...)
for i, d := range disks {
wrapped[i] = lateCommitDisk{d, i, earlyDone, releaseRename, renameReady, releaseUndo}
}
swap(wrapped)
aDone := make(chan error, 1)
go func() { _, e := put(oldCtx.Context(), bodyA); aDone <- e }()
for range 1 {
select {
case <-earlyDone:
case <-time.After(5 * time.Second):
t.Fatal("A early commit barrier")
}
}
// Expire actual dsync lease records, allowing another holder to take over.
for _, l := range lockers {
l.mutex.Lock()
for key, entries := range l.lockMap {
for i := range entries {
entries[i].TimeLastRefresh = time.Now().Add(-lockValidityDuration - time.Second).UnixNano()
}
l.lockMap[key] = entries
}
l.mutex.Unlock()
l.expireOldLocks(lockValidityDuration)
}
newHolder := newLock()
newCtx, err := newHolder.GetLock(t.Context(), newDynamicTimeout(time.Second, time.Second))
if err != nil {
t.Fatal(err)
}
swap(disks)
b, err := put(newCtx.Context(), bodyB)
newHolder.Unlock(newCtx)
if err != nil {
t.Fatalf("B must acknowledge success: %v", err)
}
counts := func() (older, newer, unknown int) {
for _, d := range disks {
fi, e := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
if e != nil {
unknown++
continue
}
switch {
case fi.ModTime.Equal(b.ModTime):
newer++
case fi.ModTime.Before(b.ModTime):
older++
default:
t.Fatalf("unexpected generation %v", fi.ModTime)
}
}
return
}
o, n, u := counts()
if o != 0 || n != 16 || u != 0 {
t.Fatalf("B ACK evidence: %d %d %d", o, n, u)
}
t.Logf("B successfully committed; observed B=16/16, all disks online; ModTime=%s", b.ModTime)
unblockRename()
for range len(disks) {
select {
case <-renameReady:
case <-time.After(5 * time.Second):
t.Fatal("rename completion barrier")
}
}
o, n, u = counts()
fi, e := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
if e != nil {
t.Fatal(e)
}
qr, qw := fi.Erasure.DataBlocks, fi.Erasure.DataBlocks
if fi.Erasure.DataBlocks == fi.Erasure.ParityBlocks {
qw++
}
if qr != 12 || qw != 12 {
t.Fatalf("unexpected quorum %d/%d", qr, qw)
}
t.Logf("Before A undo: old A=%d, newer acknowledged B=%d, U=%d, Qr=%d, Qw=%d", o, n, u, qr, qw)
get := func() (string, error) {
rd, e := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
if e != nil {
return "", e
}
defer rd.Close()
body, e := io.ReadAll(rd)
return string(body), e
}
first, firstErr := get()
t.Logf("R1 after B ACK and before A undo: body=%q error=%v", label(first), firstErr)
if firstErr != nil && !isErrReadQuorum(firstErr) {
t.Errorf("unexpected first read error: %v", firstErr)
}
unblockUndo()
select {
case e := <-aDone:
if e == nil {
t.Fatal("A unexpectedly succeeded")
}
t.Logf("A failed after completing UndoWrite: %v", e)
case <-time.After(5 * time.Second):
t.Fatal("A undo did not finish")
}
second, secondErr := get()
if secondErr != nil || second != bodyB {
t.Fatalf("B must remain after A fails: %q %v", label(second), secondErr)
}
o, n, u = counts()
t.Logf("R2 after A failure, with no new PUT: body=%q; old A=%d, B=%d, U=%d", label(second), o, n, u)
if first == bodyA {
t.Error("NON-LINEARIZABLE: R1 exposed A, then R2 returned B with no second B PUT; B completed before R1, and R1 completed before R2. Current fragment counts do not prove B was never committed.")
}
}
+289
View File
@@ -0,0 +1,289 @@
//go:build consistency_strict
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/minio/minio/internal/dsync"
"github.com/minio/minio/internal/grid"
)
type scopeLeaseLocker struct {
*localLocker
partition *atomic.Bool
refresh chan<- struct{}
}
func (l scopeLeaseLocker) Refresh(ctx context.Context, a dsync.LockArgs) (bool, error) {
if l.partition.Load() {
select {
case l.refresh <- struct{}{}:
default:
}
return false, errors.New("injected lock-plane partition")
}
return l.localLocker.Refresh(ctx, a)
}
func (l scopeLeaseLocker) ForceUnlock(ctx context.Context, a dsync.LockArgs) (bool, error) {
if l.partition.Load() {
return false, errors.New("partition")
}
return l.localLocker.ForceUnlock(ctx, a)
}
type scopeStagedContext struct {
context.Context
meta string
entered chan<- struct{}
release <-chan struct{}
once sync.Once
tracker healthDiskCtxValue
}
func (c *scopeStagedContext) Value(k any) any {
if _, ok := k.(healthDiskCtxKey); ok {
if _, e := os.Stat(c.meta); e == nil {
c.once.Do(func() { c.entered <- struct{}{}; <-c.release })
}
return &c.tracker
}
return c.Context.Value(k)
}
type scopeRPCDisk struct {
StorageAPI
client *storageRESTClient
}
func (d scopeRPCDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
return d.client.RenameData(ctx, sv, sp, fi, dv, dp, o)
}
// Controlled fault schedule: real dsync lockers lose their expired lease,
// while a real disk RenameData is paused after reading and staging xl.meta.
// Only the clock age of lease records and the disk completion schedule are injected.
// No object data or metadata is rewritten by the test after the seed PUT.
func TestMutationLeaseLossAndPartition(t *testing.T) {
for _, partitioned := range []bool{false, true} {
t.Run(fmt.Sprintf("network-partition-%t", partitioned), func(t *testing.T) {
size, _ := strconv.Atoi(os.Getenv("SILO_TEST_MUTATION_BYTES"))
payload := func(s string) string {
if size <= len(s) {
return s
}
return s + strings.Repeat(".", size-len(s))
}
z, bucket := mutationPools(t)
const object = "lease-key"
base := time.Now().Add(-time.Hour)
mutationPut(t, z, bucket, object, 0, payload("initial-0"), base)
var partition atomic.Bool
refreshed := make(chan struct{}, 16)
locals := make([]*localLocker, 4)
oldClients := make([]dsync.NetLocker, 4)
newClients := make([]dsync.NetLocker, 4)
for i := range locals {
locals[i] = newLocker()
oldClients[i] = scopeLeaseLocker{locals[i], &partition, refreshed}
newClients[i] = locals[i]
}
mklock := func(clients []dsync.NetLocker) *distLockInstance {
return &distLockInstance{rwMutex: dsync.NewDRWMutex(&dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, "scope-owner" }, Timeouts: dsync.DefaultTimeouts}, bucket+"/"+object), opsID: mustGetUUID()}
}
a := mklock(oldClients)
requestCtx, cancelRequest := context.WithCancel(context.Background())
defer cancelRequest()
alc, err := a.GetLock(requestCtx, newDynamicTimeout(time.Second, time.Second))
if err != nil {
t.Fatal(err)
}
defer a.Unlock(alc)
tg, err := grid.SetupTestGrid(2)
if err != nil {
t.Fatal(err)
}
defer tg.Cleanup()
entered, release := make(chan struct{}, 8), make(chan struct{})
physicalDone := make(chan error, 8)
var once sync.Once
unblock := func() { once.Do(func() { close(release) }) }
defer unblock()
set := z.serverPools[0].getHashedSet(object)
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
wrapped := append([]StorageAPI(nil), disks...)
for i, disk := range disks {
route := fmt.Sprintf("/lease/%d", i)
handler := func(p *RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
var success int64
gate := &scopeStagedContext{Context: context.Background(), meta: filepath.Join(disk.Endpoint().Path, p.SrcVolume, p.SrcPath, xlStorageFormatFile), entered: entered, release: release, tracker: healthDiskCtxValue{lastSuccess: &success}}
resp, e := disk.RenameData(gate, p.SrcVolume, p.SrcPath, p.FI, p.DstVolume, p.DstPath, p.Opts)
physicalDone <- e
return &resp, grid.NewRemoteErr(e)
}
if e := storageRenameDataRPC.Register(tg.Managers[1], handler, route); e != nil {
t.Fatal(e)
}
if e := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
defer p.Recycle()
return handler(&p.RenameDataHandlerParams)
}, route); e != nil {
t.Fatal(e)
}
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
client.diskID.Store(new(string))
wrapped[i] = scopeRPCDisk{disk, client}
}
swap(wrapped)
done := make(chan error, 1)
go func() {
body := []byte(payload("pending-1000"))
_, e := z.serverPools[0].PutObject(alc.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Minute)})
done <- e
}()
for range disks {
select {
case <-entered:
case e := <-done:
t.Fatalf("A returned before disk barrier: %v", e)
case <-time.After(5 * time.Second):
t.Fatal("disk barrier timeout")
}
}
t.Log("OBS A staged old metadata on all eight real disks; no final rename yet")
partition.Store(partitioned)
for _, l := range locals {
l.mutex.Lock()
for key, entries := range l.lockMap {
for i := range entries {
entries[i].TimeLastRefresh = time.Now().Add(-lockValidityDuration - time.Second).UnixNano()
}
l.lockMap[key] = entries
}
l.mutex.Unlock()
l.expireOldLocks(lockValidityDuration)
}
if partitioned {
select {
case <-refreshed:
case <-time.After(13 * time.Second):
t.Fatal("refresh not attempted")
}
time.Sleep(20 * time.Millisecond)
t.Logf("OBS after failed refresh old context error=%v", alc.Context().Err())
} else {
select {
case <-alc.Context().Done():
case <-time.After(13 * time.Second):
t.Fatal("confirmed lost lock did not cancel")
}
t.Logf("OBS confirmed lease loss canceled old context: %v", alc.Context().Err())
}
b := mklock(newClients)
blc, e := b.GetLock(context.Background(), newDynamicTimeout(time.Second, time.Second))
if e != nil {
t.Fatal(e)
}
swap(disks)
bdone := make(chan error, 1)
go func() {
body := []byte(payload("ack-9999"))
_, e := z.serverPools[0].PutObject(blc.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(2 * time.Minute)})
b.Unlock(blc)
bdone <- e
}()
bearly := false
select {
case e := <-bdone:
if e != nil {
t.Fatal(e)
}
bearly = true
t.Log("OBS B acquired the same lock and ACKed while A remained physically staged")
case <-time.After(150 * time.Millisecond):
t.Log("OBS B is waiting for the earlier disk transaction")
}
if partitioned {
cancelRequest()
}
unblock()
select {
case e := <-done:
t.Logf("OBS A finished: %v", e)
case <-time.After(5 * time.Second):
t.Fatal("A did not finish")
}
if !bearly {
select {
case e := <-bdone:
if e != nil {
t.Fatal(e)
}
t.Log("OBS B ACKed after disk serialization")
case <-time.After(5 * time.Second):
t.Fatal("B did not finish")
}
}
for range disks {
select {
case e := <-physicalDone:
t.Logf("OBS physical handler completed: %v", e)
case <-time.After(5 * time.Second):
t.Fatal("physical handler did not finish")
}
}
for _, fast := range []bool{false, true} {
r, e := z.GetObjectNInfo(context.Background(), bucket, object, nil, nil, ObjectOptions{FastGetObjInfo: fast})
if e != nil {
t.Errorf("OBS latest acknowledged payload unreadable fast=%t error=%v", fast, e)
continue
}
data, e := io.ReadAll(r)
r.Close()
if e != nil {
t.Fatal(e)
}
shown := string(data)
if len(shown) > 32 {
shown = shown[:32]
}
t.Logf("OBS final GET fast=%t bytes=%d prefix=%q", fast, len(data), shown)
if string(data) != payload("ack-9999") && string(data) != payload("pending-1000") {
t.Errorf("STRICT STALE VALUE: prefix %q is neither latest ACK nor pending write", shown)
}
}
})
}
}
+218
View File
@@ -0,0 +1,218 @@
//go:build consistency_strict
// Preserved diagnostic for the deferred strict-read policy. This is not a
// maintenance release gate; failures record intentionally unresolved risks.
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
xhttp "github.com/minio/minio/internal/http"
"io"
"strconv"
"strings"
"sync"
"testing"
"time"
)
func TestMutationReadPreservesAcknowledgedTags(t *testing.T) {
z, bucket := mutationPools(t)
const object = "tag-read"
mutationPut(t, z, bucket, object, 0, "payload", time.Now())
set := z.serverPools[0].getHashedSet(object)
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
partial := append([]StorageAPI(nil), disks...)
partial[0] = nil
swap(partial)
const tags = "owner=acknowledged-B"
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil {
t.Fatal(err)
}
swap(disks)
before, err := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
if err != nil {
t.Fatal(err)
}
after, err := disks[1].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
if err != nil {
t.Fatal(err)
}
t.Logf("old/new tags=%q/%q; equal tuple=%t", before.Metadata[xhttp.AmzObjectTagging], after.Metadata[xhttp.AmzObjectTagging], before.ModTime.Equal(after.ModTime) && before.DataDir == after.DataDir && before.Metadata["etag"] == after.Metadata["etag"])
for _, fast := range []bool{false, true} {
for _, version := range []string{"", nullVersionID} {
oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version})
if err != nil {
t.Fatal(err)
}
if oi.UserTags != tags {
t.Fatalf("acknowledged tags replaced by stale metadata (fast=%t version=%q): %q", fast, version, oi.UserTags)
}
}
}
}
func TestMutationVersionHistoryRead(t *testing.T) {
z, bucket := mutationPools(t)
const object = "history-read"
set := z.serverPools[0].getHashedSet(object)
var versions []string
for i := range 12 {
id := mustGetUUID()
body := "version-" + strconv.Itoa(i)
_, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, VersionID: id})
if err != nil {
t.Fatal(err)
}
versions = append(versions, id)
}
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
uniform := func() bool {
raw := make([]RawFileInfo, len(disks))
meta := make([]FileInfo, len(disks))
errs := make([]error, len(disks))
for i, disk := range disks {
var err error
raw[i], err = disk.ReadXL(t.Context(), bucket, object, false)
if err != nil {
t.Fatal(err)
}
meta[i], errs[i] = fileInfoFromRaw(raw[i], bucket, object, false, false)
}
return uniformLatestVersion(meta, errs)
}
if !uniform() {
t.Fatal("healthy version history requires redundant full metadata decoding")
}
partial := append([]StorageAPI(nil), disks...)
partial[0] = nil
swap(partial)
const tags = "owner=latest-acknowledged"
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil {
t.Fatal(err)
}
swap(disks)
if uniform() {
t.Fatal("stale same-version metadata incorrectly accepted as uniform")
}
for _, fast := range []bool{false, true} {
for _, version := range []string{"", versions[len(versions)-1]} {
oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version})
if err != nil || oi.VersionID != versions[len(versions)-1] || oi.UserTags != tags {
t.Fatalf("latest version/tags changed: version=%q tags=%q err=%v", oi.VersionID, oi.UserTags, err)
}
}
}
for i, id := range versions {
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: id})
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(r)
r.Close()
if err != nil || string(body) != "version-"+strconv.Itoa(i) {
t.Fatalf("history version %d changed: %q err=%v", i, body, err)
}
}
}
func TestMutationCrossDiskOrderHasStableRead(t *testing.T) {
z, bucket := mutationPools(t)
const object = "cross-disk-order"
base := time.Now().Add(-time.Hour)
mutationPut(t, z, bucket, object, 0, "initial", base)
set := z.serverPools[0].getHashedSet(object)
disks, swap := installListingDiskSwitch(set)
defer swap(disks)
earlyDone, release := make(chan struct{}, 4), make(chan struct{})
var once sync.Once
unblock := func() { once.Do(func() { close(release) }) }
defer unblock()
aDisks := append([]StorageAPI(nil), disks...)
for i, d := range disks {
aDisks[i] = orderedRenameDisk{d, i < 4, earlyDone, release}
}
swap(aDisks)
put := func(body string, mtime time.Time) error {
_, e := z.serverPools[0].PutObject(t.Context(), bucket, object,
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
ObjectOptions{NoLock: true, MTime: mtime})
return e
}
aDone := make(chan error, 1)
go func() { aDone <- put("writer-A", base.Add(time.Minute)) }()
for range 4 {
select {
case <-earlyDone:
case <-time.After(5 * time.Second):
t.Fatal("A did not reach first-half barrier")
}
}
swap(disks)
if err := put("writer-B", base.Add(2*time.Minute)); err != nil {
t.Fatalf("B failed: %v", err)
}
unblock()
select {
case err := <-aDone:
if err != nil {
t.Fatalf("A failed: %v", err)
}
case <-time.After(5 * time.Second):
t.Fatal("A did not finish")
}
t.Log("both PUTs succeeded; final per-disk order is A→B on four disks and B→A on four disks; no writer remains")
swap(disks)
for range 8 {
for _, version := range []string{"", nullVersionID} {
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{FastGetObjInfo: true, VersionID: version})
if err != nil {
t.Fatalf("quiescent successful writes became unreadable: %v", err)
}
body, err := io.ReadAll(r)
r.Close()
if err != nil || string(body) != "writer-B" {
t.Fatalf("completed reads changed value: %q %v", body, err)
}
}
}
}
func uniformLatestVersion(meta []FileInfo, errs []error) bool {
if len(meta) == 0 {
return false
}
first := meta[0]
for i, fi := range meta {
if errs[i] != nil || !fi.IsValid() || fi.XLV1 || fi.Deleted || !fi.IsLatest || fi.NumVersions < 1 ||
!fi.ModTime.Equal(first.ModTime) || !sameFileInfoVersion(first, fi) ||
fi.NumVersions != first.NumVersions || !fi.SuccessorModTime.Equal(first.SuccessorModTime) {
return false
}
}
return true
}
type orderedRenameDisk struct {
StorageAPI
early bool
earlyDone chan<- struct{}
release <-chan struct{}
}
func (d orderedRenameDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
if !d.early {
<-d.release
}
r, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
if d.early {
d.earlyDone <- struct{}{}
<-d.release
}
return r, err
}
+22 -16
View File
@@ -244,6 +244,26 @@ func TestPoolsObjectLockOverwrite(t *testing.T) {
} }
} }
defer conditionalPutPool(t, z, key, 0)() defer conditionalPutPool(t, z, key, 0)()
var source string
var copyInfo ObjectInfo
if op == "copy" || op == "self-copy" {
source = key
if op == "copy" {
source = "source-" + state
putConsistencyObject(t, z, bucket, source, 0, "replacement", ObjectOptions{Versioned: true})
}
// Resolve the source before the pool becomes unreadable.
// The destination must still reject a previously read source.
gr, gerr := z.GetObjectNInfo(ctx, bucket, source, nil, nil, ObjectOptions{})
if gerr != nil {
t.Fatal(gerr)
}
defer gr.Close()
copyInfo = gr.ObjInfo
copyInfo.metadataOnly = op == "self-copy"
copyInfo.PutObjReader = mustGetPutObjReader(t, gr, copyInfo.Size, "", "")
copyInfo.Reader = copyInfo.PutObjReader.Reader
}
var restore func() var restore func()
if state == "unreadable" { if state == "unreadable" {
restore = conditionalPutSwapDisks(z.serverPools[1], key, func(d StorageAPI) StorageAPI { restore = conditionalPutSwapDisks(z.serverPools[1], key, func(d StorageAPI) StorageAPI {
@@ -257,22 +277,8 @@ func TestPoolsObjectLockOverwrite(t *testing.T) {
case "put": case "put":
_, writeErr = z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), opts) _, writeErr = z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), opts)
case "copy", "self-copy": case "copy", "self-copy":
source := key copyInfo.UserDefined = opts.UserDefined
if op == "copy" { _, writeErr = z.CopyObject(ctx, bucket, source, bucket, key, copyInfo, ObjectOptions{VersionID: nullVersionID}, opts)
source = "source-" + state
putConsistencyObject(t, z, bucket, source, 0, "replacement", ObjectOptions{Versioned: true})
}
gr, gerr := z.GetObjectNInfo(ctx, bucket, source, nil, nil, ObjectOptions{})
if gerr != nil {
t.Fatal(gerr)
}
defer gr.Close()
info := gr.ObjInfo
info.UserDefined = opts.UserDefined
info.metadataOnly = op == "self-copy"
info.PutObjReader = mustGetPutObjReader(t, gr, info.Size, "", "")
info.Reader = info.PutObjReader.Reader
_, writeErr = z.CopyObject(ctx, bucket, source, bucket, key, info, ObjectOptions{VersionID: nullVersionID}, opts)
case "multipart": case "multipart":
id, parts := multipartConditionUpload(t, z, bucket, key, 0, opts) id, parts := multipartConditionUpload(t, z, bucket, key, 0, opts)
_, writeErr = z.CompleteMultipartUpload(ctx, bucket, key, id, parts, ObjectOptions{Versioned: true}) _, writeErr = z.CompleteMultipartUpload(ctx, bucket, key, id, parts, ObjectOptions{Versioned: true})
+522
View File
@@ -0,0 +1,522 @@
//go:build consistency_strict
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/minio/minio/internal/grid"
)
// Pause at the real disk health callback after staging xl.meta. Unlike a sleep,
// this fixes cancellation between staging and the final metadata rename.
type stagedRenameContext struct {
context.Context
meta string
entered, release chan struct{}
once sync.Once
tracker healthDiskCtxValue
}
func (c *stagedRenameContext) Value(key any) any {
if _, ok := key.(healthDiskCtxKey); ok {
if _, err := os.Stat(c.meta); err == nil {
c.once.Do(func() {
close(c.entered)
<-c.release
})
}
return &c.tracker
}
return c.Context.Value(key)
}
func TestRenameCancellationLocalWaitsForDisk(t *testing.T) {
disk, root, err := newXLStorageTestSetup(t)
if err != nil {
t.Fatal(err)
}
defer disk.Close()
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
t.Fatal(err)
}
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
var lastSuccess int64
gate := &stagedRenameContext{
Context: ctx, meta: filepath.Join(root, minioMetaTmpBucket, "staged", xlStorageFormatFile),
entered: make(chan struct{}), release: make(chan struct{}),
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
}
fi := newFileInfo("object", 2, 2)
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
fi.Fresh = true
done := make(chan error, 1)
go func() {
_, err := disk.RenameData(gate, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
done <- err
}()
<-gate.entered
cancel()
synctest.Wait()
select {
case err := <-done:
t.Errorf("RenameData returned %v while its disk worker was still running", err)
default:
}
close(gate.release)
synctest.Wait()
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Errorf("canceled staged write: %v", err)
}
default:
}
if _, err := os.Stat(filepath.Join(root, "bucket", "object", xlStorageFormatFile)); !os.IsNotExist(err) {
t.Fatalf("canceled staged write committed metadata: %v", err)
}
})
}
func TestRenameCancellationRPCWaitsForDisk(t *testing.T) {
for _, operation := range []string{"regular", "inline", "write-metadata", "update-metadata", "undo"} {
t.Run(operation, func(t *testing.T) {
tg, err := grid.SetupTestGrid(2)
if err != nil {
t.Fatal(err)
}
defer tg.Cleanup()
entered, release := make(chan struct{}), make(chan struct{})
var committed atomic.Bool
handle := func(*RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
close(entered)
<-release
committed.Store(true)
return &RenameDataResp{OldDataDir: "previous-generation"}, nil
}
const route = "/cancel-commit"
if err := storageRenameDataRPC.Register(tg.Managers[1], handle, route); err != nil {
t.Fatal(err)
}
if err := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
defer p.Recycle()
return handle(&p.RenameDataHandlerParams)
}, route); err != nil {
t.Fatal(err)
}
meta := func(*MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
_, err := handle(nil)
return grid.NewNoPayload(), err
}
if err := storageWriteMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
t.Fatal(err)
}
if err := storageUpdateMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
t.Fatal(err)
}
if err := storageDeleteVersionRPC.Register(tg.Managers[1], func(*DeleteVersionHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
return meta(nil)
}, route); err != nil {
t.Fatal(err)
}
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
client.diskID.Store(new(string))
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
fi := FileInfo{}
if operation == "inline" {
fi.Data = []byte("payload")
}
type result struct {
resp RenameDataResp
err error
}
done := make(chan result, 1)
go func() {
var resp RenameDataResp
var err error
switch operation {
case "write-metadata":
err = client.WriteMetadata(ctx, "", "bucket", "object", fi)
case "update-metadata":
err = client.UpdateMetadata(ctx, "bucket", "object", fi, UpdateMetadataOpts{})
case "undo":
err = client.DeleteVersion(ctx, "bucket", "object", fi, false, DeleteOptions{UndoWrite: true})
default:
resp, err = client.RenameData(ctx, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
}
done <- result{resp, err}
}()
<-entered
cancel()
var early *result
select {
case r := <-done:
early = &r
t.Errorf("RPC returned %v before the remote disk completed", r.err)
case <-time.After(100 * time.Millisecond):
}
close(release)
if early == nil {
select {
case r := <-done:
wantOld := ""
if operation == "regular" || operation == "inline" {
wantOld = "previous-generation"
}
if r.err != nil || r.resp.OldDataDir != wantOld || !committed.Load() {
t.Errorf("lost completed result needed for rollback: %+v", r)
}
case <-time.After(5 * time.Second):
t.Fatal("RPC did not finish after disk completion")
}
}
})
}
}
func TestRenameCancellationAlreadyCanceledRPC(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
cancel()
// A canceled call must not access the connection or send a new mutation.
client := &storageRESTClient{}
client.diskID.Store(new(string))
defer func() {
if p := recover(); p != nil {
t.Errorf("canceled mutation reached the connection: %v", p)
}
}()
_, err := client.RenameData(ctx, minioMetaTmpBucket, "staged", FileInfo{}, "bucket", "object", RenameOptions{})
if !errors.Is(err, context.Canceled) {
t.Fatalf("pre-canceled RenameData: %v", err)
}
}
type gatedRenameDisk struct {
StorageAPI
client *storageRESTClient
}
func (d gatedRenameDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
return d.client.RenameData(ctx, sv, sp, fi, dv, dp, opts)
}
func (d gatedRenameDisk) WriteMetadata(ctx context.Context, ov, volume, path string, fi FileInfo) error {
return d.client.WriteMetadata(ctx, ov, volume, path, fi)
}
func (d gatedRenameDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, force bool, opts DeleteOptions) error {
return d.client.DeleteVersion(ctx, volume, path, fi, force, opts)
}
// Use real disk commits behind the real RPC client. The barrier fixes the
// order: every commit has started, then cancellation, then remote completion.
func TestRenameCancellationKeepsLockUntilRollback(t *testing.T) {
for _, tc := range []struct {
size int
versioned bool
copy bool
shared bool
}{
{32, false, false, false},
{2 << 20, false, false, false},
{32, true, false, false},
{2 << 20, true, false, false},
{32, false, true, false},
{2 << 20, false, true, false},
{32, false, false, true},
{2 << 20, false, false, true},
} {
t.Run(fmt.Sprintf("bytes-%d/other-versions-%t/copy-%t/shared-%t", tc.size, tc.versioned, tc.copy, tc.shared), func(t *testing.T) {
size := tc.size
z, bucket := consistencyPools(t)
object := "cancel-rollback"
old := strings.Repeat("a", size)
base := UTCNow().Add(-time.Hour)
var sharedVersions [2]string
for pool := range 2 {
if tc.versioned {
putConsistencyObject(t, z, bucket, object, pool, "retained-version", ObjectOptions{Versioned: true, MTime: base.Add(-time.Minute)})
}
putConsistencyObject(t, z, bucket, object, pool, old, ObjectOptions{MTime: base})
if tc.shared {
info, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
info.metadataOnly, info.versionOnly = true, true
copied, err := z.serverPools[pool].getHashedSet(object).CopyObject(t.Context(), bucket, object, bucket, object, info,
ObjectOptions{}, ObjectOptions{Versioned: true, MTime: base.Add(time.Minute)})
if err != nil {
t.Fatal(err)
}
sharedVersions[pool] = copied.VersionID
}
}
copyInfo, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
copyInfo.metadataOnly = true
copyInfo.UserDefined["copy-test"] = "canceled"
// Exercise the production rollback handler and wire options as
// well as the client. A local DeleteVersion bypasses that boundary.
globalLocalDrivesMu.Lock()
previousSets, previousMap := globalLocalSetDrives, globalLocalDrivesMap
globalLocalSetDrives = nil
globalLocalDrivesMap = make(map[string]StorageAPI)
for _, pool := range z.serverPools {
for _, disk := range pool.getHashedSet(object).getDisks() {
globalLocalDrivesMap[disk.Endpoint().String()] = disk
}
}
globalLocalDrivesMu.Unlock()
t.Cleanup(func() {
globalLocalDrivesMu.Lock()
globalLocalSetDrives, globalLocalDrivesMap = previousSets, previousMap
globalLocalDrivesMu.Unlock()
})
tg, err := grid.SetupTestGrid(2)
if err != nil {
t.Fatal(err)
}
defer tg.Cleanup()
entered := make(chan struct{}, 32)
release := make(chan struct{})
var releaseOnce sync.Once
unblock := func() { releaseOnce.Do(func() { close(release) }) }
defer unblock()
var completed atomic.Int32
var injectCommitFailure atomic.Bool
injectCommitFailure.Store(true)
for pi, pool := range z.serverPools {
set := pool.getHashedSet(object)
disks, switchDisks := installListingDiskSwitch(set)
defer switchDisks(disks)
wrapped := append([]StorageAPI(nil), disks...)
for i, disk := range disks {
route := fmt.Sprintf("/cancel-rollback/%d/%d", pi, i)
server := storageRESTServer{endpoint: disk.Endpoint()}
if err := storageDeleteVersionRPC.Register(tg.Managers[1], server.DeleteVersionHandler, route); err != nil {
t.Fatal(err)
}
handle := func(p *RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
entered <- struct{}{}
<-release
// Cancellation alone must not undo a successful quorum.
// This rollback fixture also fails half the disks, so its
// original payload/lock assertions require a real failed commit.
defer completed.Add(1)
if i >= len(disks)/2 && injectCommitFailure.Load() {
return nil, grid.NewRemoteErr(errDiskNotFound)
}
resp, err := disk.RenameData(context.Background(), p.SrcVolume, p.SrcPath, p.FI, p.DstVolume, p.DstPath, p.Opts)
if err != nil {
return nil, grid.NewRemoteErr(err)
}
return &resp, nil
}
if err := storageRenameDataRPC.Register(tg.Managers[1], handle, route); err != nil {
t.Fatal(err)
}
if err := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
defer p.Recycle()
return handle(&p.RenameDataHandlerParams)
}, route); err != nil {
t.Fatal(err)
}
if err := storageWriteMetadataRPC.Register(tg.Managers[1], func(p *MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
if p.Volume == bucket && p.FilePath == object {
entered <- struct{}{}
<-release
defer completed.Add(1)
}
err := disk.WriteMetadata(context.Background(), p.OrigVolume, p.Volume, p.FilePath, p.FI)
return grid.NewNoPayload(), grid.NewRemoteErr(err)
}, route); err != nil {
t.Fatal(err)
}
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
client.diskID.Store(new(string))
wrapped[i] = gatedRenameDisk{disk, client}
}
switchDisks(wrapped)
}
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
pending := bytes.Repeat([]byte("b"), size)
reader := mustGetPutObjReader(t, bytes.NewReader(pending), int64(size), "", "")
done := make(chan error, 1)
go func() {
var err error
if tc.copy {
_, err = z.CopyObject(ctx, bucket, object, bucket, object, copyInfo, ObjectOptions{}, ObjectOptions{})
} else {
_, err = z.PutObject(ctx, bucket, object, reader, ObjectOptions{})
}
done <- err
}()
n := z.serverPools[0].getHashedSet(object).setDriveCount
for range n {
select {
case <-entered:
case err := <-done:
t.Fatalf("write ended before commit barrier: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("write did not reach every disk")
}
}
cancel()
var early bool
select {
case err := <-done:
early = true
t.Errorf("mutation returned before remote commits completed: %v", err)
case <-time.After(100 * time.Millisecond):
}
lock := z.NewNSLock(bucket, object)
lockCtx, cancelLock := context.WithTimeout(t.Context(), 50*time.Millisecond)
lc, lockErr := lock.GetLock(lockCtx, newDynamicTimeout(50*time.Millisecond, 50*time.Millisecond))
cancelLock()
if lockErr == nil {
lock.Unlock(lc)
t.Error("namespace lock released before remote commits and rollback")
}
unblock()
if early {
// Keep the old-code failure deterministic; don't race cleanup
// against handlers intentionally still running at the barrier.
deadline := time.Now().Add(5 * time.Second)
for completed.Load() != int32(n) {
if time.Now().After(deadline) {
t.Fatal("remote handlers did not finish")
}
time.Sleep(time.Millisecond)
}
return
}
select {
case err := <-done:
if err == nil {
t.Fatal("canceled write unexpectedly acknowledged")
}
case <-time.After(5 * time.Second):
t.Fatal("canceled write did not finish rollback")
}
if completed.Load() != int32(n) {
t.Fatal("returned without accounting for every remote commit")
}
injectCommitFailure.Store(false)
for pi, pool := range z.serverPools {
for _, version := range []string{"", nullVersionID} {
r, err := pool.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: version})
if err != nil {
t.Fatalf("pool %d lost its acknowledged copy (version %q): %v", pi, version, err)
}
body, err := io.ReadAll(r)
r.Close()
if err != nil || string(body) != old {
t.Errorf("pool %d rollback lost acknowledged payload (version %q): bytes=%d err=%v", pi, version, len(body), err)
}
}
}
got, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(got)
got.Close()
if err != nil || string(body) != old {
t.Fatalf("rollback lost acknowledged payload: bytes=%d err=%v", len(body), err)
}
newBody := bytes.Repeat([]byte("c"), size)
if _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(newBody), int64(size), "", ""), ObjectOptions{}); err != nil {
t.Fatal(err)
}
got, err = z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
body, err = io.ReadAll(got)
got.Close()
if err != nil || !bytes.Equal(body, newBody) {
t.Fatalf("late commit replaced subsequent successful PUT: bytes=%d err=%v", len(body), err)
}
if tc.shared {
for pi, pool := range z.serverPools {
r, err := pool.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: sharedVersions[pi]})
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(r)
r.Close()
if err != nil || string(body) != old {
t.Errorf("commit cleanup destroyed shared version on pool %d: bytes=%d err=%v", pi, len(body), err)
}
}
}
})
}
}
func TestRenameCancellationStaleRollbackPreservesNewWrite(t *testing.T) {
z, bucket := consistencyPools(t)
const object = "stale-rollback"
putConsistencyObject(t, z, bucket, object, 0, "canceled-generation", ObjectOptions{})
disks := z.serverPools[0].getHashedSet(object).getDisks()
stale := make([]FileInfo, len(disks))
for i, disk := range disks {
var err error
stale[i], err = disk.ReadVersion(t.Context(), "", bucket, object, nullVersionID, ReadOptions{})
if err != nil {
t.Fatal(err)
}
}
putConsistencyObject(t, z, bucket, object, 0, "new-acknowledged-generation", ObjectOptions{})
for i, disk := range disks {
err := disk.DeleteVersion(t.Context(), bucket, object, stale[i], false, DeleteOptions{UndoWrite: true})
if err != nil && !errors.Is(err, errFileVersionNotFound) {
t.Fatal(err)
}
}
got, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
if err != nil {
t.Fatalf("stale rollback removed the new acknowledged write: %v", err)
}
defer got.Close()
body, err := io.ReadAll(got)
if err != nil || string(body) != "new-acknowledged-generation" {
t.Fatalf("stale rollback changed the new acknowledged payload: %q, %v", body, err)
}
}
+333
View File
@@ -0,0 +1,333 @@
// Copyright (c) 2015-2026 MinIO, Inc.
// Copyright (c) 2026 PGSTY
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio-go/v7"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/bucket/replication"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/once"
)
// ilmDeleteTarget models one replication destination for ILM expiry tests.
type ilmDeleteTarget struct {
arn, bucket string
deletes atomic.Int32
}
func newILMTestTarget(t *testing.T) *ilmDeleteTarget {
return &ilmDeleteTarget{arn: "arn:minio:replication::" + mustGetUUID() + ":bucket", bucket: getRandomBucketName()}
}
func registerILMTestTarget(t *testing.T, obj ObjectLayer, bucket string, target *ilmDeleteTarget, handler http.HandlerFunc) {
t.Helper()
if err := obj.MakeBucket(t.Context(), target.bucket, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
remote := httptest.NewServer(handler)
t.Cleanup(remote.Close)
client, err := minio.New(strings.TrimPrefix(remote.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
if err != nil {
t.Fatal(err)
}
globalBucketTargetSys.Lock()
globalBucketTargetSys.arnRemotesMap[target.arn] = arnTarget{Client: &TargetClient{Client: client, ARN: target.arn, Bucket: target.bucket}, lastRefresh: UTCNow()}
globalBucketTargetSys.targetsMap[bucket] = append(globalBucketTargetSys.targetsMap[bucket], madmin.BucketTarget{Arn: target.arn, TargetBucket: target.bucket})
globalBucketTargetSys.Unlock()
globalBucketTargetSys.hMutex.Lock()
globalBucketTargetSys.hc[client.EndpointURL().Host] = epHealth{Online: true}
globalBucketTargetSys.hMutex.Unlock()
}
func setILMTestReplicationConfig(t *testing.T, bucket, arn, targetBucket string) {
t.Helper()
cfg := replication.Config{}
rule := configs[0].Rules[0]
rule.Priority = 1
rule.Destination = replication.Destination{ARN: arn, Bucket: targetBucket}
cfg.Rules = append(cfg.Rules, rule)
cfg.RoleArn = arn
meta, err := globalBucketMetadataSys.Get(bucket)
if err != nil {
t.Fatal(err)
}
meta.replicationConfig = &cfg
globalBucketMetadataSys.Set(bucket, meta)
}
func newILMTestReplicationPool(ctx context.Context, obj ObjectLayer, stats *ReplicationStats) (restore func(), receive func() DeletedObjectReplicationInfo) {
oldPool := globalReplicationPool
p := &ReplicationPool{ctx: ctx, objLayer: obj, workers: []chan ReplicationWorkerOperation{make(chan ReplicationWorkerOperation, 8)}, stats: stats, mrfSaveCh: make(chan MRFReplicateEntry, 8)}
globalReplicationPool = once.NewSingleton[ReplicationPool]()
globalReplicationPool.Set(p)
return func() { globalReplicationPool = oldPool }, func() DeletedObjectReplicationInfo {
select {
case op := <-p.workers[0]:
d, ok := op.(DeletedObjectReplicationInfo)
if !ok {
return DeletedObjectReplicationInfo{}
}
return d
case <-time.After(3 * time.Second):
return DeletedObjectReplicationInfo{}
}
}
}
// TestReplicateILMExpiryDeleteMarker verifies that a delete marker created by
// ILM expiry carries delete replication state and is queued for replication,
// exactly like a marker created through the S3 DELETE API.
func TestReplicateILMExpiryDeleteMarker(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testReplicateILMExpiryDeleteMarker,
})
}
func testReplicateILMExpiryDeleteMarker(obj ObjectLayer, instanceType, bucket string, _ http.Handler, creds auth.Credentials, t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
stats := NewReplicationStats(ctx, nil)
oldStats := globalReplicationStats.Swap(stats)
defer globalReplicationStats.Store(oldStats)
const name = "ilm-object"
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
t.Fatal(err)
}
// DeleteObject refuses ILM expiry (Expiration.Expire) without lifecycle
// rules; a date in the past makes every version eligible immediately.
ilmXML := []byte(`<LifecycleConfiguration><Rule><ID>ilm-expire</ID><Status>Enabled</Status><Filter/><Expiration><Date>2020-01-01T00:00:00Z</Date></Expiration></Rule></LifecycleConfiguration>`)
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, ilmXML); err != nil {
t.Fatal(err)
}
oi, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{Versioned: true})
if err != nil {
t.Fatal(err)
}
target := newILMTestTarget(t)
registerILMTestTarget(t, obj, bucket, target, func(w http.ResponseWriter, r *http.Request) {
topts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true}
switch r.Method {
case http.MethodHead:
roi, rerr := obj.GetObjectInfo(r.Context(), target.bucket, name, topts)
if roi.DeleteMarker {
w.Header().Set(xhttp.AmzDeleteMarker, "true")
w.Header().Set(xhttp.AmzVersionID, roi.VersionID)
}
if rerr != nil {
writeErrorResponseHeadersOnly(w, toAPIError(r.Context(), rerr))
return
}
w.WriteHeader(http.StatusOK)
case http.MethodDelete:
target.deletes.Add(1)
if topts.VersionID == "" {
t.Error("missing remote versionId")
}
topts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"
topts.ReplicationRequest = true
topts.SetReplicaStatus(replication.Replica)
if _, err := obj.DeleteObject(r.Context(), target.bucket, name, topts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
return
}
w.WriteHeader(http.StatusNoContent)
default:
t.Errorf("unexpected remote method %s", r.Method)
}
})
setILMTestReplicationConfig(t, bucket, target.arn, target.bucket)
restorePool, receive := newILMTestReplicationPool(ctx, obj, stats)
defer restorePool()
if ok := applyExpiryOnNonTransitionedObjects(ctx, obj, oi, lifecycle.Event{Action: lifecycle.DeleteAction}, lcEventSrc_Scanner); !ok {
t.Fatalf("%s: ILM expiry did not apply", instanceType)
}
deletion := receive()
if deletion.ObjectName == "" {
t.Fatalf("%s: ILM delete marker was not queued for replication", instanceType)
}
if !deletion.DeleteMarker || deletion.VersionID != "" || deletion.DeleteMarkerVersionID == "" {
t.Fatalf("%s: unexpected queued delete: %+v", instanceType, deletion)
}
if deletion.EventType != ReplicateILMDelete {
t.Fatalf("%s: queued event type = %q, want %q", instanceType, deletion.EventType, ReplicateILMDelete)
}
// The ILM-created marker must carry pending replication state.
markerVID := deletion.DeleteMarkerVersionID
moi, merr := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: markerVID, Versioned: true})
if !isErrMethodNotAllowed(merr) || !moi.DeleteMarker {
t.Fatalf("%s: ILM marker missing: %v %v", instanceType, moi, merr)
}
if moi.ReplicationStatus != replication.Pending {
t.Fatalf("%s: ILM marker replication status = %q, want PENDING", instanceType, moi.ReplicationStatus)
}
result := replicateDelete(ctx, deletion, obj)
if result.ReplicationStatus() != replication.Completed {
t.Fatalf("%s: marker replication result: %+v", instanceType, result)
}
if target.deletes.Load() != 1 {
t.Fatalf("%s: target saw %d deletes, want 1", instanceType, target.deletes.Load())
}
// The marker must exist on the target.
roi, rerr := obj.GetObjectInfo(ctx, target.bucket, name, ObjectOptions{VersionID: markerVID, Versioned: true})
if !isErrMethodNotAllowed(rerr) || !roi.DeleteMarker {
t.Fatalf("%s: target marker missing: %v %v", instanceType, roi, rerr)
}
// The source marker records the completed replication per target. The
// composite ReplicationStatus field is unreliable on read-back for
// markers carrying a replication timestamp, so assert the raw state.
moi, merr = obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: markerVID})
if !isErrMethodNotAllowed(merr) || !strings.Contains(moi.ReplicationStatusInternal, "=COMPLETED;") {
t.Fatalf("%s: source marker state = %q (err %v), want COMPLETED", instanceType, moi.ReplicationStatusInternal, merr)
}
}
// TestReplicateILMExpiryNoncurrentPurge verifies that a noncurrent version
// removed by ILM carries a pending version purge state, is queued for delete
// replication and converges on the target.
func TestReplicateILMExpiryNoncurrentPurge(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testReplicateILMExpiryNoncurrentPurge,
})
}
func testReplicateILMExpiryNoncurrentPurge(obj ObjectLayer, instanceType, bucket string, _ http.Handler, creds auth.Credentials, t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
stats := NewReplicationStats(ctx, nil)
oldStats := globalReplicationStats.Swap(stats)
defer globalReplicationStats.Store(oldStats)
const name = "ilm-object"
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
t.Fatal(err)
}
if _, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, strings.NewReader("v1"), 2, "", ""), ObjectOptions{Versioned: true}); err != nil {
t.Fatal(err)
}
v1Info, err := obj.PutObject(ctx, bucket, name, mustGetPutObjReader(t, strings.NewReader("v2"), 2, "", ""), ObjectOptions{Versioned: true})
if err != nil {
t.Fatal(err)
}
v1 := v1Info.VersionID
if v1 == "" {
t.Fatalf("%s: missing version ID on second version", instanceType)
}
target := newILMTestTarget(t)
registerILMTestTarget(t, obj, bucket, target, func(w http.ResponseWriter, r *http.Request) {
topts := ObjectOptions{VersionID: r.URL.Query().Get("versionId"), Versioned: true}
switch r.Method {
case http.MethodDelete:
target.deletes.Add(1)
if topts.VersionID == "" {
t.Error("missing remote versionId")
}
topts.DeleteMarker = r.Header.Get(xhttp.MinIOSourceDeleteMarker) == "true"
topts.ReplicationRequest = true
topts.SetReplicaStatus(replication.Replica)
if _, err := obj.DeleteObject(r.Context(), target.bucket, name, topts); err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
return
}
w.WriteHeader(http.StatusNoContent)
default:
t.Errorf("unexpected remote method %s", r.Method)
}
})
// Pre-create a marker version on the target with the same version ID as
// the noncurrent version being purged, so the purge can be observed.
if _, err := obj.DeleteObject(ctx, target.bucket, name, ObjectOptions{VersionID: v1, Versioned: true, DeleteMarker: true, ReplicationRequest: true}); err != nil {
t.Fatal(err)
}
setILMTestReplicationConfig(t, bucket, target.arn, target.bucket)
restorePool, receive := newILMTestReplicationPool(ctx, obj, stats)
defer restorePool()
// The scanner evaluates noncurrent version expiry per version; build the
// delete entry exactly the way the scanner does for the oldest version.
vi, verr := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: v1, Versioned: true})
if verr != nil {
t.Fatalf("%s: missing noncurrent version: %v", instanceType, verr)
}
otd := ilmPurgeToDelete(ctx, bucket, vi, true)
if otd.VersionPurgeStatus != replication.VersionPurgePending || otd.VersionPurgeStatuses == "" || otd.ReplicateDecisionStr == "" {
t.Fatalf("%s: ILM purge entry carries no replication state: %+v", instanceType, otd)
}
if plain := ilmPurgeToDelete(ctx, bucket, vi, false); plain.VersionPurgeStatus != "" || plain.ReplicateDecisionStr != "" {
t.Fatalf("%s: non-replicating purge entry carries state: %+v", instanceType, plain)
}
deleteObjectVersions(ctx, obj, bucket, []ObjectToDelete{otd}, []lifecycle.Event{{Action: lifecycle.DeleteVersionAction}})
// The version must remain locally as a tombstone with pending purge
// state until replication completes.
pi, perr := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: v1, Versioned: true})
if !isErrMethodNotAllowed(perr) {
t.Fatalf("%s: purged version metadata lost: %v", instanceType, perr)
}
if pi.VersionPurgeStatus != replication.VersionPurgePending {
t.Fatalf("%s: purged version state = %q, want PENDING", instanceType, pi.VersionPurgeStatus)
}
deletion := receive()
if deletion.ObjectName == "" {
t.Fatalf("%s: ILM noncurrent purge was not queued for replication", instanceType)
}
if deletion.VersionID != v1 || deletion.DeleteMarkerVersionID != "" || deletion.DeleteMarker {
t.Fatalf("%s: unexpected queued purge: %+v", instanceType, deletion)
}
result := replicateDelete(ctx, deletion, obj)
if result.VersionPurgeStatus() != replication.VersionPurgeComplete {
t.Fatalf("%s: purge replication result: %+v", instanceType, result)
}
if target.deletes.Load() != 1 {
t.Fatalf("%s: target saw %d deletes, want 1", instanceType, target.deletes.Load())
}
// The purge must have converged on the target.
if _, terr := obj.GetObjectInfo(ctx, target.bucket, name, ObjectOptions{VersionID: v1, Versioned: true}); !isErrVersionNotFound(terr) && !isErrObjectNotFound(terr) {
t.Fatalf("%s: target version still present: %v", instanceType, terr)
}
// Once the purge completes the local tombstone is freed and the version
// is gone on both sides.
if _, terr := obj.GetObjectInfo(ctx, bucket, name, ObjectOptions{VersionID: v1}); !isErrVersionNotFound(terr) && !isErrObjectNotFound(terr) {
t.Fatalf("%s: purged version still present locally: %v", instanceType, terr)
}
}
+187
View File
@@ -0,0 +1,187 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
"context"
"fmt"
"io"
"net"
"net/http"
"os"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
)
// Exercise the production RequestServer driver against a real S3 backend.
// The library's filesystem Server and its ReadOnly option are a different path.
func TestSFTPRequestServerObjectOperations(t *testing.T) {
for _, secure := range []bool{false, true} {
t.Run(fmt.Sprintf("TLS=%t", secure), func(t *testing.T) {
c := &check{t, "ErasureSD"}
suite := newTestSuiteIAM(TestSuiteCommon{serverType: "ErasureSD", signer: signerV4, secure: secure}, false)
suite.SetUpSuite(c)
t.Cleanup(func() { suite.TearDownSuite(c) })
previousTLS, previousTransport := globalIsTLS, globalRemoteFTPClientTransport
globalIsTLS = secure
globalRemoteFTPClientTransport = suite.TestSuiteCommon.client.Transport.(*http.Transport)
t.Cleanup(func() {
globalIsTLS, globalRemoteFTPClientTransport = previousTLS, previousTransport
})
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
defer cancel()
connect := func(policy string) *sftp.Client {
t.Helper()
accessKey, secretKey := mustGenerateCredentials(c)
if err := suite.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
t.Fatal(err)
}
if _, err := suite.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{Policies: []string{policy}, User: accessKey}); err != nil {
t.Fatal(err)
}
perms, err := sshPasswordAuth(newSSHConnMock(accessKey), []byte(secretKey))
if err != nil {
t.Fatal(err)
}
return newSFTPRequestTestClient(t, perms)
}
client := connect("readwrite")
bucket := "/" + getRandomBucketName()
if err := client.Mkdir(bucket); err != nil {
t.Fatal(err)
}
read := func(client *sftp.Client, name string) []byte {
t.Helper()
f, err := client.Open(name)
if err != nil {
t.Fatal(err)
}
defer f.Close()
data, err := io.ReadAll(f)
if err != nil {
t.Fatal(err)
}
return data
}
for _, size := range []int{0, 32767, 32768, 65537} {
name := fmt.Sprintf("%s/中文-café-%d", bucket, size)
data := bytes.Repeat([]byte{byte(size)}, size)
f, err := client.Create(name)
if err != nil {
t.Fatal(err)
}
if n, err := f.ReadFrom(bytes.NewReader(data)); err != nil || n != int64(size) {
f.Close()
t.Fatalf("upload %s: n=%d, err=%v", name, n, err)
}
if err := f.Close(); err != nil {
t.Fatal(err)
}
if got := read(client, name); !bytes.Equal(got, data) {
t.Fatalf("download %s: byte mismatch", name)
}
info, err := client.Stat(name)
if err != nil || info.Size() != int64(size) {
t.Fatalf("stat %s: info=%v, err=%v", name, info, err)
}
}
entries, err := client.ReadDir(bucket)
if err != nil || len(entries) != 4 {
t.Fatalf("list: %d entries, err=%v", len(entries), err)
}
name := bucket + "/中文-café-65537"
original := read(client, name)
// Include list permission because Client.Remove also tries Rmdir;
// both requests must observe the denied delete, not a failed listing.
readonly := connect("consolereadonly")
if !bytes.Equal(read(readonly, name), original) {
t.Fatal("readonly download changed content")
}
if f, err := readonly.OpenFile(name, os.O_WRONLY|os.O_TRUNC); err == nil {
_, writeErr := f.Write([]byte("forbidden"))
closeErr := f.Close()
// The streaming driver can acknowledge buffered bytes before S3
// rejects the PUT. Verify the object, not just the write status.
t.Logf("readonly write: write=%v close=%v", writeErr, closeErr)
}
if !bytes.Equal(read(client, name), original) {
t.Fatal("readonly upload modified content")
}
if err := readonly.Remove(name); err == nil {
t.Fatal("readonly delete succeeded")
}
if !bytes.Equal(read(client, name), original) {
t.Fatal("denied operation modified content")
}
// Remove retries Rmdir, which can mask a listing error for a user
// without ListBucket. Check the stored object even if it returns nil.
withoutList := connect("readonly")
withoutList.Remove(name)
if !bytes.Equal(read(client, name), original) {
t.Fatal("readonly user without listing permission deleted content")
}
for operation, err := range map[string]error{
"hardlink": client.Link(name, bucket+"/hardlink"),
"symlink": client.Symlink(name, bucket+"/symlink"),
"rename": client.Rename(name, bucket+"/renamed"),
} {
if status, ok := err.(*sftp.StatusError); !ok || status.Code != uint32(sftp.ErrSSHFxOpUnsupported) {
t.Fatalf("%s: expected unsupported operation, got %v", operation, err)
}
}
for _, entry := range entries {
if err := client.Remove(bucket + "/" + entry.Name()); err != nil {
t.Fatal(err)
}
}
entries, err = client.ReadDir(bucket)
if err != nil || len(entries) != 0 {
t.Fatalf("list after deletion: %d entries, err=%v", len(entries), err)
}
// SILO exposes a missing object as a directory for stat-then-list
// clients. Keep that existing virtual-directory behavior.
if info, err := client.Stat(name); err != nil || !info.IsDir() {
t.Fatalf("stat virtual directory: info=%v, err=%v", info, err)
}
if err := client.RemoveDirectory(bucket); err != nil {
t.Fatal(err)
}
})
}
}
func newSFTPRequestTestClient(t *testing.T, perms *ssh.Permissions) *sftp.Client {
t.Helper()
serverConn, clientConn := net.Pipe()
deadline := time.Now().Add(time.Minute)
serverConn.SetDeadline(deadline)
clientConn.SetDeadline(deadline)
server := sftp.NewRequestServer(serverConn, NewSFTPDriver(perms, "127.0.0.1"), sftp.WithRSAllocator())
done := make(chan struct{})
go func() {
defer close(done)
server.Serve()
}()
t.Cleanup(func() {
clientConn.Close()
server.Close()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Error("SFTP request server did not stop")
}
})
client, err := sftp.NewClientPipe(clientConn, clientConn)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { client.Close() })
return client
}
+3 -8
View File
@@ -102,14 +102,9 @@ func TestGuardedStorageCoversEveryPathMethod(t *testing.T) {
} }
} }
// TestGuardChecksUnreachableFields pins the guard on path fields that exist on // OldDataDir can reach renameAll during rollback. Guard it independently of
// the wire but that no handler currently reads, so an end-to-end test cannot // which handlers currently forward deletion options.
// observe them. DeleteVersionHandler hardcodes `opts := DeleteOptions{}` and func TestGuardChecksDeleteOptionPaths(t *testing.T) {
// discards p.Opts, which means DeleteOptions.OldDataDir - a value that reaches
// renameAll() with no containment of its own - is unreachable by accident
// rather than by design. If someone later plumbs p.Opts through, the guard is
// already there; this test is what stops it from being removed as "dead".
func TestGuardChecksUnreachableFields(t *testing.T) {
g := guardedStorage{poisonStorage{}} g := guardedStorage{poisonStorage{}}
ctx := context.Background() ctx := context.Background()
+1 -2
View File
@@ -415,8 +415,7 @@ func (s *storageRESTServer) DeleteVersionHandler(p *DeleteVersionHandlerParams)
filePath := p.FilePath filePath := p.FilePath
forceDelMarker := p.ForceDelMarker forceDelMarker := p.ForceDelMarker
opts := DeleteOptions{} err := s.getStorage().DeleteVersion(context.Background(), volume, filePath, p.FI, forceDelMarker, p.Opts)
err := s.getStorage().DeleteVersion(context.Background(), volume, filePath, p.FI, forceDelMarker, opts)
return np, grid.NewRemoteErr(err) return np, grid.NewRemoteErr(err)
} }
+3 -6
View File
@@ -231,12 +231,9 @@ func TestStorageRESTTraversalRejected(t *testing.T) {
{"DeleteVersion-DataDir", func() error { {"DeleteVersion-DataDir", func() error {
return restClient.DeleteVersion(ctx, "foo", "obj", badFI(p), false, DeleteOptions{}) return restClient.DeleteVersion(ctx, "foo", "obj", badFI(p), false, DeleteOptions{})
}}, }},
// DeleteOptions.OldDataDir is guarded too, but cannot be exercised {"DeleteVersion-OldDataDir", func() error {
// from here: DeleteVersionHandler hardcodes `opts := DeleteOptions{}` return restClient.DeleteVersion(ctx, "foo", "obj", badFI(""), false, DeleteOptions{UndoWrite: true, OldDataDir: p})
// and never reads the wire value, so the field is unreachable today. }},
// That is an accidental mitigation, not a control - see
// TestGuardChecksUnreachableFields for the unit-level assertion that
// the guard is ready if anyone ever plumbs it through.
{"RenameData-srcPath", func() error { {"RenameData-srcPath", func() error {
_, err := restClient.RenameData(ctx, "foo", p, badFI(""), "bar", "dst", RenameOptions{}) _, err := restClient.RenameData(ctx, "foo", p, badFI(""), "bar", "dst", RenameOptions{})
return err return err
+139
View File
@@ -0,0 +1,139 @@
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"sync"
"testing"
"testing/synctest"
"github.com/minio/minio/internal/grid"
)
type cancelableRenameDisk struct {
StorageAPI
entered chan<- struct{}
release <-chan struct{}
}
func (d cancelableRenameDisk) RenameData(ctx context.Context, _, _ string, _ FileInfo, _, _ string, _ RenameOptions) (RenameDataResp, error) {
d.entered <- struct{}{}
select {
case <-ctx.Done():
return RenameDataResp{}, ctx.Err()
case <-d.release:
return RenameDataResp{}, errDiskNotFound
}
}
func TestStorageTimeoutCoordinatorCancellation(t *testing.T) {
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
entered, release := make(chan struct{}, 4), make(chan struct{})
defer close(release)
disks := make([]StorageAPI, 4)
metadata := make([]FileInfo, 4)
for i := range disks {
disks[i] = cancelableRenameDisk{entered: entered, release: release}
metadata[i] = FileInfo{Deleted: true}
}
done := make(chan error, 1)
go func() {
_, _, _, err := renameData(ctx, disks, minioMetaTmpBucket, "staged", metadata, "bucket", "object", 3)
done <- err
}()
for range disks {
<-entered
}
cancel()
synctest.Wait()
select {
case err := <-done:
if err == nil {
t.Fatal("canceled commit unexpectedly succeeded")
}
default:
t.Fatal("coordinator did not propagate cancellation to disk calls")
}
})
}
type rollbackOptionsDisk struct {
StorageAPI
mu sync.Mutex
opts DeleteOptions
}
func (d *rollbackOptionsDisk) DeleteVersion(_ context.Context, _, _ string, _ FileInfo, _ bool, opts DeleteOptions) error {
d.mu.Lock()
defer d.mu.Unlock()
d.opts = opts
return nil
}
func TestStorageRollbackRPCOptions(t *testing.T) {
endpoint, err := NewEndpoint(t.TempDir())
if err != nil {
t.Fatal(err)
}
disk := &rollbackOptionsDisk{}
globalLocalDrivesMu.Lock()
oldDrives, oldSets := globalLocalDrivesMap, globalLocalSetDrives
globalLocalDrivesMap = map[string]StorageAPI{endpoint.String(): disk}
globalLocalSetDrives = nil
globalLocalDrivesMu.Unlock()
defer func() {
globalLocalDrivesMu.Lock()
globalLocalDrivesMap, globalLocalSetDrives = oldDrives, oldSets
globalLocalDrivesMu.Unlock()
}()
tg, err := grid.SetupTestGrid(2)
if err != nil {
t.Fatal(err)
}
defer tg.Cleanup()
server := storageRESTServer{endpoint: endpoint}
const route = "/rollback-options"
if err := storageDeleteVersionRPC.Register(tg.Managers[1], server.DeleteVersionHandler, route); err != nil {
t.Fatal(err)
}
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
client.diskID.Store(new(string))
want := DeleteOptions{UndoWrite: true, OldDataDir: mustGetUUID()}
if err := client.DeleteVersion(t.Context(), "bucket", "object", FileInfo{}, false, want); err != nil {
t.Fatal(err)
}
disk.mu.Lock()
defer disk.mu.Unlock()
if disk.opts != want {
t.Fatalf("remote handler lost rollback options: got %+v, want %+v", disk.opts, want)
}
}
func TestStoragePrereleaseMetadataHidden(t *testing.T) {
disk, _, err := newXLStorageTestSetup(t)
if err != nil {
t.Fatal(err)
}
defer disk.Close()
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
t.Fatal(err)
}
fi := newFileInfo("object", 2, 2)
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
fi.Metadata = map[string]string{rollbackIDKey: mustGetUUID(), "user-key": "value"}
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
t.Fatal(err)
}
got, err := disk.ReadVersion(t.Context(), "", "bucket", "object", "", ReadOptions{})
if err != nil {
t.Fatal(err)
}
public := cleanMetadata(got.Metadata)
if _, ok := public[rollbackIDKey]; ok || public["user-key"] != "value" {
t.Fatalf("pre-release metadata exposed or user metadata lost: %+v", public)
}
}
+228
View File
@@ -0,0 +1,228 @@
// Copyright (c) 2026 Feng Ruohang
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"errors"
"sync"
"sync/atomic"
"testing"
"testing/synctest"
"time"
"github.com/minio/minio/internal/grid"
)
// TrackDiskHealth makes the first lookup. Pause the real disk worker at its
// next health callback, without requiring a broken drive or a wall-clock sleep.
type timeoutDiskContext struct {
context.Context
calls atomic.Int32
entered, release chan struct{}
tracker healthDiskCtxValue
pauseDeadline bool
once sync.Once
}
func (c *timeoutDiskContext) Deadline() (time.Time, bool) {
if c.pauseDeadline {
// Metadata operations create a nested deadline in their disk worker.
c.once.Do(func() { close(c.entered); <-c.release })
}
return c.Context.Deadline()
}
func (c *timeoutDiskContext) Value(key any) any {
if _, ok := key.(healthDiskCtxKey); ok {
if !c.pauseDeadline && c.calls.Add(1) == 2 {
close(c.entered)
<-c.release
}
return &c.tracker
}
return c.Context.Value(key)
}
func TestStorageTimeoutLocal(t *testing.T) {
for _, operation := range []string{"rename", "rename-cancel", "write-metadata", "update-metadata", "undo"} {
t.Run(operation, func(t *testing.T) {
disk, _, err := newXLStorageTestSetup(t)
if err != nil {
t.Fatal(err)
}
defer disk.Close()
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
t.Fatal(err)
}
fi := newFileInfo("object", 2, 2)
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
fi.Metadata = map[string]string{"test": "original"}
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
t.Fatal(err)
}
synctest.Test(t, func(t *testing.T) {
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
var lastSuccess int64
gate := &timeoutDiskContext{
Context: ctx, entered: make(chan struct{}), release: make(chan struct{}),
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
pauseDeadline: operation != "rename" && operation != "rename-cancel",
}
unblock := sync.OnceFunc(func() { close(gate.release) })
defer unblock()
done := make(chan error, 1)
go func() {
var err error
switch operation {
case "rename", "rename-cancel":
_, err = disk.RenameData(gate, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
case "write-metadata":
err = disk.WriteMetadata(gate, "", "bucket", "object", fi)
case "update-metadata":
err = disk.UpdateMetadata(gate, "bucket", "object", fi, UpdateMetadataOpts{})
case "undo":
err = disk.DeleteVersion(gate, "bucket", "object", fi, false, DeleteOptions{UndoWrite: true})
}
done <- err
}()
select {
case <-gate.entered:
case err := <-done:
t.Fatalf("operation did not reach the disk worker: %v", err)
}
want := context.DeadlineExceeded
if operation == "rename-cancel" {
cancel()
want = context.Canceled
} else {
time.Sleep(globalDriveConfig.GetMaxTimeout() + time.Second)
}
synctest.Wait()
select {
case err := <-done:
if !errors.Is(err, want) {
t.Fatalf("expected %v, got %v", want, err)
}
default:
t.Fatal("operation still waits for the blocked worker after its deadline")
}
// A timed-out worker must not leave a new physical mutation lock
// preventing subsequent work on this object.
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
t.Fatalf("subsequent metadata write: %v", err)
}
unblock()
// synctest waits for the released worker before disk teardown.
})
})
}
}
func TestStorageTimeoutRPC(t *testing.T) {
for _, operation := range []string{"rename", "inline", "write-metadata", "update-metadata", "undo"} {
for _, deadline := range []bool{false, true} {
mode := "cancel"
if deadline {
mode = "deadline"
}
t.Run(operation+"/"+mode, func(t *testing.T) {
tg, err := grid.SetupTestGrid(2)
if err != nil {
t.Fatal(err)
}
defer tg.Cleanup()
entered, release, finished := make(chan struct{}), make(chan struct{}), make(chan struct{})
unblock := sync.OnceFunc(func() { close(release) })
defer unblock()
handle := func() {
close(entered)
<-release
close(finished)
}
const route = "/storage-timeout"
if err := storageRenameDataRPC.Register(tg.Managers[1], func(*RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
handle()
return &RenameDataResp{}, nil
}, route); err != nil {
t.Fatal(err)
}
if err := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
defer p.Recycle()
handle()
return &RenameDataResp{}, nil
}, route); err != nil {
t.Fatal(err)
}
meta := func(*MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
handle()
return grid.NewNoPayload(), nil
}
if err := storageWriteMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
t.Fatal(err)
}
if err := storageUpdateMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
t.Fatal(err)
}
if err := storageDeleteVersionRPC.Register(tg.Managers[1], func(p *DeleteVersionHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
if !p.Opts.UndoWrite || p.Opts.OldDataDir != "previous" {
t.Error("rollback options were lost on the wire")
}
return meta(nil)
}, route); err != nil {
t.Fatal(err)
}
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
client.diskID.Store(new(string))
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
if deadline {
var stop context.CancelFunc
ctx, stop = context.WithTimeout(ctx, 500*time.Millisecond)
defer stop()
}
done := make(chan error, 1)
go func() {
var err error
fi := FileInfo{}
switch operation {
case "write-metadata":
err = client.WriteMetadata(ctx, "", "bucket", "object", fi)
case "update-metadata":
err = client.UpdateMetadata(ctx, "bucket", "object", fi, UpdateMetadataOpts{})
case "undo":
err = client.DeleteVersion(ctx, "bucket", "object", fi, false, DeleteOptions{UndoWrite: true, OldDataDir: "previous"})
default:
if operation == "inline" {
fi.Data = []byte("payload")
}
_, err = client.RenameData(ctx, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
}
done <- err
}()
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("remote worker did not start")
}
want := context.DeadlineExceeded
if !deadline {
cancel()
want = context.Canceled
}
select {
case err := <-done:
if !errors.Is(err, want) {
t.Fatalf("expected %v, got %v", want, err)
}
case <-time.After(5 * time.Second):
t.Fatal("RPC still waits for the remote worker")
}
unblock()
<-finished
})
}
}
}
+108
View File
@@ -36,6 +36,7 @@ import (
"testing" "testing"
"time" "time"
goldap "github.com/go-ldap/ldap/v3"
"github.com/klauspost/compress/zip" "github.com/klauspost/compress/zip"
"github.com/minio/madmin-go/v3" "github.com/minio/madmin-go/v3"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
@@ -1095,6 +1096,7 @@ func TestIAMWithLDAPServerSuite(t *testing.T) {
suite.TestLDAPSTSServiceAccountsWithGroups(c) suite.TestLDAPSTSServiceAccountsWithGroups(c)
suite.TestLDAPAttributesLookup(c) suite.TestLDAPAttributesLookup(c)
suite.TestLDAPCyrillicUser(c) suite.TestLDAPCyrillicUser(c)
suite.TestLDAPLongNonASCIIDN(c)
suite.TestLDAPSlashDN(c) suite.TestLDAPSlashDN(c)
suite.TearDownSuite(c) suite.TearDownSuite(c)
}, },
@@ -3291,6 +3293,112 @@ func (s *TestSuiteIAM) TestLDAPCyrillicUser(c *check) {
} }
} }
// Normalized DNs hex-escape every non-ASCII byte, so a user in nested Cyrillic
// OUs has a DN longer than NAME_MAX: 160 bytes of UTF-8 become 384 characters.
// Records named after such a DN can never be stored by the object backend.
// The user is entitled only through a group, as is typical for AD deployments.
func (s *TestSuiteIAM) TestLDAPLongNonASCIIDN(c *check) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
const (
username = "Сотрудник"
parentOU = "ou=Дирекция информационных технологий,ou=people,ou=swengg,dc=min,dc=io"
childOU = "ou=Отдел разработки," + parentOU
userDN = "uid=" + username + "," + childOU
groupDN = "cn=long-dn-staff,ou=groups,ou=swengg,dc=min,dc=io"
)
normDN, err := ldap.NormalizeDN(userDN)
if err != nil {
c.Fatal(err)
}
if len(normDN) <= 255 {
c.Fatalf("normalized test DN must exceed 255 characters, got %d", len(normDN))
}
conn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
if err != nil {
c.Fatalf("LDAP connect failed: %v", err)
}
defer conn.Close()
if err = conn.Bind("cn=admin,dc=min,dc=io", "admin"); err != nil {
c.Fatalf("LDAP admin bind failed: %v", err)
}
newEntry := func(dn string, attrs map[string]string) *goldap.AddRequest {
req := goldap.NewAddRequest(dn, nil)
for name, value := range attrs {
req.Attribute(name, []string{value})
}
return req
}
entries := []*goldap.AddRequest{
newEntry(parentOU, map[string]string{"objectClass": "organizationalUnit", "ou": "Дирекция информационных технологий"}),
newEntry(childOU, map[string]string{"objectClass": "organizationalUnit", "ou": "Отдел разработки"}),
newEntry(userDN, map[string]string{"objectClass": "inetOrgPerson", "uid": username, "cn": username, "sn": username, "userPassword": "example"}),
newEntry(groupDN, map[string]string{"objectClass": "groupOfNames", "cn": "long-dn-staff", "member": userDN}),
}
removeEntries := func() {
for i := len(entries) - 1; i >= 0; i-- {
err := conn.Del(goldap.NewDelRequest(entries[i].DN, nil))
if err != nil && !goldap.IsErrorWithCode(err, goldap.LDAPResultNoSuchObject) {
c.Errorf("Unable to remove LDAP entry %s: %v", entries[i].DN, err)
}
}
}
removeEntries() // left over from an interrupted run
defer removeEntries()
for _, entry := range entries {
if err = conn.Add(entry); err != nil {
c.Fatalf("Unable to add LDAP entry %s: %v", entry.DN, err)
}
}
groupReq := madmin.PolicyAssociationReq{
Policies: []string{"readwrite"},
Group: groupDN,
}
if _, err = s.adm.AttachPolicyLDAP(ctx, groupReq); err != nil {
c.Fatalf("Unable to attach group policy: %v", err)
}
defer func() {
if _, err := s.adm.DetachPolicyLDAP(ctx, groupReq); err != nil {
c.Errorf("Unable to detach group policy: %v", err)
}
}()
bucket := getRandomBucketName()
if err = s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}); err != nil {
c.Fatalf("bucket create error: %v", err)
}
ldapID := cr.LDAPIdentity{
Client: s.TestSuiteCommon.client,
STSEndpoint: s.endPoint,
LDAPUsername: username,
LDAPPassword: "example",
}
value, err := ldapID.Retrieve()
if err != nil {
c.Fatalf("Expected to generate STS creds, got err: %#v", err)
}
u, ok := globalIAMSys.GetUser(ctx, value.AccessKeyID)
if !ok {
c.Fatalf("Expected to find user %s", value.AccessKeyID)
}
if u.Credentials.ParentUser != normDN {
c.Fatalf("unexpected parent user: %s", u.Credentials.ParentUser)
}
c.mustListObjects(ctx, s.getUserClient(c, value.AccessKeyID, value.SecretAccessKey, value.SessionToken), bucket)
// An access key created by this user inherits the same parent.
userAdmClient := s.getAdminClient(c, value.AccessKeyID, value.SecretAccessKey, value.SessionToken)
svc := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient)
c.assertSvcAccS3Access(ctx, s, svc, bucket)
if err = s.adm.DeleteServiceAccount(ctx, svc.AccessKey); err != nil {
c.Fatalf("Unable to delete service account: %v", err)
}
}
func (s *TestSuiteIAM) TestLDAPSlashDN(c *check) { func (s *TestSuiteIAM) TestLDAPSlashDN(c *check) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel() defer cancel()
+2
View File
@@ -503,6 +503,8 @@ func resetTestGlobals() {
resetGlobalHealState() resetGlobalHealState()
// Reset globalIAMSys to `nil` // Reset globalIAMSys to `nil`
resetGlobalIAMSys() resetGlobalIAMSys()
// Object-layer fixtures must not reuse the preceding IAM suite's etcd store.
globalEtcdClient = nil
} }
// Configure the server for the test run. // Configure the server for the test run.

Some files were not shown because too many files have changed in this diff Show More