mirror of
https://github.com/pgsty/minio.git
synced 2026-10-28 11:37:19 +03:00
Compare commits
43 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 39278e068e | |||
| 263ee0510d | |||
| 641eb51a40 | |||
| 45d6e5d065 | |||
| dabc894132 | |||
| fb08ca7b6a | |||
| 034f7bbf73 | |||
| aec7ddac46 | |||
| 6921f8c186 | |||
| 25d928c9df | |||
| ee3e7ea716 | |||
| 2a04d5f485 | |||
| 9449e6a4a8 | |||
| 9c4bb39571 | |||
| 86ce8d6012 | |||
| b15aa97c94 | |||
| f54c04846e | |||
| 6fdea3b8d9 | |||
| 380fcbfe47 | |||
| a02fa9644c | |||
| 652a1c09d3 | |||
| c86df14f4d | |||
| 03875d61e3 | |||
| 1865985aa1 | |||
| 076a13f9d2 | |||
| 9a4e089f56 | |||
| 9255f443a6 | |||
| ea7c794a94 | |||
| 16c21428d9 | |||
| 7e83f9950a | |||
| 1e3ab56b06 | |||
| 5ad1f344b7 | |||
| 8a4097309f | |||
| 0501a198e3 | |||
| d2eb0ade91 | |||
| 5c074efd1d | |||
| 03fe5e9c21 | |||
| 0b1c21cb4b | |||
| b55495aef0 | |||
| d8b77e5f3b | |||
| cd1f6cec3c | |||
| 4f17a23e33 | |||
| be44ff2669 |
@@ -118,6 +118,11 @@ jobs:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd -run '^Test(APIPutObjectRetention|APIVersionAddressedWrite|PutObjectRetention|DirectoryObjectLock|ObjectLockOverwrite|APICopyObjectPreservesVersionAfterSuspension|MultipartNullObjectLock|PoolsObjectLockOverwrite|MultipartVersioningTransitions|PlanReview|RecoveredMultipartUpload)' -count=1 -timeout=5m
|
||||
|
||||
- name: Run maintenance write, rollback and cleanup tests under race detector
|
||||
env:
|
||||
MINIO_API_REQUESTS_MAX: "10000"
|
||||
run: go test -race -tags kqueue,dev ./cmd ./internal/grid ./internal/ioutil -run '^Test(Mutation|RenameCancellation|PutObjectGenerationFollowsCommitLock|MaintenancePartialFailedOverwriteAvailability|SingleMutationWaitsBeyondDefaultTimeout|WithoutDefaultTimeoutStillEndsOnDisconnectOrCancellation|WithDeadlineAndWait)' -count=1 -timeout=5m
|
||||
|
||||
crosscompile:
|
||||
name: Cross Compile
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
+34
-3
@@ -1,6 +1,9 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
## RELEASE.2026-10-25T00-00-00Z
|
||||
|
||||
Package version: `20261025000000.0.0`. This source is prepared for a Draft
|
||||
Release; public Server artifacts and images remain at 20261018 until promotion.
|
||||
|
||||
- Fix LDAP login for long normalized DNs without hiding unreadable IAM
|
||||
revocation records. Only length errors for valid keys with an ASCII path
|
||||
@@ -11,9 +14,10 @@
|
||||
Go dependencies, including silo-pkg v3.15.1, the published mcli 20261025
|
||||
source and upstream minio-go `ab93db7ba435` for multipart, read/seek and
|
||||
batch-delete fixes. Retain output-format and NetBSD compatibility selections.
|
||||
- Embed the unreleased Console 2.5.0 candidate `258717be077b`, including its
|
||||
- Embed the published Console v2.5.0 source `989fa7150792`, including its
|
||||
GUI update, diagnostic WebSocket closing handshake and complete health-report
|
||||
decoding. This selects a source commit, not a new Console release.
|
||||
decoding. Bundle published mcli 20261025 in container builds and select it
|
||||
for the verified client installer and Helm client jobs.
|
||||
- Read current bucket inventory in account-info responses so Console refreshes
|
||||
immediately reflect successful creation and deletion. This management endpoint
|
||||
now performs a live list instead of returning a cached inventory.
|
||||
@@ -21,11 +25,38 @@
|
||||
Peer and etcd notifications reload committed state, so delayed delete events
|
||||
preserve subsequent edits. Keep deletion revisions, policy-in-use checks and
|
||||
custom-policy deletion behavior; storage failures retain the previous policy.
|
||||
- Reload policies without holding the authentication-cache lock, retain IAM
|
||||
write ordering and publish replacements only after a complete successful load.
|
||||
Propagate cancellation through IAM/KMS and SSE-S3 decryption without shortening
|
||||
the caller's policy-query deadline.
|
||||
- Preserve authoritative copies when a multi-pool delete or bulk cleanup fails,
|
||||
choose an available pool for a new delete marker, and keep tier data referenced
|
||||
by surviving copies. This does not introduce strict cross-pool read recovery.
|
||||
- Preserve remote tier data when lifecycle expiry only adds a delete marker,
|
||||
including non-null versions in suspended buckets. Forward `FULL_OBJECT`
|
||||
checksum type for multipart replicas. Thanks to
|
||||
[@botdeniska](https://github.com/botdeniska) for these lifecycle, replication
|
||||
and multi-pool deletion contributions.
|
||||
- Preserve Kafka's default broker protocol 2.1 and terminal-width behavior;
|
||||
fix legacy DARE v1 key unsealing and verify DARE v1/v2 and MinKMS TLS compatibility.
|
||||
- Wait for admitted storage mutations to finish before releasing local guards,
|
||||
deciding rollback or cleaning staging data. Rollback verifies ownership and
|
||||
restores only its overwritten version, preserving later writes and metadata
|
||||
edits. Abandoned-data cleanup retains backups still needed by commit/undo.
|
||||
Generate PUT timestamps under the commit lock. Keep the prior read-quorum
|
||||
and healing policy, including readability after a failed partial overwrite.
|
||||
- Update bundled curl to 8.23.0 and require patched static libraries. Apply
|
||||
[Red Hat's compromised-CA mitigation](https://access.redhat.com/security/vulnerabilities/RHSB-2026-012);
|
||||
the root blocklist also rejects legitimate chains under the affected roots.
|
||||
- Ship Windows platform archives as ZIP; Linux and macOS retain tar.gz.
|
||||
|
||||
Storage limits remain: this does not provide distributed fencing after lease
|
||||
loss or network partition, eliminate stale reads after partial metadata writes,
|
||||
or repair previously inconsistent objects. Stricter read refusal and additional
|
||||
cross-pool recovery remain deferred; their diagnostic tests are retained under
|
||||
`consistency_strict`. Waiting for admitted I/O can extend same-object waits on
|
||||
a stuck drive, and interrupted cleanup can retain rollback backups. Upgrade all
|
||||
writers before relying on the new rollback ownership checks.
|
||||
|
||||
These source changes are not included in the published 20261018 artifacts.
|
||||
|
||||
|
||||
@@ -53,6 +53,16 @@ Before opening a pull request:
|
||||
- Explain any preserved `MINIO_*`, `minio_*`, `x-minio-*`, `/minio/*`,
|
||||
`.minio.sys`, ARN, module/import-path, or serialized compatibility name.
|
||||
|
||||
The maintenance branch retains the existing read-quorum and healing policy.
|
||||
Deferred strict-read and cross-pool recovery tests are preserved behind the
|
||||
`consistency_strict` build tag. Run them explicitly with
|
||||
`go test -tags kqueue,dev,consistency_strict ./cmd`; known failures must be
|
||||
reported separately, not described as fixed or included in a passing result.
|
||||
They cover late commits after lease loss, stale metadata reads, and stricter
|
||||
read refusal. The normal suite covers the retained write/undo/cleanup fixes
|
||||
and continued readability after a failed partial overwrite. This scope does
|
||||
not provide distributed fencing or repair previously inconsistent objects.
|
||||
|
||||
### Commit changes
|
||||
|
||||
After verification, commit your changes with a concise message and a DCO
|
||||
|
||||
+32
-28
@@ -2,11 +2,11 @@
|
||||
|
||||
<!-- Generated by silo.pgsty.com/bin/contributors.py. -->
|
||||
|
||||
As of **2026-10-18**, **63 community contributors** are credited in SILO and related projects. The roll includes maintainers and every human issue / PR author, along with security reporters who have agreed to be acknowledged.
|
||||
As of **2026-10-25**, **66 community contributors** are credited in SILO and related projects. The roll includes maintainers and every human issue / PR author, along with security reporters who have agreed to be acknowledged.
|
||||
|
||||
Opening an issue or PR counts, whether open, closed, draft, or unmerged. Each account appears once. Merged PR authors come first, followed by other PR authors and reporters. Within each group, significant fixes, adopted proposals, and reports that led to fixes take priority, followed by first participation. Gold rings highlight significant contributions.
|
||||
|
||||
When a proposal was incorporated through a later repair, its adoption is credited separately from the original PR status. A closed issue does not necessarily mean a fix, and merged source does not imply a published release. Automated accounts are excluded. Upstream authorship remains in Git history and copyright notices.
|
||||
When a proposal was incorporated through a later repair, its adoption is credited alongside the original PR status. Automated accounts are excluded. Upstream authorship remains in Git history and copyright notices.
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/Vonng"><img src="https://silo.pgsty.com/images/contributors/Vonng.svg" width="60" height="60" alt="@Vonng" title="@Vonng — Maintains SILO, Console, mcli, shared packages, releases, and documentation"></a>
|
||||
@@ -25,7 +25,8 @@ When a proposal was incorporated through a later repair, its adoption is credite
|
||||
<a href="https://github.com/AleksaMCode"><img src="https://silo.pgsty.com/images/contributors/AleksaMCode.svg" width="60" height="60" alt="@AleksaMCode" title="@AleksaMCode — Fixed the Lambda target-set emptiness check and added a regression test"></a>
|
||||
<a href="https://github.com/yupoet"><img src="https://silo.pgsty.com/images/contributors/yupoet.svg" width="60" height="60" alt="@yupoet" title="@yupoet — Improved checksum mismatch messages, verified erasure-coding documentation, and contributed a single-node migration case"></a>
|
||||
<a href="https://github.com/clown15"><img src="https://silo.pgsty.com/images/contributors/clown15.svg" width="60" height="60" alt="@clown15" title="@clown15 — Fixed native multipart-upload timestamp parsing during stale-upload cleanup"></a>
|
||||
<a href="https://github.com/botdeniska"><img src="https://silo.pgsty.com/images/contributors/botdeniska.svg" width="60" height="60" alt="@botdeniska" title="@botdeniska — Contributed multi-pool migration safety repairs and faster policy evaluation in silo-pkg"></a>
|
||||
<a href="https://github.com/botdeniska"><img src="https://silo.pgsty.com/images/contributors/botdeniska.svg" width="60" height="60" alt="@botdeniska" title="@botdeniska — Contributed multi-pool migration and deletion safety, tiered expiry and replication checksum repairs, and faster policy evaluation in silo-pkg"></a>
|
||||
<a href="https://github.com/910patin"><img src="https://silo.pgsty.com/images/contributors/910patin.svg" width="60" height="60" alt="@910patin" title="@910patin — Fixed long-DN LDAP login while preserving IAM revocation-record errors"></a>
|
||||
<a href="https://github.com/magicxor"><img src="https://silo.pgsty.com/images/contributors/magicxor.svg" width="60" height="60" alt="@magicxor" title="@magicxor — Reported and proposed conditional DELETE support for If-Match"></a>
|
||||
<a href="https://github.com/davinkevin"><img src="https://silo.pgsty.com/images/contributors/davinkevin.svg" width="60" height="60" alt="@davinkevin" title="@davinkevin — Proposed the distroless container image and dependency automation"></a>
|
||||
<a href="https://github.com/metaneutrons"><img src="https://silo.pgsty.com/images/contributors/metaneutrons.svg" width="60" height="60" alt="@metaneutrons" title="@metaneutrons — Reported and proposed explicit-version delete authorization"></a>
|
||||
@@ -72,13 +73,15 @@ When a proposal was incorporated through a later repair, its adoption is credite
|
||||
<a href="https://github.com/Crowley723"><img src="https://silo.pgsty.com/images/contributors/Crowley723.svg" width="48" height="48" alt="@Crowley723" title="@Crowley723 — Requested friendly names for users"></a>
|
||||
<a href="https://github.com/geraldcombs"><img src="https://silo.pgsty.com/images/contributors/geraldcombs.svg" width="48" height="48" alt="@geraldcombs" title="@geraldcombs — Suggested zip or 7z packaging for Windows artifacts"></a>
|
||||
<a href="https://github.com/devTeam-AiReg"><img src="https://silo.pgsty.com/images/contributors/devTeam-AiReg.svg" width="48" height="48" alt="@devTeam-AiReg" title="@devTeam-AiReg — Asked about availability of the release containing IAM security fixes"></a>
|
||||
<a href="https://github.com/azataiot"><img src="https://silo.pgsty.com/images/contributors/azataiot.svg" width="48" height="48" alt="@azataiot" title="@azataiot — Reported and analyzed high-latency Console health-report truncation during WebSocket close"></a>
|
||||
<a href="https://github.com/GhaziTriki"><img src="https://silo.pgsty.com/images/contributors/GhaziTriki.svg" width="60" height="60" alt="@GhaziTriki" title="@GhaziTriki — Reported policy-query and Console performance degradation"></a>
|
||||
</p>
|
||||
|
||||
## Merged pull requests
|
||||
|
||||
| Contributor | Contribution | Record |
|
||||
| :-- | :-- | :-- |
|
||||
| [@Vonng](https://github.com/Vonng) | Maintains SILO, Console, mcli, shared packages, releases, and documentation | [pgsty/silo: Merged PRs (89)](https://github.com/pgsty/silo/issues?q=author%3AVonng+is%3Apr+is%3Amerged)<br>PRs closed without merging: [pgsty/silo#155](https://github.com/pgsty/silo/pull/155), [pgsty/silo#195](https://github.com/pgsty/silo/pull/195)<br>[pgsty/silo: Closed issues (65)](https://github.com/pgsty/silo/issues?q=author%3AVonng+is%3Aissue+is%3Aclosed)<br>Open issues: [pgsty/silo#203](https://github.com/pgsty/silo/issues/203), [pgsty/silo#217](https://github.com/pgsty/silo/issues/217), [pgsty/silo#246](https://github.com/pgsty/silo/issues/246)<br>[pgsty/silo-console: Merged PRs (26)](https://github.com/pgsty/silo-console/issues?q=author%3AVonng+is%3Apr+is%3Amerged)<br>[pgsty/silo-console: Closed issues (33)](https://github.com/pgsty/silo-console/issues?q=author%3AVonng+is%3Aissue+is%3Aclosed)<br>Open issues: [pgsty/silo-console#34](https://github.com/pgsty/silo-console/issues/34)<br>[pgsty/mc: Merged PRs (26)](https://github.com/pgsty/mc/issues?q=author%3AVonng+is%3Apr+is%3Amerged)<br>[pgsty/mc: Closed issues (18)](https://github.com/pgsty/mc/issues?q=author%3AVonng+is%3Aissue+is%3Aclosed)<br>Merged PRs: [pgsty/silo-pkg#1](https://github.com/pgsty/silo-pkg/pull/1), [pgsty/silo-pkg#2](https://github.com/pgsty/silo-pkg/pull/2), [pgsty/silo-pkg#3](https://github.com/pgsty/silo-pkg/pull/3), [pgsty/silo-pkg#4](https://github.com/pgsty/silo-pkg/pull/4), [pgsty/silo-pkg#5](https://github.com/pgsty/silo-pkg/pull/5), [pgsty/silo-pkg#6](https://github.com/pgsty/silo-pkg/pull/6), [pgsty/silo-pkg#7](https://github.com/pgsty/silo-pkg/pull/7), [pgsty/silo-pkg#8](https://github.com/pgsty/silo-pkg/pull/8), [pgsty/silo-pkg#9](https://github.com/pgsty/silo-pkg/pull/9), [pgsty/silo-pkg#10](https://github.com/pgsty/silo-pkg/pull/10), [pgsty/silo-pkg#12](https://github.com/pgsty/silo-pkg/pull/12)<br>Merged PRs: [pgsty/kes#1](https://github.com/pgsty/kes/pull/1)<br>[pgsty/silo.pgsty.com: Merged PRs (41)](https://github.com/pgsty/silo.pgsty.com/issues?q=author%3AVonng+is%3Apr+is%3Amerged)<br>PRs closed without merging: [pgsty/silo.pgsty.com#36](https://github.com/pgsty/silo.pgsty.com/pull/36) |
|
||||
| [@Vonng](https://github.com/Vonng) | Maintains SILO, Console, mcli, shared packages, releases, and documentation | [pgsty/silo: Merged PRs (89)](https://github.com/pgsty/silo/issues?q=author%3AVonng+is%3Apr+is%3Amerged)<br>PRs closed without merging: [pgsty/silo#155](https://github.com/pgsty/silo/pull/155), [pgsty/silo#195](https://github.com/pgsty/silo/pull/195)<br>[pgsty/silo: Closed issues (67)](https://github.com/pgsty/silo/issues?q=author%3AVonng+is%3Aissue+is%3Aclosed)<br>Open issues: [pgsty/silo#217](https://github.com/pgsty/silo/issues/217)<br>[pgsty/silo-console: Merged PRs (29)](https://github.com/pgsty/silo-console/issues?q=author%3AVonng+is%3Apr+is%3Amerged)<br>[pgsty/silo-console: Closed issues (34)](https://github.com/pgsty/silo-console/issues?q=author%3AVonng+is%3Aissue+is%3Aclosed)<br>[pgsty/mc: Merged PRs (26)](https://github.com/pgsty/mc/issues?q=author%3AVonng+is%3Apr+is%3Amerged)<br>[pgsty/mc: Closed issues (18)](https://github.com/pgsty/mc/issues?q=author%3AVonng+is%3Aissue+is%3Aclosed)<br>Merged PRs: [pgsty/silo-pkg#1](https://github.com/pgsty/silo-pkg/pull/1), [pgsty/silo-pkg#2](https://github.com/pgsty/silo-pkg/pull/2), [pgsty/silo-pkg#3](https://github.com/pgsty/silo-pkg/pull/3), [pgsty/silo-pkg#4](https://github.com/pgsty/silo-pkg/pull/4), [pgsty/silo-pkg#5](https://github.com/pgsty/silo-pkg/pull/5), [pgsty/silo-pkg#6](https://github.com/pgsty/silo-pkg/pull/6), [pgsty/silo-pkg#7](https://github.com/pgsty/silo-pkg/pull/7), [pgsty/silo-pkg#8](https://github.com/pgsty/silo-pkg/pull/8), [pgsty/silo-pkg#9](https://github.com/pgsty/silo-pkg/pull/9), [pgsty/silo-pkg#10](https://github.com/pgsty/silo-pkg/pull/10), [pgsty/silo-pkg#12](https://github.com/pgsty/silo-pkg/pull/12)<br>Merged PRs: [pgsty/kes#1](https://github.com/pgsty/kes/pull/1)<br>[pgsty/silo.pgsty.com: Merged PRs (41)](https://github.com/pgsty/silo.pgsty.com/issues?q=author%3AVonng+is%3Apr+is%3Amerged)<br>PRs closed without merging: [pgsty/silo.pgsty.com#36](https://github.com/pgsty/silo.pgsty.com/pull/36) |
|
||||
| [@ZouhairCharef](https://github.com/ZouhairCharef) | Patched CVE-2026-34986 in go-jose | Merged PRs: [pgsty/silo#18](https://github.com/pgsty/silo/pull/18) |
|
||||
| [@mfredenhagen](https://github.com/mfredenhagen) | Patched CVE-2026-39883 in OpenTelemetry | Merged PRs: [pgsty/silo#19](https://github.com/pgsty/silo/pull/19) |
|
||||
| [@cbornet](https://github.com/cbornet) | Reported checksum and missing-bucket defects and aligned S3 checksum error messages | Merged PRs: [pgsty/silo#225](https://github.com/pgsty/silo/pull/225)<br>Closed issues: [pgsty/silo#31](https://github.com/pgsty/silo/issues/31), [pgsty/silo#32](https://github.com/pgsty/silo/issues/32), [pgsty/silo#107](https://github.com/pgsty/silo/issues/107), [pgsty/silo#226](https://github.com/pgsty/silo/issues/226) |
|
||||
@@ -92,9 +95,10 @@ When a proposal was incorporated through a later repair, its adoption is credite
|
||||
| [@julienlau](https://github.com/julienlau) | Fixed missing-commit-metadata error handling and stale metadata reuse; contributed replication reports and proposals | Merged PRs: [pgsty/silo#232](https://github.com/pgsty/silo/pull/232), [pgsty/silo#233](https://github.com/pgsty/silo/pull/233)<br>Open PRs: [pgsty/silo#240](https://github.com/pgsty/silo/pull/240), [pgsty/silo#241](https://github.com/pgsty/silo/pull/241)<br>PRs closed without merging: [pgsty/silo#184](https://github.com/pgsty/silo/pull/184)<br>Closed issues: [pgsty/silo#152](https://github.com/pgsty/silo/issues/152), [pgsty/silo#153](https://github.com/pgsty/silo/issues/153)<br>The original PR was not merged directly. Its purge-state and recovery analysis, proposed fix, tests, and documentation helped shape [#196](https://github.com/pgsty/silo/pull/196); see the [maintainer acknowledgement](https://github.com/pgsty/silo/pull/184#issuecomment-5689295222). |
|
||||
| [@Aeirx](https://github.com/Aeirx) | Preserved Object Lock legal-hold headers in federated CopyObject | Merged PRs: [pgsty/silo#172](https://github.com/pgsty/silo/pull/172) |
|
||||
| [@AleksaMCode](https://github.com/AleksaMCode) | Fixed the Lambda target-set emptiness check and added a regression test | Merged PRs: [pgsty/silo#220](https://github.com/pgsty/silo/pull/220) |
|
||||
| [@yupoet](https://github.com/yupoet) | Improved checksum mismatch messages, verified erasure-coding documentation, and contributed a single-node migration case | Merged PRs: [pgsty/silo#228](https://github.com/pgsty/silo/pull/228), [pgsty/silo#229](https://github.com/pgsty/silo/pull/229)<br>Open PRs: [pgsty/silo#230](https://github.com/pgsty/silo/pull/230)<br>Also [cross-checked the two-drive EC:1 documentation](https://github.com/pgsty/silo/issues/197#issuecomment-5846341939). The single-node migration report in #228 was merged on 2026-10-23; it records a contributor-reported forward upgrade, not a downgrade guarantee. The registry-mirror proposal remains open. |
|
||||
| [@yupoet](https://github.com/yupoet) | Improved checksum mismatch messages, verified erasure-coding documentation, and contributed a single-node migration case | Merged PRs: [pgsty/silo#228](https://github.com/pgsty/silo/pull/228), [pgsty/silo#229](https://github.com/pgsty/silo/pull/229)<br>PRs closed without merging: [pgsty/silo#230](https://github.com/pgsty/silo/pull/230)<br>Also [cross-checked the two-drive EC:1 documentation](https://github.com/pgsty/silo/issues/197#issuecomment-5846341939). The single-node migration report in #228 was merged on 2026-10-23; the registry-mirror proposal in #230 remains open. |
|
||||
| [@clown15](https://github.com/clown15) | Fixed native multipart-upload timestamp parsing during stale-upload cleanup | Merged PRs: [pgsty/silo#236](https://github.com/pgsty/silo/pull/236) |
|
||||
| [@botdeniska](https://github.com/botdeniska) | Contributed multi-pool migration safety repairs and faster policy evaluation in silo-pkg | Open PRs: [pgsty/silo#245](https://github.com/pgsty/silo/pull/245), [pgsty/silo#248](https://github.com/pgsty/silo/pull/248)<br>Merged PRs: [pgsty/silo-pkg#11](https://github.com/pgsty/silo-pkg/pull/11)<br>The migration repair from [#248](https://github.com/pgsty/silo/pull/248) was incorporated in [60c7179dd](https://github.com/pgsty/silo/commit/60c7179ddbfcbd41a1fe67707230f5b360a43020), retaining the original authorship and DCO. The original PR remains open; this does not imply that #245 shipped. |
|
||||
| [@botdeniska](https://github.com/botdeniska) | Contributed multi-pool migration and deletion safety, tiered expiry and replication checksum repairs, and faster policy evaluation in silo-pkg | Merged PRs: [pgsty/silo#253](https://github.com/pgsty/silo/pull/253), [pgsty/silo#255](https://github.com/pgsty/silo/pull/255), [pgsty/silo#257](https://github.com/pgsty/silo/pull/257)<br>PRs closed without merging: [pgsty/silo#245](https://github.com/pgsty/silo/pull/245), [pgsty/silo#248](https://github.com/pgsty/silo/pull/248)<br>Closed issues: [pgsty/silo#252](https://github.com/pgsty/silo/issues/252), [pgsty/silo#254](https://github.com/pgsty/silo/issues/254), [pgsty/silo#256](https://github.com/pgsty/silo/issues/256)<br>Merged PRs: [pgsty/silo-pkg#11](https://github.com/pgsty/silo-pkg/pull/11)<br>The migration repair from [#248](https://github.com/pgsty/silo/pull/248) was incorporated in [60c7179dd](https://github.com/pgsty/silo/commit/60c7179ddbfcbd41a1fe67707230f5b360a43020), retaining the original authorship and DCO. |
|
||||
| [@910patin](https://github.com/910patin) | Fixed long-DN LDAP login while preserving IAM revocation-record errors | Merged PRs: [pgsty/silo#250](https://github.com/pgsty/silo/pull/250) |
|
||||
|
||||
## Other pull-request authors
|
||||
|
||||
@@ -116,7 +120,7 @@ When a proposal was incorporated through a later repair, its adoption is credite
|
||||
| [@jiri-pejchal](https://github.com/jiri-pejchal) | Reported the RabbitMQ client vulnerability and the Console share proxy's exposure of internal metrics | Closed issues: [pgsty/silo#176](https://github.com/pgsty/silo/issues/176)<br>Closed issues: [pgsty/silo-console#52](https://github.com/pgsty/silo-console/issues/52) |
|
||||
| [@AEGEGE](https://github.com/AEGEGE) | Reported the slow-HTTP denial-of-service vulnerability | Closed issues: [pgsty/silo#183](https://github.com/pgsty/silo/issues/183)<br>The reproduced slow-header defect was fixed on main through [#196](https://github.com/pgsty/silo/pull/196). |
|
||||
| [@naiveHobo](https://github.com/naiveHobo) | Reported an IAM admin API access-control issue and the unavailable private reporting channel | Closed issues: [pgsty/silo#222](https://github.com/pgsty/silo/issues/222) |
|
||||
| [@orenyomtov](https://github.com/orenyomtov) | Reported the unsigned-header CopyObject cross-object read (SN-2026-011) | Security disclosure credited in [SN-2026-011](https://silo.pgsty.com/about/security-advisories/#sn-2026-011), with the source fix in [#173](https://github.com/pgsty/silo/pull/173). This credit is retained separately from public issue/PR authorship. |
|
||||
| [@orenyomtov](https://github.com/orenyomtov) | Reported the unsigned-header CopyObject cross-object read (SN-2026-011) | Security disclosure credited in [SN-2026-011](https://silo.pgsty.com/about/security-advisories/#sn-2026-011), fixed in [#173](https://github.com/pgsty/silo/pull/173). |
|
||||
| [@nik0xFF](https://github.com/nik0xFF) | Reported a security issue privately by email | Dominik Klug is credited under his full name with his permission. |
|
||||
| [@szopi00](https://github.com/szopi00) | Reported the Object Lock GOVERNANCE retention authorization bypass | The report led to the Object Lock repairs in [#234](https://github.com/pgsty/silo/pull/234) and [#235](https://github.com/pgsty/silo/pull/235). |
|
||||
| [@mosesdd](https://github.com/mosesdd) | Requested a maintained Helm chart | Closed issues: [pgsty/silo#1](https://github.com/pgsty/silo/issues/1) |
|
||||
@@ -137,7 +141,7 @@ When a proposal was incorporated through a later repair, its adoption is credite
|
||||
| [@meesudzu](https://github.com/meesudzu) | Requested the migration guide from upstream MinIO | Closed issues: [pgsty/silo#42](https://github.com/pgsty/silo/issues/42) |
|
||||
| [@pmezhuev](https://github.com/pmezhuev) | Reported missing RPM package signatures | Closed issues: [pgsty/silo#43](https://github.com/pgsty/silo/issues/43) |
|
||||
| [@kh0mka](https://github.com/kh0mka) | Reported inter-node I/O timeouts and OIDC configuration-fetch failures | Closed issues: [pgsty/silo#51](https://github.com/pgsty/silo/issues/51), [pgsty/silo#154](https://github.com/pgsty/silo/issues/154) |
|
||||
| [@bagutzu](https://github.com/bagutzu) | Requested KES-compatible external KMS and OpenBao support | Open issues: [pgsty/silo#61](https://github.com/pgsty/silo/issues/61) |
|
||||
| [@bagutzu](https://github.com/bagutzu) | Requested KES-compatible external KMS and OpenBao support | Closed issues: [pgsty/silo#61](https://github.com/pgsty/silo/issues/61) |
|
||||
| [@liuhaodongliu990-cmyk](https://github.com/liuhaodongliu990-cmyk) | Reported indeterminate progress for prefix downloads | Closed issues: [pgsty/silo#62](https://github.com/pgsty/silo/issues/62) |
|
||||
| [@DestroyLee](https://github.com/DestroyLee) | Reported the missing documentation navigation | Closed issues: [pgsty/silo.pgsty.com#4](https://github.com/pgsty/silo.pgsty.com/issues/4) |
|
||||
| [@sargarass](https://github.com/sargarass) | Reported ListMultipartUploads prefix and pagination semantics | Open issues: [pgsty/silo#79](https://github.com/pgsty/silo/issues/79) |
|
||||
@@ -147,26 +151,28 @@ When a proposal was incorporated through a later repair, its adoption is credite
|
||||
| [@aschyolkin](https://github.com/aschyolkin) | Reported a data race in CPU metrics collection | Closed issues: [pgsty/silo#210](https://github.com/pgsty/silo/issues/210) |
|
||||
| [@0mp](https://github.com/0mp) | Requested and discussed FreeBSD support | Open issues: [pgsty/silo#219](https://github.com/pgsty/silo/issues/219) |
|
||||
| [@ztarbug](https://github.com/ztarbug) | Requested publication of the Helm chart | Open issues: [pgsty/silo#223](https://github.com/pgsty/silo/issues/223) |
|
||||
| [@zmstone](https://github.com/zmstone) | Requested a GitHub Container Registry mirror | Open issues: [pgsty/silo#224](https://github.com/pgsty/silo/issues/224) |
|
||||
| [@zmstone](https://github.com/zmstone) | Requested a GitHub Container Registry mirror | Closed issues: [pgsty/silo#224](https://github.com/pgsty/silo/issues/224) |
|
||||
| [@Crowley723](https://github.com/Crowley723) | Requested friendly names for users | Open issues: [pgsty/silo#227](https://github.com/pgsty/silo/issues/227) |
|
||||
| [@geraldcombs](https://github.com/geraldcombs) | Suggested zip or 7z packaging for Windows artifacts | Open issues: [pgsty/mc#45](https://github.com/pgsty/mc/issues/45) |
|
||||
| [@devTeam-AiReg](https://github.com/devTeam-AiReg) | Asked about availability of the release containing IAM security fixes | Open issues: [pgsty/silo#247](https://github.com/pgsty/silo/issues/247) |
|
||||
| [@devTeam-AiReg](https://github.com/devTeam-AiReg) | Asked about availability of the release containing IAM security fixes | Closed issues: [pgsty/silo#247](https://github.com/pgsty/silo/issues/247) |
|
||||
| [@azataiot](https://github.com/azataiot) | Reported and analyzed high-latency Console health-report truncation during WebSocket close | Closed issues: [pgsty/silo-console#63](https://github.com/pgsty/silo-console/issues/63) |
|
||||
| [@GhaziTriki](https://github.com/GhaziTriki) | Reported policy-query and Console performance degradation | Open issues: [pgsty/silo#251](https://github.com/pgsty/silo/issues/251)<br>IAM policy reload and KMS cancellation repairs have landed; #251 remains open for the broader performance symptoms. |
|
||||
|
||||
## Audit scope
|
||||
|
||||
Counts below come from every page of the GitHub issue / PR records, across all states, including automated accounts. There are 60 distinct human public issue / PR authors, plus 3 credited security reporters. Excluded automated accounts: `Copilot`, `dependabot[bot]`.
|
||||
Counts below come from every page of the GitHub issue / PR records, across all states, including automated accounts. Review dates are listed per repository; records not rechecked retain their earlier dates and states. There are 63 distinct human public issue / PR authors, plus 3 credited security reporters. Excluded automated accounts: `Copilot`, `dependabot[bot]`.
|
||||
|
||||
| Repository | Issues | Pull requests |
|
||||
| :-- | --: | --: |
|
||||
| [pgsty/silo](https://github.com/pgsty/silo) | 119 | 126 |
|
||||
| [pgsty/silo-console](https://github.com/pgsty/silo-console) | 35 | 26 |
|
||||
| [pgsty/mc](https://github.com/pgsty/mc) | 19 | 28 |
|
||||
| [pgsty/silo-pkg](https://github.com/pgsty/silo-pkg) | 0 | 12 |
|
||||
| [pgsty/silo-go](https://github.com/pgsty/silo-go) | 0 | 0 |
|
||||
| [pgsty/kes](https://github.com/pgsty/kes) | 0 | 1 |
|
||||
| [pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com) | 1 | 43 |
|
||||
| [pgsty/minio-docs](https://github.com/pgsty/minio-docs) | 0 | 0 |
|
||||
| **Total** | **174** | **236** |
|
||||
| Repository | Issues | Pull requests | Reviewed |
|
||||
| :-- | --: | --: | :-- |
|
||||
| [pgsty/silo](https://github.com/pgsty/silo) | 123 | 130 | 2026-10-25 |
|
||||
| [pgsty/silo-console](https://github.com/pgsty/silo-console) | 36 | 29 | 2026-10-25 |
|
||||
| [pgsty/mc](https://github.com/pgsty/mc) | 19 | 28 | 2026-10-18 |
|
||||
| [pgsty/silo-pkg](https://github.com/pgsty/silo-pkg) | 0 | 12 | 2026-10-18 |
|
||||
| [pgsty/silo-go](https://github.com/pgsty/silo-go) | 0 | 0 | 2026-10-18 |
|
||||
| [pgsty/kes](https://github.com/pgsty/kes) | 0 | 1 | 2026-10-18 |
|
||||
| [pgsty/silo.pgsty.com](https://github.com/pgsty/silo.pgsty.com) | 1 | 43 | 2026-10-18 |
|
||||
| [pgsty/minio-docs](https://github.com/pgsty/minio-docs) | 0 | 0 | 2026-10-18 |
|
||||
| **Total** | **179** | **243** | |
|
||||
|
||||
## Maintainer record
|
||||
|
||||
@@ -177,15 +183,13 @@ Merged PRs: [pgsty/silo#44](https://github.com/pgsty/silo/pull/44), [pgsty/silo#
|
||||
|
||||
PRs closed without merging: [pgsty/silo#155](https://github.com/pgsty/silo/pull/155), [pgsty/silo#195](https://github.com/pgsty/silo/pull/195)
|
||||
|
||||
Closed issues: [pgsty/silo#22](https://github.com/pgsty/silo/issues/22), [pgsty/silo#23](https://github.com/pgsty/silo/issues/23), [pgsty/silo#24](https://github.com/pgsty/silo/issues/24), [pgsty/silo#25](https://github.com/pgsty/silo/issues/25), [pgsty/silo#26](https://github.com/pgsty/silo/issues/26), [pgsty/silo#27](https://github.com/pgsty/silo/issues/27), [pgsty/silo#28](https://github.com/pgsty/silo/issues/28), [pgsty/silo#46](https://github.com/pgsty/silo/issues/46), [pgsty/silo#47](https://github.com/pgsty/silo/issues/47), [pgsty/silo#48](https://github.com/pgsty/silo/issues/48), [pgsty/silo#49](https://github.com/pgsty/silo/issues/49), [pgsty/silo#50](https://github.com/pgsty/silo/issues/50), [pgsty/silo#52](https://github.com/pgsty/silo/issues/52), [pgsty/silo#53](https://github.com/pgsty/silo/issues/53), [pgsty/silo#55](https://github.com/pgsty/silo/issues/55), [pgsty/silo#63](https://github.com/pgsty/silo/issues/63), [pgsty/silo#64](https://github.com/pgsty/silo/issues/64), [pgsty/silo#65](https://github.com/pgsty/silo/issues/65), [pgsty/silo#67](https://github.com/pgsty/silo/issues/67), [pgsty/silo#68](https://github.com/pgsty/silo/issues/68), [pgsty/silo#75](https://github.com/pgsty/silo/issues/75), [pgsty/silo#76](https://github.com/pgsty/silo/issues/76), [pgsty/silo#77](https://github.com/pgsty/silo/issues/77), [pgsty/silo#78](https://github.com/pgsty/silo/issues/78), [pgsty/silo#82](https://github.com/pgsty/silo/issues/82), [pgsty/silo#83](https://github.com/pgsty/silo/issues/83), [pgsty/silo#84](https://github.com/pgsty/silo/issues/84), [pgsty/silo#99](https://github.com/pgsty/silo/issues/99), [pgsty/silo#100](https://github.com/pgsty/silo/issues/100), [pgsty/silo#102](https://github.com/pgsty/silo/issues/102), [pgsty/silo#105](https://github.com/pgsty/silo/issues/105), [pgsty/silo#109](https://github.com/pgsty/silo/issues/109), [pgsty/silo#110](https://github.com/pgsty/silo/issues/110), [pgsty/silo#111](https://github.com/pgsty/silo/issues/111), [pgsty/silo#112](https://github.com/pgsty/silo/issues/112), [pgsty/silo#113](https://github.com/pgsty/silo/issues/113), [pgsty/silo#114](https://github.com/pgsty/silo/issues/114), [pgsty/silo#115](https://github.com/pgsty/silo/issues/115), [pgsty/silo#116](https://github.com/pgsty/silo/issues/116), [pgsty/silo#117](https://github.com/pgsty/silo/issues/117), [pgsty/silo#118](https://github.com/pgsty/silo/issues/118), [pgsty/silo#119](https://github.com/pgsty/silo/issues/119), [pgsty/silo#120](https://github.com/pgsty/silo/issues/120), [pgsty/silo#133](https://github.com/pgsty/silo/issues/133), [pgsty/silo#136](https://github.com/pgsty/silo/issues/136), [pgsty/silo#137](https://github.com/pgsty/silo/issues/137), [pgsty/silo#139](https://github.com/pgsty/silo/issues/139), [pgsty/silo#141](https://github.com/pgsty/silo/issues/141), [pgsty/silo#144](https://github.com/pgsty/silo/issues/144), [pgsty/silo#147](https://github.com/pgsty/silo/issues/147), [pgsty/silo#148](https://github.com/pgsty/silo/issues/148), [pgsty/silo#158](https://github.com/pgsty/silo/issues/158), [pgsty/silo#165](https://github.com/pgsty/silo/issues/165), [pgsty/silo#166](https://github.com/pgsty/silo/issues/166), [pgsty/silo#167](https://github.com/pgsty/silo/issues/167), [pgsty/silo#168](https://github.com/pgsty/silo/issues/168), [pgsty/silo#169](https://github.com/pgsty/silo/issues/169), [pgsty/silo#170](https://github.com/pgsty/silo/issues/170), [pgsty/silo#171](https://github.com/pgsty/silo/issues/171), [pgsty/silo#199](https://github.com/pgsty/silo/issues/199), [pgsty/silo#200](https://github.com/pgsty/silo/issues/200), [pgsty/silo#201](https://github.com/pgsty/silo/issues/201), [pgsty/silo#202](https://github.com/pgsty/silo/issues/202), [pgsty/silo#204](https://github.com/pgsty/silo/issues/204), [pgsty/silo#218](https://github.com/pgsty/silo/issues/218)
|
||||
Closed issues: [pgsty/silo#22](https://github.com/pgsty/silo/issues/22), [pgsty/silo#23](https://github.com/pgsty/silo/issues/23), [pgsty/silo#24](https://github.com/pgsty/silo/issues/24), [pgsty/silo#25](https://github.com/pgsty/silo/issues/25), [pgsty/silo#26](https://github.com/pgsty/silo/issues/26), [pgsty/silo#27](https://github.com/pgsty/silo/issues/27), [pgsty/silo#28](https://github.com/pgsty/silo/issues/28), [pgsty/silo#46](https://github.com/pgsty/silo/issues/46), [pgsty/silo#47](https://github.com/pgsty/silo/issues/47), [pgsty/silo#48](https://github.com/pgsty/silo/issues/48), [pgsty/silo#49](https://github.com/pgsty/silo/issues/49), [pgsty/silo#50](https://github.com/pgsty/silo/issues/50), [pgsty/silo#52](https://github.com/pgsty/silo/issues/52), [pgsty/silo#53](https://github.com/pgsty/silo/issues/53), [pgsty/silo#55](https://github.com/pgsty/silo/issues/55), [pgsty/silo#63](https://github.com/pgsty/silo/issues/63), [pgsty/silo#64](https://github.com/pgsty/silo/issues/64), [pgsty/silo#65](https://github.com/pgsty/silo/issues/65), [pgsty/silo#67](https://github.com/pgsty/silo/issues/67), [pgsty/silo#68](https://github.com/pgsty/silo/issues/68), [pgsty/silo#75](https://github.com/pgsty/silo/issues/75), [pgsty/silo#76](https://github.com/pgsty/silo/issues/76), [pgsty/silo#77](https://github.com/pgsty/silo/issues/77), [pgsty/silo#78](https://github.com/pgsty/silo/issues/78), [pgsty/silo#82](https://github.com/pgsty/silo/issues/82), [pgsty/silo#83](https://github.com/pgsty/silo/issues/83), [pgsty/silo#84](https://github.com/pgsty/silo/issues/84), [pgsty/silo#99](https://github.com/pgsty/silo/issues/99), [pgsty/silo#100](https://github.com/pgsty/silo/issues/100), [pgsty/silo#102](https://github.com/pgsty/silo/issues/102), [pgsty/silo#105](https://github.com/pgsty/silo/issues/105), [pgsty/silo#109](https://github.com/pgsty/silo/issues/109), [pgsty/silo#110](https://github.com/pgsty/silo/issues/110), [pgsty/silo#111](https://github.com/pgsty/silo/issues/111), [pgsty/silo#112](https://github.com/pgsty/silo/issues/112), [pgsty/silo#113](https://github.com/pgsty/silo/issues/113), [pgsty/silo#114](https://github.com/pgsty/silo/issues/114), [pgsty/silo#115](https://github.com/pgsty/silo/issues/115), [pgsty/silo#116](https://github.com/pgsty/silo/issues/116), [pgsty/silo#117](https://github.com/pgsty/silo/issues/117), [pgsty/silo#118](https://github.com/pgsty/silo/issues/118), [pgsty/silo#119](https://github.com/pgsty/silo/issues/119), [pgsty/silo#120](https://github.com/pgsty/silo/issues/120), [pgsty/silo#133](https://github.com/pgsty/silo/issues/133), [pgsty/silo#136](https://github.com/pgsty/silo/issues/136), [pgsty/silo#137](https://github.com/pgsty/silo/issues/137), [pgsty/silo#139](https://github.com/pgsty/silo/issues/139), [pgsty/silo#141](https://github.com/pgsty/silo/issues/141), [pgsty/silo#144](https://github.com/pgsty/silo/issues/144), [pgsty/silo#147](https://github.com/pgsty/silo/issues/147), [pgsty/silo#148](https://github.com/pgsty/silo/issues/148), [pgsty/silo#158](https://github.com/pgsty/silo/issues/158), [pgsty/silo#165](https://github.com/pgsty/silo/issues/165), [pgsty/silo#166](https://github.com/pgsty/silo/issues/166), [pgsty/silo#167](https://github.com/pgsty/silo/issues/167), [pgsty/silo#168](https://github.com/pgsty/silo/issues/168), [pgsty/silo#169](https://github.com/pgsty/silo/issues/169), [pgsty/silo#170](https://github.com/pgsty/silo/issues/170), [pgsty/silo#171](https://github.com/pgsty/silo/issues/171), [pgsty/silo#199](https://github.com/pgsty/silo/issues/199), [pgsty/silo#200](https://github.com/pgsty/silo/issues/200), [pgsty/silo#201](https://github.com/pgsty/silo/issues/201), [pgsty/silo#202](https://github.com/pgsty/silo/issues/202), [pgsty/silo#203](https://github.com/pgsty/silo/issues/203), [pgsty/silo#204](https://github.com/pgsty/silo/issues/204), [pgsty/silo#218](https://github.com/pgsty/silo/issues/218), [pgsty/silo#246](https://github.com/pgsty/silo/issues/246)
|
||||
|
||||
Open issues: [pgsty/silo#203](https://github.com/pgsty/silo/issues/203), [pgsty/silo#217](https://github.com/pgsty/silo/issues/217), [pgsty/silo#246](https://github.com/pgsty/silo/issues/246)
|
||||
Open issues: [pgsty/silo#217](https://github.com/pgsty/silo/issues/217)
|
||||
|
||||
Merged PRs: [pgsty/silo-console#9](https://github.com/pgsty/silo-console/pull/9), [pgsty/silo-console#10](https://github.com/pgsty/silo-console/pull/10), [pgsty/silo-console#11](https://github.com/pgsty/silo-console/pull/11), [pgsty/silo-console#38](https://github.com/pgsty/silo-console/pull/38), [pgsty/silo-console#39](https://github.com/pgsty/silo-console/pull/39), [pgsty/silo-console#40](https://github.com/pgsty/silo-console/pull/40), [pgsty/silo-console#41](https://github.com/pgsty/silo-console/pull/41), [pgsty/silo-console#42](https://github.com/pgsty/silo-console/pull/42), [pgsty/silo-console#43](https://github.com/pgsty/silo-console/pull/43), [pgsty/silo-console#44](https://github.com/pgsty/silo-console/pull/44), [pgsty/silo-console#45](https://github.com/pgsty/silo-console/pull/45), [pgsty/silo-console#46](https://github.com/pgsty/silo-console/pull/46), [pgsty/silo-console#47](https://github.com/pgsty/silo-console/pull/47), [pgsty/silo-console#48](https://github.com/pgsty/silo-console/pull/48), [pgsty/silo-console#49](https://github.com/pgsty/silo-console/pull/49), [pgsty/silo-console#50](https://github.com/pgsty/silo-console/pull/50), [pgsty/silo-console#51](https://github.com/pgsty/silo-console/pull/51), [pgsty/silo-console#53](https://github.com/pgsty/silo-console/pull/53), [pgsty/silo-console#54](https://github.com/pgsty/silo-console/pull/54), [pgsty/silo-console#55](https://github.com/pgsty/silo-console/pull/55), [pgsty/silo-console#56](https://github.com/pgsty/silo-console/pull/56), [pgsty/silo-console#57](https://github.com/pgsty/silo-console/pull/57), [pgsty/silo-console#58](https://github.com/pgsty/silo-console/pull/58), [pgsty/silo-console#59](https://github.com/pgsty/silo-console/pull/59), [pgsty/silo-console#60](https://github.com/pgsty/silo-console/pull/60), [pgsty/silo-console#61](https://github.com/pgsty/silo-console/pull/61)
|
||||
Merged PRs: [pgsty/silo-console#9](https://github.com/pgsty/silo-console/pull/9), [pgsty/silo-console#10](https://github.com/pgsty/silo-console/pull/10), [pgsty/silo-console#11](https://github.com/pgsty/silo-console/pull/11), [pgsty/silo-console#38](https://github.com/pgsty/silo-console/pull/38), [pgsty/silo-console#39](https://github.com/pgsty/silo-console/pull/39), [pgsty/silo-console#40](https://github.com/pgsty/silo-console/pull/40), [pgsty/silo-console#41](https://github.com/pgsty/silo-console/pull/41), [pgsty/silo-console#42](https://github.com/pgsty/silo-console/pull/42), [pgsty/silo-console#43](https://github.com/pgsty/silo-console/pull/43), [pgsty/silo-console#44](https://github.com/pgsty/silo-console/pull/44), [pgsty/silo-console#45](https://github.com/pgsty/silo-console/pull/45), [pgsty/silo-console#46](https://github.com/pgsty/silo-console/pull/46), [pgsty/silo-console#47](https://github.com/pgsty/silo-console/pull/47), [pgsty/silo-console#48](https://github.com/pgsty/silo-console/pull/48), [pgsty/silo-console#49](https://github.com/pgsty/silo-console/pull/49), [pgsty/silo-console#50](https://github.com/pgsty/silo-console/pull/50), [pgsty/silo-console#51](https://github.com/pgsty/silo-console/pull/51), [pgsty/silo-console#53](https://github.com/pgsty/silo-console/pull/53), [pgsty/silo-console#54](https://github.com/pgsty/silo-console/pull/54), [pgsty/silo-console#55](https://github.com/pgsty/silo-console/pull/55), [pgsty/silo-console#56](https://github.com/pgsty/silo-console/pull/56), [pgsty/silo-console#57](https://github.com/pgsty/silo-console/pull/57), [pgsty/silo-console#58](https://github.com/pgsty/silo-console/pull/58), [pgsty/silo-console#59](https://github.com/pgsty/silo-console/pull/59), [pgsty/silo-console#60](https://github.com/pgsty/silo-console/pull/60), [pgsty/silo-console#61](https://github.com/pgsty/silo-console/pull/61), [pgsty/silo-console#62](https://github.com/pgsty/silo-console/pull/62), [pgsty/silo-console#64](https://github.com/pgsty/silo-console/pull/64), [pgsty/silo-console#65](https://github.com/pgsty/silo-console/pull/65)
|
||||
|
||||
Closed issues: [pgsty/silo-console#1](https://github.com/pgsty/silo-console/issues/1), [pgsty/silo-console#2](https://github.com/pgsty/silo-console/issues/2), [pgsty/silo-console#3](https://github.com/pgsty/silo-console/issues/3), [pgsty/silo-console#4](https://github.com/pgsty/silo-console/issues/4), [pgsty/silo-console#5](https://github.com/pgsty/silo-console/issues/5), [pgsty/silo-console#6](https://github.com/pgsty/silo-console/issues/6), [pgsty/silo-console#7](https://github.com/pgsty/silo-console/issues/7), [pgsty/silo-console#8](https://github.com/pgsty/silo-console/issues/8), [pgsty/silo-console#12](https://github.com/pgsty/silo-console/issues/12), [pgsty/silo-console#13](https://github.com/pgsty/silo-console/issues/13), [pgsty/silo-console#14](https://github.com/pgsty/silo-console/issues/14), [pgsty/silo-console#15](https://github.com/pgsty/silo-console/issues/15), [pgsty/silo-console#16](https://github.com/pgsty/silo-console/issues/16), [pgsty/silo-console#17](https://github.com/pgsty/silo-console/issues/17), [pgsty/silo-console#18](https://github.com/pgsty/silo-console/issues/18), [pgsty/silo-console#19](https://github.com/pgsty/silo-console/issues/19), [pgsty/silo-console#20](https://github.com/pgsty/silo-console/issues/20), [pgsty/silo-console#21](https://github.com/pgsty/silo-console/issues/21), [pgsty/silo-console#22](https://github.com/pgsty/silo-console/issues/22), [pgsty/silo-console#23](https://github.com/pgsty/silo-console/issues/23), [pgsty/silo-console#24](https://github.com/pgsty/silo-console/issues/24), [pgsty/silo-console#25](https://github.com/pgsty/silo-console/issues/25), [pgsty/silo-console#26](https://github.com/pgsty/silo-console/issues/26), [pgsty/silo-console#27](https://github.com/pgsty/silo-console/issues/27), [pgsty/silo-console#28](https://github.com/pgsty/silo-console/issues/28), [pgsty/silo-console#29](https://github.com/pgsty/silo-console/issues/29), [pgsty/silo-console#30](https://github.com/pgsty/silo-console/issues/30), [pgsty/silo-console#31](https://github.com/pgsty/silo-console/issues/31), [pgsty/silo-console#32](https://github.com/pgsty/silo-console/issues/32), [pgsty/silo-console#33](https://github.com/pgsty/silo-console/issues/33), [pgsty/silo-console#35](https://github.com/pgsty/silo-console/issues/35), [pgsty/silo-console#36](https://github.com/pgsty/silo-console/issues/36), [pgsty/silo-console#37](https://github.com/pgsty/silo-console/issues/37)
|
||||
|
||||
Open issues: [pgsty/silo-console#34](https://github.com/pgsty/silo-console/issues/34)
|
||||
Closed issues: [pgsty/silo-console#1](https://github.com/pgsty/silo-console/issues/1), [pgsty/silo-console#2](https://github.com/pgsty/silo-console/issues/2), [pgsty/silo-console#3](https://github.com/pgsty/silo-console/issues/3), [pgsty/silo-console#4](https://github.com/pgsty/silo-console/issues/4), [pgsty/silo-console#5](https://github.com/pgsty/silo-console/issues/5), [pgsty/silo-console#6](https://github.com/pgsty/silo-console/issues/6), [pgsty/silo-console#7](https://github.com/pgsty/silo-console/issues/7), [pgsty/silo-console#8](https://github.com/pgsty/silo-console/issues/8), [pgsty/silo-console#12](https://github.com/pgsty/silo-console/issues/12), [pgsty/silo-console#13](https://github.com/pgsty/silo-console/issues/13), [pgsty/silo-console#14](https://github.com/pgsty/silo-console/issues/14), [pgsty/silo-console#15](https://github.com/pgsty/silo-console/issues/15), [pgsty/silo-console#16](https://github.com/pgsty/silo-console/issues/16), [pgsty/silo-console#17](https://github.com/pgsty/silo-console/issues/17), [pgsty/silo-console#18](https://github.com/pgsty/silo-console/issues/18), [pgsty/silo-console#19](https://github.com/pgsty/silo-console/issues/19), [pgsty/silo-console#20](https://github.com/pgsty/silo-console/issues/20), [pgsty/silo-console#21](https://github.com/pgsty/silo-console/issues/21), [pgsty/silo-console#22](https://github.com/pgsty/silo-console/issues/22), [pgsty/silo-console#23](https://github.com/pgsty/silo-console/issues/23), [pgsty/silo-console#24](https://github.com/pgsty/silo-console/issues/24), [pgsty/silo-console#25](https://github.com/pgsty/silo-console/issues/25), [pgsty/silo-console#26](https://github.com/pgsty/silo-console/issues/26), [pgsty/silo-console#27](https://github.com/pgsty/silo-console/issues/27), [pgsty/silo-console#28](https://github.com/pgsty/silo-console/issues/28), [pgsty/silo-console#29](https://github.com/pgsty/silo-console/issues/29), [pgsty/silo-console#30](https://github.com/pgsty/silo-console/issues/30), [pgsty/silo-console#31](https://github.com/pgsty/silo-console/issues/31), [pgsty/silo-console#32](https://github.com/pgsty/silo-console/issues/32), [pgsty/silo-console#33](https://github.com/pgsty/silo-console/issues/33), [pgsty/silo-console#34](https://github.com/pgsty/silo-console/issues/34), [pgsty/silo-console#35](https://github.com/pgsty/silo-console/issues/35), [pgsty/silo-console#36](https://github.com/pgsty/silo-console/issues/36), [pgsty/silo-console#37](https://github.com/pgsty/silo-console/issues/37)
|
||||
|
||||
Merged PRs: [pgsty/mc#1](https://github.com/pgsty/mc/pull/1), [pgsty/mc#2](https://github.com/pgsty/mc/pull/2), [pgsty/mc#3](https://github.com/pgsty/mc/pull/3), [pgsty/mc#4](https://github.com/pgsty/mc/pull/4), [pgsty/mc#8](https://github.com/pgsty/mc/pull/8), [pgsty/mc#9](https://github.com/pgsty/mc/pull/9), [pgsty/mc#10](https://github.com/pgsty/mc/pull/10), [pgsty/mc#11](https://github.com/pgsty/mc/pull/11), [pgsty/mc#13](https://github.com/pgsty/mc/pull/13), [pgsty/mc#22](https://github.com/pgsty/mc/pull/22), [pgsty/mc#24](https://github.com/pgsty/mc/pull/24), [pgsty/mc#27](https://github.com/pgsty/mc/pull/27), [pgsty/mc#32](https://github.com/pgsty/mc/pull/32), [pgsty/mc#33](https://github.com/pgsty/mc/pull/33), [pgsty/mc#34](https://github.com/pgsty/mc/pull/34), [pgsty/mc#35](https://github.com/pgsty/mc/pull/35), [pgsty/mc#36](https://github.com/pgsty/mc/pull/36), [pgsty/mc#37](https://github.com/pgsty/mc/pull/37), [pgsty/mc#38](https://github.com/pgsty/mc/pull/38), [pgsty/mc#39](https://github.com/pgsty/mc/pull/39), [pgsty/mc#40](https://github.com/pgsty/mc/pull/40), [pgsty/mc#41](https://github.com/pgsty/mc/pull/41), [pgsty/mc#42](https://github.com/pgsty/mc/pull/42), [pgsty/mc#43](https://github.com/pgsty/mc/pull/43), [pgsty/mc#46](https://github.com/pgsty/mc/pull/46), [pgsty/mc#47](https://github.com/pgsty/mc/pull/47)
|
||||
|
||||
|
||||
@@ -17,9 +17,9 @@ ENV GOPATH=/go
|
||||
ENV CGO_ENABLED=0
|
||||
|
||||
ARG MC_REPO=pgsty/mc
|
||||
ARG MC_VERSION=RELEASE.2026-10-18T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=b5392a642dbc9685b517ee901f032d1d5e88f41704436f0f08dde6060b577db3
|
||||
ARG MC_ARM64_SHA256=17f3eb694ed34e155b331b1215832140749656d8d4b60e2e3a4d0573b8f320d7
|
||||
ARG MC_VERSION=RELEASE.2026-10-25T00-00-00Z
|
||||
ARG MC_AMD64_SHA256=54bd131039dad6677f083a75207b28f82818c067c8e65fae1973f51510d268e1
|
||||
ARG MC_ARM64_SHA256=9bb294f1912f8c6b9ac08c9a0b8d2964b86306a7c440b987ee5f0570d18de3c5
|
||||
|
||||
RUN apk add -U --no-cache \
|
||||
ca-certificates \
|
||||
|
||||
@@ -43,11 +43,12 @@ This release strengthens Object Lock and IAM authorization, makes scoped STS rev
|
||||
|
||||
Read the [release notes](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-18T00-00-00Z) before upgrading, especially the removal of strict multipart listing, coordinated server upgrades, and measured performance costs. See [CONTRIBUTORS.md](CONTRIBUTORS.md) for the people behind the code, reports, and documentation.
|
||||
|
||||
Current source is unreleased: Go 1.27.3, silo-pkg v3.15.1, mcli 20261025,
|
||||
upstream minio-go `ab93db7ba435`, and the Console 2.5.0 candidate `258717be077b`.
|
||||
It includes the Server IAM cache repair needed by Console's **Reset to default**
|
||||
action. See [CHANGELOG.md](CHANGELOG.md#unreleased); the published artifacts above
|
||||
retain their original component versions.
|
||||
The 20261025 source is prepared for a Draft Release with Go 1.27.3,
|
||||
silo-pkg v3.15.1, released mcli 20261025, Console v2.5.0 (`989fa7150792`),
|
||||
and upstream minio-go `ab93db7ba435`. It includes the Server IAM cache repair
|
||||
needed by Console's **Reset to default** action and bounded write/rollback/cleanup
|
||||
repairs. Read the [changes and remaining storage limits](CHANGELOG.md#release2026-10-25t00-00-00z).
|
||||
The published artifacts above retain their original component versions.
|
||||
|
||||
## Overview
|
||||
|
||||
|
||||
+6
-4
@@ -43,10 +43,12 @@
|
||||
|
||||
升级前请阅读[发行说明](https://github.com/pgsty/silo/releases/tag/RELEASE.2026-10-18T00-00-00Z),特别是 strict 分片列表的移除、服务端协调升级要求与实测性能成本。代码、问题报告和文档贡献者见 [CONTRIBUTORS.md](CONTRIBUTORS.md)。
|
||||
|
||||
当前源码尚未发布:使用 Go 1.27.3、silo-pkg v3.15.1、mcli 20261025、上游
|
||||
minio-go `ab93db7ba435`,并内嵌 Console 2.5.0 候选提交 `258717be077b`。
|
||||
其中包含 Console **Reset to default(恢复默认)** 所需的 Server IAM 缓存修复。
|
||||
详见 [CHANGELOG.md](CHANGELOG.md#unreleased);上述已发布制品仍保留原有组件版本。
|
||||
20261025 源码正在准备 Draft Release:使用 Go 1.27.3、silo-pkg v3.15.1、
|
||||
已发布的 mcli 20261025、Console v2.5.0(`989fa7150792`)与上游 minio-go
|
||||
`ab93db7ba435`。其中包含 Console **Reset to default(恢复默认)** 所需的
|
||||
Server IAM 缓存修复,以及限定范围的写入、回滚与清理修复。
|
||||
请阅读[变更与仍存的存储限制](CHANGELOG.md#release2026-10-25t00-00-00z);
|
||||
上述已发布制品仍保留原有组件版本。
|
||||
|
||||
## 概述
|
||||
|
||||
|
||||
@@ -40,7 +40,7 @@ if [ -n "${MCLI_BIN:-}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-10-18T00-00-00Z}
|
||||
release=${MCLI_RELEASE:-RELEASE.2026-10-25T00-00-00Z}
|
||||
version_hyphen=${release#RELEASE.}
|
||||
package_version=$(printf '%s\n' "${version_hyphen}" | sed -E 's/^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2})-([0-9]{2})-([0-9]{2})Z$/\1\2\3\4\5\6.0.0/')
|
||||
if [ "${package_version}" = "${version_hyphen}" ]; then
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type abandonedCleanupCommitDisk struct {
|
||||
StorageAPI
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d abandonedCleanupCommitDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
|
||||
var success int64
|
||||
gate := &scopeStagedContext{
|
||||
Context: ctx, meta: filepath.Join(d.Endpoint().Path, dv, dp, fi.DataDir, "part.1"),
|
||||
entered: d.entered, release: d.release,
|
||||
tracker: healthDiskCtxValue{lastSuccess: &success},
|
||||
}
|
||||
return d.StorageAPI.RenameData(gate, sv, sp, fi, dv, dp, o)
|
||||
}
|
||||
|
||||
// A lost namespace lease can admit cleanup while a disk transaction is between
|
||||
// moving its external shards and replacing xl.meta. Those shards are live work,
|
||||
// even though the old metadata does not refer to them yet. Cleanup must wait.
|
||||
func TestMutationAbandonedCleanupWaitsForCommit(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cleanup-during-commit"
|
||||
body := strings.Repeat("new payload", 200000)
|
||||
mutationPut(t, z, bucket, object, 0, strings.Repeat("old payload", 200000), time.Now().Add(-time.Minute))
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = abandonedCleanupCommitDisk{d, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true})
|
||||
committed <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-committed:
|
||||
t.Fatalf("PUT returned before shard barrier: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("PUT did not move all external shards")
|
||||
}
|
||||
}
|
||||
t.Log("All eight new external shard directories exist; old xl.meta is still active")
|
||||
cleaned := make(chan error, len(disks))
|
||||
for _, d := range disks {
|
||||
go func(d StorageAPI) { cleaned <- d.CleanAbandonedData(context.Background(), bucket, object) }(d)
|
||||
}
|
||||
early := 0
|
||||
timer := time.NewTimer(150 * time.Millisecond)
|
||||
wait:
|
||||
for {
|
||||
select {
|
||||
case err := <-cleaned:
|
||||
early++
|
||||
t.Errorf("abandoned-data cleanup overtook unfinished commit: %v", err)
|
||||
if early == len(disks) {
|
||||
break wait
|
||||
}
|
||||
case <-timer.C:
|
||||
break wait
|
||||
}
|
||||
}
|
||||
timer.Stop()
|
||||
unblock()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err != nil {
|
||||
t.Fatalf("PUT failed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("PUT did not finish")
|
||||
}
|
||||
for i := early; i < len(disks); i++ {
|
||||
select {
|
||||
case err := <-cleaned:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cleanup did not finish")
|
||||
}
|
||||
}
|
||||
swap(disks)
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("acknowledged PUT unreadable after abandoned-data cleanup: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil || string(data) != body {
|
||||
t.Fatalf("acknowledged external shards lost: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup can also arrive after a disk has committed but before its delayed
|
||||
// reply lets the coordinator decide to undo a failed quorum. The previous
|
||||
// generation's backup is still owned by that unfinished transaction.
|
||||
func TestMutationAbandonedCleanupPreservesUndoBackup(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cleanup-before-undo"
|
||||
body := strings.Repeat("last acknowledged", 150000)
|
||||
base := time.Now().Add(-time.Minute)
|
||||
mutationPut(t, z, bucket, object, 0, body, base)
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, 4), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = delayedRenameResult{d, i >= 4, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
next := strings.Repeat("failed update", 180000)
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(next), int64(len(next)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Second)})
|
||||
committed <- err
|
||||
}()
|
||||
for range 4 {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-committed:
|
||||
t.Fatalf("PUT returned before reply barrier: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("disk commit did not reach reply barrier")
|
||||
}
|
||||
}
|
||||
for _, d := range disks[:4] {
|
||||
if err := d.CleanAbandonedData(t.Context(), bucket, object); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err == nil {
|
||||
t.Fatal("four-of-eight commit unexpectedly reached write quorum")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("failed PUT did not finish undo")
|
||||
}
|
||||
swap(disks)
|
||||
for i, d := range disks {
|
||||
fi, err := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil || !fi.ModTime.Equal(base) {
|
||||
t.Errorf("disk%d lost last acknowledged generation after failed PUT: mtime=%v err=%v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("last acknowledged value unreadable after cleanup and undo: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
data, err := io.ReadAll(r)
|
||||
if err != nil || string(data) != body {
|
||||
t.Fatalf("last acknowledged value lost after cleanup and undo: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
@@ -511,14 +511,13 @@ func testSiteReplicationStatusDetectsCorsTimestampMismatch(obj ObjectLayer, _ st
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
serviceCred, err := auth.CreateCredentials("cors-status-service", "cors-status-service-secret-key")
|
||||
serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "cors-status-service",
|
||||
secretKey: "cors-status-service-secret-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = credentials.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
@@ -585,14 +584,13 @@ func testSiteReplicationStatusCountsCorsPerSite(obj ObjectLayer, _ string, bucke
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
serviceCred, err := auth.CreateCredentials("cors-count-service", "cors-count-service-secret-key")
|
||||
serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "cors-count-service",
|
||||
secretKey: "cors-count-service-secret-key",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = credentials.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
@@ -1190,14 +1188,13 @@ func testCorsReplicationDispatchStatusHealReload(obj ObjectLayer, _ string, buck
|
||||
}))
|
||||
defer remote.Close()
|
||||
|
||||
serviceCred, err := auth.CreateCredentials("cors-integration-svc", "cors-integration-service-secret")
|
||||
serviceCred, _, err := globalIAMSys.NewServiceAccount(ctx, credentials.AccessKey, nil, newServiceAccountOpts{
|
||||
accessKey: "cors-integration-svc",
|
||||
secretKey: "cors-integration-service-secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = credentials.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
|
||||
// Tests create endpoints with ellipses and without.
|
||||
func TestCreateServerEndpoints(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
args []string
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
@@ -25,9 +26,42 @@ import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/dnscache"
|
||||
)
|
||||
|
||||
type endpointTestResolver struct {
|
||||
dnscache.DNSResolver
|
||||
}
|
||||
|
||||
func (r endpointTestResolver) LookupHost(ctx context.Context, host string) ([]string, error) {
|
||||
// These endpoint fixtures test locality and parsing, not public DNS.
|
||||
for i, name := range []string{"example.org", "example.com", "example.net", "example.edu"} {
|
||||
if host == name {
|
||||
return []string{fmt.Sprintf("203.0.113.%d", i+1)}, nil
|
||||
}
|
||||
}
|
||||
return r.DNSResolver.LookupHost(ctx, host)
|
||||
}
|
||||
|
||||
func useTestEndpointDNS(t *testing.T) {
|
||||
t.Helper()
|
||||
previous := globalDNSCache
|
||||
globalDNSCache = &dnscache.Resolver{Resolver: endpointTestResolver{net.DefaultResolver}}
|
||||
t.Cleanup(func() { globalDNSCache = previous })
|
||||
}
|
||||
|
||||
func useStaticTestEndpoints(t *testing.T) {
|
||||
t.Helper()
|
||||
useTestEndpointDNS(t)
|
||||
// These fixtures deliberately use localhost, including inside containers.
|
||||
previous := orchestrated
|
||||
orchestrated = false
|
||||
t.Cleanup(func() { orchestrated = previous })
|
||||
}
|
||||
|
||||
func TestNewEndpoint(t *testing.T) {
|
||||
useTestEndpointDNS(t)
|
||||
u2, _ := url.Parse("https://example.org/path")
|
||||
u4, _ := url.Parse("http://192.168.253.200/path")
|
||||
rootSlashFoo, _ := filepath.Abs("/foo")
|
||||
@@ -130,6 +164,7 @@ func TestNewEndpoints(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCreateEndpoints(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
@@ -353,6 +388,7 @@ func TestCreateEndpoints(t *testing.T) {
|
||||
// So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
|
||||
// is considered a remote service from localhost:9000 perspective.
|
||||
func TestGetLocalPeer(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
@@ -393,6 +429,7 @@ func TestGetLocalPeer(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestGetRemotePeers(t *testing.T) {
|
||||
useStaticTestEndpoints(t)
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
|
||||
+20
-10
@@ -1094,11 +1094,19 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
|
||||
|
||||
// Similar to rename but renames data from srcEntry to dstEntry at dataDir
|
||||
func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, []byte, string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, nil, "", err
|
||||
}
|
||||
// Admission is cancelable. After admission use actual disk results for
|
||||
// commit/rollback, even if the client or namespace lease is canceled.
|
||||
ctx = context.WithoutCancel(ctx)
|
||||
g := errgroup.WithNErrs(len(disks))
|
||||
|
||||
fvID := mustGetUUID()
|
||||
for index := range disks {
|
||||
metadata[index].Metadata = maps.Clone(metadata[index].Metadata)
|
||||
metadata[index].SetTierFreeVersionID(fvID)
|
||||
metadata[index].Metadata[rollbackIDKey] = fvID
|
||||
}
|
||||
|
||||
diskVersions := make([][]byte, len(disks))
|
||||
@@ -1556,6 +1564,18 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
compIndex = opts.IndexCB()
|
||||
}
|
||||
|
||||
if !opts.NoLock {
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
// Mint the generation while holding the commit lock. Staged uploads may
|
||||
// acquire the lock in a different order from their encoding completion.
|
||||
modTime := opts.MTime
|
||||
if opts.MTime.IsZero() {
|
||||
modTime = UTCNow()
|
||||
@@ -1652,16 +1672,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.NoLock {
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
}
|
||||
|
||||
if !opts.ReplicationRequest && !opts.ReplicaLockReconcile && !opts.DataMovement &&
|
||||
(fi.VersionID == "" || fi.VersionID == nullVersionID) {
|
||||
getObjectInfo := er.getObjectInfo
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
// All metadata is valid and readable. Only the completed generations differ.
|
||||
// No timing, missing files, synthetic read error or malformed metadata is needed.
|
||||
func TestReadGenerationConflictIsNotAbsence(t *testing.T) {
|
||||
raw := make([]RawFileInfo, 8)
|
||||
for i := range raw {
|
||||
fi := newFileInfo("object", 4, 4)
|
||||
fi.Erasure.Index = i + 1
|
||||
fi.ModTime = time.Unix(100+int64(i%4), 0)
|
||||
fi.DataDir = mustGetUUID()
|
||||
fi.Metadata = map[string]string{"etag": fmt.Sprint(i % 4)}
|
||||
var meta xlMetaV2
|
||||
if err := meta.AddVersion(fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
buf, err := meta.AppendTo(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw[i] = RawFileInfo{Buf: buf}
|
||||
}
|
||||
_, errs := pickLatestQuorumFilesInfo(t.Context(), raw, make([]error, len(raw)), "bucket", "object", true, false)
|
||||
err := reduceReadQuorumErrs(t.Context(), errs, objectOpIgnoredErrs, 4)
|
||||
if api := toAPIError(t.Context(), toObjectErr(err, "bucket", "object")); api.HTTPStatusCode != http.StatusServiceUnavailable {
|
||||
t.Fatalf("readable conflicting generations reported as absence: %v (%+v)", err, api)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPoolsReadGenerationConflictDoesNotExposeOldValue(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
router := initTestAPIEndPoints(z, []string{"GetObject", "HeadObject"})
|
||||
for _, tc := range []struct {
|
||||
pool int
|
||||
mode string
|
||||
}{
|
||||
{0, "split"},
|
||||
{1, "split"},
|
||||
{0, "older-quorum"},
|
||||
{1, "older-quorum"},
|
||||
{0, "newer-quorum"},
|
||||
{1, "newer-quorum"},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("%s/pool-%d", tc.mode, tc.pool), func(t *testing.T) {
|
||||
broken := tc.pool
|
||||
object := fmt.Sprintf("conflict-%s-%d", tc.mode, broken)
|
||||
base := UTCNow().Add(-time.Hour)
|
||||
putConsistencyObject(t, z, bucket, object, 1-broken, "old-acknowledged", ObjectOptions{MTime: base})
|
||||
set := z.serverPools[broken].getHashedSet(object)
|
||||
disks := set.getDisks()
|
||||
saved := make([][]byte, len(disks))
|
||||
savedParts := make([]map[string][]byte, len(disks))
|
||||
for generation := range 4 {
|
||||
putConsistencyObject(t, z, bucket, object, broken, fmt.Sprintf("newer-%d", generation),
|
||||
ObjectOptions{MTime: base.Add(time.Duration(generation+1) * time.Minute)})
|
||||
for i, disk := range disks {
|
||||
g := i % 4
|
||||
switch tc.mode {
|
||||
case "older-quorum":
|
||||
g = 0
|
||||
if i >= len(disks)-3 {
|
||||
g = i - (len(disks) - 3) + 1
|
||||
}
|
||||
case "newer-quorum":
|
||||
g = 3
|
||||
if i < 3 {
|
||||
g = i
|
||||
}
|
||||
}
|
||||
if g != generation {
|
||||
continue
|
||||
}
|
||||
buf, err := disk.ReadAll(ctx, bucket, pathJoin(object, xlStorageFormatFile))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved[i] = bytes.Clone(buf)
|
||||
savedParts[i] = readGenerationParts(t, disk, bucket, object, buf)
|
||||
}
|
||||
}
|
||||
for i, disk := range disks {
|
||||
for path, data := range savedParts[i] {
|
||||
if err := disk.WriteAll(ctx, bucket, path, data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := disk.WriteAll(ctx, bucket, pathJoin(object, xlStorageFormatFile), saved[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
for _, method := range []string{http.MethodHead, http.MethodGet} {
|
||||
rec := consistencyRequest(t, router, method, bucket, object, version)
|
||||
want := http.StatusServiceUnavailable
|
||||
if tc.mode == "newer-quorum" {
|
||||
want = http.StatusOK
|
||||
}
|
||||
if rec.Code != want {
|
||||
t.Errorf("%s version=%q: want status=%d, got=%d body=%q", method, version, want, rec.Code, rec.Body.String())
|
||||
}
|
||||
if want == http.StatusOK && method == http.MethodGet && rec.Body.String() != "newer-3" {
|
||||
t.Errorf("latest quorum did not return its payload: %q", rec.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
if tc.mode != "newer-quorum" {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
_, err := set.CopyObject(ctx, bucket, object, bucket, object,
|
||||
ObjectInfo{metadataOnly: true, UserDefined: map[string]string{"copy-test": "conflicting-source"}},
|
||||
ObjectOptions{VersionID: version}, ObjectOptions{})
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Errorf("COPY-to-self version=%q: want read-quorum error, got %v", version, err)
|
||||
}
|
||||
for _, scan := range []madmin.HealScanMode{madmin.HealNormalScan, madmin.HealDeepScan} {
|
||||
_, err := set.HealObject(ctx, bucket, object, version, madmin.HealOpts{ScanMode: scan, Remove: true})
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Errorf("heal version=%q scan=%v: want read-quorum error, got %v", version, scan, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, disk := range disks {
|
||||
buf, err := disk.ReadAll(ctx, bucket, pathJoin(object, xlStorageFormatFile))
|
||||
if err != nil || !bytes.Equal(buf, saved[i]) {
|
||||
t.Errorf("read or heal destroyed conflicting generation on disk %d: %v", i, err)
|
||||
}
|
||||
if tc.mode != "newer-quorum" {
|
||||
for path, data := range savedParts[i] {
|
||||
got, err := disk.ReadAll(ctx, bucket, path)
|
||||
if err != nil || !bytes.Equal(got, data) {
|
||||
t.Errorf("read or heal destroyed generation shard on disk %d: %s: %v", i, path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func readGenerationParts(t *testing.T, disk StorageAPI, bucket, object string, metadata []byte) map[string][]byte {
|
||||
t.Helper()
|
||||
fi, err := fileInfoFromRaw(RawFileInfo{Buf: metadata}, bucket, object, false, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts := make(map[string][]byte)
|
||||
if fi.InlineData() || fi.Size == 0 {
|
||||
return parts
|
||||
}
|
||||
// Later PUTs remove the previous data directory. Save its real bitrot
|
||||
// shards too, so restoring metadata never fabricates missing-part damage.
|
||||
for _, part := range fi.Parts {
|
||||
path := pathJoin(object, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
|
||||
data, err := disk.ReadAll(t.Context(), bucket, path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
parts[path] = data
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
func TestPoolsReadUnavailablePoolDoesNotExposeOldValue(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
for broken := range 2 {
|
||||
t.Run(fmt.Sprintf("broken-pool-%d", broken), func(t *testing.T) {
|
||||
object := fmt.Sprintf("unavailable-%d", broken)
|
||||
putConsistencyObject(t, z, bucket, object, 1-broken, "old-acknowledged", ObjectOptions{})
|
||||
set := z.serverPools[broken].getHashedSet(object)
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
defer switchDisks(disks)
|
||||
faulty := append([]StorageAPI(nil), disks...)
|
||||
for i := range faulty {
|
||||
faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: object}
|
||||
}
|
||||
switchDisks(faulty)
|
||||
if _, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err == nil || isErrObjectNotFound(err) {
|
||||
t.Errorf("unknown pool state allowed an old value: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type generationReadGateDisk struct {
|
||||
StorageAPI
|
||||
ready chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d generationReadGateDisk) ReadXL(ctx context.Context, volume, path string, data bool) (RawFileInfo, error) {
|
||||
fi, err := d.StorageAPI.ReadXL(ctx, volume, path, data)
|
||||
d.ready <- struct{}{}
|
||||
if d.release != nil {
|
||||
<-d.release
|
||||
}
|
||||
return fi, err
|
||||
}
|
||||
|
||||
func (d generationReadGateDisk) ReadVersion(ctx context.Context, orig, volume, path, version string, opts ReadOptions) (FileInfo, error) {
|
||||
fi, err := d.StorageAPI.ReadVersion(ctx, orig, volume, path, version, opts)
|
||||
d.ready <- struct{}{}
|
||||
if d.release != nil {
|
||||
<-d.release
|
||||
}
|
||||
return fi, err
|
||||
}
|
||||
|
||||
func TestReadGenerationFastInfoWaitsForSlowDisk(t *testing.T) {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
t.Run("version="+version, func(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const object = "slow-newer-generation"
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
defer switchDisks(disks)
|
||||
base := UTCNow().Add(-time.Hour)
|
||||
putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: base})
|
||||
saved := make([][]byte, len(disks)-1)
|
||||
for i := range saved {
|
||||
saved[i] = mustReadNullQuorumMeta(t, disks[i], bucket, object)
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, 0, "new", ObjectOptions{MTime: base.Add(time.Minute)})
|
||||
for i, meta := range saved {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ready, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range disks {
|
||||
gate := generationReadGateDisk{StorageAPI: disk, ready: ready}
|
||||
if i == len(disks)-1 {
|
||||
gate.release = release
|
||||
}
|
||||
wrapped[i] = gate
|
||||
}
|
||||
switchDisks(wrapped)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: true, VersionID: version}, false)
|
||||
done <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-ready:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("metadata reads did not reach the barrier")
|
||||
}
|
||||
}
|
||||
early := false
|
||||
select {
|
||||
case err := <-done:
|
||||
early = true
|
||||
t.Errorf("fast read returned before the newer disk: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
unblock()
|
||||
if !early {
|
||||
select {
|
||||
case err := <-done:
|
||||
if !isErrReadQuorum(err) {
|
||||
t.Fatalf("fast read exposed the old generation: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("fast read did not finish after the newer disk")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// An acknowledged recovery overwrite must resolve the unreadable pool, even
|
||||
// when another pool already has a readable copy of the same object.
|
||||
// Real inline metadata is combined in the same 15-old/1-new state used by
|
||||
// that regression. Capacity adapters only make destination choice deterministic.
|
||||
func TestPoolsMinorityOverwriteRecovery(t *testing.T) {
|
||||
for _, versioning := range []string{"unversioned", "versioned", "suspended"} {
|
||||
for _, mode := range []string{"single-pool", "dual-pool-only-inconsistent-copy", "dual-pool-readable-other-copy"} {
|
||||
t.Run(versioning+"/"+mode, func(t *testing.T) {
|
||||
testPoolsMinorityOverwriteRecovery(t, mode, versioning, "")
|
||||
})
|
||||
}
|
||||
}
|
||||
for _, encryption := range []string{"SSE-C", "SSE-S3", "SSE-KMS"} {
|
||||
t.Run(encryption, func(t *testing.T) {
|
||||
testPoolsMinorityOverwriteRecovery(t, "dual-pool-readable-other-copy", "unversioned", encryption)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func testPoolsMinorityOverwriteRecovery(t *testing.T, mode, versioning, encryption string) {
|
||||
t.Helper()
|
||||
var z *erasureServerPools
|
||||
if mode == "single-pool" {
|
||||
z, _ = mutationPools(t)
|
||||
} else {
|
||||
z, _ = consistencyPools(t)
|
||||
}
|
||||
const object = "review-retry"
|
||||
for _, pool := range z.serverPools {
|
||||
restore := conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI { return conditionalPutCapacityDisk{StorageAPI: d} })
|
||||
defer restore()
|
||||
}
|
||||
bucket, router := conditionalPutBucket(t, z, versioning)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
oldBody := bytes.Repeat([]byte{'a'}, 8192)
|
||||
pendingBody := bytes.Repeat([]byte{'b'}, 8192)
|
||||
recoveryBody := bytes.Repeat([]byte{'c'}, 8192)
|
||||
seed := func(pool int, body []byte, when time.Time) {
|
||||
t.Helper()
|
||||
putConsistencyObject(t, z, bucket, object, pool, string(body), ObjectOptions{MTime: when})
|
||||
}
|
||||
seed(0, oldBody, oldTime)
|
||||
disks := z.serverPools[0].getHashedSet(object).getDisks()
|
||||
oldMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
oldMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
seed(0, pendingBody, oldTime.Add(time.Minute))
|
||||
for i := 0; i < len(disks)-1; i++ {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if mode == "dual-pool-readable-other-copy" {
|
||||
seed(1, oldBody, oldTime.Add(-time.Minute))
|
||||
// Ordinary allocation can select this healthy other pool.
|
||||
// Force that permitted choice without altering metadata reads.
|
||||
restore := conditionalPutPool(t, z, object, 1)
|
||||
defer restore()
|
||||
}
|
||||
before := consistencyRequest(t, router, http.MethodGet, bucket, object, "")
|
||||
if before.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("fixture must refuse ambiguous read; got %d", before.Code)
|
||||
}
|
||||
var headers, readHeaders map[string]string
|
||||
if encryption != "" {
|
||||
oldKMS, oldTLS := GlobalKMS, globalIsTLS
|
||||
GlobalKMS, globalIsTLS = kms.NewStub("recovery-key"), true
|
||||
defer func() { GlobalKMS, globalIsTLS = oldKMS, oldTLS }()
|
||||
headers = map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
|
||||
switch encryption {
|
||||
case "SSE-C":
|
||||
key := bytes.Repeat([]byte{0x42}, 32)
|
||||
digest := md5.Sum(key)
|
||||
headers = map[string]string{
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(digest[:]),
|
||||
}
|
||||
readHeaders = headers
|
||||
case "SSE-KMS":
|
||||
headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||
headers[xhttp.AmzServerSideEncryptionKmsID] = "recovery-key"
|
||||
}
|
||||
}
|
||||
url := getPutObjectURL("", bucket, object)
|
||||
result := multipartConditionRequest(t, router, http.MethodPut, url, string(recoveryBody), headers)
|
||||
t.Logf("recovery PUT HTTP=%d response=%s", result.Code, result.Body.String())
|
||||
if result.Code != http.StatusOK {
|
||||
t.Fatalf("unconditional recovery PUT: %d %s", result.Code, result.Body.String())
|
||||
}
|
||||
after := multipartConditionRequest(t, router, http.MethodGet, url, "", readHeaders)
|
||||
t.Logf("after acknowledged recovery PUT: GET HTTP=%d matches_new=%t", after.Code, bytes.Equal(after.Body.Bytes(), recoveryBody))
|
||||
for i, pool := range z.serverPools {
|
||||
oi, err := pool.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
t.Logf("pool=%d mtime=%s err=%v", i, oi.ModTime, err)
|
||||
}
|
||||
if after.Code != http.StatusOK || !bytes.Equal(after.Body.Bytes(), recoveryBody) {
|
||||
t.Fatalf("acknowledged recovery overwrite did not restore reads: HTTP=%d body=%s", after.Code, after.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Read failures activated by a completed disk mutation verify that a recovery
|
||||
// ACK is not issued from preflight information alone.
|
||||
type recoveryReadFaultDisk struct {
|
||||
StorageAPI
|
||||
bucket, object string
|
||||
fail *atomic.Bool
|
||||
afterRename bool
|
||||
}
|
||||
|
||||
func (d recoveryReadFaultDisk) ReadXL(ctx context.Context, bucket, object string, data bool) (RawFileInfo, error) {
|
||||
if bucket == d.bucket && object == d.object && d.fail.Load() {
|
||||
return RawFileInfo{}, errDiskNotFound
|
||||
}
|
||||
return d.StorageAPI.ReadXL(ctx, bucket, object, data)
|
||||
}
|
||||
|
||||
func (d recoveryReadFaultDisk) ReadVersion(ctx context.Context, orig, bucket, object, version string, opts ReadOptions) (FileInfo, error) {
|
||||
if bucket == d.bucket && object == d.object && d.fail.Load() {
|
||||
return FileInfo{}, errDiskNotFound
|
||||
}
|
||||
return d.StorageAPI.ReadVersion(ctx, orig, bucket, object, version, opts)
|
||||
}
|
||||
|
||||
func (d recoveryReadFaultDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
res, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
if err == nil && d.afterRename && dv == d.bucket && dp == d.object {
|
||||
d.fail.Store(true)
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
||||
func TestPoolsRecoveryRefusesUnresolvedPools(t *testing.T) {
|
||||
for _, mode := range []string{"two-inconsistent", "suspended", "rebalancing", "offline", "read-fails-after-commit", "newer-readable-other-copy"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
z, _ := consistencyPools(t)
|
||||
const object = "refused-recovery"
|
||||
bucket, router := conditionalPutBucket(t, z, "unversioned")
|
||||
for _, pool := range z.serverPools {
|
||||
restore := conditionalPutSwapDisks(pool, object, func(d StorageAPI) StorageAPI { return conditionalPutCapacityDisk{StorageAPI: d} })
|
||||
defer restore()
|
||||
}
|
||||
base := time.Now().UTC().Add(-time.Hour)
|
||||
old := bytes.Repeat([]byte{'a'}, 8192)
|
||||
pending := bytes.Repeat([]byte{'b'}, 8192)
|
||||
replacement := bytes.Repeat([]byte{'c'}, 8192)
|
||||
inconsistent := func(pool int) {
|
||||
t.Helper()
|
||||
putConsistencyObject(t, z, bucket, object, pool, string(old), ObjectOptions{MTime: base})
|
||||
disks := z.serverPools[pool].getHashedSet(object).getDisks()
|
||||
saved := make([][]byte, len(disks))
|
||||
for i, d := range disks {
|
||||
saved[i] = mustReadNullQuorumMeta(t, d, bucket, object)
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, pool, string(pending), ObjectOptions{MTime: base.Add(time.Minute)})
|
||||
for i := 0; i < len(disks)-1; i++ {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, saved[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
inconsistent(0)
|
||||
otherTime := base.Add(-time.Minute)
|
||||
if mode == "newer-readable-other-copy" {
|
||||
otherTime = base.Add(2 * time.Hour)
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, 1, string(old), ObjectOptions{MTime: otherTime})
|
||||
if mode == "two-inconsistent" {
|
||||
inconsistent(1)
|
||||
}
|
||||
// Select a healthy destination if the resolver incorrectly ignores pool0.
|
||||
postCommitFailure := mode == "read-fails-after-commit" || mode == "newer-readable-other-copy"
|
||||
if !postCommitFailure {
|
||||
restore := conditionalPutPool(t, z, object, 1)
|
||||
defer restore()
|
||||
}
|
||||
if mode == "suspended" {
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta.Pools[0].Decommission = &PoolDecommissionInfo{}
|
||||
z.poolMetaMutex.Unlock()
|
||||
}
|
||||
if mode == "rebalancing" {
|
||||
z.rebalMu.Lock()
|
||||
z.rebalMeta = &rebalanceMeta{PoolStats: []*rebalanceStats{{Participating: true, Info: rebalanceInfo{Status: rebalStarted}}, {}}}
|
||||
z.rebalMu.Unlock()
|
||||
}
|
||||
var fail atomic.Bool
|
||||
if mode == "offline" || mode == "read-fails-after-commit" {
|
||||
fail.Store(mode == "offline")
|
||||
restore := conditionalPutSwapDisks(z.serverPools[0], object, func(d StorageAPI) StorageAPI {
|
||||
return recoveryReadFaultDisk{StorageAPI: d, bucket: bucket, object: object, fail: &fail, afterRename: mode == "read-fails-after-commit"}
|
||||
})
|
||||
defer restore()
|
||||
}
|
||||
snapshot := func() [][32]byte {
|
||||
var result [][32]byte
|
||||
for _, pool := range z.serverPools {
|
||||
for _, disk := range pool.getHashedSet(object).getDisks() {
|
||||
b, err := disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result = append(result, sha256.Sum256(b))
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
before := snapshot()
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getGetObjectURL("", bucket, object), int64(len(replacement)), bytes.NewReader(replacement), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result := httptest.NewRecorder()
|
||||
router.ServeHTTP(result, req.WithContext(t.Context()))
|
||||
if result.Code != http.StatusServiceUnavailable {
|
||||
t.Errorf("unresolved recovery returned HTTP=%d: %s", result.Code, result.Body.String())
|
||||
}
|
||||
switch {
|
||||
case !postCommitFailure:
|
||||
after := snapshot()
|
||||
for i := range before {
|
||||
if before[i] != after[i] {
|
||||
t.Errorf("refused preflight modified shard %d", i)
|
||||
}
|
||||
}
|
||||
case mode == "read-fails-after-commit":
|
||||
fail.Store(false)
|
||||
got := consistencyRequest(t, router, http.MethodGet, bucket, object, "")
|
||||
if got.Code != http.StatusOK || !bytes.Equal(got.Body.Bytes(), replacement) {
|
||||
t.Errorf("read validation failure must not undo committed recovery: HTTP=%d", got.Code)
|
||||
}
|
||||
default:
|
||||
// A readable but different latest copy cannot confirm recovery.
|
||||
// The committed replacement must still remain in its target pool.
|
||||
r, err := z.serverPools[0].GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || !bytes.Equal(body, replacement) {
|
||||
t.Fatalf("failed recovery validation must preserve committed data: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPutObjectGenerationFollowsCommitLock(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
for _, preserve := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "generated", true: "preserved"}[preserve], func(t *testing.T) {
|
||||
const object = "ordered-generation"
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
lock := set.NewNSLock(bucket, object)
|
||||
lc, err := lock.GetLock(t.Context(), globalOperationTimeout)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
locked := true
|
||||
defer func() {
|
||||
if locked {
|
||||
lock.Unlock(lc)
|
||||
}
|
||||
}()
|
||||
|
||||
opts := ObjectOptions{}
|
||||
if preserve {
|
||||
opts.MTime = time.Unix(1234567890, 0)
|
||||
}
|
||||
reader := mustGetPutObjReader(t, strings.NewReader("payload"), 7, "", "")
|
||||
type result struct {
|
||||
info ObjectInfo
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
info, err := set.PutObject(t.Context(), bucket, object, reader, opts)
|
||||
done <- result{info, err}
|
||||
}()
|
||||
|
||||
// Wait for the real writer to queue behind the held namespace
|
||||
// lock. No elapsed delay is used to infer its progress.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
set.nsMutex.lockMapMutex.Lock()
|
||||
entry := set.nsMutex.lockMap[pathJoin(bucket, object)]
|
||||
waiting := entry != nil && entry.ref == 2
|
||||
set.nsMutex.lockMapMutex.Unlock()
|
||||
if waiting {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case got := <-done:
|
||||
t.Fatalf("PUT bypassed the held commit lock: %v", got.err)
|
||||
default:
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("PUT did not queue for the commit lock")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
beforeUnlock := UTCNow()
|
||||
lock.Unlock(lc)
|
||||
locked = false
|
||||
got := <-done
|
||||
if got.err != nil {
|
||||
t.Fatal(got.err)
|
||||
}
|
||||
if preserve {
|
||||
if !got.info.ModTime.Equal(opts.MTime) {
|
||||
t.Fatalf("explicit modification time changed: %s", got.info.ModTime)
|
||||
}
|
||||
} else if got.info.ModTime.Before(beforeUnlock) {
|
||||
t.Fatalf("generation %s predates commit lock release %s", got.info.ModTime, beforeUnlock)
|
||||
}
|
||||
for i, disk := range set.getDisks() {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, nullVersionID, ReadOptions{})
|
||||
if err != nil || !fi.ModTime.Equal(got.info.ModTime) {
|
||||
t.Errorf("disk %d stored a different generation: %s, %v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
)
|
||||
|
||||
type compatibilityFailedOverwrite struct {
|
||||
StorageAPI
|
||||
commit bool
|
||||
}
|
||||
|
||||
func (d compatibilityFailedOverwrite) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
if d.commit {
|
||||
if _, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts); err != nil {
|
||||
return RenameDataResp{}, err
|
||||
}
|
||||
}
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
|
||||
// Maintenance compatibility requirement: after a completely joined failed
|
||||
// overwrite with one residue, keep the previously acknowledged quorum readable.
|
||||
// This tests a known history; it does not claim the resulting snapshot proves it.
|
||||
func TestMaintenancePartialFailedOverwriteAvailability(t *testing.T) {
|
||||
// Lock RPC tests configure a single-drive backend (EC:0). Pin this
|
||||
// fixture to 12+4 so its fifteen old shards really have read quorum.
|
||||
previousStorageClass := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(previousStorageClass) })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
dirs, err := getRandomDisks(16)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, dirs...))
|
||||
if err != nil {
|
||||
removeRoots(dirs)
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
defer func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) }()
|
||||
bucket := "maintenance-compatibility"
|
||||
key := "partial-overwrite"
|
||||
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldBody := bytes.Repeat([]byte("old-A"), 65536)
|
||||
newBody := bytes.Repeat([]byte("new-B"), 65536)
|
||||
put := func(body []byte) error {
|
||||
_, err := z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{})
|
||||
return err
|
||||
}
|
||||
if err := put(oldBody); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet(key)
|
||||
getDisks := set.getDisks
|
||||
disks := getDisks()
|
||||
fi, err := disks[0].ReadVersion(ctx, "", bucket, key, "", ReadOptions{})
|
||||
if err != nil || fi.Erasure.DataBlocks != 12 || fi.Erasure.ParityBlocks != 4 {
|
||||
t.Fatalf("compatibility fixture must have 12+4 erasure coding: %+v, %v", fi.Erasure, err)
|
||||
}
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = compatibilityFailedOverwrite{d, i == 0}
|
||||
}
|
||||
set.getDisks = func() []StorageAPI { return wrapped }
|
||||
err = put(newBody)
|
||||
set.getDisks = getDisks
|
||||
if err == nil {
|
||||
t.Fatal("partial overwrite must fail")
|
||||
}
|
||||
t.Logf("Overwrite completed with failure; one new residue, fifteen original shards: %v", err)
|
||||
r, err := z.GetObjectNInfo(ctx, bucket, key, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("MAINTENANCE COMPATIBILITY: previously readable acknowledged value now refused: %v", err)
|
||||
}
|
||||
data, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || !bytes.Equal(data, oldBody) {
|
||||
t.Fatalf("old acknowledged payload mismatch: bytes=%d err=%v", len(data), err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStrictListFallbackRejectsSplitGenerations(t *testing.T) {
|
||||
z, bucket, _ := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
mtime := time.Now().UTC().Add(-time.Hour)
|
||||
var generations [3][][]byte
|
||||
for generation := range generations {
|
||||
body := bytes.Repeat([]byte{byte('a' + generation)}, 8192)
|
||||
nullQuorumPut(t, z, bucket, object, body, ObjectOptions{MTime: mtime.Add(time.Duration(generation) * time.Minute)})
|
||||
generations[generation] = make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
generations[generation][i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
}
|
||||
// All sixteen drives are online, but the 7/5/4 split leaves no generation
|
||||
// with the eight matching snapshots required by the listing resolver.
|
||||
for i, disk := range disks {
|
||||
generation := 0
|
||||
if i >= 7 {
|
||||
generation = 1
|
||||
}
|
||||
if i >= 12 {
|
||||
generation = 2
|
||||
}
|
||||
if err := disk.WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, generations[generation][i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{}, false); !isErrReadQuorum(err) {
|
||||
t.Fatalf("split metadata must remain a read-quorum failure: %v", err)
|
||||
}
|
||||
entries := make(metaCacheEntries, len(disks))
|
||||
for i, disk := range disks {
|
||||
entries[i] = metaCacheEntry{name: object, metadata: mustReadNullQuorumMeta(t, disk, bucket, object)}
|
||||
if _, err := entries[i].xlmeta(); err != nil {
|
||||
t.Fatalf("drive %d no longer has valid metadata: %v", i, err)
|
||||
}
|
||||
}
|
||||
resolver := metadataResolutionParams{dirQuorum: 8, objQuorum: 8, bucket: bucket, requestedVersions: 1}
|
||||
entry, err := z.resolveListingGroup(t.Context(), bucket, []*metaCacheEntry{set.resolveListEntry(entries, nil, &resolver)})
|
||||
if entry != nil || !isErrReadQuorum(err) {
|
||||
t.Fatalf("split live metadata became absence: entry=%v err=%v", entry, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
madmin "github.com/minio/madmin-go/v3"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStrictSingleNullQuorumReadAndConsumers(t *testing.T) {
|
||||
z, bucket, router := nullQuorumBackend(t)
|
||||
set := z.serverPools[0].sets[0]
|
||||
disks := set.getDisks()
|
||||
const object = "object"
|
||||
oldBody, newBody := bytes.Repeat([]byte{'a'}, 8192), bytes.Repeat([]byte{'b'}, 8192)
|
||||
oldTime := time.Now().UTC().Add(-time.Hour)
|
||||
put := func(body []byte, modTime time.Time) {
|
||||
t.Helper()
|
||||
_, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: modTime})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(oldBody, oldTime)
|
||||
oldMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
var err error
|
||||
oldMeta[i], err = disk.ReadAll(t.Context(), bucket, object+"/"+xlStorageFormatFile)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put(newBody, oldTime.Add(time.Minute))
|
||||
newMeta := make([][]byte, len(disks))
|
||||
for i, disk := range disks {
|
||||
newMeta[i] = mustReadNullQuorumMeta(t, disk, bucket, object)
|
||||
}
|
||||
// Model an incomplete overwrite using real inline shards: fifteen old
|
||||
// copies retain a read quorum; the final disk contains the newer minority.
|
||||
// The old quorum does not prove that the newer overwrite was uncommitted.
|
||||
for i := range 15 {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
checkRead := func(body []byte, modTime time.Time) {
|
||||
t.Helper()
|
||||
get := nullQuorumRequest(t, router, http.MethodGet, getGetObjectURL("", bucket, object))
|
||||
head := nullQuorumRequest(t, router, http.MethodHead, getGetObjectURL("", bucket, object))
|
||||
if get.Code != 200 || !bytes.Equal(get.Body.Bytes(), body) || head.Code != 200 || head.Header().Get("Content-Length") != "8192" {
|
||||
t.Fatalf("GET/HEAD lost readable quorum: GET=%d HEAD=%d body=%q", get.Code, head.Code, get.Body.String())
|
||||
}
|
||||
oi, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil || !oi.ModTime.Equal(modTime) {
|
||||
t.Fatalf("wrong readable generation: %v, %v", oi.ModTime, err)
|
||||
}
|
||||
}
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
||||
rec := nullQuorumRequest(t, router, method, getGetObjectURL("", bucket, object))
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("%s exposed an unresolved old quorum: %d", method, rec.Code)
|
||||
}
|
||||
}
|
||||
if _, err := z.HealObject(t.Context(), bucket, object, "", madmin.HealOpts{Remove: true}); !isErrReadQuorum(err) {
|
||||
t.Fatalf("heal accepted an unresolved old quorum: %v", err)
|
||||
}
|
||||
// Reading must not reinterpret the minority as absent or delete it.
|
||||
minority, err := disks[15].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil || !minority.ModTime.Equal(oldTime.Add(time.Minute)) {
|
||||
t.Fatalf("read mutated the minority: %+v, %v", minority, err)
|
||||
}
|
||||
// Now model the safe direction: the latest generation has the quorum.
|
||||
// Keep the migration, Walk and scanner assertions on that readable state.
|
||||
for i := range 15 {
|
||||
if err := disks[i].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, newMeta[i]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := disks[15].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[15]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newTime := oldTime.Add(time.Minute)
|
||||
checkRead(newBody, newTime)
|
||||
for _, kind := range []string{"rebalance", "decommission"} {
|
||||
t.Run(kind, func(t *testing.T) {
|
||||
var names []string
|
||||
consume := func(entry metaCacheEntry) {
|
||||
versions, err := entry.fileInfoVersions(bucket)
|
||||
if err != nil || len(versions.Versions) != 1 || !versions.Versions[0].ModTime.Equal(newTime) {
|
||||
t.Errorf("invalid migration metadata: %+v, %v", versions, err)
|
||||
return
|
||||
}
|
||||
// Migration reopens the selected name/version on all drives;
|
||||
// it must not treat the selected listing metadata as disk agreement.
|
||||
reader, err := set.GetObjectNInfo(t.Context(), bucket, entry.name, nil, nil, ObjectOptions{VersionID: nullVersionID, NoLock: true, NoDecryption: true})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(reader)
|
||||
reader.Close()
|
||||
if err != nil || !bytes.Equal(body, newBody) {
|
||||
t.Errorf("migration could not read selected object data: %v", err)
|
||||
}
|
||||
names = append(names, entry.name)
|
||||
}
|
||||
var err error
|
||||
if kind == "rebalance" {
|
||||
err = set.listObjectsToRebalance(t.Context(), bucket, consume)
|
||||
} else {
|
||||
err = set.listObjectsToDecommission(t.Context(), decomBucketInfo{Name: bucket}, consume)
|
||||
}
|
||||
if err != nil || !slices.Equal(names, []string{object}) {
|
||||
t.Fatalf("migration listing dropped the quorum: %v, %v", names, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, latestOnly := range []bool{false, true} {
|
||||
results := make(chan itemOrErr[ObjectInfo], 16)
|
||||
if err := z.Walk(t.Context(), bucket, "", results, WalkOptions{AskDisks: "strict", LatestOnly: latestOnly}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var names []string
|
||||
for result := range results {
|
||||
if result.Err != nil || !result.Item.ModTime.Equal(newTime) {
|
||||
t.Fatalf("Walk returned invalid metadata: %+v", result)
|
||||
}
|
||||
names = append(names, result.Item.Name)
|
||||
}
|
||||
if !slices.Equal(names, []string{object}) {
|
||||
t.Fatalf("Walk dropped the quorum: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// Exercise the scanner's actual abandoned-child path. Make the scan drive
|
||||
// miss the object, while keeping the latest quorum and an older minority on
|
||||
// the remaining disks. Its queued heal must read all disks and repair both.
|
||||
if err := disks[14].WriteAll(t.Context(), bucket, object+"/"+xlStorageFormatFile, oldMeta[14]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(disks[15].Endpoint().Path, bucket, object)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
scanNullQuorumAbandoned(t, z, bucket, object, newTime)
|
||||
for i, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{Healing: true})
|
||||
if err != nil || !fi.ModTime.Equal(newTime) {
|
||||
t.Fatalf("scanner heal did not reconcile disk %d: %v, %v", i, fi.ModTime, err)
|
||||
}
|
||||
}
|
||||
checkRead(newBody, newTime)
|
||||
put(newBody, oldTime.Add(2*time.Minute))
|
||||
checkRead(newBody, oldTime.Add(2*time.Minute))
|
||||
}
|
||||
@@ -815,11 +815,22 @@ func scanNullQuorumAbandoned(t *testing.T, z *erasureServerPools, bucket, object
|
||||
disks := z.serverPools[0].sets[0].getDisks()
|
||||
seq := newBgHealSequence()
|
||||
defer seq.cancelCtx()
|
||||
previousState, previousRoutine := globalBackgroundHealState, globalBackgroundHealRoutine
|
||||
globalBackgroundHealState = &allHealState{healSeqMap: map[string]*healSequence{"test": seq}}
|
||||
// Disk monitors may still read the shared state between object-layer tests.
|
||||
// Install the test sequence under its lock without replacing the state.
|
||||
state := globalBackgroundHealState
|
||||
state.Lock()
|
||||
previousSequences := state.healSeqMap
|
||||
state.healSeqMap = map[string]*healSequence{"test": seq}
|
||||
state.Unlock()
|
||||
previousRoutine := globalBackgroundHealRoutine
|
||||
routine := &healRoutine{tasks: make(chan healTask)}
|
||||
globalBackgroundHealRoutine = routine
|
||||
defer func() { globalBackgroundHealState, globalBackgroundHealRoutine = previousState, previousRoutine }()
|
||||
defer func() {
|
||||
globalBackgroundHealRoutine = previousRoutine
|
||||
state.Lock()
|
||||
state.healSeqMap = previousSequences
|
||||
state.Unlock()
|
||||
}()
|
||||
workerDone := make(chan struct{})
|
||||
var healed []string
|
||||
go func() {
|
||||
|
||||
@@ -186,6 +186,7 @@ func TestGetAPIEndpoints(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCheckLocalServerAddr(t *testing.T) {
|
||||
useTestEndpointDNS(t)
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
expectedErr error
|
||||
|
||||
@@ -404,7 +404,7 @@ func cleanMetadata(metadata map[string]string) map[string]string {
|
||||
// Remove STANDARD StorageClass
|
||||
metadata = removeStandardStorageClass(metadata)
|
||||
// Clean meta etag keys 'md5Sum', 'etag', "expires", "x-amz-tagging".
|
||||
return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging, "last-modified", VersionPurgeStatusKey)
|
||||
return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging, "last-modified", VersionPurgeStatusKey, rollbackIDKey)
|
||||
}
|
||||
|
||||
// Filter X-Amz-Storage-Class field only if it is set to STANDARD.
|
||||
|
||||
@@ -0,0 +1,650 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/cachevalue"
|
||||
"github.com/minio/minio/internal/config/drive"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
)
|
||||
|
||||
func mutationPools(t testing.TB) (*erasureServerPools, string) {
|
||||
t.Helper()
|
||||
// These scenarios require a 4+4 set: five physical commits form a
|
||||
// readable generation, while four successful replies fail write quorum.
|
||||
// Other server tests can leave EC:0 configured before this fixture runs.
|
||||
previousStorageClass := globalStorageClass
|
||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 4}})
|
||||
t.Cleanup(func() { globalStorageClass.Update(previousStorageClass) })
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
dirs, err := getRandomDisks(8)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoints := mustGetPoolEndpoints(0, dirs[:8]...)
|
||||
|
||||
obj, _, err := initObjectLayer(ctx, endpoints)
|
||||
if err != nil {
|
||||
cancel()
|
||||
removeRoots(dirs)
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
old := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(old) })
|
||||
bucket := "scope-probe"
|
||||
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return z, bucket
|
||||
}
|
||||
|
||||
func mutationPut(t testing.TB, z *erasureServerPools, bucket, obj string, pool int, body string, when time.Time) {
|
||||
t.Helper()
|
||||
_, err := z.serverPools[pool].PutObject(context.Background(), bucket, obj, mustGetPutObjReader(t, bytes.NewBufferString(body), int64(len(body)), "", ""), ObjectOptions{MTime: when})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type mutationAmbiguousRename struct {
|
||||
StorageAPI
|
||||
index int
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d mutationAmbiguousRename) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
|
||||
if d.index >= 5 {
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
r, e := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, o)
|
||||
d.entered <- struct{}{}
|
||||
<-d.release
|
||||
if d.index == 4 && e == nil {
|
||||
return r, context.DeadlineExceeded
|
||||
}
|
||||
return r, e
|
||||
}
|
||||
|
||||
// A physical fifth commit loses its reply. B successfully updates tags on all
|
||||
// five copies of A. A then undoes four known successes. B's tag update retains
|
||||
// the DataDir and ModTime, so the proposed ownership predicate cannot see it.
|
||||
func TestMutationRollbackPreservesAcknowledgedTags(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "tag-undo"
|
||||
base := time.Now().Add(-time.Hour)
|
||||
mutationPut(t, z, bucket, object, 0, "initial", base)
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, 5), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = mutationAmbiguousRename{d, i, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
aDone := make(chan error, 1)
|
||||
go func() {
|
||||
body := "pending-A"
|
||||
_, e := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Minute)})
|
||||
aDone <- e
|
||||
}()
|
||||
for range 5 {
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not finish physical commits")
|
||||
}
|
||||
}
|
||||
swap(disks)
|
||||
const tags = "owner=acknowledged-B"
|
||||
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{NoLock: true}); err != nil {
|
||||
t.Fatalf("B tags failed: %v", err)
|
||||
}
|
||||
count := func() int {
|
||||
n := 0
|
||||
for _, d := range disks {
|
||||
fi, e := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if e == nil && fi.Metadata[xhttp.AmzObjectTagging] == tags {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
if n := count(); n != 5 {
|
||||
t.Fatalf("B did not update all five A copies: %d", n)
|
||||
}
|
||||
t.Log("B tag update acknowledged on five physical copies; fifth A reply will report timeout")
|
||||
unblock()
|
||||
select {
|
||||
case e := <-aDone:
|
||||
if e == nil {
|
||||
t.Fatal("A unexpectedly succeeded")
|
||||
}
|
||||
t.Logf("A result=%v", e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not finish")
|
||||
}
|
||||
if n := count(); n != 5 {
|
||||
t.Errorf("acknowledged tags lost physical copies: got %d, want 5", n)
|
||||
}
|
||||
oi, e := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if oi.UserTags != tags {
|
||||
t.Fatalf("version-scoped rollback lost acknowledged tag update: tags=%q", oi.UserTags)
|
||||
}
|
||||
}
|
||||
|
||||
type delayedRenameResult struct {
|
||||
StorageAPI
|
||||
fail bool
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d delayedRenameResult) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
if d.fail {
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
r, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
d.entered <- struct{}{}
|
||||
<-d.release // Delay the reply, after the physical mutation and local unlock.
|
||||
return r, err
|
||||
}
|
||||
|
||||
// One pool, eight real disk directories, no edited object metadata. A commits
|
||||
// on four disks (below write quorum five); its replies are delayed. Model a lost
|
||||
// namespace lease by letting B execute with NoLock, like the previous lease
|
||||
// fixture. B adds an explicit version and succeeds on seven disks. A then
|
||||
// fails quorum and rolls back. A must not erase the separately acknowledged B.
|
||||
func TestMutationRollbackPreservesConcurrentVersion(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "rollback-and-version"
|
||||
size, _ := strconv.Atoi(os.Getenv("SILO_TEST_MUTATION_BYTES"))
|
||||
payload := func(v string) string {
|
||||
if size > len(v) {
|
||||
return v + strings.Repeat(".", size-len(v))
|
||||
}
|
||||
return v
|
||||
}
|
||||
base := time.Now().Add(-time.Hour)
|
||||
mutationPut(t, z, bucket, object, 0, payload("initial-null"), base)
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, 4), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
aDisks := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
aDisks[i] = delayedRenameResult{d, i >= 4, entered, release}
|
||||
}
|
||||
swap(aDisks)
|
||||
aDone := make(chan error, 1)
|
||||
go func() {
|
||||
body := payload("pending-null-A")
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object,
|
||||
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
|
||||
ObjectOptions{NoLock: true, MTime: base.Add(time.Minute)})
|
||||
aDone <- err
|
||||
}()
|
||||
for range 4 {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-aDone:
|
||||
t.Fatalf("A returned early: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not reach the reply barrier")
|
||||
}
|
||||
}
|
||||
t.Log("A physically committed on four disks; replies held; other four RenameData calls failed")
|
||||
bDisks := append([]StorageAPI(nil), disks...)
|
||||
bDisks[7] = nil
|
||||
swap(bDisks)
|
||||
bVersion := mustGetUUID()
|
||||
body := payload("acknowledged-version-B")
|
||||
_, err := z.serverPools[0].PutObject(t.Context(), bucket, object,
|
||||
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
|
||||
ObjectOptions{NoLock: true, Versioned: true, VersionID: bVersion, MTime: base.Add(2 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("B failed: %v", err)
|
||||
}
|
||||
countB := func() int {
|
||||
n := 0
|
||||
for _, d := range disks {
|
||||
if _, e := d.ReadVersion(t.Context(), "", bucket, object, bVersion, ReadOptions{}); e == nil {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
if n := countB(); n != 7 {
|
||||
t.Fatalf("B was not committed on the expected seven drives: %d", n)
|
||||
}
|
||||
t.Log("B succeeded on seven drives and its version is present on all seven")
|
||||
unblock()
|
||||
select {
|
||||
case err := <-aDone:
|
||||
if err == nil {
|
||||
t.Fatal("A unexpectedly succeeded")
|
||||
}
|
||||
t.Logf("A failed and rolled back: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not finish")
|
||||
}
|
||||
swap(disks)
|
||||
t.Logf("B metadata survives on %d of eight drives after A's rollback", countB())
|
||||
if n := countB(); n != 7 {
|
||||
t.Fatalf("unrelated rollback removed B's acknowledged copies: got %d, want 7", n)
|
||||
}
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: bVersion})
|
||||
if err != nil {
|
||||
t.Fatalf("acknowledged B became unreadable after unrelated rollback: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
got, err := io.ReadAll(r)
|
||||
if err != nil || !bytes.Equal(got, []byte(body)) {
|
||||
t.Fatalf("B payload changed: readErr=%v length=%d", err, len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationDeleteMarkerDoesNotReenter(t *testing.T) {
|
||||
disk, _, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi := FileInfo{Volume: "bucket", Name: "absent", VersionID: mustGetUUID(), Deleted: true, ModTime: UTCNow()}
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- disk.storage.DeleteVersion(t.Context(), "bucket", "absent", fi, true, DeleteOptions{}) }()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("DeleteVersion(forceDelMarker) blocked acquiring the same guard again through WriteMetadata")
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup can arrive after an RPC reply is lost while the real disk commit is
|
||||
// still running. Both single and bulk cleanup must retain its staged metadata.
|
||||
func TestMutationStagingCleanupWaitsForCommit(t *testing.T) {
|
||||
for _, bulk := range []bool{false, true} {
|
||||
t.Run(strconv.FormatBool(bulk), func(t *testing.T) {
|
||||
disk, root, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var lastSuccess int64
|
||||
gate := &stagedRenameContext{
|
||||
Context: context.Background(), meta: filepath.Join(root, minioMetaTmpBucket, "staged", xlStorageFormatFile),
|
||||
entered: make(chan struct{}), release: make(chan struct{}),
|
||||
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
|
||||
}
|
||||
unblock := sync.OnceFunc(func() { close(gate.release) })
|
||||
defer unblock()
|
||||
fi := newFileInfo("object", 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Fresh = true
|
||||
committed := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := disk.RenameData(gate, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
|
||||
committed <- err
|
||||
}()
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("commit did not reach staging barrier")
|
||||
}
|
||||
cleaned := make(chan error, 1)
|
||||
go func() {
|
||||
if bulk {
|
||||
cleaned <- disk.DeleteBulk(t.Context(), minioMetaTmpBucket, pathJoin("staged", xlStorageFormatFile))
|
||||
} else {
|
||||
cleaned <- disk.Delete(t.Context(), minioMetaTmpBucket, "staged", DeleteOptions{Recursive: true})
|
||||
}
|
||||
}()
|
||||
early := false
|
||||
select {
|
||||
case err := <-cleaned:
|
||||
early = true
|
||||
t.Errorf("cleanup overtook admitted disk commit: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case err := <-committed:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("commit did not finish")
|
||||
}
|
||||
if !early {
|
||||
select {
|
||||
case <-cleaned:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("cleanup did not finish")
|
||||
}
|
||||
}
|
||||
got, err := disk.ReadVersion(t.Context(), "", "bucket", "object", "", ReadOptions{})
|
||||
if err != nil || got.DataDir != fi.DataDir || !got.ModTime.Equal(fi.ModTime) {
|
||||
t.Fatalf("cleanup lost committed metadata: got=%+v err=%v", got, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type reviewReadGate struct {
|
||||
StorageAPI
|
||||
ready chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d reviewReadGate) ReadXL(ctx context.Context, volume, path string, data bool) (RawFileInfo, error) {
|
||||
r, err := d.StorageAPI.ReadXL(ctx, volume, path, data)
|
||||
d.ready <- struct{}{}
|
||||
if d.release != nil {
|
||||
<-d.release
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
|
||||
func TestReviewFastReadAlreadyWaitsForSlowDisk(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "healthy-slow-read"
|
||||
mutationPut(t, z, bucket, object, 0, "healthy-inline", time.Now())
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
ready, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
g := reviewReadGate{StorageAPI: d, ready: ready}
|
||||
if i == len(disks)-1 {
|
||||
g.release = release
|
||||
}
|
||||
wrapped[i] = g
|
||||
}
|
||||
swap(wrapped)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, _, _, err := set.getObjectFileInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: true}, true)
|
||||
done <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-ready:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("reads did not reach barrier")
|
||||
}
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("read returned before the final disk: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
t.Log("seven healthy replies are insufficient to return; final unbuffered mrfCheck send is still waiting")
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("read did not finish after slow disk release")
|
||||
}
|
||||
}
|
||||
|
||||
// Pause the real trash worker at its disk-space callback, after moving the
|
||||
// external shards but before DeleteVersion can replace xl.meta. An outer join
|
||||
// must also contain this nested worker, and later mutations must remain excluded.
|
||||
func TestMutationDeleteJoinsNestedWorker(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "nested-timeout"
|
||||
mutationPut(t, z, bucket, object, 0, strings.Repeat("x", 2<<20), time.Now())
|
||||
disk := z.serverPools[0].getHashedSet(object).getDisks()[0].(*xlStorageDiskIDCheck)
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
oldCache := disk.storage.diskInfoCache
|
||||
disk.storage.diskInfoCache = cachevalue.NewFromFunc(time.Hour, cachevalue.Opts{}, func(context.Context) (DiskInfo, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
return DiskInfo{Used: 100, Free: 100, UsedInodes: 100, FreeInodes: 100}, nil
|
||||
})
|
||||
defer func() { disk.storage.diskInfoCache = oldCache }()
|
||||
oldTimeout := globalDriveConfig.GetMaxTimeout()
|
||||
globalDriveConfig.Update(drive.Config{MaxTimeout: 1200 * time.Millisecond})
|
||||
defer globalDriveConfig.Update(drive.Config{MaxTimeout: oldTimeout})
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, e := xioutil.WithDeadlineAndWait(context.Background(), globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, disk.storage.DeleteVersion(ctx, bucket, object, fi, false, DeleteOptions{UndoWrite: true})
|
||||
})
|
||||
done <- e
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("trash callback not reached")
|
||||
}
|
||||
early := false
|
||||
select {
|
||||
case e := <-done:
|
||||
early = true
|
||||
t.Errorf("outer join returned before nested disk worker completed: %v", e)
|
||||
case <-time.After(1500 * time.Millisecond):
|
||||
}
|
||||
updated := make(chan error, 1)
|
||||
go func() {
|
||||
updated <- disk.storage.UpdateMetadata(context.Background(), bucket, object, FileInfo{VersionID: fi.VersionID, Metadata: map[string]string{"after-timeout": "yes"}}, UpdateMetadataOpts{})
|
||||
}()
|
||||
writerEarly := false
|
||||
select {
|
||||
case e := <-updated:
|
||||
writerEarly = true
|
||||
t.Errorf("later metadata writer passed unfinished deletion: %v", e)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
unblock()
|
||||
if !early {
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("delete did not finish")
|
||||
}
|
||||
}
|
||||
if !writerEarly {
|
||||
select {
|
||||
case <-updated:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("later writer did not finish")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Even an update that leaves all public fields unchanged is a later successful
|
||||
// operation. An old writer must not regain undo ownership through an ABA cycle.
|
||||
func TestMutationNoopUpdateRevokesUndo(t *testing.T) {
|
||||
for _, method := range []string{"update", "write", "aba"} {
|
||||
t.Run(method, func(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "no-op-update"
|
||||
mutationPut(t, z, bucket, object, 0, "acknowledged", time.Now())
|
||||
disks := z.serverPools[0].getHashedSet(object).getDisks()
|
||||
for _, disk := range disks {
|
||||
fi, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch method {
|
||||
case "update":
|
||||
err = disk.UpdateMetadata(t.Context(), bucket, object, fi, UpdateMetadataOpts{})
|
||||
case "write":
|
||||
err = disk.WriteMetadata(t.Context(), "", bucket, object, fi)
|
||||
case "aba":
|
||||
for _, etag := range []string{"temporary", fi.Metadata["etag"]} {
|
||||
err = disk.UpdateMetadata(t.Context(), bucket, object, FileInfo{Metadata: map[string]string{"etag": etag}}, UpdateMetadataOpts{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := disk.DeleteVersion(t.Context(), bucket, object, fi, false, DeleteOptions{UndoWrite: true}); err != errFileVersionNotFound {
|
||||
t.Fatalf("old writer retained undo after %s: %v", method, err)
|
||||
}
|
||||
}
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
body, err := io.ReadAll(r)
|
||||
if err != nil || string(body) != "acknowledged" {
|
||||
t.Fatalf("later acknowledged state lost: %q, %v", body, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationCanceledSuccessfulCommitIsNotUndone(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cancel-after-commit"
|
||||
mutationPut(t, z, bucket, object, 0, "initial", time.Now().Add(-time.Hour))
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
entered, release := make(chan struct{}, len(disks)), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range disks {
|
||||
wrapped[i] = delayedRenameResult{disk, false, entered, release}
|
||||
}
|
||||
swap(wrapped)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := set.PutObject(ctx, bucket, object, mustGetPutObjReader(t, strings.NewReader("committed"), 9, "", ""), ObjectOptions{})
|
||||
done <- err
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("physical commit barrier not reached")
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
unblock()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("commit did not finish")
|
||||
}
|
||||
swap(disks)
|
||||
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer r.Close()
|
||||
body, err := io.ReadAll(r)
|
||||
if err != nil || string(body) != "committed" {
|
||||
t.Fatalf("cancellation rolled back an actual successful quorum: %q, %v", body, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Partial no-op updates may revoke undo on only some disks. That internal
|
||||
// capability must not split the quorum of otherwise identical object contents.
|
||||
func TestMutationPartialNoopPreservesReadability(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "partial-noop"
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
const body = "acknowledged-data"
|
||||
const tags = "owner=unchanged"
|
||||
_, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{UserDefined: map[string]string{xhttp.AmzObjectTagging: tags}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
partial := append([]StorageAPI(nil), disks...)
|
||||
for i := 4; i < len(partial); i++ {
|
||||
partial[i] = nil
|
||||
}
|
||||
swap(partial)
|
||||
if _, err = set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err == nil {
|
||||
t.Fatal("four drives unexpectedly gave write quorum")
|
||||
}
|
||||
swap(disks)
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatalf("unchanged acknowledged object became unreadable after no-op partial update: %v", err)
|
||||
}
|
||||
got, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(got) != body || r.ObjInfo.UserTags != tags {
|
||||
t.Fatalf("body/tags changed: %q %v %+v", got, err, r.ObjInfo)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Keep the old slot calculation here to reproduce the false sharing between
|
||||
// distinct resources, including resources on separate storage drives.
|
||||
func mutationGuardLegacySlot(drive, path string) uint64 {
|
||||
h := uint64(14695981039346656037)
|
||||
for _, part := range []string{drive, path} {
|
||||
for i := range part {
|
||||
h = (h ^ uint64(part[i])) * 1099511628211
|
||||
}
|
||||
h *= 1099511628211
|
||||
}
|
||||
return h % 65536
|
||||
}
|
||||
|
||||
func mutationGuardCollision(t *testing.T, drive string, target uint64) string {
|
||||
t.Helper()
|
||||
for i := 0; i < 2000000; i++ {
|
||||
key := fmt.Sprintf("independent-%d", i)
|
||||
if mutationGuardLegacySlot(drive, pathJoin("bucket", key)) == target {
|
||||
return key
|
||||
}
|
||||
}
|
||||
t.Fatal("no collision found in bounded search")
|
||||
return ""
|
||||
}
|
||||
|
||||
func TestMutationGuardIsolatesStorageResources(t *testing.T) {
|
||||
blocked, rootA, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer blocked.Close()
|
||||
healthy, rootB, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer healthy.Close()
|
||||
defer mutationGuardCheckEmpty(t, rootA, rootB)
|
||||
for _, disk := range []*xlStorageDiskIDCheck{blocked, healthy} {
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
const object = "blocked"
|
||||
target := mutationGuardLegacySlot(rootA, pathJoin("bucket", object))
|
||||
type mutation struct {
|
||||
name, key string
|
||||
disk *xlStorageDiskIDCheck
|
||||
done chan error
|
||||
}
|
||||
mutations := []mutation{
|
||||
{name: "same-key", key: object, disk: blocked},
|
||||
{name: "different-key-same-drive-collision", key: mutationGuardCollision(t, rootA, target), disk: blocked},
|
||||
{name: "different-key-healthy-drive-collision", key: mutationGuardCollision(t, rootB, target), disk: healthy},
|
||||
{name: "same-path-healthy-drive", key: object, disk: healthy},
|
||||
}
|
||||
fixture := func(name string) FileInfo {
|
||||
fi := newFileInfo(name, 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Fresh = true
|
||||
fi.Metadata = map[string]string{"x-amz-meta-probe": "before"}
|
||||
return fi
|
||||
}
|
||||
for _, m := range mutations {
|
||||
if err := m.disk.WriteMetadata(t.Context(), "", "bucket", m.key, fixture(m.key)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
var lastSuccess int64
|
||||
gate := &stagedRenameContext{
|
||||
Context: ctx, meta: filepath.Join(rootA, minioMetaTmpBucket, "staged", xlStorageFormatFile),
|
||||
entered: make(chan struct{}), release: make(chan struct{}),
|
||||
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
|
||||
}
|
||||
var once sync.Once
|
||||
release := func() { once.Do(func() { close(gate.release) }) }
|
||||
var workers sync.WaitGroup
|
||||
defer func() {
|
||||
release()
|
||||
workers.Wait()
|
||||
}()
|
||||
holderDone := make(chan error, 1)
|
||||
workers.Go(func() {
|
||||
_, err := blocked.RenameData(gate, minioMetaTmpBucket, "staged", fixture(object), "bucket", object, RenameOptions{})
|
||||
holderDone <- err
|
||||
})
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("RenameData did not reach the staged metadata barrier")
|
||||
}
|
||||
// Cancellation cannot release the guard while the admitted worker can
|
||||
// still commit. Unrelated mutations must nevertheless be able to finish.
|
||||
cancel()
|
||||
for i := range mutations {
|
||||
m := &mutations[i]
|
||||
m.done = make(chan error, 1)
|
||||
workers.Go(func() {
|
||||
m.done <- m.disk.UpdateMetadata(t.Context(), "bucket", m.key, FileInfo{
|
||||
Metadata: map[string]string{"x-amz-meta-probe": "after"},
|
||||
}, UpdateMetadataOpts{})
|
||||
})
|
||||
}
|
||||
mutationGuardWaitRefs(t, rootA, pathJoin("bucket", object), 2)
|
||||
for _, m := range mutations[1:] {
|
||||
select {
|
||||
case err := <-m.done:
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", m.name, err)
|
||||
}
|
||||
t.Logf("%s completed while RenameData remained paused", m.name)
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Errorf("%s blocked by an unrelated paused mutation", m.name)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case err := <-holderDone:
|
||||
t.Fatalf("canceled RenameData returned before its worker finished: %v", err)
|
||||
default:
|
||||
}
|
||||
select {
|
||||
case err := <-mutations[0].done:
|
||||
t.Fatalf("same-key mutation escaped the held guard: %v", err)
|
||||
default:
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case err := <-holderDone:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("canceled RenameData result: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("RenameData did not finish after releasing the barrier")
|
||||
}
|
||||
select {
|
||||
case err := <-mutations[0].done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("same-key mutation did not finish after releasing the holder")
|
||||
}
|
||||
workers.Wait()
|
||||
for _, m := range mutations {
|
||||
fi, err := m.disk.ReadVersion(t.Context(), "", "bucket", m.key, "", ReadOptions{})
|
||||
if err != nil || fi.Metadata["x-amz-meta-probe"] != "after" {
|
||||
t.Errorf("%s persisted metadata: %q, %v", m.name, fi.Metadata["x-amz-meta-probe"], err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mutationGuardCheckEmpty(t *testing.T, drives ...string) {
|
||||
t.Helper()
|
||||
storageMutationLocks.Lock()
|
||||
defer storageMutationLocks.Unlock()
|
||||
for key, lock := range storageMutationLocks.locks {
|
||||
if slices.Contains(drives, key.drive) {
|
||||
t.Errorf("unused guard retained for %+v with %d references", key, lock.refs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func mutationGuardWaitRefs(t *testing.T, drive, path string, refs int) *storageMutationLock {
|
||||
t.Helper()
|
||||
deadline := time.NewTimer(5 * time.Second)
|
||||
defer deadline.Stop()
|
||||
tick := time.NewTicker(time.Millisecond)
|
||||
defer tick.Stop()
|
||||
key := storageMutationKey{drive, path}
|
||||
for {
|
||||
storageMutationLocks.Lock()
|
||||
lock := storageMutationLocks.locks[key]
|
||||
ready := lock != nil && lock.refs == refs
|
||||
storageMutationLocks.Unlock()
|
||||
if ready {
|
||||
return lock
|
||||
}
|
||||
select {
|
||||
case <-deadline.C:
|
||||
t.Fatalf("guard %+v did not reach %d references", key, refs)
|
||||
case <-tick.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationGuardRetainsWaitersAndRemovesIdleEntries(t *testing.T) {
|
||||
s := &xlStorage{drivePath: t.TempDir()}
|
||||
defer mutationGuardCheckEmpty(t, s.drivePath)
|
||||
unlock := sync.OnceFunc(s.lockMutations("object"))
|
||||
defer unlock()
|
||||
original := mutationGuardWaitRefs(t, s.drivePath, "object", 1)
|
||||
acquired, release, finished := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
go func() {
|
||||
defer close(finished)
|
||||
defer s.lockMutations("object", "object")()
|
||||
close(acquired)
|
||||
<-release
|
||||
}()
|
||||
if lock := mutationGuardWaitRefs(t, s.drivePath, "object", 2); lock != original {
|
||||
t.Error("waiting operation registered a second mutex for the same resource")
|
||||
}
|
||||
select {
|
||||
case <-acquired:
|
||||
t.Error("waiter acquired before the holder released")
|
||||
default:
|
||||
}
|
||||
unlock()
|
||||
select {
|
||||
case <-acquired:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("waiter did not acquire after release")
|
||||
}
|
||||
if lock := mutationGuardWaitRefs(t, s.drivePath, "object", 1); lock != original {
|
||||
t.Error("holder release discarded the queued operation's mutex")
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case <-finished:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("waiter did not release")
|
||||
}
|
||||
mutationGuardCheckEmpty(t, s.drivePath)
|
||||
// The registry must not grow with the historical object count.
|
||||
for i := 0; i < 1000; i++ {
|
||||
s.lockMutations(fmt.Sprintf("object-%d", i))()
|
||||
}
|
||||
s.lockMutations()()
|
||||
}
|
||||
|
||||
func TestMutationGuardOrdersAndDeduplicatesPaths(t *testing.T) {
|
||||
s := &xlStorage{drivePath: t.TempDir()}
|
||||
defer mutationGuardCheckEmpty(t, s.drivePath)
|
||||
paths := []string{"b", "a", "b", "a"}
|
||||
before := slices.Clone(paths)
|
||||
unlock := s.lockMutations(paths...)
|
||||
mutationGuardWaitRefs(t, s.drivePath, "a", 1)
|
||||
mutationGuardWaitRefs(t, s.drivePath, "b", 1)
|
||||
unlock()
|
||||
if !slices.Equal(paths, before) {
|
||||
t.Fatalf("changed caller's path slice: %v", paths)
|
||||
}
|
||||
const workers, iterations = 8, 500
|
||||
start, done := make(chan struct{}), make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
var protected atomic.Int64
|
||||
var active atomic.Int32
|
||||
for i := 0; i < workers; i++ {
|
||||
worker := i
|
||||
wg.Go(func() {
|
||||
<-start
|
||||
order := []string{"a", "b", "a"}
|
||||
if worker%2 != 0 {
|
||||
order = []string{"b", "a", "b"}
|
||||
}
|
||||
for j := 0; j < iterations; j++ {
|
||||
unlock := s.lockMutations(order...)
|
||||
if active.Add(1) != 1 {
|
||||
t.Error("overlapping guards entered concurrently")
|
||||
}
|
||||
protected.Add(1)
|
||||
active.Add(-1)
|
||||
unlock()
|
||||
}
|
||||
})
|
||||
}
|
||||
go func() { wg.Wait(); close(done) }()
|
||||
close(start)
|
||||
select {
|
||||
case <-done:
|
||||
if got := protected.Load(); got != workers*iterations {
|
||||
t.Fatalf("overlapping guards lost updates: got %d, want %d", got, workers*iterations)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("opposing path orders or duplicate paths deadlocked")
|
||||
}
|
||||
}
|
||||
|
||||
// Measure registry/allocation cost separately from storage I/O. Independent
|
||||
// workers use two resources, as RenameData does, and shared workers use one.
|
||||
func BenchmarkMutationGuard(b *testing.B) {
|
||||
s := &xlStorage{drivePath: "/benchmark/drive"}
|
||||
b.Run("single", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
for b.Loop() {
|
||||
s.lockMutations("bucket/object")()
|
||||
}
|
||||
})
|
||||
b.Run("parallel-independent", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
var worker atomic.Uint64
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
id := worker.Add(1)
|
||||
paths := []string{fmt.Sprintf("staging/%d", id), fmt.Sprintf("bucket/object-%d", id)}
|
||||
for pb.Next() {
|
||||
s.lockMutations(paths...)()
|
||||
}
|
||||
})
|
||||
})
|
||||
b.Run("parallel-shared", func(b *testing.B) {
|
||||
b.ReportAllocs()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
s.lockMutations("bucket/object")()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// A call must retain paths it has not reached yet, even across xlStorage
|
||||
// instances for the same drive. No resource wait may hold the registry mutex.
|
||||
func TestMutationGuardRetainsUnacquiredPaths(t *testing.T) {
|
||||
s := &xlStorage{drivePath: t.TempDir()}
|
||||
sameDrive := &xlStorage{drivePath: s.drivePath}
|
||||
defer mutationGuardCheckEmpty(t, s.drivePath)
|
||||
unlockA := sync.OnceFunc(s.lockMutations("a"))
|
||||
defer unlockA()
|
||||
acquired, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{})
|
||||
unblock := sync.OnceFunc(func() { close(release) })
|
||||
defer unblock()
|
||||
go func() {
|
||||
defer close(done)
|
||||
defer sameDrive.lockMutations("b", "a")()
|
||||
close(acquired)
|
||||
<-release
|
||||
}()
|
||||
mutationGuardWaitRefs(t, s.drivePath, "a", 2)
|
||||
original := mutationGuardWaitRefs(t, s.drivePath, "b", 1)
|
||||
// The queued call has not locked b yet: sorting must put a first.
|
||||
unlockB := sync.OnceFunc(s.lockMutations("b"))
|
||||
defer unlockB()
|
||||
if got := mutationGuardWaitRefs(t, s.drivePath, "b", 2); got != original {
|
||||
t.Fatal("unacquired path got a different mutex")
|
||||
}
|
||||
unlockA()
|
||||
select {
|
||||
case <-acquired:
|
||||
t.Fatal("multi-path call escaped the independently held second path")
|
||||
default:
|
||||
}
|
||||
unlockB()
|
||||
select {
|
||||
case <-acquired:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("multi-path call did not acquire after both paths were released")
|
||||
}
|
||||
if got := mutationGuardWaitRefs(t, s.drivePath, "b", 1); got != original {
|
||||
t.Fatal("release replaced an entry retained by a pending multi-path call")
|
||||
}
|
||||
unblock()
|
||||
<-done
|
||||
// The variadic helper has no hidden four-path limit.
|
||||
s.lockMutations("e", "d", "c", "b", "a", "a")()
|
||||
}
|
||||
|
||||
func TestMutationGuardCanceledWaiterRetainsSerialization(t *testing.T) {
|
||||
disk, root, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
defer mutationGuardCheckEmpty(t, root)
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fi := newFileInfo("object", 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Fresh = true
|
||||
fi.Metadata = map[string]string{"x-amz-meta-probe": "before"}
|
||||
if err := disk.WriteMetadata(t.Context(), "", "bucket", "object", fi); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Use a separate instance with the same drive identity as the API wrapper.
|
||||
s := &xlStorage{drivePath: root}
|
||||
unlock := sync.OnceFunc(s.lockMutations("bucket/object"))
|
||||
defer unlock()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- disk.UpdateMetadata(ctx, "bucket", "object", FileInfo{
|
||||
Metadata: map[string]string{"x-amz-meta-probe": "canceled"},
|
||||
}, UpdateMetadataOpts{})
|
||||
}()
|
||||
mutationGuardWaitRefs(t, root, "bucket/object", 2)
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("canceled API returned with worker still queued: %v", err)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
mutationGuardWaitRefs(t, root, "bucket/object", 2)
|
||||
unlock()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("canceled waiter: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("canceled waiter did not finish after holder release")
|
||||
}
|
||||
got, err := disk.ReadVersion(t.Context(), "", "bucket", "object", "", ReadOptions{})
|
||||
if err != nil || got.Metadata["x-amz-meta-probe"] != "before" {
|
||||
t.Fatalf("canceled metadata update persisted: %v, %v", got.Metadata, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
)
|
||||
|
||||
func lateCommitPools(t *testing.T) (*erasureServerPools, string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
dirs, err := getRandomDisks(16)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, dirs...))
|
||||
if err != nil {
|
||||
cancel()
|
||||
removeRoots(dirs)
|
||||
t.Fatal(err)
|
||||
}
|
||||
z := obj.(*erasureServerPools)
|
||||
previous := newObjectLayerFn()
|
||||
setObjectLayer(z)
|
||||
t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs); setObjectLayer(previous) })
|
||||
bucket := "late-commit-regression"
|
||||
if err := z.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return z, bucket
|
||||
}
|
||||
|
||||
func lateCommitSwitch(set *erasureObjects) ([]StorageAPI, func([]StorageAPI)) {
|
||||
original := set.getDisks()
|
||||
var active atomic.Value
|
||||
active.Store(original)
|
||||
set.getDisks = func() []StorageAPI { return active.Load().([]StorageAPI) }
|
||||
return original, func(disks []StorageAPI) { active.Store(disks) }
|
||||
}
|
||||
|
||||
type lateCommitDisk struct {
|
||||
StorageAPI
|
||||
index int
|
||||
earlyDone chan<- struct{}
|
||||
releaseRename <-chan struct{}
|
||||
renameReady chan<- struct{}
|
||||
releaseUndo <-chan struct{}
|
||||
}
|
||||
|
||||
func (d lateCommitDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
defer func() { d.renameReady <- struct{}{} }()
|
||||
if d.index >= 1 {
|
||||
<-d.releaseRename
|
||||
}
|
||||
result, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if d.index < 1 {
|
||||
d.earlyDone <- struct{}{}
|
||||
<-d.releaseRename
|
||||
}
|
||||
// Model a lost successful response. Real disk commit has already happened.
|
||||
if d.index < 1 || d.index >= 12 {
|
||||
return RenameDataResp{}, errDiskNotFound
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (d lateCommitDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, force bool, opts DeleteOptions) error {
|
||||
if opts.UndoWrite {
|
||||
<-d.releaseUndo
|
||||
}
|
||||
return d.StorageAPI.DeleteVersion(ctx, volume, path, fi, force, opts)
|
||||
}
|
||||
|
||||
// All metadata changes use production PutObject/RenameData/UndoWrite on real disks.
|
||||
// Fault injection only controls timing, namespace lease age, and transport results.
|
||||
func TestMutationReadDuringLateCommitAndUndo(t *testing.T) {
|
||||
z, bucket := lateCommitPools(t)
|
||||
object := "late-undo-16"
|
||||
bodyA, bodyB := strings.Repeat("writer-A", 262144), strings.Repeat("writer-B", 262144)
|
||||
label := func(s string) string {
|
||||
if s == bodyA {
|
||||
return "A (2 MiB, exact payload match)"
|
||||
}
|
||||
if s == bodyB {
|
||||
return "B (2 MiB, exact payload match)"
|
||||
}
|
||||
if s == "" {
|
||||
return "empty"
|
||||
}
|
||||
return "UNEXPECTED PAYLOAD"
|
||||
}
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
put := func(ctx context.Context, body string) (ObjectInfo, error) {
|
||||
return set.PutObject(ctx, bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true})
|
||||
}
|
||||
if _, err := put(t.Context(), "seed"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lockers := make([]*localLocker, 4)
|
||||
clients := make([]dsync.NetLocker, 4)
|
||||
for i := range lockers {
|
||||
lockers[i] = newLocker()
|
||||
clients[i] = lockers[i]
|
||||
}
|
||||
newLock := func() *distLockInstance {
|
||||
return &distLockInstance{rwMutex: dsync.NewDRWMutex(&dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, "late-commit" }, Timeouts: dsync.DefaultTimeouts}, bucket+"/"+object), opsID: mustGetUUID()}
|
||||
}
|
||||
oldLock := newLock()
|
||||
oldCtx, err := oldLock.GetLock(t.Context(), newDynamicTimeout(time.Second, time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer oldLock.Unlock(oldCtx)
|
||||
disks, swap := lateCommitSwitch(set)
|
||||
defer swap(disks)
|
||||
earlyDone, renameReady := make(chan struct{}, 16), make(chan struct{}, 16)
|
||||
releaseRename, releaseUndo := make(chan struct{}), make(chan struct{})
|
||||
var renameOnce, undoOnce sync.Once
|
||||
unblockRename := func() { renameOnce.Do(func() { close(releaseRename) }) }
|
||||
unblockUndo := func() { undoOnce.Do(func() { close(releaseUndo) }) }
|
||||
defer unblockUndo()
|
||||
defer unblockRename()
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
wrapped[i] = lateCommitDisk{d, i, earlyDone, releaseRename, renameReady, releaseUndo}
|
||||
}
|
||||
swap(wrapped)
|
||||
aDone := make(chan error, 1)
|
||||
go func() { _, e := put(oldCtx.Context(), bodyA); aDone <- e }()
|
||||
for range 1 {
|
||||
select {
|
||||
case <-earlyDone:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A early commit barrier")
|
||||
}
|
||||
}
|
||||
// Expire actual dsync lease records, allowing another holder to take over.
|
||||
for _, l := range lockers {
|
||||
l.mutex.Lock()
|
||||
for key, entries := range l.lockMap {
|
||||
for i := range entries {
|
||||
entries[i].TimeLastRefresh = time.Now().Add(-lockValidityDuration - time.Second).UnixNano()
|
||||
}
|
||||
l.lockMap[key] = entries
|
||||
}
|
||||
l.mutex.Unlock()
|
||||
l.expireOldLocks(lockValidityDuration)
|
||||
}
|
||||
newHolder := newLock()
|
||||
newCtx, err := newHolder.GetLock(t.Context(), newDynamicTimeout(time.Second, time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
swap(disks)
|
||||
b, err := put(newCtx.Context(), bodyB)
|
||||
newHolder.Unlock(newCtx)
|
||||
if err != nil {
|
||||
t.Fatalf("B must acknowledge success: %v", err)
|
||||
}
|
||||
counts := func() (older, newer, unknown int) {
|
||||
for _, d := range disks {
|
||||
fi, e := d.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if e != nil {
|
||||
unknown++
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case fi.ModTime.Equal(b.ModTime):
|
||||
newer++
|
||||
case fi.ModTime.Before(b.ModTime):
|
||||
older++
|
||||
default:
|
||||
t.Fatalf("unexpected generation %v", fi.ModTime)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
o, n, u := counts()
|
||||
if o != 0 || n != 16 || u != 0 {
|
||||
t.Fatalf("B ACK evidence: %d %d %d", o, n, u)
|
||||
}
|
||||
t.Logf("B successfully committed; observed B=16/16, all disks online; ModTime=%s", b.ModTime)
|
||||
unblockRename()
|
||||
for range len(disks) {
|
||||
select {
|
||||
case <-renameReady:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("rename completion barrier")
|
||||
}
|
||||
}
|
||||
o, n, u = counts()
|
||||
|
||||
fi, e := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
qr, qw := fi.Erasure.DataBlocks, fi.Erasure.DataBlocks
|
||||
if fi.Erasure.DataBlocks == fi.Erasure.ParityBlocks {
|
||||
qw++
|
||||
}
|
||||
if qr != 12 || qw != 12 {
|
||||
t.Fatalf("unexpected quorum %d/%d", qr, qw)
|
||||
}
|
||||
t.Logf("Before A undo: old A=%d, newer acknowledged B=%d, U=%d, Qr=%d, Qw=%d", o, n, u, qr, qw)
|
||||
get := func() (string, error) {
|
||||
rd, e := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if e != nil {
|
||||
return "", e
|
||||
}
|
||||
defer rd.Close()
|
||||
body, e := io.ReadAll(rd)
|
||||
return string(body), e
|
||||
}
|
||||
first, firstErr := get()
|
||||
t.Logf("R1 after B ACK and before A undo: body=%q error=%v", label(first), firstErr)
|
||||
if firstErr != nil && !isErrReadQuorum(firstErr) {
|
||||
t.Errorf("unexpected first read error: %v", firstErr)
|
||||
}
|
||||
unblockUndo()
|
||||
select {
|
||||
case e := <-aDone:
|
||||
if e == nil {
|
||||
t.Fatal("A unexpectedly succeeded")
|
||||
}
|
||||
t.Logf("A failed after completing UndoWrite: %v", e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A undo did not finish")
|
||||
}
|
||||
second, secondErr := get()
|
||||
if secondErr != nil || second != bodyB {
|
||||
t.Fatalf("B must remain after A fails: %q %v", label(second), secondErr)
|
||||
}
|
||||
o, n, u = counts()
|
||||
t.Logf("R2 after A failure, with no new PUT: body=%q; old A=%d, B=%d, U=%d", label(second), o, n, u)
|
||||
if first == bodyA {
|
||||
t.Error("NON-LINEARIZABLE: R1 exposed A, then R2 returned B with no second B PUT; B completed before R1, and R1 completed before R2. Current fragment counts do not prove B was never committed.")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
type scopeLeaseLocker struct {
|
||||
*localLocker
|
||||
partition *atomic.Bool
|
||||
refresh chan<- struct{}
|
||||
}
|
||||
|
||||
func (l scopeLeaseLocker) Refresh(ctx context.Context, a dsync.LockArgs) (bool, error) {
|
||||
if l.partition.Load() {
|
||||
select {
|
||||
case l.refresh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return false, errors.New("injected lock-plane partition")
|
||||
}
|
||||
return l.localLocker.Refresh(ctx, a)
|
||||
}
|
||||
|
||||
func (l scopeLeaseLocker) ForceUnlock(ctx context.Context, a dsync.LockArgs) (bool, error) {
|
||||
if l.partition.Load() {
|
||||
return false, errors.New("partition")
|
||||
}
|
||||
return l.localLocker.ForceUnlock(ctx, a)
|
||||
}
|
||||
|
||||
type scopeStagedContext struct {
|
||||
context.Context
|
||||
meta string
|
||||
entered chan<- struct{}
|
||||
release <-chan struct{}
|
||||
once sync.Once
|
||||
tracker healthDiskCtxValue
|
||||
}
|
||||
|
||||
func (c *scopeStagedContext) Value(k any) any {
|
||||
if _, ok := k.(healthDiskCtxKey); ok {
|
||||
if _, e := os.Stat(c.meta); e == nil {
|
||||
c.once.Do(func() { c.entered <- struct{}{}; <-c.release })
|
||||
}
|
||||
return &c.tracker
|
||||
}
|
||||
return c.Context.Value(k)
|
||||
}
|
||||
|
||||
type scopeRPCDisk struct {
|
||||
StorageAPI
|
||||
client *storageRESTClient
|
||||
}
|
||||
|
||||
func (d scopeRPCDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, o RenameOptions) (RenameDataResp, error) {
|
||||
return d.client.RenameData(ctx, sv, sp, fi, dv, dp, o)
|
||||
}
|
||||
|
||||
// Controlled fault schedule: real dsync lockers lose their expired lease,
|
||||
// while a real disk RenameData is paused after reading and staging xl.meta.
|
||||
// Only the clock age of lease records and the disk completion schedule are injected.
|
||||
// No object data or metadata is rewritten by the test after the seed PUT.
|
||||
func TestMutationLeaseLossAndPartition(t *testing.T) {
|
||||
for _, partitioned := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("network-partition-%t", partitioned), func(t *testing.T) {
|
||||
size, _ := strconv.Atoi(os.Getenv("SILO_TEST_MUTATION_BYTES"))
|
||||
payload := func(s string) string {
|
||||
if size <= len(s) {
|
||||
return s
|
||||
}
|
||||
return s + strings.Repeat(".", size-len(s))
|
||||
}
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "lease-key"
|
||||
base := time.Now().Add(-time.Hour)
|
||||
mutationPut(t, z, bucket, object, 0, payload("initial-0"), base)
|
||||
var partition atomic.Bool
|
||||
refreshed := make(chan struct{}, 16)
|
||||
locals := make([]*localLocker, 4)
|
||||
oldClients := make([]dsync.NetLocker, 4)
|
||||
newClients := make([]dsync.NetLocker, 4)
|
||||
for i := range locals {
|
||||
locals[i] = newLocker()
|
||||
oldClients[i] = scopeLeaseLocker{locals[i], &partition, refreshed}
|
||||
newClients[i] = locals[i]
|
||||
}
|
||||
mklock := func(clients []dsync.NetLocker) *distLockInstance {
|
||||
return &distLockInstance{rwMutex: dsync.NewDRWMutex(&dsync.Dsync{GetLockers: func() ([]dsync.NetLocker, string) { return clients, "scope-owner" }, Timeouts: dsync.DefaultTimeouts}, bucket+"/"+object), opsID: mustGetUUID()}
|
||||
}
|
||||
a := mklock(oldClients)
|
||||
requestCtx, cancelRequest := context.WithCancel(context.Background())
|
||||
defer cancelRequest()
|
||||
alc, err := a.GetLock(requestCtx, newDynamicTimeout(time.Second, time.Second))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer a.Unlock(alc)
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tg.Cleanup()
|
||||
entered, release := make(chan struct{}, 8), make(chan struct{})
|
||||
physicalDone := make(chan error, 8)
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range disks {
|
||||
route := fmt.Sprintf("/lease/%d", i)
|
||||
handler := func(p *RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
var success int64
|
||||
gate := &scopeStagedContext{Context: context.Background(), meta: filepath.Join(disk.Endpoint().Path, p.SrcVolume, p.SrcPath, xlStorageFormatFile), entered: entered, release: release, tracker: healthDiskCtxValue{lastSuccess: &success}}
|
||||
resp, e := disk.RenameData(gate, p.SrcVolume, p.SrcPath, p.FI, p.DstVolume, p.DstPath, p.Opts)
|
||||
physicalDone <- e
|
||||
return &resp, grid.NewRemoteErr(e)
|
||||
}
|
||||
if e := storageRenameDataRPC.Register(tg.Managers[1], handler, route); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
if e := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
defer p.Recycle()
|
||||
return handler(&p.RenameDataHandlerParams)
|
||||
}, route); e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
|
||||
client.diskID.Store(new(string))
|
||||
wrapped[i] = scopeRPCDisk{disk, client}
|
||||
}
|
||||
swap(wrapped)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
body := []byte(payload("pending-1000"))
|
||||
_, e := z.serverPools[0].PutObject(alc.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(time.Minute)})
|
||||
done <- e
|
||||
}()
|
||||
for range disks {
|
||||
select {
|
||||
case <-entered:
|
||||
case e := <-done:
|
||||
t.Fatalf("A returned before disk barrier: %v", e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("disk barrier timeout")
|
||||
}
|
||||
}
|
||||
t.Log("OBS A staged old metadata on all eight real disks; no final rename yet")
|
||||
partition.Store(partitioned)
|
||||
for _, l := range locals {
|
||||
l.mutex.Lock()
|
||||
for key, entries := range l.lockMap {
|
||||
for i := range entries {
|
||||
entries[i].TimeLastRefresh = time.Now().Add(-lockValidityDuration - time.Second).UnixNano()
|
||||
}
|
||||
l.lockMap[key] = entries
|
||||
}
|
||||
l.mutex.Unlock()
|
||||
l.expireOldLocks(lockValidityDuration)
|
||||
}
|
||||
if partitioned {
|
||||
select {
|
||||
case <-refreshed:
|
||||
case <-time.After(13 * time.Second):
|
||||
t.Fatal("refresh not attempted")
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
t.Logf("OBS after failed refresh old context error=%v", alc.Context().Err())
|
||||
} else {
|
||||
select {
|
||||
case <-alc.Context().Done():
|
||||
case <-time.After(13 * time.Second):
|
||||
t.Fatal("confirmed lost lock did not cancel")
|
||||
}
|
||||
t.Logf("OBS confirmed lease loss canceled old context: %v", alc.Context().Err())
|
||||
}
|
||||
b := mklock(newClients)
|
||||
blc, e := b.GetLock(context.Background(), newDynamicTimeout(time.Second, time.Second))
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
swap(disks)
|
||||
bdone := make(chan error, 1)
|
||||
go func() {
|
||||
body := []byte(payload("ack-9999"))
|
||||
_, e := z.serverPools[0].PutObject(blc.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{NoLock: true, MTime: base.Add(2 * time.Minute)})
|
||||
b.Unlock(blc)
|
||||
bdone <- e
|
||||
}()
|
||||
bearly := false
|
||||
select {
|
||||
case e := <-bdone:
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
bearly = true
|
||||
t.Log("OBS B acquired the same lock and ACKed while A remained physically staged")
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
t.Log("OBS B is waiting for the earlier disk transaction")
|
||||
}
|
||||
if partitioned {
|
||||
cancelRequest()
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case e := <-done:
|
||||
t.Logf("OBS A finished: %v", e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not finish")
|
||||
}
|
||||
if !bearly {
|
||||
select {
|
||||
case e := <-bdone:
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
t.Log("OBS B ACKed after disk serialization")
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("B did not finish")
|
||||
}
|
||||
}
|
||||
|
||||
for range disks {
|
||||
select {
|
||||
case e := <-physicalDone:
|
||||
t.Logf("OBS physical handler completed: %v", e)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("physical handler did not finish")
|
||||
}
|
||||
}
|
||||
for _, fast := range []bool{false, true} {
|
||||
r, e := z.GetObjectNInfo(context.Background(), bucket, object, nil, nil, ObjectOptions{FastGetObjInfo: fast})
|
||||
if e != nil {
|
||||
t.Errorf("OBS latest acknowledged payload unreadable fast=%t error=%v", fast, e)
|
||||
continue
|
||||
}
|
||||
data, e := io.ReadAll(r)
|
||||
r.Close()
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
shown := string(data)
|
||||
if len(shown) > 32 {
|
||||
shown = shown[:32]
|
||||
}
|
||||
t.Logf("OBS final GET fast=%t bytes=%d prefix=%q", fast, len(data), shown)
|
||||
if string(data) != payload("ack-9999") && string(data) != payload("pending-1000") {
|
||||
t.Errorf("STRICT STALE VALUE: prefix %q is neither latest ACK nor pending write", shown)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
//go:build consistency_strict
|
||||
|
||||
// Preserved diagnostic for the deferred strict-read policy. This is not a
|
||||
// maintenance release gate; failures record intentionally unresolved risks.
|
||||
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMutationReadPreservesAcknowledgedTags(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "tag-read"
|
||||
mutationPut(t, z, bucket, object, 0, "payload", time.Now())
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
partial := append([]StorageAPI(nil), disks...)
|
||||
partial[0] = nil
|
||||
swap(partial)
|
||||
const tags = "owner=acknowledged-B"
|
||||
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
swap(disks)
|
||||
before, err := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
after, err := disks[1].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("old/new tags=%q/%q; equal tuple=%t", before.Metadata[xhttp.AmzObjectTagging], after.Metadata[xhttp.AmzObjectTagging], before.ModTime.Equal(after.ModTime) && before.DataDir == after.DataDir && before.Metadata["etag"] == after.Metadata["etag"])
|
||||
for _, fast := range []bool{false, true} {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if oi.UserTags != tags {
|
||||
t.Fatalf("acknowledged tags replaced by stale metadata (fast=%t version=%q): %q", fast, version, oi.UserTags)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationVersionHistoryRead(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "history-read"
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
var versions []string
|
||||
for i := range 12 {
|
||||
id := mustGetUUID()
|
||||
body := "version-" + strconv.Itoa(i)
|
||||
_, err := set.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""), ObjectOptions{Versioned: true, VersionID: id})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
versions = append(versions, id)
|
||||
}
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
uniform := func() bool {
|
||||
raw := make([]RawFileInfo, len(disks))
|
||||
meta := make([]FileInfo, len(disks))
|
||||
errs := make([]error, len(disks))
|
||||
for i, disk := range disks {
|
||||
var err error
|
||||
raw[i], err = disk.ReadXL(t.Context(), bucket, object, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta[i], errs[i] = fileInfoFromRaw(raw[i], bucket, object, false, false)
|
||||
}
|
||||
return uniformLatestVersion(meta, errs)
|
||||
}
|
||||
if !uniform() {
|
||||
t.Fatal("healthy version history requires redundant full metadata decoding")
|
||||
}
|
||||
partial := append([]StorageAPI(nil), disks...)
|
||||
partial[0] = nil
|
||||
swap(partial)
|
||||
const tags = "owner=latest-acknowledged"
|
||||
if _, err := set.PutObjectTags(t.Context(), bucket, object, tags, ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
swap(disks)
|
||||
if uniform() {
|
||||
t.Fatal("stale same-version metadata incorrectly accepted as uniform")
|
||||
}
|
||||
for _, fast := range []bool{false, true} {
|
||||
for _, version := range []string{"", versions[len(versions)-1]} {
|
||||
oi, err := set.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{FastGetObjInfo: fast, VersionID: version})
|
||||
if err != nil || oi.VersionID != versions[len(versions)-1] || oi.UserTags != tags {
|
||||
t.Fatalf("latest version/tags changed: version=%q tags=%q err=%v", oi.VersionID, oi.UserTags, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, id := range versions {
|
||||
r, err := set.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: id})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(body) != "version-"+strconv.Itoa(i) {
|
||||
t.Fatalf("history version %d changed: %q err=%v", i, body, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMutationCrossDiskOrderHasStableRead(t *testing.T) {
|
||||
z, bucket := mutationPools(t)
|
||||
const object = "cross-disk-order"
|
||||
base := time.Now().Add(-time.Hour)
|
||||
mutationPut(t, z, bucket, object, 0, "initial", base)
|
||||
set := z.serverPools[0].getHashedSet(object)
|
||||
disks, swap := installListingDiskSwitch(set)
|
||||
defer swap(disks)
|
||||
earlyDone, release := make(chan struct{}, 4), make(chan struct{})
|
||||
var once sync.Once
|
||||
unblock := func() { once.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
aDisks := append([]StorageAPI(nil), disks...)
|
||||
for i, d := range disks {
|
||||
aDisks[i] = orderedRenameDisk{d, i < 4, earlyDone, release}
|
||||
}
|
||||
swap(aDisks)
|
||||
put := func(body string, mtime time.Time) error {
|
||||
_, e := z.serverPools[0].PutObject(t.Context(), bucket, object,
|
||||
mustGetPutObjReader(t, strings.NewReader(body), int64(len(body)), "", ""),
|
||||
ObjectOptions{NoLock: true, MTime: mtime})
|
||||
return e
|
||||
}
|
||||
aDone := make(chan error, 1)
|
||||
go func() { aDone <- put("writer-A", base.Add(time.Minute)) }()
|
||||
for range 4 {
|
||||
select {
|
||||
case <-earlyDone:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not reach first-half barrier")
|
||||
}
|
||||
}
|
||||
swap(disks)
|
||||
if err := put("writer-B", base.Add(2*time.Minute)); err != nil {
|
||||
t.Fatalf("B failed: %v", err)
|
||||
}
|
||||
unblock()
|
||||
select {
|
||||
case err := <-aDone:
|
||||
if err != nil {
|
||||
t.Fatalf("A failed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("A did not finish")
|
||||
}
|
||||
t.Log("both PUTs succeeded; final per-disk order is A→B on four disks and B→A on four disks; no writer remains")
|
||||
swap(disks)
|
||||
for range 8 {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
r, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{FastGetObjInfo: true, VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatalf("quiescent successful writes became unreadable: %v", err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(body) != "writer-B" {
|
||||
t.Fatalf("completed reads changed value: %q %v", body, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func uniformLatestVersion(meta []FileInfo, errs []error) bool {
|
||||
if len(meta) == 0 {
|
||||
return false
|
||||
}
|
||||
first := meta[0]
|
||||
for i, fi := range meta {
|
||||
if errs[i] != nil || !fi.IsValid() || fi.XLV1 || fi.Deleted || !fi.IsLatest || fi.NumVersions < 1 ||
|
||||
!fi.ModTime.Equal(first.ModTime) || !sameFileInfoVersion(first, fi) ||
|
||||
fi.NumVersions != first.NumVersions || !fi.SuccessorModTime.Equal(first.SuccessorModTime) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type orderedRenameDisk struct {
|
||||
StorageAPI
|
||||
early bool
|
||||
earlyDone chan<- struct{}
|
||||
release <-chan struct{}
|
||||
}
|
||||
|
||||
func (d orderedRenameDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
if !d.early {
|
||||
<-d.release
|
||||
}
|
||||
r, err := d.StorageAPI.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
if d.early {
|
||||
d.earlyDone <- struct{}{}
|
||||
<-d.release
|
||||
}
|
||||
return r, err
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// This is an undo capability, not a distributed fencing token. Any subsequent
|
||||
// mutation of the same version revokes it, including a no-op metadata update.
|
||||
const rollbackIDKey = ReservedMetadataPrefixLower + "rollback-id"
|
||||
|
||||
type storageMutationKey struct {
|
||||
drive, path string
|
||||
}
|
||||
|
||||
type storageMutationLock struct {
|
||||
mu sync.Mutex
|
||||
refs int // Holders and waiters; protected by the registry mutex.
|
||||
}
|
||||
|
||||
// Keep exact resource locks only while they have holders or waiters. The
|
||||
// registry mutex protects references, never storage I/O or resource waits.
|
||||
var storageMutationLocks = struct {
|
||||
sync.Mutex
|
||||
locks map[storageMutationKey]*storageMutationLock
|
||||
}{locks: make(map[storageMutationKey]*storageMutationLock)}
|
||||
|
||||
func (s *xlStorage) lockMutations(paths ...string) func() {
|
||||
// Each call uses one drive. Sorting its unique paths gives all overlapping
|
||||
// calls the same acquisition order without changing the caller's slice.
|
||||
paths = slices.Clone(paths)
|
||||
slices.Sort(paths)
|
||||
paths = slices.Compact(paths)
|
||||
drive := s.drivePath
|
||||
locks := make([]*storageMutationLock, len(paths))
|
||||
// Retain every entry before waiting so unlock cannot remove a mutex that
|
||||
// a queued operation still needs.
|
||||
storageMutationLocks.Lock()
|
||||
for i, path := range paths {
|
||||
key := storageMutationKey{drive, path}
|
||||
lock := storageMutationLocks.locks[key]
|
||||
if lock == nil {
|
||||
lock = &storageMutationLock{}
|
||||
storageMutationLocks.locks[key] = lock
|
||||
}
|
||||
lock.refs++
|
||||
locks[i] = lock
|
||||
}
|
||||
storageMutationLocks.Unlock()
|
||||
for _, lock := range locks {
|
||||
lock.mu.Lock()
|
||||
}
|
||||
return func() {
|
||||
for i := len(locks) - 1; i >= 0; i-- {
|
||||
locks[i].mu.Unlock()
|
||||
}
|
||||
storageMutationLocks.Lock()
|
||||
for i, path := range paths {
|
||||
locks[i].refs--
|
||||
if locks[i].refs == 0 {
|
||||
delete(storageMutationLocks.locks, storageMutationKey{drive, path})
|
||||
}
|
||||
}
|
||||
storageMutationLocks.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// Compare logical version state, excluding per-shard checksums/index, physical
|
||||
// DataDir placement and properties of other versions in the same xl.meta.
|
||||
// Different DataDir values remain readable for the existing rebalance recovery
|
||||
// contract; rollback separately requires an exact DataDir match.
|
||||
func sameFileInfoVersion(a, b FileInfo) bool {
|
||||
if a.VersionID != b.VersionID || a.Deleted != b.Deleted || a.XLV1 != b.XLV1 || a.Size != b.Size ||
|
||||
a.TransitionStatus != b.TransitionStatus || a.TransitionTier != b.TransitionTier ||
|
||||
a.TransitionedObjName != b.TransitionedObjName || a.TransitionVersionID != b.TransitionVersionID ||
|
||||
!sameVersionMetadata(a.Metadata, b.Metadata) || !bytes.Equal(a.Checksum, b.Checksum) || len(a.Parts) != len(b.Parts) {
|
||||
return false
|
||||
}
|
||||
if !a.Deleted && a.Size != 0 && !a.Erasure.Equal(b.Erasure) {
|
||||
return false
|
||||
}
|
||||
for i, p := range a.Parts {
|
||||
q := b.Parts[i]
|
||||
if p.Number != q.Number || p.Size != q.Size || p.ActualSize != q.ActualSize || p.ETag != q.ETag || !bytes.Equal(p.Index, q.Index) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// The undo capability controls rollback ownership, not the logical contents.
|
||||
// A partial no-op update can revoke it without changing a single public field.
|
||||
func sameVersionMetadata(a, b map[string]string) bool {
|
||||
if a[rollbackIDKey] == b[rollbackIDKey] {
|
||||
return maps.Equal(a, b)
|
||||
}
|
||||
for k, v := range a {
|
||||
if k != rollbackIDKey {
|
||||
if other, ok := b[k]; !ok || other != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
for k := range b {
|
||||
if k != rollbackIDKey {
|
||||
if _, ok := a[k]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Legacy callers without an undo capability must match the persisted state,
|
||||
// rather than transient coordinator-only fields in their original FileInfo.
|
||||
func samePersistedVersion(current, expected FileInfo) bool {
|
||||
if current.Metadata[rollbackIDKey] != expected.Metadata[rollbackIDKey] {
|
||||
return false
|
||||
}
|
||||
var meta xlMetaV2
|
||||
if err := meta.AddVersion(expected); err != nil {
|
||||
return false
|
||||
}
|
||||
fi, err := meta.ToFileInfo(current.Volume, current.Name, expected.VersionID, true, true)
|
||||
return err == nil && sameFileInfoVersion(current, fi)
|
||||
}
|
||||
|
||||
// Caller holds the object's mutation guard. Restore just the overwritten
|
||||
// version, and persist that decision before removing any external shards.
|
||||
func (s *xlStorage) undoWrite(ctx context.Context, volume, path string, fi FileInfo, opts DeleteOptions, meta *xlMetaV2) error {
|
||||
versionID := fi.VersionID
|
||||
if versionID == "" {
|
||||
versionID = nullVersionID
|
||||
}
|
||||
current, err := meta.ToFileInfo(volume, path, versionID, true, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if current.DataDir != fi.DataDir || !current.ModTime.Equal(fi.ModTime) || !samePersistedVersion(current, fi) {
|
||||
return errFileVersionNotFound
|
||||
}
|
||||
var previous xlMetaV2
|
||||
var oldVersion *xlMetaV2Version
|
||||
var oldData []byte
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if opts.OldDataDir != "" {
|
||||
buf, err := s.readAllData(ctx, volume, volumeDir, pathJoin(volumeDir, path, opts.OldDataDir, xlStorageFormatFileBackup))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer metaDataPoolPut(buf)
|
||||
if err := previous.LoadOrConvert(buf); err != nil {
|
||||
return err
|
||||
}
|
||||
_, oldVersion, err = previous.findVersionStr(versionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldData = previous.data.find(versionID)
|
||||
if oldData == nil {
|
||||
oldData = previous.data.find(oldVersion.getDataDir())
|
||||
}
|
||||
}
|
||||
dataDir, err := meta.DeleteVersion(fi)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dataDir != "" {
|
||||
meta.data.remove(versionID, dataDir)
|
||||
}
|
||||
if oldVersion != nil {
|
||||
if err := meta.addVersion(*oldVersion); err != nil {
|
||||
return err
|
||||
}
|
||||
if oldData != nil {
|
||||
meta.data.replace(versionID, oldData)
|
||||
}
|
||||
if dataDir == oldVersion.getDataDir() {
|
||||
dataDir = ""
|
||||
}
|
||||
}
|
||||
if len(meta.versions) > 0 {
|
||||
buf, err := meta.AppendTo(metaDataPoolGet())
|
||||
defer metaDataPoolPut(buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.writeAllMeta(ctx, volume, pathJoin(path, xlStorageFormatFile), buf, true); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := s.deleteFile(volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile), true, false); err != nil {
|
||||
return err
|
||||
}
|
||||
if dataDir != "" {
|
||||
if err := s.moveToTrash(pathJoin(volumeDir, path, dataDir), true, false); err != nil && !errors.Is(err, errFileNotFound) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if opts.OldDataDir != "" {
|
||||
backupDir := pathJoin(volumeDir, path, opts.OldDataDir)
|
||||
Remove(pathJoin(backupDir, xlStorageFormatFileBackup))
|
||||
if strings.HasSuffix(opts.OldDataDir, "-rollback") {
|
||||
Remove(backupDir)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -244,6 +244,26 @@ func TestPoolsObjectLockOverwrite(t *testing.T) {
|
||||
}
|
||||
}
|
||||
defer conditionalPutPool(t, z, key, 0)()
|
||||
var source string
|
||||
var copyInfo ObjectInfo
|
||||
if op == "copy" || op == "self-copy" {
|
||||
source = key
|
||||
if op == "copy" {
|
||||
source = "source-" + state
|
||||
putConsistencyObject(t, z, bucket, source, 0, "replacement", ObjectOptions{Versioned: true})
|
||||
}
|
||||
// Resolve the source before the pool becomes unreadable.
|
||||
// The destination must still reject a previously read source.
|
||||
gr, gerr := z.GetObjectNInfo(ctx, bucket, source, nil, nil, ObjectOptions{})
|
||||
if gerr != nil {
|
||||
t.Fatal(gerr)
|
||||
}
|
||||
defer gr.Close()
|
||||
copyInfo = gr.ObjInfo
|
||||
copyInfo.metadataOnly = op == "self-copy"
|
||||
copyInfo.PutObjReader = mustGetPutObjReader(t, gr, copyInfo.Size, "", "")
|
||||
copyInfo.Reader = copyInfo.PutObjReader.Reader
|
||||
}
|
||||
var restore func()
|
||||
if state == "unreadable" {
|
||||
restore = conditionalPutSwapDisks(z.serverPools[1], key, func(d StorageAPI) StorageAPI {
|
||||
@@ -257,22 +277,8 @@ func TestPoolsObjectLockOverwrite(t *testing.T) {
|
||||
case "put":
|
||||
_, writeErr = z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewBufferString("replacement"), 11, "", ""), opts)
|
||||
case "copy", "self-copy":
|
||||
source := key
|
||||
if op == "copy" {
|
||||
source = "source-" + state
|
||||
putConsistencyObject(t, z, bucket, source, 0, "replacement", ObjectOptions{Versioned: true})
|
||||
}
|
||||
gr, gerr := z.GetObjectNInfo(ctx, bucket, source, nil, nil, ObjectOptions{})
|
||||
if gerr != nil {
|
||||
t.Fatal(gerr)
|
||||
}
|
||||
defer gr.Close()
|
||||
info := gr.ObjInfo
|
||||
info.UserDefined = opts.UserDefined
|
||||
info.metadataOnly = op == "self-copy"
|
||||
info.PutObjReader = mustGetPutObjReader(t, gr, info.Size, "", "")
|
||||
info.Reader = info.PutObjReader.Reader
|
||||
_, writeErr = z.CopyObject(ctx, bucket, source, bucket, key, info, ObjectOptions{VersionID: nullVersionID}, opts)
|
||||
copyInfo.UserDefined = opts.UserDefined
|
||||
_, writeErr = z.CopyObject(ctx, bucket, source, bucket, key, copyInfo, ObjectOptions{VersionID: nullVersionID}, opts)
|
||||
case "multipart":
|
||||
id, parts := multipartConditionUpload(t, z, bucket, key, 0, opts)
|
||||
_, writeErr = z.CompleteMultipartUpload(ctx, bucket, key, id, parts, ObjectOptions{Versioned: true})
|
||||
|
||||
@@ -0,0 +1,520 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/grid"
|
||||
)
|
||||
|
||||
// Pause at the real disk health callback after staging xl.meta. Unlike a sleep,
|
||||
// this fixes cancellation between staging and the final metadata rename.
|
||||
type stagedRenameContext struct {
|
||||
context.Context
|
||||
meta string
|
||||
entered, release chan struct{}
|
||||
once sync.Once
|
||||
tracker healthDiskCtxValue
|
||||
}
|
||||
|
||||
func (c *stagedRenameContext) Value(key any) any {
|
||||
if _, ok := key.(healthDiskCtxKey); ok {
|
||||
if _, err := os.Stat(c.meta); err == nil {
|
||||
c.once.Do(func() {
|
||||
close(c.entered)
|
||||
<-c.release
|
||||
})
|
||||
}
|
||||
return &c.tracker
|
||||
}
|
||||
return c.Context.Value(key)
|
||||
}
|
||||
|
||||
func TestRenameCancellationLocalWaitsForDisk(t *testing.T) {
|
||||
disk, root, err := newXLStorageTestSetup(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer disk.Close()
|
||||
if err := disk.MakeVol(t.Context(), "bucket"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
var lastSuccess int64
|
||||
gate := &stagedRenameContext{
|
||||
Context: ctx, meta: filepath.Join(root, minioMetaTmpBucket, "staged", xlStorageFormatFile),
|
||||
entered: make(chan struct{}), release: make(chan struct{}),
|
||||
tracker: healthDiskCtxValue{lastSuccess: &lastSuccess},
|
||||
}
|
||||
fi := newFileInfo("object", 2, 2)
|
||||
fi.DataDir, fi.ModTime, fi.Erasure.Index = mustGetUUID(), UTCNow(), 1
|
||||
fi.Fresh = true
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := disk.RenameData(gate, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
|
||||
done <- err
|
||||
}()
|
||||
<-gate.entered
|
||||
cancel()
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Errorf("RenameData returned %v while its disk worker was still running", err)
|
||||
default:
|
||||
}
|
||||
close(gate.release)
|
||||
synctest.Wait()
|
||||
select {
|
||||
case err := <-done:
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("canceled staged write: %v", err)
|
||||
}
|
||||
default:
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(root, "bucket", "object", xlStorageFormatFile)); !os.IsNotExist(err) {
|
||||
t.Fatalf("canceled staged write committed metadata: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRenameCancellationRPCWaitsForDisk(t *testing.T) {
|
||||
for _, operation := range []string{"regular", "inline", "write-metadata", "update-metadata", "undo"} {
|
||||
t.Run(operation, func(t *testing.T) {
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tg.Cleanup()
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
var committed atomic.Bool
|
||||
handle := func(*RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
close(entered)
|
||||
<-release
|
||||
committed.Store(true)
|
||||
return &RenameDataResp{OldDataDir: "previous-generation"}, nil
|
||||
}
|
||||
const route = "/cancel-commit"
|
||||
if err := storageRenameDataRPC.Register(tg.Managers[1], handle, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
defer p.Recycle()
|
||||
return handle(&p.RenameDataHandlerParams)
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta := func(*MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
|
||||
_, err := handle(nil)
|
||||
return grid.NewNoPayload(), err
|
||||
}
|
||||
if err := storageWriteMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageUpdateMetadataRPC.Register(tg.Managers[1], meta, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageDeleteVersionRPC.Register(tg.Managers[1], func(*DeleteVersionHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
|
||||
return meta(nil)
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
|
||||
client.diskID.Store(new(string))
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
fi := FileInfo{}
|
||||
if operation == "inline" {
|
||||
fi.Data = []byte("payload")
|
||||
}
|
||||
type result struct {
|
||||
resp RenameDataResp
|
||||
err error
|
||||
}
|
||||
done := make(chan result, 1)
|
||||
go func() {
|
||||
var resp RenameDataResp
|
||||
var err error
|
||||
switch operation {
|
||||
case "write-metadata":
|
||||
err = client.WriteMetadata(ctx, "", "bucket", "object", fi)
|
||||
case "update-metadata":
|
||||
err = client.UpdateMetadata(ctx, "bucket", "object", fi, UpdateMetadataOpts{})
|
||||
case "undo":
|
||||
err = client.DeleteVersion(ctx, "bucket", "object", fi, false, DeleteOptions{UndoWrite: true})
|
||||
default:
|
||||
resp, err = client.RenameData(ctx, minioMetaTmpBucket, "staged", fi, "bucket", "object", RenameOptions{})
|
||||
}
|
||||
done <- result{resp, err}
|
||||
}()
|
||||
<-entered
|
||||
cancel()
|
||||
var early *result
|
||||
select {
|
||||
case r := <-done:
|
||||
early = &r
|
||||
t.Errorf("RPC returned %v before the remote disk completed", r.err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
close(release)
|
||||
if early == nil {
|
||||
select {
|
||||
case r := <-done:
|
||||
wantOld := ""
|
||||
if operation == "regular" || operation == "inline" {
|
||||
wantOld = "previous-generation"
|
||||
}
|
||||
if r.err != nil || r.resp.OldDataDir != wantOld || !committed.Load() {
|
||||
t.Errorf("lost completed result needed for rollback: %+v", r)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("RPC did not finish after disk completion")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameCancellationAlreadyCanceledRPC(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
cancel()
|
||||
// A canceled call must not access the connection or send a new mutation.
|
||||
client := &storageRESTClient{}
|
||||
client.diskID.Store(new(string))
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
t.Errorf("canceled mutation reached the connection: %v", p)
|
||||
}
|
||||
}()
|
||||
_, err := client.RenameData(ctx, minioMetaTmpBucket, "staged", FileInfo{}, "bucket", "object", RenameOptions{})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("pre-canceled RenameData: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type gatedRenameDisk struct {
|
||||
StorageAPI
|
||||
client *storageRESTClient
|
||||
}
|
||||
|
||||
func (d gatedRenameDisk) RenameData(ctx context.Context, sv, sp string, fi FileInfo, dv, dp string, opts RenameOptions) (RenameDataResp, error) {
|
||||
return d.client.RenameData(ctx, sv, sp, fi, dv, dp, opts)
|
||||
}
|
||||
|
||||
func (d gatedRenameDisk) WriteMetadata(ctx context.Context, ov, volume, path string, fi FileInfo) error {
|
||||
return d.client.WriteMetadata(ctx, ov, volume, path, fi)
|
||||
}
|
||||
|
||||
func (d gatedRenameDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, force bool, opts DeleteOptions) error {
|
||||
return d.client.DeleteVersion(ctx, volume, path, fi, force, opts)
|
||||
}
|
||||
|
||||
// Use real disk commits behind the real RPC client. The barrier fixes the
|
||||
// order: every commit has started, then cancellation, then remote completion.
|
||||
func TestRenameCancellationKeepsLockUntilRollback(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
size int
|
||||
versioned bool
|
||||
copy bool
|
||||
shared bool
|
||||
}{
|
||||
{32, false, false, false},
|
||||
{2 << 20, false, false, false},
|
||||
{32, true, false, false},
|
||||
{2 << 20, true, false, false},
|
||||
{32, false, true, false},
|
||||
{2 << 20, false, true, false},
|
||||
{32, false, false, true},
|
||||
{2 << 20, false, false, true},
|
||||
} {
|
||||
t.Run(fmt.Sprintf("bytes-%d/other-versions-%t/copy-%t/shared-%t", tc.size, tc.versioned, tc.copy, tc.shared), func(t *testing.T) {
|
||||
size := tc.size
|
||||
z, bucket := consistencyPools(t)
|
||||
object := "cancel-rollback"
|
||||
old := strings.Repeat("a", size)
|
||||
base := UTCNow().Add(-time.Hour)
|
||||
var sharedVersions [2]string
|
||||
for pool := range 2 {
|
||||
if tc.versioned {
|
||||
putConsistencyObject(t, z, bucket, object, pool, "retained-version", ObjectOptions{Versioned: true, MTime: base.Add(-time.Minute)})
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, pool, old, ObjectOptions{MTime: base})
|
||||
if tc.shared {
|
||||
info, err := z.serverPools[pool].GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
info.metadataOnly, info.versionOnly = true, true
|
||||
copied, err := z.serverPools[pool].getHashedSet(object).CopyObject(t.Context(), bucket, object, bucket, object, info,
|
||||
ObjectOptions{}, ObjectOptions{Versioned: true, MTime: base.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sharedVersions[pool] = copied.VersionID
|
||||
}
|
||||
}
|
||||
copyInfo, err := z.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
copyInfo.metadataOnly = true
|
||||
copyInfo.UserDefined["copy-test"] = "canceled"
|
||||
// Exercise the production rollback handler and wire options as
|
||||
// well as the client. A local DeleteVersion bypasses that boundary.
|
||||
globalLocalDrivesMu.Lock()
|
||||
previousSets, previousMap := globalLocalSetDrives, globalLocalDrivesMap
|
||||
globalLocalSetDrives = nil
|
||||
globalLocalDrivesMap = make(map[string]StorageAPI)
|
||||
for _, pool := range z.serverPools {
|
||||
for _, disk := range pool.getHashedSet(object).getDisks() {
|
||||
globalLocalDrivesMap[disk.Endpoint().String()] = disk
|
||||
}
|
||||
}
|
||||
globalLocalDrivesMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
globalLocalDrivesMu.Lock()
|
||||
globalLocalSetDrives, globalLocalDrivesMap = previousSets, previousMap
|
||||
globalLocalDrivesMu.Unlock()
|
||||
})
|
||||
tg, err := grid.SetupTestGrid(2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer tg.Cleanup()
|
||||
entered := make(chan struct{}, 32)
|
||||
release := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
unblock := func() { releaseOnce.Do(func() { close(release) }) }
|
||||
defer unblock()
|
||||
var completed atomic.Int32
|
||||
var injectCommitFailure atomic.Bool
|
||||
injectCommitFailure.Store(true)
|
||||
for pi, pool := range z.serverPools {
|
||||
set := pool.getHashedSet(object)
|
||||
disks, switchDisks := installListingDiskSwitch(set)
|
||||
defer switchDisks(disks)
|
||||
wrapped := append([]StorageAPI(nil), disks...)
|
||||
for i, disk := range disks {
|
||||
route := fmt.Sprintf("/cancel-rollback/%d/%d", pi, i)
|
||||
server := storageRESTServer{endpoint: disk.Endpoint()}
|
||||
if err := storageDeleteVersionRPC.Register(tg.Managers[1], server.DeleteVersionHandler, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handle := func(p *RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
entered <- struct{}{}
|
||||
<-release
|
||||
// Cancellation alone must not undo a successful quorum.
|
||||
// This rollback fixture also fails half the disks, so its
|
||||
// original payload/lock assertions require a real failed commit.
|
||||
defer completed.Add(1)
|
||||
if i >= len(disks)/2 && injectCommitFailure.Load() {
|
||||
return nil, grid.NewRemoteErr(errDiskNotFound)
|
||||
}
|
||||
resp, err := disk.RenameData(context.Background(), p.SrcVolume, p.SrcPath, p.FI, p.DstVolume, p.DstPath, p.Opts)
|
||||
if err != nil {
|
||||
return nil, grid.NewRemoteErr(err)
|
||||
}
|
||||
return &resp, nil
|
||||
}
|
||||
if err := storageRenameDataRPC.Register(tg.Managers[1], handle, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageRenameDataInlineRPC.Register(tg.Managers[1], func(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
|
||||
defer p.Recycle()
|
||||
return handle(&p.RenameDataHandlerParams)
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := storageWriteMetadataRPC.Register(tg.Managers[1], func(p *MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
|
||||
if p.Volume == bucket && p.FilePath == object {
|
||||
entered <- struct{}{}
|
||||
<-release
|
||||
defer completed.Add(1)
|
||||
}
|
||||
err := disk.WriteMetadata(context.Background(), p.OrigVolume, p.Volume, p.FilePath, p.FI)
|
||||
return grid.NewNoPayload(), grid.NewRemoteErr(err)
|
||||
}, route); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &storageRESTClient{gridConn: tg.Managers[0].Connection(tg.Hosts[1]).Subroute(route)}
|
||||
client.diskID.Store(new(string))
|
||||
wrapped[i] = gatedRenameDisk{disk, client}
|
||||
}
|
||||
switchDisks(wrapped)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
pending := bytes.Repeat([]byte("b"), size)
|
||||
reader := mustGetPutObjReader(t, bytes.NewReader(pending), int64(size), "", "")
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
var err error
|
||||
if tc.copy {
|
||||
_, err = z.CopyObject(ctx, bucket, object, bucket, object, copyInfo, ObjectOptions{}, ObjectOptions{})
|
||||
} else {
|
||||
_, err = z.PutObject(ctx, bucket, object, reader, ObjectOptions{})
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
n := z.serverPools[0].getHashedSet(object).setDriveCount
|
||||
for range n {
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("write ended before commit barrier: %v", err)
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("write did not reach every disk")
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
var early bool
|
||||
select {
|
||||
case err := <-done:
|
||||
early = true
|
||||
t.Errorf("mutation returned before remote commits completed: %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
lock := z.NewNSLock(bucket, object)
|
||||
lockCtx, cancelLock := context.WithTimeout(t.Context(), 50*time.Millisecond)
|
||||
lc, lockErr := lock.GetLock(lockCtx, newDynamicTimeout(50*time.Millisecond, 50*time.Millisecond))
|
||||
cancelLock()
|
||||
if lockErr == nil {
|
||||
lock.Unlock(lc)
|
||||
t.Error("namespace lock released before remote commits and rollback")
|
||||
}
|
||||
unblock()
|
||||
if early {
|
||||
// Keep the old-code failure deterministic; don't race cleanup
|
||||
// against handlers intentionally still running at the barrier.
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for completed.Load() != int32(n) {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("remote handlers did not finish")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
t.Fatal("canceled write unexpectedly acknowledged")
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("canceled write did not finish rollback")
|
||||
}
|
||||
if completed.Load() != int32(n) {
|
||||
t.Fatal("returned without accounting for every remote commit")
|
||||
}
|
||||
injectCommitFailure.Store(false)
|
||||
for pi, pool := range z.serverPools {
|
||||
for _, version := range []string{"", nullVersionID} {
|
||||
r, err := pool.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: version})
|
||||
if err != nil {
|
||||
t.Fatalf("pool %d lost its acknowledged copy (version %q): %v", pi, version, err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(body) != old {
|
||||
t.Errorf("pool %d rollback lost acknowledged payload (version %q): bytes=%d err=%v", pi, version, len(body), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
got, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(got)
|
||||
got.Close()
|
||||
if err != nil || string(body) != old {
|
||||
t.Fatalf("rollback lost acknowledged payload: bytes=%d err=%v", len(body), err)
|
||||
}
|
||||
newBody := bytes.Repeat([]byte("c"), size)
|
||||
if _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(newBody), int64(size), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err = z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err = io.ReadAll(got)
|
||||
got.Close()
|
||||
if err != nil || !bytes.Equal(body, newBody) {
|
||||
t.Fatalf("late commit replaced subsequent successful PUT: bytes=%d err=%v", len(body), err)
|
||||
}
|
||||
if tc.shared {
|
||||
for pi, pool := range z.serverPools {
|
||||
r, err := pool.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{VersionID: sharedVersions[pi]})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil || string(body) != old {
|
||||
t.Errorf("commit cleanup destroyed shared version on pool %d: bytes=%d err=%v", pi, len(body), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameCancellationStaleRollbackPreservesNewWrite(t *testing.T) {
|
||||
z, bucket := consistencyPools(t)
|
||||
const object = "stale-rollback"
|
||||
putConsistencyObject(t, z, bucket, object, 0, "canceled-generation", ObjectOptions{})
|
||||
disks := z.serverPools[0].getHashedSet(object).getDisks()
|
||||
stale := make([]FileInfo, len(disks))
|
||||
for i, disk := range disks {
|
||||
var err error
|
||||
stale[i], err = disk.ReadVersion(t.Context(), "", bucket, object, nullVersionID, ReadOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
putConsistencyObject(t, z, bucket, object, 0, "new-acknowledged-generation", ObjectOptions{})
|
||||
for i, disk := range disks {
|
||||
err := disk.DeleteVersion(t.Context(), bucket, object, stale[i], false, DeleteOptions{UndoWrite: true})
|
||||
if err != nil && !errors.Is(err, errFileVersionNotFound) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
got, err := z.GetObjectNInfo(t.Context(), bucket, object, nil, nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("stale rollback removed the new acknowledged write: %v", err)
|
||||
}
|
||||
defer got.Close()
|
||||
body, err := io.ReadAll(got)
|
||||
if err != nil || string(body) != "new-acknowledged-generation" {
|
||||
t.Fatalf("stale rollback changed the new acknowledged payload: %q, %v", body, err)
|
||||
}
|
||||
}
|
||||
@@ -408,8 +408,11 @@ func (client *storageRESTClient) CreateFile(ctx context.Context, origvolume, vol
|
||||
}
|
||||
|
||||
func (client *storageRESTClient) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
|
||||
defer cancel()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
// As with RenameData, admitted metadata writes must finish before unlock.
|
||||
ctx = grid.WithoutDefaultTimeout(context.WithoutCancel(ctx))
|
||||
|
||||
_, err := storageWriteMetadataRPC.Call(ctx, client.gridConn, &MetadataHandlerParams{
|
||||
DiskID: *client.diskID.Load(),
|
||||
@@ -422,8 +425,10 @@ func (client *storageRESTClient) WriteMetadata(ctx context.Context, origvolume,
|
||||
}
|
||||
|
||||
func (client *storageRESTClient) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
|
||||
defer cancel()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = grid.WithoutDefaultTimeout(context.WithoutCancel(ctx))
|
||||
|
||||
_, err := storageUpdateMetadataRPC.Call(ctx, client.gridConn, &MetadataHandlerParams{
|
||||
DiskID: *client.diskID.Load(),
|
||||
@@ -436,8 +441,14 @@ func (client *storageRESTClient) UpdateMetadata(ctx context.Context, volume, pat
|
||||
}
|
||||
|
||||
func (client *storageRESTClient) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) (err error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
|
||||
defer cancel()
|
||||
if opts.UndoWrite {
|
||||
// Rollback is part of the commit and must finish before unlocking.
|
||||
ctx = grid.WithoutDefaultTimeout(context.WithoutCancel(ctx))
|
||||
} else {
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
|
||||
defer cancel()
|
||||
}
|
||||
|
||||
_, err = storageDeleteVersionRPC.Call(ctx, client.gridConn, &DeleteVersionHandlerParams{
|
||||
DiskID: *client.diskID.Load(),
|
||||
@@ -487,6 +498,13 @@ func (client *storageRESTClient) CheckParts(ctx context.Context, volume string,
|
||||
func (client *storageRESTClient) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo,
|
||||
dstVolume, dstPath string, opts RenameOptions,
|
||||
) (res RenameDataResp, err error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, err
|
||||
}
|
||||
// The remote handler cannot cancel an admitted RenameData. Keep waiting
|
||||
// for its result so renameData can roll back every completed mutation
|
||||
// before the caller cleans staging files and releases the namespace lock.
|
||||
ctx = grid.WithoutDefaultTimeout(context.WithoutCancel(ctx))
|
||||
params := RenameDataHandlerParams{
|
||||
DiskID: *client.diskID.Load(),
|
||||
SrcVolume: srcVolume,
|
||||
|
||||
@@ -102,14 +102,9 @@ func TestGuardedStorageCoversEveryPathMethod(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestGuardChecksUnreachableFields pins the guard on path fields that exist on
|
||||
// the wire but that no handler currently reads, so an end-to-end test cannot
|
||||
// observe them. DeleteVersionHandler hardcodes `opts := DeleteOptions{}` and
|
||||
// discards p.Opts, which means DeleteOptions.OldDataDir - a value that reaches
|
||||
// renameAll() with no containment of its own - is unreachable by accident
|
||||
// rather than by design. If someone later plumbs p.Opts through, the guard is
|
||||
// already there; this test is what stops it from being removed as "dead".
|
||||
func TestGuardChecksUnreachableFields(t *testing.T) {
|
||||
// OldDataDir can reach renameAll during rollback. Guard it independently of
|
||||
// which handlers currently forward deletion options.
|
||||
func TestGuardChecksDeleteOptionPaths(t *testing.T) {
|
||||
g := guardedStorage{poisonStorage{}}
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
@@ -415,8 +415,7 @@ func (s *storageRESTServer) DeleteVersionHandler(p *DeleteVersionHandlerParams)
|
||||
filePath := p.FilePath
|
||||
forceDelMarker := p.ForceDelMarker
|
||||
|
||||
opts := DeleteOptions{}
|
||||
err := s.getStorage().DeleteVersion(context.Background(), volume, filePath, p.FI, forceDelMarker, opts)
|
||||
err := s.getStorage().DeleteVersion(context.Background(), volume, filePath, p.FI, forceDelMarker, p.Opts)
|
||||
return np, grid.NewRemoteErr(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -231,12 +231,9 @@ func TestStorageRESTTraversalRejected(t *testing.T) {
|
||||
{"DeleteVersion-DataDir", func() error {
|
||||
return restClient.DeleteVersion(ctx, "foo", "obj", badFI(p), false, DeleteOptions{})
|
||||
}},
|
||||
// DeleteOptions.OldDataDir is guarded too, but cannot be exercised
|
||||
// from here: DeleteVersionHandler hardcodes `opts := DeleteOptions{}`
|
||||
// and never reads the wire value, so the field is unreachable today.
|
||||
// That is an accidental mitigation, not a control - see
|
||||
// TestGuardChecksUnreachableFields for the unit-level assertion that
|
||||
// the guard is ready if anyone ever plumbs it through.
|
||||
{"DeleteVersion-OldDataDir", func() error {
|
||||
return restClient.DeleteVersion(ctx, "foo", "obj", badFI(""), false, DeleteOptions{UndoWrite: true, OldDataDir: p})
|
||||
}},
|
||||
{"RenameData-srcPath", func() error {
|
||||
_, err := restClient.RenameData(ctx, "foo", p, badFI(""), "bar", "dst", RenameOptions{})
|
||||
return err
|
||||
|
||||
@@ -503,6 +503,8 @@ func resetTestGlobals() {
|
||||
resetGlobalHealState()
|
||||
// Reset globalIAMSys to `nil`
|
||||
resetGlobalIAMSys()
|
||||
// Object-layer fixtures must not reuse the preceding IAM suite's etcd store.
|
||||
globalEtcdClient = nil
|
||||
}
|
||||
|
||||
// Configure the server for the test run.
|
||||
|
||||
@@ -495,7 +495,7 @@ func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPat
|
||||
if len(fi.Data) > 0 {
|
||||
fi.Data = append(grid.GetByteBufferCap(len(fi.Data))[:0], fi.Data...)
|
||||
}
|
||||
return xioutil.WithDeadline[RenameDataResp](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (res RenameDataResp, err error) {
|
||||
return xioutil.WithDeadlineAndWait[RenameDataResp](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (res RenameDataResp, err error) {
|
||||
if len(fi.Data) > 0 {
|
||||
defer grid.PutByteBuffer(fi.Data)
|
||||
}
|
||||
@@ -634,6 +634,12 @@ func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path s
|
||||
}
|
||||
}()
|
||||
|
||||
if opts.UndoWrite {
|
||||
_, err = xioutil.WithDeadlineAndWait(ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
|
||||
})
|
||||
return err
|
||||
}
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
return w.Run(func() error { return p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts) })
|
||||
}
|
||||
@@ -645,8 +651,10 @@ func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path
|
||||
}
|
||||
defer done(0, &err)
|
||||
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
return w.Run(func() error { return p.storage.UpdateMetadata(ctx, volume, path, fi, opts) })
|
||||
_, err = xioutil.WithDeadlineAndWait(ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, p.storage.UpdateMetadata(ctx, volume, path, fi, opts)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) (err error) {
|
||||
@@ -656,8 +664,10 @@ func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, origvolume, vo
|
||||
}
|
||||
defer done(0, &err)
|
||||
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
return w.Run(func() error { return p.storage.WriteMetadata(ctx, origvolume, volume, path, fi) })
|
||||
_, err = xioutil.WithDeadlineAndWait(ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (struct{}, error) {
|
||||
return struct{}{}, p.storage.WriteMetadata(ctx, origvolume, volume, path, fi)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (fi FileInfo, err error) {
|
||||
|
||||
@@ -1570,7 +1570,11 @@ func (x *xlMetaV2) UpdateObjectVersion(fi FileInfo) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(ver.ObjectV2.MetaSys, rollbackIDKey)
|
||||
for k, v := range fi.Metadata {
|
||||
if k == rollbackIDKey {
|
||||
continue
|
||||
}
|
||||
if len(k) > len(ReservedMetadataPrefixLower) && strings.EqualFold(k[:len(ReservedMetadataPrefixLower)], ReservedMetadataPrefixLower) {
|
||||
ver.ObjectV2.MetaSys[k] = []byte(v)
|
||||
} else {
|
||||
|
||||
+45
-11
@@ -25,6 +25,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
pathutil "path"
|
||||
"path/filepath"
|
||||
@@ -1100,6 +1101,7 @@ func (s *xlStorage) ListDir(ctx context.Context, origvolume, volume, dirPath str
|
||||
}
|
||||
|
||||
func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis ...FileInfo) error {
|
||||
defer s.lockMutations(pathJoin(volume, path))()
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1308,11 +1310,10 @@ func (s *xlStorage) readAllData(ctx context.Context, volume, volumeDir string, f
|
||||
})
|
||||
}
|
||||
|
||||
func (s *xlStorage) moveToTrash(filePath string, recursive, immediatePurge bool) (err error) {
|
||||
w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
|
||||
return w.Run(func() (err error) {
|
||||
return s.moveToTrashNoDeadline(filePath, recursive, immediatePurge)
|
||||
})
|
||||
func (s *xlStorage) moveToTrash(filePath string, recursive, immediatePurge bool) error {
|
||||
// The caller must retain its mutation guard until this rename completes.
|
||||
// Only purging the already detached unique trash directory is asynchronous.
|
||||
return s.moveToTrashNoDeadline(filePath, recursive, immediatePurge)
|
||||
}
|
||||
|
||||
// DeleteVersion - deletes FileInfo metadata for path at `xl.meta`. forceDelMarker
|
||||
@@ -1325,6 +1326,8 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
})
|
||||
}
|
||||
|
||||
defer s.lockMutations(pathJoin(volume, path))()
|
||||
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1345,7 +1348,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
metaDataPoolPut(buf) // Never used, return it
|
||||
if fi.Deleted && forceDelMarker {
|
||||
// Create a new xl.meta with a delete marker in it
|
||||
return s.WriteMetadata(ctx, "", volume, path, fi)
|
||||
return s.writeMetadataNoLock(ctx, "", volume, path, fi)
|
||||
}
|
||||
|
||||
s.RLock()
|
||||
@@ -1381,6 +1384,10 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
return err
|
||||
}
|
||||
|
||||
if opts.UndoWrite {
|
||||
return s.undoWrite(ctx, volume, path, fi, opts, &xlMeta)
|
||||
}
|
||||
|
||||
dataDir, err := xlMeta.DeleteVersion(fi)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1421,15 +1428,12 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
|
||||
return s.writeAllMeta(ctx, volume, pathJoin(path, xlStorageFormatFile), buf, true)
|
||||
}
|
||||
|
||||
if opts.UndoWrite && opts.OldDataDir != "" {
|
||||
return renameAll(pathJoin(filePath, opts.OldDataDir, xlStorageFormatFileBackup), pathJoin(filePath, xlStorageFormatFile), filePath)
|
||||
}
|
||||
|
||||
return s.deleteFile(volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile), true, false)
|
||||
}
|
||||
|
||||
// Updates only metadata for a given version.
|
||||
func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) (err error) {
|
||||
defer s.lockMutations(pathJoin(volume, path))()
|
||||
if len(fi.Metadata) == 0 {
|
||||
return errInvalidArgument
|
||||
}
|
||||
@@ -1478,6 +1482,16 @@ func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi
|
||||
|
||||
// WriteMetadata - writes FileInfo metadata for path at `xl.meta`
|
||||
func (s *xlStorage) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) (err error) {
|
||||
defer s.lockMutations(pathJoin(volume, path))()
|
||||
return s.writeMetadataNoLock(ctx, origvolume, volume, path, fi)
|
||||
}
|
||||
|
||||
func (s *xlStorage) writeMetadataNoLock(ctx context.Context, origvolume, volume, path string, fi FileInfo) (err error) {
|
||||
// Any later write of this version revokes the original writer's undo.
|
||||
if _, ok := fi.Metadata[rollbackIDKey]; ok {
|
||||
fi.Metadata = maps.Clone(fi.Metadata)
|
||||
delete(fi.Metadata, rollbackIDKey)
|
||||
}
|
||||
if fi.Fresh {
|
||||
if origvolume != "" {
|
||||
origvolumeDir, err := s.getVolDir(origvolume)
|
||||
@@ -2540,7 +2554,10 @@ func (s *xlStorage) DeleteBulk(ctx context.Context, volume string, paths ...stri
|
||||
return err
|
||||
}
|
||||
|
||||
if err = s.moveToTrash(filePath, false, false); err != nil {
|
||||
unlock := s.lockMutations(pathJoin(volume, fp), pathJoin(volume, pathutil.Dir(fp)))
|
||||
err = s.moveToTrash(filePath, false, false)
|
||||
unlock()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -2550,6 +2567,7 @@ func (s *xlStorage) DeleteBulk(ctx context.Context, volume string, paths ...stri
|
||||
|
||||
// Delete - delete a file at path.
|
||||
func (s *xlStorage) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
|
||||
defer s.lockMutations(pathJoin(volume, path), pathJoin(volume, pathutil.Dir(path)))()
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -2579,6 +2597,7 @@ func skipAccessChecks(volume string) (ok bool) {
|
||||
|
||||
// RenameData - rename source path to destination path atomically, metadata and data directory.
|
||||
func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (res RenameDataResp, err error) {
|
||||
defer s.lockMutations(pathJoin(srcVolume, srcPath), pathJoin(dstVolume, dstPath))()
|
||||
defer func() {
|
||||
ignoredErrs := []error{
|
||||
errFileNotFound,
|
||||
@@ -2822,6 +2841,12 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
|
||||
xlMeta.data.remove(reqVID, dataDir)
|
||||
}
|
||||
}
|
||||
if !healing && res.OldDataDir == "" {
|
||||
// Shared data must survive both commit and rollback. Keep the
|
||||
// previous metadata in a separate transaction directory instead
|
||||
// of using a data directory still owned by another version.
|
||||
res.OldDataDir = fi.DataDir + "-rollback"
|
||||
}
|
||||
}
|
||||
|
||||
if err = xlMeta.AddVersion(fi); err != nil {
|
||||
@@ -3274,6 +3299,7 @@ func (s *xlStorage) CleanAbandonedData(ctx context.Context, volume string, path
|
||||
if volume == "" || path == "" {
|
||||
return nil // Ignore
|
||||
}
|
||||
defer s.lockMutations(pathJoin(volume, path))()
|
||||
|
||||
volumeDir, err := s.getVolDir(volume)
|
||||
if err != nil {
|
||||
@@ -3325,6 +3351,14 @@ func (s *xlStorage) CleanAbandonedData(ctx context.Context, volume string, path
|
||||
// Do not abort on context errors.
|
||||
for dir := range foundDirs {
|
||||
toRemove := pathJoin(volumeDir, path, dir+SlashSeparator)
|
||||
// A completed disk mutation may still be awaiting its quorum result.
|
||||
// Its previous generation is needed if that transaction must undo.
|
||||
// Only commit/undo cleanup owns a directory containing this backup.
|
||||
if _, err := Lstat(pathJoin(toRemove, xlStorageFormatFileBackup)); err == nil {
|
||||
continue
|
||||
} else if !osIsNotExist(err) {
|
||||
return osErrToFileErr(err)
|
||||
}
|
||||
err = s.deleteFile(volumeDir, toRemove, true, true)
|
||||
diskHealthCheckOK(ctx, err)
|
||||
if err != nil {
|
||||
|
||||
@@ -4,7 +4,7 @@ go 1.27.3
|
||||
|
||||
// Console and MC retain their historical module paths for best-effort upstream
|
||||
// compatibility. Pin the maintained PGSTY implementations used by SILO.
|
||||
replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20261025140954-258717be077b
|
||||
replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20261025145732-989fa7150792
|
||||
|
||||
replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20261025104502-dcc78d506f94
|
||||
|
||||
|
||||
@@ -508,8 +508,8 @@ github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwp
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
github.com/pgsty/mc v0.0.0-20261025104502-dcc78d506f94 h1:M5n0rCPOTyo3RiQBeSCQW/klx08efFpiVYmPq6eoM3w=
|
||||
github.com/pgsty/mc v0.0.0-20261025104502-dcc78d506f94/go.mod h1:H3Uwlm1Qj3kPsbmojhNnv3aj5a+QdjdTKwv4w4gMSF8=
|
||||
github.com/pgsty/silo-console v0.0.0-20261025140954-258717be077b h1:/RQ+D5TRP8/GGZaY5cbwIixd1RFE0eQjEVLBpIl3+8k=
|
||||
github.com/pgsty/silo-console v0.0.0-20261025140954-258717be077b/go.mod h1:fMxNcVxan3mu9mc7HCRiczSyQoFrbZEGumEmpatxtds=
|
||||
github.com/pgsty/silo-console v0.0.0-20261025145732-989fa7150792 h1:wPnltE28wxotYQIKeg5OykplBcrhaWeTnWRxbYXOTtQ=
|
||||
github.com/pgsty/silo-console v0.0.0-20261025145732-989fa7150792/go.mod h1:fMxNcVxan3mu9mc7HCRiczSyQoFrbZEGumEmpatxtds=
|
||||
github.com/pgsty/silo-pkg/v3 v3.15.1 h1:T5rqY3HR1WGybcjnx/ghBvliEgo7yNCA7JAC3f1LpTk=
|
||||
github.com/pgsty/silo-pkg/v3 v3.15.1/go.mod h1:GmTF6fIEkTrzgM+ysWWfDZIlVRD2DYAJkKjA5bjiWpQ=
|
||||
github.com/philhofer/fwd v1.2.1 h1:UtpS73q8bFgy/yH3gCq8LSNvj5gv/FowNOiWZa7uKwQ=
|
||||
|
||||
@@ -25,7 +25,7 @@ imagePullSecrets: []
|
||||
##
|
||||
mcImage:
|
||||
repository: pgsty/mc
|
||||
tag: RELEASE.2026-10-18T00-00-00Z
|
||||
tag: RELEASE.2026-10-25T00-00-00Z
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
## Silo mode, i.e. standalone or distributed.
|
||||
|
||||
@@ -344,7 +344,8 @@ func (c *Subroute) newMuxClient(ctx context.Context) (*muxClient, error) {
|
||||
|
||||
// Request allows to do a single remote request.
|
||||
// 'req' will not be used after the call and caller can reuse.
|
||||
// If no deadline is set on ctx, a 1-minute deadline will be added.
|
||||
// If no deadline is set on ctx, a 1-minute deadline will be added unless
|
||||
// WithoutDefaultTimeout was requested.
|
||||
func (c *Connection) Request(ctx context.Context, h HandlerID, req []byte) ([]byte, error) {
|
||||
if !h.valid() {
|
||||
return nil, ErrUnknownHandler
|
||||
@@ -370,7 +371,8 @@ func (c *Connection) Request(ctx context.Context, h HandlerID, req []byte) ([]by
|
||||
|
||||
// Request allows to do a single remote request.
|
||||
// 'req' will not be used after the call and caller can reuse.
|
||||
// If no deadline is set on ctx, a 1-minute deadline will be added.
|
||||
// If no deadline is set on ctx, a 1-minute deadline will be added unless
|
||||
// WithoutDefaultTimeout was requested.
|
||||
func (c *Subroute) Request(ctx context.Context, h HandlerID, req []byte) ([]byte, error) {
|
||||
if !h.valid() {
|
||||
return nil, ErrUnknownHandler
|
||||
|
||||
@@ -550,7 +550,8 @@ type Requester interface {
|
||||
|
||||
// Call the remote with the request and return the response.
|
||||
// The response should be returned with PutResponse when no error.
|
||||
// If no deadline is set, a 1-minute deadline is added.
|
||||
// If no deadline is set, a 1-minute deadline is added unless
|
||||
// WithoutDefaultTimeout was requested.
|
||||
func (h *SingleHandler[Req, Resp]) Call(ctx context.Context, c Requester, req Req) (resp Resp, err error) {
|
||||
if c == nil {
|
||||
if h.ignoreNilConn {
|
||||
|
||||
@@ -59,6 +59,16 @@ type Response struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
type withoutDefaultTimeoutKey struct{}
|
||||
|
||||
// WithoutDefaultTimeout disables the implicit single-request timeout.
|
||||
// Explicit context cancellation and connection loss still end the request.
|
||||
// Use this for admitted mutations that must return their actual result before
|
||||
// the caller releases its lock.
|
||||
func WithoutDefaultTimeout(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, withoutDefaultTimeoutKey{}, true)
|
||||
}
|
||||
|
||||
func newMuxClient(ctx context.Context, muxID uint64, parent *Connection) *muxClient {
|
||||
ctx, cancelFn := context.WithCancelCause(ctx)
|
||||
return &muxClient{
|
||||
@@ -101,8 +111,10 @@ func (m *muxClient) roundtrip(h HandlerID, req []byte) ([]byte, error) {
|
||||
m.respMu.Unlock()
|
||||
ctx := m.ctx
|
||||
|
||||
// Add deadline if none.
|
||||
if msg.DeadlineMS == 0 {
|
||||
// Admitted storage mutations must not time out while the handler can
|
||||
// still commit. A zero wire deadline also keeps their late response.
|
||||
noDefaultTimeout, _ := ctx.Value(withoutDefaultTimeoutKey{}).(bool)
|
||||
if msg.DeadlineMS == 0 && !noDefaultTimeout {
|
||||
msg.DeadlineMS = uint32(defaultSingleRequestTimeout / time.Millisecond)
|
||||
var cancel context.CancelFunc
|
||||
ctx, cancel = context.WithTimeout(ctx, defaultSingleRequestTimeout)
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
//
|
||||
// This file is part of Silo Object Storage.
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package grid
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestSingleMutationWaitsBeyondDefaultTimeout(t *testing.T) {
|
||||
for _, admitted := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "ordinary", true: "admitted"}[admitted], func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
if admitted {
|
||||
ctx = WithoutDefaultTimeout(ctx)
|
||||
}
|
||||
conn := &Connection{outQueue: make(chan []byte, 1)}
|
||||
client := newMuxClient(ctx, 1, conn)
|
||||
defer client.cancelFn(context.Canceled)
|
||||
done := make(chan Response, 1)
|
||||
go func() {
|
||||
msg, err := client.roundtrip(handlerTest, nil)
|
||||
done <- Response{Msg: msg, Err: err}
|
||||
}()
|
||||
wire := <-conn.outQueue
|
||||
var request message
|
||||
if _, err := request.UnmarshalMsg(wire); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
PutByteBuffer(wire)
|
||||
if admitted && request.DeadlineMS != 0 {
|
||||
t.Errorf("admitted mutation tells the server to discard a late result after %dms", request.DeadlineMS)
|
||||
}
|
||||
time.Sleep(defaultSingleRequestTimeout + time.Second)
|
||||
synctest.Wait()
|
||||
if !admitted {
|
||||
select {
|
||||
case got := <-done:
|
||||
if !errors.Is(got.Err, context.DeadlineExceeded) {
|
||||
t.Fatalf("ordinary timeout: %v", got.Err)
|
||||
}
|
||||
default:
|
||||
t.Fatal("ordinary request lost its default timeout")
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case got := <-done:
|
||||
t.Fatalf("mutation returned before its commit completed: %v", got.Err)
|
||||
default:
|
||||
}
|
||||
client.response(0, Response{Msg: []byte("committed")})
|
||||
got := <-done
|
||||
if got.Err != nil || string(got.Msg) != "committed" {
|
||||
t.Fatalf("lost late commit result: %q, %v", got.Msg, got.Err)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithoutDefaultTimeoutStillEndsOnDisconnectOrCancellation(t *testing.T) {
|
||||
for _, disconnect := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "explicit-cancel", true: "disconnect"}[disconnect], func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
conn := &Connection{outQueue: make(chan []byte, 1)}
|
||||
client := newMuxClient(WithoutDefaultTimeout(ctx), 1, conn)
|
||||
defer client.cancelFn(context.Canceled)
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := client.roundtrip(handlerTest, nil)
|
||||
done <- err
|
||||
}()
|
||||
PutByteBuffer(<-conn.outQueue)
|
||||
want := context.Canceled
|
||||
if disconnect {
|
||||
client.close()
|
||||
want = ErrDisconnected
|
||||
} else {
|
||||
cancel()
|
||||
}
|
||||
if got := <-done; !errors.Is(got, want) {
|
||||
t.Fatalf("request did not terminate correctly: want %v, got %v", want, got)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -127,6 +127,17 @@ var deadlineWorkPanics atomic.Int64
|
||||
// If the deadline/context passes before the function finishes executing,
|
||||
// the zero value and the context error is returned.
|
||||
func WithDeadline[V any](ctx context.Context, timeout time.Duration, work func(ctx context.Context) (result V, err error)) (result V, err error) {
|
||||
return withDeadline(ctx, timeout, work, false)
|
||||
}
|
||||
|
||||
// WithDeadlineAndWait cancels work at the deadline, but waits for it to finish
|
||||
// and returns its actual result. Use it for mutations whose caller must not
|
||||
// release a lock or start cleanup while the work can still change storage.
|
||||
func WithDeadlineAndWait[V any](ctx context.Context, timeout time.Duration, work func(ctx context.Context) (result V, err error)) (result V, err error) {
|
||||
return withDeadline(ctx, timeout, work, true)
|
||||
}
|
||||
|
||||
func withDeadline[V any](ctx context.Context, timeout time.Duration, work func(ctx context.Context) (result V, err error), wait bool) (result V, err error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -163,6 +174,10 @@ func WithDeadline[V any](ctx context.Context, timeout time.Duration, work func(c
|
||||
case v := <-c:
|
||||
return v.val, v.err
|
||||
case <-ctx.Done():
|
||||
if wait {
|
||||
v := <-c
|
||||
return v.val, v.err
|
||||
}
|
||||
var zero V
|
||||
return zero, ctx.Err()
|
||||
}
|
||||
|
||||
@@ -25,9 +25,62 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWithDeadlineAndWait(t *testing.T) {
|
||||
for _, deadline := range []bool{false, true} {
|
||||
name := "cancel"
|
||||
if deadline {
|
||||
name = "deadline"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
defer cancel()
|
||||
entered, release := make(chan struct{}), make(chan struct{})
|
||||
done := make(chan ioret[int], 1)
|
||||
go func() {
|
||||
v, err := WithDeadlineAndWait(ctx, time.Second, func(context.Context) (int, error) {
|
||||
close(entered)
|
||||
<-release
|
||||
return 42, nil
|
||||
})
|
||||
done <- ioret[int]{v, err}
|
||||
}()
|
||||
<-entered
|
||||
if deadline {
|
||||
time.Sleep(2 * time.Second)
|
||||
} else {
|
||||
cancel()
|
||||
}
|
||||
synctest.Wait()
|
||||
select {
|
||||
case r := <-done:
|
||||
t.Fatalf("returned before mutation completed: %+v", r)
|
||||
default:
|
||||
}
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
r := <-done
|
||||
if r.err != nil || r.val != 42 {
|
||||
t.Fatalf("lost actual commit result: %+v", r)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWithDeadlineAndWaitPanic(t *testing.T) {
|
||||
_, err := WithDeadlineAndWait(t.Context(), time.Second, func(context.Context) (int, error) {
|
||||
panic("mutation failure")
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "mutation failure") {
|
||||
t.Fatalf("worker panic was not returned: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type sleepWriter struct {
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user